diff --git a/README.md b/README.md index 5f565a3..aad8cf2 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ Currently, these API calls are available: * Instruments * Instrument-event mapping * File +* File Repository * Logging * Metadata * Project Info @@ -68,6 +69,7 @@ Currently, these API calls are available: * Data Access Groups * Events * File +* File Repository * Instrument-event mapping * Metadata * Records @@ -83,6 +85,7 @@ Currently, these API calls are available: * Data Access Groups * Events * File +* File Repository * Records * Users * User Roles diff --git a/docs/api_reference/file_repository.md b/docs/api_reference/file_repository.md new file mode 100644 index 0000000..b4e2b51 --- /dev/null +++ b/docs/api_reference/file_repository.md @@ -0,0 +1,5 @@ +# File Repository + +::: redcap.methods.file_repository + selection: + inherited_members: true diff --git a/mkdocs.yml b/mkdocs.yml index f956601..0fdf4b5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - Data Access Groups: api_reference/data_access_groups.md - Events: api_reference/events.md - Field Names: api_reference/field_names.md + - File Repository: api_reference/file_repository.md - Files: api_reference/files.md - Instruments: api_reference/instruments.md - Logging: api_reference/logging.md diff --git a/pytest.ini b/pytest.ini index d00b7cb..064e25f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS FAIL_FAST REPORT_NDIFF addopts = -rsxX -l --tb=short --strict --pylint --black --cov=redcap --cov-report=xml --mypy markers = - integration: test connects to redcapdemo.vanderbilt.edu server + integration: test connects to redcapdemo.vumc.org server # Keep current format for future version of pytest junit_family=xunit1 # Ignore unimportant warnings diff --git a/redcap/conftest.py b/redcap/conftest.py index 713e37e..7b58ae6 100644 --- a/redcap/conftest.py +++ b/redcap/conftest.py @@ -10,8 +10,10 @@ from redcap.project import Project from tests.integration.conftest import ( + add_files_to_repository, create_project, grant_superuser_rights, + redcapdemo_url, SUPER_TOKEN, ) @@ -19,14 +21,15 @@ @pytest.fixture(scope="session", autouse=True) def add_doctest_objects(doctest_namespace): """Add the doctest project instance to the doctest_namespace""" - url = "https://redcapdemo.vanderbilt.edu/api/" doctest_project_xml = Path("tests/data/doctest_project.xml") doctest_token = create_project( - url=url, + url=redcapdemo_url(), super_token=SUPER_TOKEN, project_xml_path=doctest_project_xml, ) - doctest_project = Project(url, doctest_token) + doctest_project = Project(redcapdemo_url(), doctest_token) doctest_project = grant_superuser_rights(doctest_project) + doctest_project = add_files_to_repository(doctest_project) + doctest_namespace["proj"] = doctest_project doctest_namespace["TOKEN"] = doctest_token diff --git a/redcap/methods/__init__.py b/redcap/methods/__init__.py index 491aac0..ca0e954 100644 --- a/redcap/methods/__init__.py +++ b/redcap/methods/__init__.py @@ -4,6 +4,7 @@ import redcap.methods.data_access_groups import redcap.methods.events import redcap.methods.field_names +import redcap.methods.file_repository import redcap.methods.files import redcap.methods.instruments import redcap.methods.logging diff --git a/redcap/methods/base.py b/redcap/methods/base.py index 5007fbf..87e197f 100644 --- a/redcap/methods/base.py +++ b/redcap/methods/base.py @@ -260,7 +260,7 @@ def _initialize_payload( format_type: Optional[Literal["json", "csv", "xml", "df"]] = None, return_format_type: Optional[Literal["json", "csv", "xml"]] = None, record_type: Literal["flat", "eav"] = "flat", - ) -> Dict[str, str]: + ) -> Dict[str, Any]: """Create the default dictionary for payloads This can be used as is for simple API requests or added to @@ -339,6 +339,7 @@ def _return_data( "dag", "event", "exportFieldNames", + "fileRepository", "formEventMapping", "instrument", "log", diff --git a/redcap/methods/file_repository.py b/redcap/methods/file_repository.py new file mode 100644 index 0000000..1373c12 --- /dev/null +++ b/redcap/methods/file_repository.py @@ -0,0 +1,248 @@ +"""REDCap API methods for Project file repository""" + +from typing import Any, Dict, IO, Literal, Optional, Union, cast + +from redcap.methods.base import Base, FileMap, Json +from redcap.request import EmptyJson, FileUpload + + +class FileRepository(Base): + """Responsible for all API methods under 'File Repository' in the API Playground""" + + def create_folder_in_repository( + self, + name: str, + folder_id: Optional[int] = None, + dag_id: Optional[int] = None, + role_id: Optional[int] = None, + format_type: Literal["json", "csv", "xml"] = "json", + return_format_type: Literal["json", "csv", "xml"] = "json", + ): + """ + Create a New Folder in the File Repository + + Args: + name: + The desired name of the folder to be created (max length = 150 characters) + folder_id: + The folder_id of a specific folder in the File Repository for which you wish + to create this sub-folder. If none is provided, the folder will be created in + the top-level directory of the File Repository. + dag_id: + The dag_id of the DAG (Data Access Group) to which you wish to restrict + access for this folder. If none is provided, the folder will accessible to + users in all DAGs and users in no DAGs. + role_id: + The role_id of the User Role to which you wish to restrict access for this + folder. If none is provided, the folder will accessible to users in all + User Roles and users in no User Roles. + format_type: + Return the metadata in native objects, csv or xml. + return_format_type: + Response format. By default, response will be json-decoded. + Returns: + Union[str, List[Dict[str, Any]]]: + List of all changes made to this project, including data exports, + data changes, and the creation or deletion of users + + Examples: + >>> proj.create_folder_in_repository(name="New Folder") + [{'folder_id': ...}] + """ + payload: Dict[str, Any] = self._initialize_payload( + content="fileRepository", + format_type=format_type, + return_format_type=return_format_type, + ) + + payload["action"] = "createFolder" + payload["name"] = name + + if folder_id: + payload["folder_id"] = folder_id + + if dag_id: + payload["dag_id"] = dag_id + + if role_id: + payload["role_id"] = role_id + + return_type = self._lookup_return_type(format_type, request_type="export") + + return cast(Union[Json, str], self._call_api(payload, return_type)) + + def export_file_repository( + self, + folder_id: Optional[int] = None, + format_type: Literal["json", "csv", "xml"] = "json", + return_format_type: Literal["json", "csv", "xml"] = "json", + ): + """ + Export of list of files/folders in the File Repository + + Only exports the top-level of files/folders. To see which files are contained + within a folder, use the `folder_id` parameter + + Args: + folder_id: + The folder_id of a specific folder in the File Repository for which you wish + to search for files/folders. If none is provided, the search will be conducted + in the top-level directory of the File Repository. + format_type: + Return the metadata in native objects, csv or xml. + return_format_type: + Response format. By default, response will be json-decoded. + Returns: + Union[str, List[Dict[str, Any]]]: + List of all changes made to this project, including data exports, + data changes, and the creation or deletion of users + + Examples: + >>> proj.export_file_repository() + [{'folder_id': ..., 'name': 'New Folder'}, ...] + """ + payload: Dict[str, Any] = self._initialize_payload( + content="fileRepository", + format_type=format_type, + return_format_type=return_format_type, + ) + + payload["action"] = "list" + + if folder_id: + payload["folder_id"] = folder_id + + return_type = self._lookup_return_type(format_type, request_type="export") + + return cast(Union[Json, str], self._call_api(payload, return_type)) + + def export_file_from_repository( + self, + doc_id: int, + return_format_type: Literal["json", "csv", "xml"] = "json", + ) -> FileMap: + """ + Export the contents of a file stored in the File Repository + + Args: + doc_id: The doc_id of the file in the File Repository + return_format_type: + Response format. By default, response will be json-decoded. + + Returns: + Content of the file and content-type dictionary + + Examples: + >>> file_dir = proj.export_file_repository() + >>> text_file = [file for file in file_dir if file["name"] == "test.txt"].pop() + >>> proj.export_file_from_repository(doc_id=text_file["doc_id"]) + (b'hello', {'name': 'test.txt', 'charset': 'UTF-8'}) + """ + payload = self._initialize_payload( + content="fileRepository", return_format_type=return_format_type + ) + # there's no format field in this call + payload["action"] = "export" + payload["doc_id"] = doc_id + + content, headers = cast( + FileMap, self._call_api(payload=payload, return_type="file_map") + ) + # REDCap adds some useful things in content-type + content_map = {} + if "content-type" in headers: + splat = [ + key_values.strip() for key_values in headers["content-type"].split(";") + ] + key_values = [ + (key_values.split("=")[0], key_values.split("=")[1].replace('"', "")) + for key_values in splat + if "=" in key_values + ] + content_map = dict(key_values) + + return content, content_map + + def import_file_into_repository( + self, + file_name: str, + file_object: IO, + folder_id: Optional[int] = None, + ) -> EmptyJson: + """ + Import the contents of a file represented by file_object into + the file repository + + Args: + file_name: File name visible in REDCap UI + file_object: File object as returned by `open` + folder_id: + The folder_id of a specific folder in the File Repository where + you wish to store the file. If none is provided, the file will + be stored in the top-level directory of the File Repository. + + Returns: + Empty JSON object + + Examples: + >>> import tempfile + >>> tmp_file = tempfile.TemporaryFile() + >>> proj.import_file_into_repository( + ... file_name="myupload.txt", + ... file_object=tmp_file, + ... ) + [{}] + """ + payload: Dict[str, Any] = self._initialize_payload(content="fileRepository") + payload["action"] = "import" + + if folder_id: + payload["folder_id"] = folder_id + + file_upload_dict: FileUpload = {"file": (file_name, file_object)} + + return cast( + EmptyJson, + self._call_api( + payload=payload, return_type="empty_json", file=file_upload_dict + ), + ) + + def delete_file_from_repository( + self, + doc_id: int, + return_format_type: Literal["json", "csv", "xml"] = "json", + ) -> EmptyJson: + # pylint: disable=line-too-long + """ + Delete a File from the File Repository + + Once deleted, the file will remain in the Recycle Bin folder for up to 30 days. + + Args: + doc_id: The doc_id of the file in the File Repository + return_format_type: + Response format. By default, response will be json-decoded. + + Returns: + Empty JSON object + + Examples: + >>> file_dir = proj.export_file_repository() + >>> test_folder = [folder for folder in file_dir if folder["name"] == "test"].pop() + >>> test_dir = proj.export_file_repository(folder_id=test_folder["folder_id"]) + >>> test_file = [file for file in test_dir if file["name"] == "test_in_folder.txt"].pop() + >>> proj.delete_file_from_repository(doc_id=test_file["doc_id"]) + [{}] + """ + # pylint: enable=line-too-long + payload = self._initialize_payload( + content="fileRepository", return_format_type=return_format_type + ) + # there's no format field in this call + payload["action"] = "delete" + payload["doc_id"] = doc_id + + return cast( + EmptyJson, self._call_api(payload=payload, return_type="empty_json") + ) diff --git a/redcap/methods/files.py b/redcap/methods/files.py index cfdee06..405e6f9 100644 --- a/redcap/methods/files.py +++ b/redcap/methods/files.py @@ -1,13 +1,10 @@ """REDCap API methods for Project files""" -from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast +from typing import Any, Dict, IO, Optional, Union, cast from redcap.methods.base import Base, FileMap from redcap.request import EmptyJson, FileUpload -if TYPE_CHECKING: - from io import TextIOWrapper - class Files(Base): """Responsible for all API methods under 'Files' in the API Playground""" @@ -90,7 +87,7 @@ def import_file( record: str, field: str, file_name: str, - file_object: "TextIOWrapper", + file_object: IO, event: Optional[str] = None, repeat_instance: Optional[Union[int, str]] = None, ) -> EmptyJson: diff --git a/redcap/methods/logging.py b/redcap/methods/logging.py index 65a94a6..1941765 100644 --- a/redcap/methods/logging.py +++ b/redcap/methods/logging.py @@ -1,4 +1,4 @@ -"""REDCap API methods for Project field names""" +"""REDCap API methods for Project logs""" from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Union, cast diff --git a/redcap/project.py b/redcap/project.py index e839d57..6674ea5 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -23,6 +23,7 @@ class Project( methods.data_access_groups.DataAccessGroups, methods.events.Events, methods.field_names.FieldNames, + methods.file_repository.FileRepository, methods.files.Files, methods.instruments.Instruments, methods.logging.Logging, @@ -50,7 +51,7 @@ class Project( Examples: >>> from redcap import Project - >>> URL = "https://redcapdemo.vanderbilt.edu/api/" + >>> URL = "https://redcapdemo.vumc.org/api/" >>> proj = Project(URL, TOKEN) >>> proj.field_names ['record_id', 'field_1', 'checkbox_field', 'upload_field'] diff --git a/redcap/request.py b/redcap/request.py index a416b37..4a58636 100644 --- a/redcap/request.py +++ b/redcap/request.py @@ -4,10 +4,10 @@ from collections import namedtuple from typing import ( - TYPE_CHECKING, Any, Dict, List, + IO, Literal, Optional, Tuple, @@ -18,8 +18,6 @@ from requests import RequestException, Response, Session -if TYPE_CHECKING: - from io import TextIOWrapper Json = List[Dict[str, Any]] EmptyJson = List[dict] @@ -36,7 +34,7 @@ class FileUpload(TypedDict): """Typing for the file upload API""" - file: Tuple[str, "TextIOWrapper"] + file: Tuple[str, IO] _ContentConfig = namedtuple("_ContentConfig", ["return_empty_json", "return_bytes"]) @@ -190,19 +188,19 @@ def execute( return_bytes=self.config.return_bytes, ) + bad_request = False + if self.fmt == "json": try: bad_request = "error" in content.keys() # type: ignore except AttributeError: # we're not dealing with an error dict - bad_request = False + pass elif self.fmt == "csv": bad_request = content.lower().startswith("error:") # type: ignore # xml is the default returnFormat for error messages elif self.fmt == "xml" or self.fmt is None: bad_request = "" in str(content).lower() - else: - raise ValueError(f"Unsupported format { self.fmt }") if bad_request: raise RedcapError(content) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 075116c..e9b82cf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,10 @@ """Test fixtures for integration tests only""" # pylint: disable=redefined-outer-name -from datetime import datetime import os +import tempfile +from datetime import datetime from pathlib import Path from typing import cast @@ -16,7 +17,7 @@ def create_project(url: str, super_token: str, project_xml_path: Path) -> str: - """Create a project for testing on redcapdemo.vanderbilt.edu + """Create a project for testing on redcapdemo.vumc.org This API method returns the token for the newly created project, which used for the integration tests """ @@ -47,19 +48,24 @@ def create_project(url: str, super_token: str, project_xml_path: Path) -> str: return res.text[-32:] -@pytest.fixture(scope="module") def redcapdemo_url() -> str: """API url for redcapdemo testing site""" - return "https://redcapdemo.vanderbilt.edu/api/" + return "https://redcapdemo.vumc.org/api/" + + +@pytest.fixture(scope="module") +def redcapdemo_url_fixture() -> str: + """API url for redcapdemo testing site, as a testing fixture""" + return redcapdemo_url() @pytest.fixture(scope="module") -def simple_project_token(redcapdemo_url) -> str: +def simple_project_token(redcapdemo_url_fixture) -> str: """Create a simple project and return it's API token""" simple_project_xml_path = Path("tests/data/test_simple_project.xml") super_token = cast(str, SUPER_TOKEN) project_token = create_project( # type: ignore - redcapdemo_url, super_token, simple_project_xml_path + redcapdemo_url_fixture, super_token, simple_project_xml_path ) return project_token @@ -82,27 +88,52 @@ def grant_superuser_rights(proj: Project) -> Project: return proj +def add_files_to_repository(proj: Project) -> Project: + """Given a project, fill out it's file repository + For some reason, this doesn't carry over in the XML file so + it has to be done after project creation + """ + new_folder = proj.create_folder_in_repository("test").pop() + + with tempfile.NamedTemporaryFile(mode="w+t") as tmp_file: + tmp_file.write("hello") + tmp_file.seek(0) + + proj.import_file_into_repository(file_name="test.txt", file_object=tmp_file) + proj.import_file_into_repository( + file_name="test_in_folder.txt", + file_object=tmp_file, + folder_id=new_folder["folder_id"], + ) + + return proj + + @pytest.fixture(scope="module") -def simple_project(redcapdemo_url, simple_project_token): +def simple_project(redcapdemo_url_fixture, simple_project_token): """A simple REDCap project""" - simple_proj = Project(redcapdemo_url, simple_project_token) + simple_proj = Project(redcapdemo_url_fixture, simple_project_token) simple_proj = grant_superuser_rights(simple_proj) + simple_proj = add_files_to_repository(simple_proj) + return simple_proj @pytest.fixture(scope="module") -def long_project_token(redcapdemo_url) -> str: +def long_project_token(redcapdemo_url_fixture) -> str: """Create a long project and return it's API token""" long_project_xml_path = Path("tests/data/test_long_project.xml") super_token = cast(str, SUPER_TOKEN) - project_token = create_project(redcapdemo_url, super_token, long_project_xml_path) + project_token = create_project( + redcapdemo_url_fixture, super_token, long_project_xml_path + ) return project_token @pytest.fixture(scope="module") -def long_project(redcapdemo_url, long_project_token): +def long_project(redcapdemo_url_fixture, long_project_token): """A long REDCap project""" - long_proj = Project(redcapdemo_url, long_project_token) + long_proj = Project(redcapdemo_url_fixture, long_project_token) long_proj = grant_superuser_rights(long_proj) return long_proj diff --git a/tests/integration/test_long_project.py b/tests/integration/test_long_project.py index 4215b09..a506674 100644 --- a/tests/integration/test_long_project.py +++ b/tests/integration/test_long_project.py @@ -5,9 +5,6 @@ import pytest -from redcap import RedcapError - - if not os.getenv("REDCAPDEMO_SUPERUSER_TOKEN"): pytest.skip( "Super user token not found, skipping integration tests", @@ -207,10 +204,8 @@ def test_arms_import_override(long_project): response = long_project.import_arms(new_arms, override=1) assert response == 2 - # Confirm that there are no events associated with new override arms - with pytest.raises(RedcapError): - response = long_project.export_arms() + # Restore project state response = long_project.import_events(state_dict["events"]) assert response == 16 diff --git a/tests/integration/test_simple_project.py b/tests/integration/test_simple_project.py index 7114ac3..4c660a3 100644 --- a/tests/integration/test_simple_project.py +++ b/tests/integration/test_simple_project.py @@ -2,6 +2,8 @@ # pylint: disable=missing-function-docstring import os +import tempfile + from io import StringIO import pandas as pd @@ -298,3 +300,48 @@ def test_export_events(simple_project): def test_export_instrument_event_mapping(simple_project): with pytest.raises(RedcapError): simple_project.export_instrument_event_mappings() + + +@pytest.mark.integration +def test_create_folder_in_repository(simple_project): + folder_name = "New Folder" + new_folder = simple_project.create_folder_in_repository(name=folder_name) + assert new_folder[0]["folder_id"] > 0 + + +@pytest.mark.integration +def test_export_file_repository(simple_project): + directory = simple_project.export_file_repository() + assert len(directory) > 0 + + +@pytest.mark.integration +def test_export_file_from_repository(simple_project): + file_dir = simple_project.export_file_repository() + text_file = [file for file in file_dir if file["name"] == "test.txt"].pop() + file_contents, _ = simple_project.export_file_from_repository( + doc_id=text_file["doc_id"] + ) + assert isinstance(file_contents, bytes) + + +@pytest.mark.integration +def test_import_file_repository(simple_project): + initial_len = len(simple_project.export_file_repository()) + + tmp_file = tempfile.TemporaryFile() + simple_project.import_file_into_repository( + file_name="new_upload.txt", file_object=tmp_file + ) + + new_len = len(simple_project.export_file_repository()) + + assert new_len > initial_len + + +@pytest.mark.integration +def test_delete_file_from_repository(simple_project): + file_dir = simple_project.export_file_repository() + text_file = [file for file in file_dir if file["name"] == "test.txt"].pop() + resp = simple_project.delete_file_from_repository(doc_id=text_file["doc_id"]) + assert resp == [{}] diff --git a/tests/unit/callback_utils.py b/tests/unit/callback_utils.py index 3175a2a..8ab167e 100644 --- a/tests/unit/callback_utils.py +++ b/tests/unit/callback_utils.py @@ -312,6 +312,42 @@ def handle_long_project_file_request(**kwargs) -> Any: return (201, headers, json.dumps(resp)) +def handle_long_project_file_repo_create_folder(data) -> Any: + """Handle the create folder request""" + assert data["name"] + return [{"folder_id": 101}] + + +def handle_long_project_export_file_repo(data) -> Any: + """Handle the export file/folder request""" + assert data + return [{"folder_id": 101, "name": "test"}] + + +def handle_long_project_export_file_from_repo(data) -> Any: + """Handle the export file from repo request""" + assert data["doc_id"] + return {} + + +def handle_long_project_file_repository_request(**kwargs) -> Any: + """Handle file repository requests""" + data = kwargs["data"] + headers = kwargs["headers"] + resp = {} + if "createFolder" in data.get("action"): + resp = handle_long_project_file_repo_create_folder(data) + elif "list" in data.get("action"): + resp = handle_long_project_export_file_repo(data) + elif "export" in data.get("action"): + resp = handle_long_project_export_file_from_repo(data) + headers["content-type"] = "text/plain;name=test.txt" + elif data.get("action") in ["import", "delete"]: + resp = [{}] + + return (201, headers, json.dumps(resp)) + + def handle_generate_next_record_name_request(**kwargs) -> Any: """Handle generating next record name""" headers = kwargs["headers"] @@ -778,6 +814,7 @@ def get_long_project_request_handler(request_type: str) -> Callable: "arm": handle_long_project_arms_request, "event": handle_long_project_events_request, "file": handle_long_project_file_request, + "fileRepository": handle_long_project_file_repository_request, "formEventMapping": handle_long_project_form_event_mapping_request, "instrument": handle_long_project_instruments_request, "repeatingFormsEvents": handle_long_project_repeating_form_request, diff --git a/tests/unit/test_long_project.py b/tests/unit/test_long_project.py index c283336..9c4a147 100644 --- a/tests/unit/test_long_project.py +++ b/tests/unit/test_long_project.py @@ -3,6 +3,7 @@ # pylint: disable=missing-function-docstring # pylint: disable=redefined-outer-name import os +import tempfile import pandas as pd import pytest @@ -254,3 +255,32 @@ def test_events_delete(long_project): response = long_project.delete_events(events) assert response == 1 + + +def test_file_repo_folder_create(long_project): + response = long_project.create_folder_in_repository( + name="test", folder_id=1, dag_id=2, role_id=3 + ) + assert response[0]["folder_id"] + + +def test_export_file_repo(long_project): + response = long_project.export_file_repository(folder_id=1) + assert is_json(response) + + +def test_export_file_from_repo(long_project): + resp, headers = long_project.export_file_from_repository(doc_id=1) + assert isinstance(resp, bytes) + assert headers["name"] == "test.txt" + + +def test_import_file_into_file_repo(long_project): + tmp_file = tempfile.TemporaryFile() + resp = long_project.import_file_into_repository("test.txt", tmp_file, folder_id=1) + assert resp + + +def test_delete_file_from_file_repo(long_project): + resp = long_project.delete_file_from_repository(doc_id=1) + assert resp