From 37fb3cd4ef1b78bc802ff5997f00f7045458d157 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 24 Oct 2024 15:20:12 +0700 Subject: [PATCH 01/14] adding latest python setup version --- .github/workflows/coverage.yaml | 2 +- .github/workflows/tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 062f87b..eec120e 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@master - name: Setup Python - uses: actions/setup-python@master + uses: actions/setup-python@v5 with: python-version: 3.7 - name: Generate coverage report diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3aa33e4..d770b92 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From a3123a0ec61d8a903198aadeeb960c6c0b63778d Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 24 Oct 2024 15:24:24 +0700 Subject: [PATCH 02/14] adding latest python version --- .github/workflows/coverage.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index eec120e..e2e9061 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -8,13 +8,13 @@ jobs: os: [ubuntu-latest] env: OS: ${{ matrix.os }} - PYTHON: '3.7' + PYTHON: '3.12' steps: - uses: actions/checkout@master - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.12 - name: Generate coverage report run: | python -m pip install --upgrade pip From 715ea0ca85e9e69262f147696107f761de5e1807 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 24 Oct 2024 15:26:00 +0700 Subject: [PATCH 03/14] testing with only local version --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d770b92..f7cbe2a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -10,7 +10,8 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: [3.12] + # ["3.6", "3.7", "3.8", "3.9"] steps: - uses: actions/checkout@v2 From 0c1fb5e1e053be7a4f5265752412847cad06437f Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 24 Oct 2024 17:43:36 +0700 Subject: [PATCH 04/14] adding new CI --- .github/workflows/version_update.yaml | 39 ++++++++++++++ .zenodo.json | 18 +++++++ tests/test_version.py | 74 +++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 .github/workflows/version_update.yaml create mode 100644 .zenodo.json create mode 100644 tests/test_version.py diff --git a/.github/workflows/version_update.yaml b/.github/workflows/version_update.yaml new file mode 100644 index 0000000..4074033 --- /dev/null +++ b/.github/workflows/version_update.yaml @@ -0,0 +1,39 @@ +name: CI + +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: [main] + tags: + - 'v*' + pull_request: + branches: [main] + # Allows you to run this workflow manually from the Actions tab + # workflow_dispatch: + +jobs: + + UploadToZenodo: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: install req + run: pip install git+https://github.com/drifter089/zenodopy.git@working#egg=zenodopy + + - name: Update Zenodo Deposition + run: | + python tests/test_version.py \ + --version_tag "${{ github.ref_name }}" \ + --zenodo_token "${{ secrets.ZENODO_TOKEN }}" \ + --dep_id "${{ secrets.DEPOSITION_ID }}" \ + --base_dir "${{ github.workspace }}" \ + --metadata_file "${{ github.workspace }}/.zenodo.json" \ + --upload_dir "${{ github.workspace }}/" \ No newline at end of file diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 0000000..f936184 --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,18 @@ +{ + "metadata": { + "title": "solver benchmarks", + "upload_type": "other", + "description": "", + "version": "0.1.0", + "access_right": "open", + "license": "Apache-2.0", + "keywords": ["zenodo", "github", "git"], + "publication_date":"", + "creators": [ + { + "name": "Jhon, Doe", + "orcid": "0000-0003-2584-3576" + } + ] + } +} diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..3a96af1 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,74 @@ +import os +import time +import json +from pathlib import Path +import argparse +import zenodopy + +def main(): + # Set up argument parsing + parser = argparse.ArgumentParser(description='Update Zenodo deposition with new version and files.') + parser.add_argument('--version_tag', required=True, help='The version tag for the new release.') + parser.add_argument('--zenodo_token', required=True, help='The Zenodo API token.') + parser.add_argument('--dep_id', required=True, type=int, help='The Zenodo deposition ID.') + parser.add_argument('--base_dir', required=True, help='The base directory path.') + parser.add_argument('--metadata_file', required=True, help='The metadata JSON file path.') + parser.add_argument('--upload_dir', required=True, help='The directory containing files to upload.') + + args = parser.parse_args() + + version_tag = args.version_tag + zenodo_token = args.zenodo_token + dep_id = args.dep_id + base_dir = Path(args.base_dir) + zenodo_metadata_file = Path(args.metadata_file) + upload_dir = Path(args.upload_dir) + + print("Version Tag:", version_tag) + + def prepare_metadata(file_path, new_version): + with open(file_path, "r") as file: + zenodo_data = json.load(file) + + zenodo_data["metadata"]["version"] = new_version + + with open(file_path, "w") as file: + json.dump(zenodo_data, file, indent=2) + + # Prepare the metadata file with the new version tag + prepare_metadata(zenodo_metadata_file, version_tag) + + max_retries = 5 + + for attempt in range(1, max_retries + 1): + try: + zeno = zenodopy.Client( + sandbox=True, + token=zenodo_token, + ) + + zeno.set_project(dep_id=dep_id) + + zeno.update( + source=str(upload_dir), + publish=True, + metadata_json=str(zenodo_metadata_file), + ) + print("Update succeeded.") + break + + except Exception as e: + print(f"Attempt {attempt} failed with error: {e}") + + time.sleep(2) # Optional: Wait before retrying + + zeno._delete_project(dep_id=dep_id) + + if attempt == max_retries: + print("Max retries reached. Exiting.") + raise + else: + time.sleep(2) + +if __name__ == "__main__": + main() From 8e40ec68dc0b87b0908ad956caa7212c2f5e31b1 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 24 Oct 2024 17:57:41 +0700 Subject: [PATCH 05/14] using new PR brnach --- .github/workflows/version_update.yaml | 2 +- .zenodo.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/version_update.yaml b/.github/workflows/version_update.yaml index 4074033..7bed4d9 100644 --- a/.github/workflows/version_update.yaml +++ b/.github/workflows/version_update.yaml @@ -26,7 +26,7 @@ jobs: python-version: 3.12 - name: install req - run: pip install git+https://github.com/drifter089/zenodopy.git@working#egg=zenodopy + run: pip install git+https://github.com/drifter089/zenodopy.git@update_Patch#egg=zenodopy - name: Update Zenodo Deposition run: | diff --git a/.zenodo.json b/.zenodo.json index f936184..cffd492 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,6 +1,6 @@ { "metadata": { - "title": "solver benchmarks", + "title": "Zenodo CI test", "upload_type": "other", "description": "", "version": "0.1.0", From bb5c86b18f2e3f33bc2575eec188a9dfaa1472f4 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Tue, 12 Nov 2024 16:15:54 +0700 Subject: [PATCH 06/14] disabled previous tests added basic test --- .github/workflows/coverage.yaml | 31 ------ .github/workflows/tests.yaml | 11 +- pyproject.toml | 2 +- requirements.txt | 3 +- requirements_dev.txt | 3 +- src/zenodopy/zenodopy.py | 158 ++++++++++++++++------------ tests/test_zenodopy.py | 180 +++++++++++++++++--------------- 7 files changed, 200 insertions(+), 188 deletions(-) delete mode 100644 .github/workflows/coverage.yaml diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml deleted file mode 100644 index e2e9061..0000000 --- a/.github/workflows/coverage.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Example workflow for Codecov -on: [push] -jobs: - run: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - env: - OS: ${{ matrix.os }} - PYTHON: '3.12' - steps: - - uses: actions/checkout@master - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Generate coverage report - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements_dev.txt - pip install -e . - pip install pytest - pip install pytest-cov - pytest --cov=./ --cov-report=xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f7cbe2a..0a30742 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -15,17 +15,20 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install -r requirements_dev.txt pip install -e . - pip install tox tox-gh-actions - - - name: Test with tox - run: tox + + - name: Run pytest + env: + ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} + run: pytest tests/test_zenodopy.py -v diff --git a/pyproject.toml b/pyproject.toml index 845c885..e5ef9f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=42.0", "wheel"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -addopts = "--cov=zenodopy" +# addopts = "--cov=zenodopy" testpaths = [ "tests", ] diff --git a/requirements.txt b/requirements.txt index 4cbe9c5..3f3cf48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ requests==2.26.0 types-requests==2.27.7 -wget==3.2 \ No newline at end of file +wget==3.2 +datetime \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index c6e34de..7f20f84 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,4 +5,5 @@ pytest-cov==2.12.1 mypy==0.910 requests==2.26.0 types-requests==2.27.7 -wget==3.2 \ No newline at end of file +wget==3.2 +datetime \ No newline at end of file diff --git a/src/zenodopy/zenodopy.py b/src/zenodopy/zenodopy.py index 02bc5cd..067e9b8 100644 --- a/src/zenodopy/zenodopy.py +++ b/src/zenodopy/zenodopy.py @@ -6,6 +6,8 @@ import warnings import tarfile import zipfile +from datetime import datetime +import time def validate_url(url): @@ -187,7 +189,7 @@ def _get_depositions(self): else: return r.raise_for_status() - def _get_depositions_by_id(self, dep_id=None): + def _get_depositions_by_id(self): """gets the deposition based on project id this provides details on the project, including metadata @@ -199,10 +201,12 @@ def _get_depositions_by_id(self, dep_id=None): dict: dictionary containing project details """ # get request, returns our response - # if dep_id is not None: - r = requests.get(f"{self._endpoint}/deposit/depositions/{dep_id}", + if self.deposition_id is not None: + r = requests.get(f"{self._endpoint}/deposit/depositions/{self.deposition_id}", auth=self._bearer_auth) - + else: + print(' ** no deposition id is set on the project ** ') + return None if r.ok: return r.json() else: @@ -217,8 +221,11 @@ def _get_depositions_files(self): dict: dictionary containing project details """ # get request, returns our response - r = requests.get(f"{self._endpoint}/deposit/depositions/{self.deposition_id}/files", + if self.deposition_id is not None: + r = requests.get(f"{self._endpoint}/deposit/depositions/{self.deposition_id}/files", auth=self._bearer_auth) + else: + print(' ** no deposition id is set on the project ** ') if r.ok: return r.json() @@ -260,7 +267,11 @@ def _get_bucket_by_id(self, dep_id=None): str: the bucket URL to upload files to """ # get request, returns our response - r = requests.get(f"{self._endpoint}/deposit/depositions/{dep_id}", + if dep_id is not None: + r = requests.get(f"{self._endpoint}/deposit/depositions/{dep_id}", + auth=self._bearer_auth) + else: + r = requests.get(f"{self._endpoint}/deposit/depositions/{self.deposition_id}", auth=self._bearer_auth) if r.ok: @@ -340,7 +351,7 @@ def list_files(self): prints filenames to screen """ dep_id = self.deposition_id - dep = self._get_depositions_by_id(dep_id) + dep = self._get_depositions_by_id() if dep is not None: print('Files') print('------------------------') @@ -351,7 +362,9 @@ def list_files(self): # except UserWarning: # warnings.warn("The object is not pointing to a project. Either create a project or explicity set the project'", UserWarning) - def create_project(self, title=None, upload_type=None, description=None): + def create_project( + self, title=None, upload_type=None, metadata_json=None, description=None + ): """Creates a new project After a project is creates the zenodopy object @@ -366,30 +379,24 @@ def create_project(self, title=None, upload_type=None, description=None): description (str, optional): new description """ - if upload_type is None: - upload_types = self._get_upload_types() - warnings.warn(f"upload_type not set, so defaulted to 'other', possible choices include {upload_types}", - UserWarning) - upload_type = 'other' - # get request, returns our response - r = requests.post(f"{self._endpoint}/deposit/depositions", - auth=self._bearer_auth, - data=json.dumps({}), - headers={'Content-Type': 'application/json'}) + r = requests.post( + f"{self._endpoint}/deposit/depositions", + auth=self._bearer_auth, + data=json.dumps({}), + headers={"Content-Type": "application/json"}, + ) if r.ok: - deposition_id = r.json()['id'] - - self.change_metadata(dep_id=deposition_id, - title=title, - upload_type=upload_type, - description=description, - ) - self.deposition_id = r.json()['id'] - self.bucket = r.json()['links']['bucket'] + self.deposition_id = r.json()["id"] + self.bucket = r.json()["links"]["bucket"] self.title = title + + self.change_metadata( + json_file_path=metadata_json, + ) + else: print("** Project not created, something went wrong. Check that your ACCESS_TOKEN is in ~/.zenodo_token ") @@ -398,21 +405,28 @@ def set_project(self, dep_id=None): projects = self._get_depositions() if projects is not None: - project_list = [d for d in projects if d['id'] == int(dep_id)] + project_list = [ + d + for d in projects + if self._check_parent_doi(dep_id=dep_id, project_obj=d) + ] if len(project_list) > 0: - self.title = project_list[0]['title'] - self.bucket = self._get_bucket_by_id(dep_id) - self.deposition_id = dep_id + self.title = project_list[0]["title"] + self.bucket = self._get_bucket_by_id(project_list[0]["id"]) + self.deposition_id = project_list[0]["id"] + else: print(f' ** Deposition ID: {dep_id} does not exist in your projects ** ') - def change_metadata(self, dep_id=None, - title=None, - upload_type=None, - description=None, - creator=None, - **kwargs - ): + def _check_parent_doi(self, dep_id, project_obj): + if project_obj["id"] == int(dep_id): + return True + concept_doi = project_obj.get("conceptdoi", None) + if concept_doi != None: + return int(dep_id) == int(concept_doi.split(".")[-1]) + return False + + def change_metadata(self, json_file_path=None): """change projects metadata ** warning ** @@ -432,30 +446,35 @@ def change_metadata(self, dep_id=None, Returns: dict: dictionary with new metadata """ - if upload_type is None: - upload_type = 'other' - if description is None: - description = "description goes here" - - if creator is None: - creator = "creator goes here" - - data = { - "metadata": { - "title": f"{title}", - "upload_type": f"{upload_type}", - "description": f"{description}", - "creators": [{"name": f"{creator}"}] - } - } - # update metadata with a new metadata dictionary - data.update(kwargs) - - r = requests.put(f"{self._endpoint}/deposit/depositions/{dep_id}", - auth=self._bearer_auth, - data=json.dumps(data), - headers={'Content-Type': 'application/json'}) + if json_file_path is None: + print("You need to supply a path") + + if not Path(os.path.expanduser(json_file_path)).exists(): + print( + f"{json_file_path} does not exist. Please check you entered the correct path" + ) + + if json_file_path: + with open(json_file_path, "rb") as json_file: + file_data = json.load(json_file) + + # if upload_type is None: + # upload_types = self._get_upload_types() + # warnings.warn( + # f"upload_type not set, so defaulted to 'other', possible choices include {upload_types}", + # UserWarning, + # ) + # upload_type = "other" + + file_data["metadata"]["publication_date"] = datetime.now().strftime("%Y-%m-%d") + + r = requests.put( + f"{self._endpoint}/deposit/depositions/{self.deposition_id}", + auth=self._bearer_auth, + data=json.dumps(file_data), + headers={"Content-Type": "application/json"}, + ) if r.ok: return r.json() @@ -610,7 +629,7 @@ def upload_tar(self, source_dir=None, output_file=None, publish=False): # remove tar file after uploading it os.remove(output_file) - def update(self, source=None, output_file=None, publish=False): + def update(self, source=None, output_file=None, metadata_json=None, publish=False): """update an existed record Args: @@ -626,8 +645,15 @@ def update(self, source=None, output_file=None, publish=False): # parse current project to the draft deposition new_dep_id = r.json()['links']['latest_draft'].split('/')[-1] + + # adding this to let new id propogate in the backend + time.sleep(2) + self.set_project(new_dep_id) + time.sleep(5) + + self.change_metadata(json_file_path=metadata_json) # invoke upload funcions if not source: print("You need to supply a path") @@ -766,8 +792,10 @@ def _delete_project(self, dep_id=None): print('') # if input("are you sure you want to delete this project? (y/n)") == "y": # delete requests, we are deleting the resource at the specified URL - r = requests.delete(f'{self._endpoint}/deposit/depositions/{dep_id}', - auth=self._bearer_auth) + r = requests.delete( + f"{self._endpoint}/deposit/depositions/{self.deposition_id}", + auth=self._bearer_auth, + ) # response status print(r.status_code) @@ -776,4 +804,4 @@ def _delete_project(self, dep_id=None): self.bucket = None self.deposition_id = None # else: - # print(f'Project title {self.title} is still available.') + # print(f'Project title {self.title} is still available.') \ No newline at end of file diff --git a/tests/test_zenodopy.py b/tests/test_zenodopy.py index f1f2d01..c284932 100644 --- a/tests/test_zenodopy.py +++ b/tests/test_zenodopy.py @@ -11,7 +11,7 @@ def test_client(): _ = zen.Client() - _ = zen.Client(ACCESS_TOKEN=ACCESS_TOKEN, sandbox=True) + _ = zen.Client(token=ACCESS_TOKEN, sandbox=True) # zeno.list_projects @@ -23,120 +23,130 @@ def test_read_config(): def test_get_baseurl(): zeno = zen.Client(sandbox=True) - assert zeno._get_baseurl() == 'https://sandbox.zenodo.org/api' + assert zeno._endpoint == 'https://sandbox.zenodo.org/api' zeno = zen.Client() - assert zeno._get_baseurl() == 'https://zenodo.org/api' + assert zeno._endpoint == 'https://zenodo.org/api' -def test_get_key(): - zeno = zen.Client(ACCESS_TOKEN=ACCESS_TOKEN, sandbox=True) - zeno._get_key() - zeno.title - zeno.bucket - zeno.deposition_id - zeno.sandbox +# def test_get_key(): +# zeno = zen.Client(token=ACCESS_TOKEN, sandbox=True) +# zeno._get_key() +# zeno.title +# zeno.bucket +# zeno.deposition_id +# zeno.sandbox - zobj = zen.Client() - if zobj._get_key() is None: - pass +# zobj = zen.Client() +# if zobj._get_key() is None: +# pass -def test_get_headers(): - zeno = zen.Client(ACCESS_TOKEN=ACCESS_TOKEN, sandbox=True) - zeno._get_headers() +# def test_get_headers(): +# zeno = zen.Client(token=ACCESS_TOKEN, sandbox=True) +# zeno._get_headers() - zeno = zen.Client() - if zeno._get_headers() is None: - pass +# zeno = zen.Client() +# if zeno._get_headers() is None: +# pass def test_get_depositions(): - zeno = zen.Client() - if zeno._get_depositions() is None: + zeno = zen.Client(sandbox=True,token=ACCESS_TOKEN) + dep_id=106299 + zeno.set_project(dep_id=dep_id) + depositions = zeno._get_depositions() + deposition_by_id = zeno._get_depositions_by_id() + deposition_files = zeno._get_depositions_files() + if len(depositions) > 0: pass - if zeno._get_depositions_by_id(dep_id='123') is None: + elif int(deposition_by_id['id']) == dep_id or int(deposition_by_id['conceptrecid']) == dep_id: pass - if zeno._get_depositions_files() is None: + elif len(deposition_files) >0: pass - + else: + raise ValueError('Depositions not found') def test_get_bucket(): - zeno = zen.Client() - if zeno._get_bucket_by_id(dep_id='123') is None: - pass - if zeno._get_bucket_by_title(title='fake title') is None: - pass + zeno = zen.Client(sandbox=True,token=ACCESS_TOKEN) + dep_id=106299 + zeno.set_project(dep_id=dep_id) + bucket_link = zeno._get_bucket_by_id() + assert bucket_link.startswith('https://sandbox.zenodo.org/api/files/') + # if zeno._get_bucket_by_title(title='fake title') is None: + # pass def test_get_projects_and_files(): - zeno = zen.Client() + zeno = zen.Client(sandbox=True,token=ACCESS_TOKEN) + dep_id=106299 + zeno.set_project(dep_id=dep_id) _ = zeno.list_projects _ = zeno.list_files -@pytest.mark.filterwarnings('ignore::UserWarning') -def test_create_project(): - zeno = zen.Client(sandbox=True) - zeno.create_project(title='test', upload_type='other') - zeno.create_project(title='test') +# @pytest.mark.filterwarnings('ignore::UserWarning') +# def test_create_project(): +# zeno = zen.Client(sandbox=True) +# zeno.create_project(title='test', upload_type='other') +# zeno.create_project(title='test') -def test_set_project(): - zeno = zen.Client() - zeno.set_project(dep_id='123') +# def test_set_project(): +# zeno = zen.Client() +# zeno.set_project(dep_id='123') -# don't know how to mock inputs -def test_delete_project(): - pass +# # don't know how to mock inputs +# def test_delete_project(): +# pass -def test_change_metadata(): - zeno = zen.Client(sandbox=True) - zeno.change_metadata(dep_id='fake_ID', title='fake_title') +# def test_change_metadata(): +# zeno = zen.Client(sandbox=True) +# zeno.change_metadata(dep_id='fake_ID', title='fake_title') -def test_upload_file(): - zeno = zen.Client(sandbox=True) - zeno.upload_file(file_path='path') +# def test_upload_file(): +# zeno = zen.Client(sandbox=True) +# zeno.upload_file(file_path='path') -def test_download_file(): - zeno = zen.Client(sandbox=True) - zeno.download_file(filename='test') - zeno.bucket = 'invalid_url' - zeno.download_file(filename='test') - - -def test_tutorial(): - zeno = zen.Client(ACCESS_TOKEN=ACCESS_TOKEN, sandbox=True) - zeno.list_projects - zeno.list_files - zeno.title - zeno.bucket - zeno.deposition_id - zeno.sandbox - - params = {'title': 'test_set', 'upload_type': 'other'} - zeno.create_project(**params) - - # params = {'title': 'test', 'upload_type': 'other'} - # zeno.create_project(**params) - zeno.list_projects - - # with open("/home/test_file.txt", "w+") as f: - # f.write("test") - - # zeno.upload_file("/home/test_file.txt") - zeno.list_files - zeno.list_projects - _ = zeno.change_metadata(zeno.deposition_id, title='test_new') - zeno.list_projects - # zeno.download_file('test_file.txt') - # zeno.delete_file('test_file.txt') - zeno.list_files - # zeno._delete_project(zeno.deposition_id) - zeno.list_projects - zeno._delete_project(zeno.deposition_id) - zeno.list_projects +# def test_download_file(): +# zeno = zen.Client(sandbox=True) +# zeno.download_file(filename='test') +# zeno.bucket = 'invalid_url' +# zeno.download_file(filename='test') + + +# def test_tutorial(): +# zeno = zen.Client(ACCESS_TOKEN=ACCESS_TOKEN, sandbox=True) +# zeno.list_projects +# zeno.list_files +# zeno.title +# zeno.bucket +# zeno.deposition_id +# zeno.sandbox + +# params = {'title': 'test_set', 'upload_type': 'other'} +# zeno.create_project(**params) + +# # params = {'title': 'test', 'upload_type': 'other'} +# # zeno.create_project(**params) +# zeno.list_projects + +# # with open("/home/test_file.txt", "w+") as f: +# # f.write("test") + +# # zeno.upload_file("/home/test_file.txt") +# zeno.list_files +# zeno.list_projects +# _ = zeno.change_metadata(zeno.deposition_id, title='test_new') +# zeno.list_projects +# # zeno.download_file('test_file.txt') +# # zeno.delete_file('test_file.txt') +# zeno.list_files +# # zeno._delete_project(zeno.deposition_id) +# zeno.list_projects +# zeno._delete_project(zeno.deposition_id) +# zeno.list_projects From 5f842b81bbd70aca1bfaca3dc242cc90d712634c Mon Sep 17 00:00:00 2001 From: Akshat Mittal Date: Thu, 28 Nov 2024 14:15:33 +0700 Subject: [PATCH 07/14] test using this branch --- .github/workflows/version_update.yaml | 2 +- src/zenodopy/zenodopy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/version_update.yaml b/.github/workflows/version_update.yaml index 7bed4d9..30dd3c5 100644 --- a/.github/workflows/version_update.yaml +++ b/.github/workflows/version_update.yaml @@ -26,7 +26,7 @@ jobs: python-version: 3.12 - name: install req - run: pip install git+https://github.com/drifter089/zenodopy.git@update_Patch#egg=zenodopy + run: pip install git+https://github.com/drifter089/zenodopy.git@basic_test#egg=zenodopy - name: Update Zenodo Deposition run: | diff --git a/src/zenodopy/zenodopy.py b/src/zenodopy/zenodopy.py index 067e9b8..011d2f1 100644 --- a/src/zenodopy/zenodopy.py +++ b/src/zenodopy/zenodopy.py @@ -448,10 +448,10 @@ def change_metadata(self, json_file_path=None): """ if json_file_path is None: - print("You need to supply a path") + raise ValueError("You need to supply a path") if not Path(os.path.expanduser(json_file_path)).exists(): - print( + raise ValueError( f"{json_file_path} does not exist. Please check you entered the correct path" ) From 222d77e0af39a9946e9a744624e0d714c006c100 Mon Sep 17 00:00:00 2001 From: Akshat Mittal Date: Thu, 28 Nov 2024 14:20:48 +0700 Subject: [PATCH 08/14] update get_deposition_by_id --- src/zenodopy/zenodopy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/zenodopy/zenodopy.py b/src/zenodopy/zenodopy.py index 011d2f1..56e2870 100644 --- a/src/zenodopy/zenodopy.py +++ b/src/zenodopy/zenodopy.py @@ -639,7 +639,7 @@ def update(self, source=None, output_file=None, metadata_json=None, publish=Fals publish (bool): whether implemente publish action or not, argument for `upload_file` """ # create a draft deposition - url_action = self._get_depositions_by_id(self.deposition_id)['links']['newversion'] + url_action = self._get_depositions_by_id()['links']['newversion'] r = requests.post(url_action, auth=self._bearer_auth) r.raise_for_status() @@ -674,7 +674,7 @@ def update(self, source=None, output_file=None, metadata_json=None, publish=Fals def publish(self): """ publish a record """ - url_action = self._get_depositions_by_id(self.deposition_id)['links']['publish'] + url_action = self._get_depositions_by_id()['links']['publish'] r = requests.post(url_action, auth=self._bearer_auth) r.raise_for_status() return r @@ -766,7 +766,7 @@ def _get_latest_record(self, record_id=None): str: the latest record id or 'None' if not found """ try: - record = self._get_depositions_by_id(record_id)['links']['latest'].split('/')[-1] + record = self._get_depositions_by_id()['links']['latest'].split('/')[-1] except: record = 'None' return record From e791404232f5c1ae7d5b9f7f5afea44eb3f3089e Mon Sep 17 00:00:00 2001 From: Akshat Mittal Date: Thu, 28 Nov 2024 14:48:33 +0700 Subject: [PATCH 09/14] adding comment to tests --- tests/test_zenodopy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_zenodopy.py b/tests/test_zenodopy.py index c284932..603b1ce 100644 --- a/tests/test_zenodopy.py +++ b/tests/test_zenodopy.py @@ -1,4 +1,18 @@ import zenodopy as zen +""" +This module contains tests for the zenodopy library using pytest. +Functions: + test_client: Tests the initialization of the zen.Client object with and without a token. + test_read_config: Tests the _read_config method of the zen.Client object to ensure it raises a TypeError. + test_get_baseurl: Tests the _endpoint attribute of the zen.Client object for both sandbox and production environments. + test_get_depositions: Tests the _get_depositions, _get_depositions_by_id, and _get_depositions_files methods of the zen.Client object. + test_get_bucket: Tests the _get_bucket_by_id method of the zen.Client object. + test_get_projects_and_files: Tests the list_projects and list_files properties of the zen.Client object. +Note: + The update and change_metadata functions have been updated to add new versions to existing depositions. + This functionality is being tested in test_version. We will bring back individual tests once these changes + have been merged upstream to keep the changes incremental. +""" import pytest # use this when using pytest From 9730e1ecdc1f958a990d040abe3337d17a9670bd Mon Sep 17 00:00:00 2001 From: Akshat Mittal Date: Thu, 28 Nov 2024 14:57:49 +0700 Subject: [PATCH 10/14] remove datetime --- requirements.txt | 3 +-- requirements_dev.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3f3cf48..4cbe9c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ requests==2.26.0 types-requests==2.27.7 -wget==3.2 -datetime \ No newline at end of file +wget==3.2 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 7f20f84..c6e34de 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,5 +5,4 @@ pytest-cov==2.12.1 mypy==0.910 requests==2.26.0 types-requests==2.27.7 -wget==3.2 -datetime \ No newline at end of file +wget==3.2 \ No newline at end of file From 140016469df30d96cbd5edaaac44857cf1ee2f17 Mon Sep 17 00:00:00 2001 From: Akshat Mittal Date: Thu, 28 Nov 2024 15:14:51 +0700 Subject: [PATCH 11/14] remove unused args from create_project function --- src/zenodopy/zenodopy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/zenodopy/zenodopy.py b/src/zenodopy/zenodopy.py index 56e2870..5f70acb 100644 --- a/src/zenodopy/zenodopy.py +++ b/src/zenodopy/zenodopy.py @@ -363,7 +363,7 @@ def list_files(self): # warnings.warn("The object is not pointing to a project. Either create a project or explicity set the project'", UserWarning) def create_project( - self, title=None, upload_type=None, metadata_json=None, description=None + self, title=None, metadata_json=None, ): """Creates a new project @@ -375,8 +375,7 @@ def create_project( Args: title (str): new title of project - upload_type (str, optional): new upload type - description (str, optional): new description + metadata_json (str): path to json file with metadata """ # get request, returns our response From 67fcd0e3cec34742932717815e4cfae2fe5a0045 Mon Sep 17 00:00:00 2001 From: Akshat Mittal Date: Thu, 28 Nov 2024 16:49:24 +0700 Subject: [PATCH 12/14] try new zenodo metadat class --- src/zenodopy/zenodopy.py | 78 ++++++++++++++++++---------------- test.py | 90 ++++++++++++++++++++++++++++++++++++++++ tests/test_version.py | 16 +++++++ 3 files changed, 148 insertions(+), 36 deletions(-) create mode 100644 test.py diff --git a/src/zenodopy/zenodopy.py b/src/zenodopy/zenodopy.py index 5f70acb..8e412e3 100644 --- a/src/zenodopy/zenodopy.py +++ b/src/zenodopy/zenodopy.py @@ -8,7 +8,8 @@ import zipfile from datetime import datetime import time - +from dataclasses import dataclass, field +from typing import Optional, List def validate_url(url): """validates if URL is formatted correctly @@ -49,7 +50,19 @@ def make_zipfile(path, ziph): ziph.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), os.path.join(path, '..'))) - + +@dataclass +class ZenodoMetadata: + title: str + upload_type: str = "other" + description: Optional[str] = None + publication_date: str = field(default_factory=lambda: datetime.now().strftime("%Y-%m-%d")) + version: str = "0.1.0" + access_right: str = "open" + license: str = "Apache-2.0" + keywords: List[str] = field(default_factory=lambda: ["zenodo", "github", "git"]) + creators: List[dict] = field(default_factory=lambda: [{"name": "Jhon, Doe", "orcid": "0000-0003-2584-3576"}]) + class BearerAuth(requests.auth.AuthBase): """Bearer Authentication""" @@ -363,7 +376,7 @@ def list_files(self): # warnings.warn("The object is not pointing to a project. Either create a project or explicity set the project'", UserWarning) def create_project( - self, title=None, metadata_json=None, + self, metadata:ZenodoMetadata, ): """Creates a new project @@ -391,9 +404,9 @@ def create_project( self.deposition_id = r.json()["id"] self.bucket = r.json()["links"]["bucket"] self.title = title - + self.change_metadata( - json_file_path=metadata_json, + metadata=metadata, ) else: @@ -425,38 +438,31 @@ def _check_parent_doi(self, dep_id, project_obj): return int(dep_id) == int(concept_doi.split(".")[-1]) return False - def change_metadata(self, json_file_path=None): - """change projects metadata + def change_metadata(self, metadata: ZenodoMetadata): + # """Change project's metadata. + # Args: + # metadata (ZenodoMetadata): The metadata to update. + # Returns: + # dict: Dictionary with the updated metadata. + # """ + metadata.publication_date = datetime.now().strftime("%Y-%m-%d") - ** warning ** - This changes everything. If nothing is supplied then - uses default values are used. + data = { + "metadata": metadata.__dict__ + } - For example. If you do not supply an upload_type - then it will default to "other" - Args: - dep_id (str): deposition to change - title (str): new title of project - upload_type (str): new upload type - description (str): new description - **kwargs: dictionary to update default metadata + # if json_file_path is None: + # raise ValueError("You need to supply a path") - Returns: - dict: dictionary with new metadata - """ - - if json_file_path is None: - raise ValueError("You need to supply a path") - - if not Path(os.path.expanduser(json_file_path)).exists(): - raise ValueError( - f"{json_file_path} does not exist. Please check you entered the correct path" - ) + # if not Path(os.path.expanduser(json_file_path)).exists(): + # raise ValueError( + # f"{json_file_path} does not exist. Please check you entered the correct path" + # ) - if json_file_path: - with open(json_file_path, "rb") as json_file: - file_data = json.load(json_file) + # if json_file_path: + # with open(json_file_path, "rb") as json_file: + # file_data = json.load(json_file) # if upload_type is None: # upload_types = self._get_upload_types() @@ -466,12 +472,12 @@ def change_metadata(self, json_file_path=None): # ) # upload_type = "other" - file_data["metadata"]["publication_date"] = datetime.now().strftime("%Y-%m-%d") + # file_data["metadata"]["publication_date"] = datetime.now().strftime("%Y-%m-%d") r = requests.put( f"{self._endpoint}/deposit/depositions/{self.deposition_id}", auth=self._bearer_auth, - data=json.dumps(file_data), + data=json.dumps(data), headers={"Content-Type": "application/json"}, ) @@ -628,7 +634,7 @@ def upload_tar(self, source_dir=None, output_file=None, publish=False): # remove tar file after uploading it os.remove(output_file) - def update(self, source=None, output_file=None, metadata_json=None, publish=False): + def update(self, metadata:ZenodoMetadata, source=None, output_file=None, publish=False): """update an existed record Args: @@ -652,7 +658,7 @@ def update(self, source=None, output_file=None, metadata_json=None, publish=Fals time.sleep(5) - self.change_metadata(json_file_path=metadata_json) + self.change_metadata(metadata=metadata) # invoke upload funcions if not source: print("You need to supply a path") diff --git a/test.py b/test.py new file mode 100644 index 0000000..b7b1f0a --- /dev/null +++ b/test.py @@ -0,0 +1,90 @@ +import os +import time +import json +from pathlib import Path +import argparse +import zenodopy +from zenodopy import ZenodoMetadata +import parse + +def parse_metadata_from_json(json_file_path: Path) -> zenodopy.ZenodoMetadata: + """Parse metadata from a JSON file into a ZenodoMetadata object.""" + json_file_path = json_file_path.expanduser() + if not json_file_path.exists(): + raise ValueError( + f"{json_file_path} does not exist. Please check you entered the correct path." + ) + + with json_file_path.open("r") as json_file: + data = json.load(json_file) + + metadata_dict = data.get("metadata", {}) + return ZenodoMetadata(**metadata_dict) + +def main(): + # Set up argument parsing + parser = argparse.ArgumentParser(description='Update Zenodo deposition with new version and files.') + parser.add_argument('--version_tag', required=True, help='The version tag for the new release.') + parser.add_argument('--zenodo_token', required=True, help='The Zenodo API token.') + parser.add_argument('--dep_id', required=True, type=int, help='The Zenodo deposition ID.') + parser.add_argument('--base_dir', required=True, help='The base directory path.') + parser.add_argument('--metadata_file', required=True, help='The metadata JSON file path.') + parser.add_argument('--upload_dir', required=True, help='The directory containing files to upload.') + + args = parser.parse_args() + + version_tag = args.version_tag + zenodo_token = args.zenodo_token + dep_id = args.dep_id + base_dir = Path(args.base_dir) + zenodo_metadata_file = Path(args.metadata_file) + upload_dir = Path(args.upload_dir) + + print("Version Tag:", version_tag) + + def prepare_metadata(file_path, new_version): + with open(file_path, "r") as file: + zenodo_data = json.load(file) + + zenodo_data["metadata"]["version"] = new_version + + with open(file_path, "w") as file: + json.dump(zenodo_data, file, indent=2) + + # Prepare the metadata file with the new version tag + prepare_metadata(zenodo_metadata_file, version_tag) + + max_retries = 5 + + for attempt in range(1, max_retries + 1): + try: + zeno = zenodopy.Client( + sandbox=True, + token=zenodo_token, + ) + + zeno.set_project(dep_id=dep_id) + + zeno.update( + source=str(upload_dir), + publish=True, + metadata_json=str(zenodo_metadata_file), + ) + print("Update succeeded.") + break + + except Exception as e: + print(f"Attempt {attempt} failed with error: {e}") + + time.sleep(2) # Optional: Wait before retrying + + zeno._delete_project(dep_id=dep_id) + + if attempt == max_retries: + print("Max retries reached. Exiting.") + raise + else: + time.sleep(2) + +if __name__ == "__main__": + main() diff --git a/tests/test_version.py b/tests/test_version.py index 3a96af1..b7b1f0a 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -4,6 +4,22 @@ from pathlib import Path import argparse import zenodopy +from zenodopy import ZenodoMetadata +import parse + +def parse_metadata_from_json(json_file_path: Path) -> zenodopy.ZenodoMetadata: + """Parse metadata from a JSON file into a ZenodoMetadata object.""" + json_file_path = json_file_path.expanduser() + if not json_file_path.exists(): + raise ValueError( + f"{json_file_path} does not exist. Please check you entered the correct path." + ) + + with json_file_path.open("r") as json_file: + data = json.load(json_file) + + metadata_dict = data.get("metadata", {}) + return ZenodoMetadata(**metadata_dict) def main(): # Set up argument parsing From ca75a86dce5bbb6658340e2fe21c93f68c99d2f3 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 28 Nov 2024 17:08:18 +0700 Subject: [PATCH 13/14] using new ZenodoMeteDataClass --- .github/workflows/version_update.yaml | 2 +- src/zenodopy/__init__.py | 3 ++- tests/test_version.py | 25 ++++++++----------------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.github/workflows/version_update.yaml b/.github/workflows/version_update.yaml index 30dd3c5..3c1df27 100644 --- a/.github/workflows/version_update.yaml +++ b/.github/workflows/version_update.yaml @@ -26,7 +26,7 @@ jobs: python-version: 3.12 - name: install req - run: pip install git+https://github.com/drifter089/zenodopy.git@basic_test#egg=zenodopy + run: pip install git+https://github.com/drifter089/zenodopy.git@new_metadata_class#egg=zenodopy - name: Update Zenodo Deposition run: | diff --git a/src/zenodopy/__init__.py b/src/zenodopy/__init__.py index a0999b6..81348a7 100644 --- a/src/zenodopy/__init__.py +++ b/src/zenodopy/__init__.py @@ -2,5 +2,6 @@ Set up module access for the base package """ from .zenodopy import Client +from .zenodopy import ZenodoMetadata -__all__ = ['Client'] +__all__ = ['Client','ZenodoMetadata'] diff --git a/tests/test_version.py b/tests/test_version.py index b7b1f0a..fc4cd7c 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -4,8 +4,6 @@ from pathlib import Path import argparse import zenodopy -from zenodopy import ZenodoMetadata -import parse def parse_metadata_from_json(json_file_path: Path) -> zenodopy.ZenodoMetadata: """Parse metadata from a JSON file into a ZenodoMetadata object.""" @@ -19,7 +17,8 @@ def parse_metadata_from_json(json_file_path: Path) -> zenodopy.ZenodoMetadata: data = json.load(json_file) metadata_dict = data.get("metadata", {}) - return ZenodoMetadata(**metadata_dict) + return zenodopy.ZenodoMetadata(**metadata_dict) + def main(): # Set up argument parsing @@ -41,19 +40,11 @@ def main(): upload_dir = Path(args.upload_dir) print("Version Tag:", version_tag) - - def prepare_metadata(file_path, new_version): - with open(file_path, "r") as file: - zenodo_data = json.load(file) - - zenodo_data["metadata"]["version"] = new_version - - with open(file_path, "w") as file: - json.dump(zenodo_data, file, indent=2) - - # Prepare the metadata file with the new version tag - prepare_metadata(zenodo_metadata_file, version_tag) - + + # Parse and update metadata with new version tag + metadata = parse_metadata_from_json(zenodo_metadata_file) + metadata.version = version_tag + max_retries = 5 for attempt in range(1, max_retries + 1): @@ -68,7 +59,7 @@ def prepare_metadata(file_path, new_version): zeno.update( source=str(upload_dir), publish=True, - metadata_json=str(zenodo_metadata_file), + metadata_json=metadata, ) print("Update succeeded.") break From eec640620bdf17aa9058f56bea0b74a55f741147 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 28 Nov 2024 17:12:37 +0700 Subject: [PATCH 14/14] using new metadata argument --- tests/test_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_version.py b/tests/test_version.py index fc4cd7c..d4bc16f 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -59,7 +59,7 @@ def main(): zeno.update( source=str(upload_dir), publish=True, - metadata_json=metadata, + metadata=metadata, ) print("Update succeeded.") break