diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yml similarity index 83% rename from .github/workflows/test.yml rename to .github/workflows/ci.yml index 1278145..bf77e7b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,28 @@ jobs: with: black_args: ". --check" + check-sync-code: + name: "Check auto-generated sync code" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "0.5.1" + enable-cache: true + + - name: Install dependencies + run: uv pip install -e '.[dev]' + + - name: Run unasync to regenerate code + run: python utils/run-unasync.py --check + integration-test: runs-on: ${{ matrix.os }} strategy: diff --git a/.github/workflows/tag-new-release.yml b/.github/workflows/tag-new-release.yml index 97e06fa..02a6059 100644 --- a/.github/workflows/tag-new-release.yml +++ b/.github/workflows/tag-new-release.yml @@ -26,8 +26,8 @@ jobs: echo "Error: 'release_version' should be in SemVer format." exit 1 fi - if [[ ! "${{ inputs.next_version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Error: 'next_version' should be in SemVer format." + if [[ ! "${{ inputs.next_version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "${{ inputs.next_version }}" != "skip" ]]; then + echo "Error: 'next_version' should be in SemVer format or 'skip'." exit 1 fi @@ -62,4 +62,4 @@ jobs: RELEASE_VERSION: ${{ inputs.release_version }} NEXT_VERSION: ${{ inputs.next_version }} run: | - python .github/scripts/release.py + python utils/run-release.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..131ce00 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,70 @@ +# Contributing to the Central Dogma Client + +First of all, thank you so much for taking your time to visit centraldogma-python. +Here, everything you do or even your mere presence is a contribution in a broad sense. +We strongly believe your contribution will enrich our community and create +a virtuous cycle that propels the project into something great. + +## Install + +``` +$ uv pip install -e ".[dev]" +``` + +## Code generation + +This project uses `unasync` to automatically generate all synchronous (sync) client code from its asynchronous (async) counterpart. + +As a contributor, you must not edit the generated synchronous code (e.g., files directly under `centraldogma/_sync/`) by hand. +All changes must be made to the asynchronous source files, which are located in the centraldogma/_async/ directory. +After you modify any code in `centraldogma/_async/`, you must run the code generation script to update the synchronous code: + +``` +$ python utils/run-unasync.py +``` + +This command regenerates the synchronous code based on your changes. +You must include both your original changes (in `centraldogma/_async/`) and the newly generated synchronous files in your Pull Request. + +## Running tests locally + +### Unit test + +``` +$ pytest +``` + +### Integration test + +1. Run local Central Dogma server with docker-compose + ``` + $ docker-compose up -d + ``` + +2. Run integration tests + ``` + $ INTEGRATION_TEST=true pytest + ``` + +3. Stop the server + ``` + $ docker-compose down + ``` + +## Lint + +- [PEP 8](https://www.python.org/dev/peps/pep-0008) + ``` + $ black . + ``` + +## Documentation + +- [PEP 257](https://www.python.org/dev/peps/pep-0257) + +### To build sphinx at local + +``` +$ pip install sphinx sphinx_rtd_theme +$ cd docs && make html +``` diff --git a/README.md b/README.md index f5b4188..5d6a4d2 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,13 @@ Python client library for [Central Dogma](https://line.github.io/centraldogma/). ## Install + ``` $ pip install centraldogma-python ``` ## Getting started + Only URL indicating CentralDogma server and access token are required. ```pycon >>> from centraldogma.dogma import Dogma @@ -29,42 +31,6 @@ It supports client configurations. Please see [`examples` folder](https://github.com/line/centraldogma-python/tree/main/examples) for more detail. ---- - -## Development -### Tests -#### Unit test -``` -$ pytest -``` - -#### Integration test -1. Run local Central Dogma server with docker-compose - ``` - $ docker-compose up -d - ``` +## Contributing -2. Run integration tests - ``` - $ INTEGRATION_TEST=true pytest - ``` - -3. Stop the server - ``` - $ docker-compose down - ``` - -### Lint -- [PEP 8](https://www.python.org/dev/peps/pep-0008) - ``` - $ black . - ``` - -### Documentation -- [PEP 257](https://www.python.org/dev/peps/pep-0257) - -#### To build sphinx at local -``` -$ pip install sphinx sphinx_rtd_theme -$ cd docs && make html -``` +See [CONTRIBUTING.md](./CONTRIBUTING.md) \ No newline at end of file diff --git a/centraldogma/_async/base_client.py b/centraldogma/_async/base_client.py new file mode 100644 index 0000000..3b1b338 --- /dev/null +++ b/centraldogma/_async/base_client.py @@ -0,0 +1,113 @@ +# Copyright 2025 LINE Corporation +# +# LINE Corporation licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from typing import Any, Dict, Union, Callable, TypeVar, Optional + +from httpx import AsyncClient, Limits, Response +from tenacity import stop_after_attempt, wait_exponential, AsyncRetrying + +from centraldogma.exceptions import to_exception + +T = TypeVar("T") + + +class BaseClient: + def __init__( + self, + base_url: str, + token: str, + http2: bool = True, + retries: int = 1, + max_connections: int = 10, + max_keepalive_connections: int = 2, + **configs, + ): + assert retries >= 0, "retries must be greater than or equal to zero" + assert max_connections > 0, "max_connections must be greater than zero" + assert ( + max_keepalive_connections > 0 + ), "max_keepalive_connections must be greater than zero" + + base_url = base_url[:-1] if base_url[-1] == "/" else base_url + + for key in ["transport", "limits"]: + if key in configs: + del configs[key] + + self.retries = retries + self.client = AsyncClient( + base_url=f"{base_url}/api/v1", + http2=http2, + limits=Limits( + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + ), + **configs, + ) + self.token = token + self.headers = self._get_headers(token) + self.patch_headers = self._get_patch_headers(token) + + async def __aexit__(self, *_: Any) -> None: + await self.client.aclose() + + async def request( + self, + method: str, + path: str, + handler: Optional[Dict[int, Callable[[Response], T]]] = None, + **kwargs, + ) -> Union[Response, T]: + kwargs = self._set_request_headers(method, **kwargs) + retryer = AsyncRetrying( + stop=stop_after_attempt(self.retries + 1), + wait=wait_exponential(max=60), + reraise=True, + ) + return retryer(self._request, method, path, handler, **kwargs) + + def _set_request_headers(self, method: str, **kwargs) -> Dict: + default_headers = self.patch_headers if method == "patch" else self.headers + kwargs["headers"] = {**default_headers, **(kwargs.get("headers") or {})} + return kwargs + + async def _request( + self, + method: str, + path: str, + handler: Optional[Dict[int, Callable[[Response], T]]] = None, + **kwargs, + ): + resp = await self.client.request(method, path, **kwargs) + if handler: + converter = handler.get(resp.status_code) + if converter: + return converter(resp) + else: # Unexpected response status + raise to_exception(resp) + return resp + + @staticmethod + def _get_headers(token: str) -> Dict: + return { + "Authorization": f"bearer {token}", + "Content-Type": "application/json", + } + + @staticmethod + def _get_patch_headers(token: str) -> Dict: + return { + "Authorization": f"bearer {token}", + "Content-Type": "application/json-patch+json", + } diff --git a/centraldogma/base_client.py b/centraldogma/_sync/base_client.py similarity index 94% rename from centraldogma/base_client.py rename to centraldogma/_sync/base_client.py index 86ce9d8..a3f760a 100644 --- a/centraldogma/base_client.py +++ b/centraldogma/_sync/base_client.py @@ -1,4 +1,4 @@ -# Copyright 2021 LINE Corporation +# Copyright 2025 LINE Corporation # # LINE Corporation licenses this file to you under the Apache License, # version 2.0 (the "License"); you may not use this file except in compliance @@ -11,9 +11,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from typing import Dict, Union, Callable, TypeVar, Optional -from httpx import Client, HTTPTransport, Limits, Response +from typing import Any, Dict, Union, Callable, TypeVar, Optional + +from httpx import Client, Limits, Response from tenacity import stop_after_attempt, wait_exponential, Retrying from centraldogma.exceptions import to_exception @@ -48,7 +49,6 @@ def __init__( self.client = Client( base_url=f"{base_url}/api/v1", http2=http2, - transport=HTTPTransport(retries=retries), limits=Limits( max_connections=max_connections, max_keepalive_connections=max_keepalive_connections, @@ -59,6 +59,9 @@ def __init__( self.headers = self._get_headers(token) self.patch_headers = self._get_patch_headers(token) + def __exit__(self, *_: Any) -> None: + self.client.close() + def request( self, method: str, diff --git a/centraldogma/content_service.py b/centraldogma/content_service.py index 6eff85a..eed9bed 100644 --- a/centraldogma/content_service.py +++ b/centraldogma/content_service.py @@ -19,7 +19,7 @@ from httpx import Response -from centraldogma.base_client import BaseClient +from centraldogma._sync.base_client import BaseClient from centraldogma.data import Content from centraldogma.data.change import Change from centraldogma.data.commit import Commit diff --git a/centraldogma/dogma.py b/centraldogma/dogma.py index 8945430..fe4646c 100644 --- a/centraldogma/dogma.py +++ b/centraldogma/dogma.py @@ -14,7 +14,7 @@ import os from typing import List, Optional, TypeVar, Callable -from centraldogma.base_client import BaseClient +from centraldogma._sync.base_client import BaseClient from centraldogma.content_service import ContentService # noinspection PyUnresolvedReferences diff --git a/centraldogma/project_service.py b/centraldogma/project_service.py index d9cd4dd..6ea9a2d 100644 --- a/centraldogma/project_service.py +++ b/centraldogma/project_service.py @@ -14,7 +14,7 @@ from http import HTTPStatus from typing import List -from centraldogma.base_client import BaseClient +from centraldogma._sync.base_client import BaseClient from centraldogma.data import Project diff --git a/centraldogma/repository_service.py b/centraldogma/repository_service.py index 688ce8d..1a29c54 100644 --- a/centraldogma/repository_service.py +++ b/centraldogma/repository_service.py @@ -14,7 +14,7 @@ from http import HTTPStatus from typing import List -from centraldogma.base_client import BaseClient +from centraldogma._sync.base_client import BaseClient from centraldogma.data import Repository diff --git a/pyproject.toml b/pyproject.toml index 95aa292..faa3f28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + [build-system] requires = ["setuptools >= 61"] build-backend = "setuptools.build_meta" @@ -59,10 +60,11 @@ dev = [ "pytest", "pytest-cov", "pytest-mock", - "respx" + "respx", + "unasync>=0.6.0" ] docs = [ - "sphinx_rtd_theme" + "sphinx_rtd_theme>=3.0" ] [tool.setuptools] diff --git a/tests/test_base_client.py b/tests/test_base_client.py index cf4d28b..8edd407 100644 --- a/tests/test_base_client.py +++ b/tests/test_base_client.py @@ -11,11 +11,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + from http import HTTPStatus import json +from centraldogma._sync.base_client import BaseClient from centraldogma.exceptions import UnauthorizedException, NotFoundException -from centraldogma.base_client import BaseClient from httpx import ConnectError, NetworkError, Response import pytest diff --git a/.github/scripts/release.py b/utils/run-release.py similarity index 100% rename from .github/scripts/release.py rename to utils/run-release.py diff --git a/utils/run-unasync.py b/utils/run-unasync.py new file mode 100644 index 0000000..9eb4544 --- /dev/null +++ b/utils/run-unasync.py @@ -0,0 +1,91 @@ +# Copyright 2025 LINE Corporation +# +# LINE Corporation licenses this file to you under the Apache License, +# version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import subprocess +import sys +from glob import glob +from pathlib import Path + +import unasync + + +def cleanup(source_dir: Path, output_dir: Path, patterns: list[str]): + for file in glob("*.py", root_dir=source_dir): + path = output_dir / file + for pattern in patterns: + subprocess.check_call(["sed", "-i.bak", pattern, str(path)]) + subprocess.check_call(["rm", f"{path}.bak"]) + + +def run( + rule: unasync.Rule, + cleanup_patterns: list[str] = [], + check: bool = False, +): + root_dir = Path(__file__).absolute().parent.parent + source_dir = root_dir / rule.fromdir.lstrip("/") + output_dir = check_dir = root_dir / rule.todir.lstrip("/") + if check: + rule.todir += "_sync_check/" + output_dir = root_dir / rule.todir.lstrip("/") + + filepaths = [] + for root, _, filenames in os.walk(source_dir): + for filename in filenames: + if filename.rpartition(".")[-1] in { + "py", + "pyi", + } and not filename.startswith("utils.py"): + filepaths.append(os.path.join(root, filename)) + + unasync.unasync_files(filepaths, [rule]) + + if cleanup_patterns: + cleanup(source_dir, output_dir, cleanup_patterns) + + if check: + subprocess.check_call(["black", output_dir]) + # Make sure there are no differences between _sync and _sync_check + for file in glob(f"{output_dir}/*.py"): + file_name = file.split("/")[-1] + subprocess.check_call( + [ + "diff", + f"{check_dir}/{file_name}", + f"{output_dir}/{file_name}", + ] + ) + subprocess.check_call(["rm", "-rf", output_dir]) + + +def main(check: bool = False): + run( + rule=unasync.Rule( + fromdir="/centraldogma/_async/", + todir="/centraldogma/_sync/", + additional_replacements={ + "AsyncClient": "Client", + "AsyncDogma": "Dogma", + "AsyncRetrying": "Retrying", + "__aexit__": "__exit__", + "aclose": "close", + }, + ), + check=check, + ) + + +if __name__ == "__main__": + main(check="--check" in sys.argv) diff --git a/uv.lock b/uv.lock index 46001a5..acc93bf 100644 --- a/uv.lock +++ b/uv.lock @@ -106,6 +106,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "respx" }, + { name = "unasync" }, ] docs = [ { name = "sphinx-rtd-theme" }, @@ -124,8 +125,9 @@ requires-dist = [ { name = "pytest-mock", marker = "extra == 'dev'" }, { name = "python-dateutil", specifier = ">=2.9.0.post0,<3.0.0" }, { name = "respx", marker = "extra == 'dev'" }, - { name = "sphinx-rtd-theme", marker = "extra == 'docs'" }, + { name = "sphinx-rtd-theme", marker = "extra == 'docs'", specifier = ">=3.0" }, { name = "tenacity", specifier = ">=9.0.0,<10.0.0" }, + { name = "unasync", specifier = ">=0.6.0" }, ] provides-extras = ["dev", "docs"] @@ -802,6 +804,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/5c/428523509b26c243c1e93aa2ae385def597ef1fbdbbd47978430ba19037d/respx-0.21.1-py2.py3-none-any.whl", hash = "sha256:05f45de23f0c785862a2c92a3e173916e8ca88e4caad715dd5f68584d6053c20", size = 25130, upload-time = "2024-03-27T20:41:55.709Z" }, ] +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + [[package]] name = "six" version = "1.16.0" @@ -947,6 +958,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/cb/b86984bed139586d01532a587464b5805f12e397594f19f931c4c2fbfa61/tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539", size = 28169, upload-time = "2024-07-29T12:12:25.825Z" }, ] +[[package]] +name = "tokenize-rt" +version = "6.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/ed/8f07e893132d5051d86a553e749d5c89b2a4776eb3a579b72ed61f8559ca/tokenize_rt-6.2.0.tar.gz", hash = "sha256:8439c042b330c553fdbe1758e4a05c0ed460dbbbb24a606f11f0dee75da4cad6", size = 5476, upload-time = "2025-05-23T23:48:00.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/f0/3fe8c6e69135a845f4106f2ff8b6805638d4e85c264e70114e8126689587/tokenize_rt-6.2.0-py2.py3-none-any.whl", hash = "sha256:a152bf4f249c847a66497a4a95f63376ed68ac6abf092a2f7cfb29d044ecff44", size = 6004, upload-time = "2025-05-23T23:47:58.812Z" }, +] + [[package]] name = "tomli" version = "2.1.0" @@ -978,6 +998,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, ] +[[package]] +name = "unasync" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools" }, + { name = "tokenize-rt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/4e/735dbc0885ca197bcd80a2479ca24035627e2e768c784261fc7f1b8d7600/unasync-0.6.0.tar.gz", hash = "sha256:a9d01ace3e1068b20550ab15b7f9723b15b8bcde728bc1770bcb578374c7ee58", size = 18755, upload-time = "2024-05-03T11:14:58.312Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/b5/d2842541718ffa12060854735587543120a31ebc339435e0bd0faf368541/unasync-0.6.0-py3-none-any.whl", hash = "sha256:9cf7aaaea9737e417d8949bf9be55dc25fdb4ef1f4edc21b58f76ff0d2b9d73f", size = 9959, upload-time = "2024-05-03T11:14:56.17Z" }, +] + [[package]] name = "urllib3" version = "2.2.3"