diff --git a/cdmtaskservice/jaws/client.py b/cdmtaskservice/jaws/client.py index cdc4e4b..4e23af7 100644 --- a/cdmtaskservice/jaws/client.py +++ b/cdmtaskservice/jaws/client.py @@ -6,9 +6,11 @@ import aiohttp +import aiohttp_socks import datetime from enum import Enum import logging +import os from typing import Any from cdmtaskservice.arg_checkers import require_string as _require_string, not_falsy as _not_falsy @@ -64,11 +66,16 @@ def __init__(self, url: str, token: str): if not url.endswith("/"): url += "/" url += "api/v2/" - self._sess = aiohttp.ClientSession( - base_url=url, - headers={"Authorization": f"Bearer {_require_string(token, 'token')}"} - ) + headers = {"Authorization": f"Bearer {_require_string(token, 'token')}"} self._logr = logging.getLogger(__name__) + pxurl = os.environ.get("JAWS_HTTPS_PROXY") + if pxurl: + # just used for testing so not advertised for now + self._logr.info(f"Using proxy {pxurl}") + proxy = aiohttp_socks.ProxyConnector.from_url(pxurl) + self._sess = aiohttp.ClientSession(base_url=url, headers=headers, connector=proxy) + else: + self._sess = aiohttp.ClientSession(base_url=url, headers=headers) async def _get(self, url, params=None) -> dict[str, Any]: return await self._req("GET", url, params=params) diff --git a/docker-compose.yaml b/docker-compose.yaml index 12caa64..f74dfd0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -77,7 +77,7 @@ services: - KBCTS_JOB_MAX_CPU_HOURS=200 - KBCTS_KAFKA_BOOTSTRAP_SERVERS=kafka:9092 - KBCTS_KAFKA_TOPIC_JOBS=cts-jobs - - KBCTS_SERVICE_ROOT_URL=https://cdm_task_service:5000 + - KBCTS_SERVICE_ROOT_URL=http://cdm_task_service:5000 - KBCTS_SERVICE_GROUP=local_development volumes: - ./tmp_creds:/creds diff --git a/pyproject.toml b/pyproject.toml index b7e7106..e034664 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,9 @@ dependencies = [ "aiobotocore==3.1.0", "aiofiles==25.1.0", "aiohttp==3.13.3", + "aiohttp-socks==0.11.0", "aiokafka==0.12.0", + "apscheduler==3.11.2", "async-lru==2.0.5", "awscrt==0.31.0", "bidict==0.23.1", diff --git a/uv.lock b/uv.lock index b8e6ef1..8fab709 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" [[package]] @@ -123,6 +123,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/63/278a98c715ae467624eafe375542d8ba9b4383a016df8fdefe0ae28382a7/aiohttp-3.13.3-cp314-cp314t-win_amd64.whl", hash = "sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344", size = 499694, upload-time = "2026-01-03T17:32:24.546Z" }, ] +[[package]] +name = "aiohttp-socks" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "python-socks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/cc/e5bbd54f76bd56291522251e47267b645dac76327b2657ade9545e30522c/aiohttp_socks-0.11.0.tar.gz", hash = "sha256:0afe51638527c79077e4bd6e57052c87c4824233d6e20bb061c53766421b10f0", size = 11196, upload-time = "2025-12-09T13:35:52.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/7d/4b633d709b8901d59444d2e512b93e72fe62d2b492a040097c3f7ba017bb/aiohttp_socks-0.11.0-py3-none-any.whl", hash = "sha256:9aacce57c931b8fbf8f6d333cf3cafe4c35b971b35430309e167a35a8aab9ec1", size = 10556, upload-time = "2025-12-09T13:35:50.18Z" }, +] + [[package]] name = "aioitertools" version = "0.12.0" @@ -202,6 +215,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] +[[package]] +name = "apscheduler" +version = "3.11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/12/3e4389e5920b4c1763390c6d371162f3784f86f85cd6d6c1bfe68eef14e2/apscheduler-3.11.2.tar.gz", hash = "sha256:2a9966b052ec805f020c8c4c3ae6e6a06e24b1bf19f2e11d91d8cca0473eef41", size = 108683, upload-time = "2025-12-22T00:39:34.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/64/2e54428beba8d9992aa478bb8f6de9e4ecaa5f8f513bcfd567ed7fb0262d/apscheduler-3.11.2-py3-none-any.whl", hash = "sha256:ce005177f741409db4e4dd40a7431b76feb856b9dd69d57e0da49d6715bfd26d", size = 64439, upload-time = "2025-12-22T00:39:33.303Z" }, +] + [[package]] name = "argparse-ext" version = "1.4.2" @@ -335,7 +360,9 @@ dependencies = [ { name = "aiobotocore" }, { name = "aiofiles" }, { name = "aiohttp" }, + { name = "aiohttp-socks" }, { name = "aiokafka" }, + { name = "apscheduler" }, { name = "async-lru" }, { name = "awscrt" }, { name = "bidict" }, @@ -371,7 +398,9 @@ requires-dist = [ { name = "aiobotocore", specifier = "==3.1.0" }, { name = "aiofiles", specifier = "==25.1.0" }, { name = "aiohttp", specifier = "==3.13.3" }, + { name = "aiohttp-socks", specifier = "==0.11.0" }, { name = "aiokafka", specifier = "==0.12.0" }, + { name = "apscheduler", specifier = "==3.11.2" }, { name = "async-lru", specifier = "==2.0.5" }, { name = "awscrt", specifier = "==0.31.0" }, { name = "bidict", specifier = "==0.23.1" }, @@ -1619,6 +1648,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, ] +[[package]] +name = "python-socks" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/07/cfdd6a846ac859e513b4e68bb6c669a90a74d89d8d405516fba7fc9c6f0c/python_socks-2.8.0.tar.gz", hash = "sha256:340f82778b20a290bdd538ee47492978d603dff7826aaf2ce362d21ad9ee6f1b", size = 273130, upload-time = "2025-12-09T12:17:05.433Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/10/e2b575faa32d1d32e5e6041fc64794fa9f09526852a06b25353b66f52cae/python_socks-2.8.0-py3-none-any.whl", hash = "sha256:57c24b416569ccea493a101d38b0c82ed54be603aa50b6afbe64c46e4a4e4315", size = 55075, upload-time = "2025-12-09T12:17:03.269Z" }, +] + [[package]] name = "pywin32" version = "311" @@ -1957,6 +1995,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, ] +[[package]] +name = "tzdata" +version = "2025.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "urllib3" version = "2.4.0"