IEBH-353: Sync with the latest changes

.env.example

@@ -1,13 +1,7 @@
-# Vault configuration
 # is sensitive/secret
-VAULT_TOKEN=
 RDS_PASSWORD=
 REDIS_PASSWORD=
 
-# contains defaults but can be overriden
-VAULT_URL=
-VAULT_CRT=
-
 # Microservices
 # needs to be set (no defaults)
 QUEUE_SERVICE=
@@ -37,7 +31,6 @@ RDS_ECHO_SQL_QUERIES=false
 
 # Service configuration
 # contains defaults but can be overriden
-CONFIG_CENTER_ENABLED=false
 APP_NAME=dataops_service
 VERSION=2.0.0
 PORT=5063

.github/workflows/hdc-pipeline.yml

@@ -0,0 +1,37 @@
+name: HDC ci/cd pipeline
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_tests_hdc:
+    uses: PilotDataPlatform/pilot-hdc-ci-tools/.github/workflows/run_tests.yml@main
+    with:
+      min_coverage_percent: 92
+      coverage_target: 'dataops'
+    secrets: inherit
+
+  build_and_publish_hdc:
+    needs: [run_tests_hdc]
+    uses: PilotDataPlatform/pilot-hdc-ci-tools/.github/workflows/build_and_publish.yml@main
+    with:
+      matrix_config: '["alembic","dataops"]'
+      service_name: 'dataops'
+    secrets: inherit
+
+  deploy_hdc:
+    needs: [build_and_publish_hdc]
+    uses: PilotDataPlatform/pilot-hdc-ci-tools/.github/workflows/trigger_deployment.yml@main
+    with:
+      hdc_service_name: 'dataops'
+    secrets: inherit

.gitignore

@@ -36,6 +36,11 @@ tests/logs
 *.manifest
 *.spec
 
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt

.pre-commit-config.yaml

@@ -67,15 +67,15 @@ repos:
           '--max-line-length=120',
         ]
 
-#  - repo: https://github.com/PyCQA/docformatter
-#    rev: v1.7.5
-#    hooks:
-#      - id: docformatter
-#        args: [
-#          '--wrap-summaries=120',
-#          '--wrap-descriptions=120',
-#          '--in-place',
-#        ]
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.7
+    hooks:
+      - id: docformatter
+        args: [
+          '--wrap-summaries=120',
+          '--wrap-descriptions=120',
+          '--in-place',
+        ]
 
   - repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.5.5

dataops/components/archive_preview/views.py

@@ -4,7 +4,6 @@
 # Version 3.0 (the "License") available at https://www.gnu.org/licenses/agpl-3.0.en.html.
 # You may not use this file except in compliance with the License.
 
-from typing import Union
 from uuid import UUID
 
 from fastapi import APIRouter
@@ -23,7 +22,7 @@
 
 @router.get('', summary='Get a archive preview given file id', response_model=ArchivePreviewResponseSchema)
 async def get_archive_preview(
-    file_id: Union[UUID, str], archive_preview_crud: ArchivePreviewCRUD = Depends(get_archive_preview_crud)
+    file_id: UUID | str, archive_preview_crud: ArchivePreviewCRUD = Depends(get_archive_preview_crud)
 ) -> ArchivePreviewResponseSchema:
     """Get an archive preview by id or code."""
 

dataops/components/cache.py

@@ -4,9 +4,6 @@
 # Version 3.0 (the "License") available at https://www.gnu.org/licenses/agpl-3.0.en.html.
 # You may not use this file except in compliance with the License.
 
-from typing import Optional
-from typing import Union
-
 from redis.asyncio.client import Redis
 
 
@@ -16,12 +13,12 @@ class Cache:
     def __init__(self, redis: Redis) -> None:
         self.redis = redis
 
-    async def set(self, key: str, value: Union[str, bytes]) -> bool:
+    async def set(self, key: str, value: str | bytes) -> bool:
         """Set the value for the key."""
 
         return await self.redis.set(key, value)
 
-    async def get(self, key: str) -> Optional[bytes]:
+    async def get(self, key: str) -> bytes | None:
         """Return the value for the key or None if the key doesn't exist."""
 
         return await self.redis.get(key)

dataops/components/crud.py

@@ -7,8 +7,6 @@
 from datetime import timedelta
 from typing import Any
 from typing import Optional
-from typing import Type
-from typing import Union
 from uuid import UUID
 
 from pydantic import BaseModel
@@ -35,7 +33,7 @@ class CRUD:
     """Base CRUD class for managing database models."""
 
     session: AsyncSession
-    model: Type[DBModel]
+    model: type[DBModel]
     db_error_codes: dict[str, ServiceException] = {
         '23503': NotFound(),  # missing foreign key
         '23505': AlreadyExists(),  # duplicated entry
@@ -65,7 +63,7 @@ def select_query(self) -> Select:
         """Create base select."""
         return select(self.model)
 
-    async def execute(self, statement: Executable, **kwds: Any) -> Union[CursorResult, Result]:
+    async def execute(self, statement: Executable, **kwds: Any) -> CursorResult | Result:
         """Execute a statement and return buffered result."""
 
         return await self.session.execute(statement, **kwds)
@@ -75,7 +73,7 @@ async def scalars(self, statement: Executable, **kwds: Any) -> ScalarResult:
 
         return await self.session.scalars(statement, **kwds)
 
-    async def _create_one(self, statement: Executable) -> Union[UUID, str]:
+    async def _create_one(self, statement: Executable) -> UUID | str:
         """Execute a statement to create one entry."""
 
         try:
@@ -155,7 +153,7 @@ async def set_by_key(self, key: str, content: str, expire: int = Optional[None])
 
     async def mget_by_prefix(self, prefix: str) -> bytes:
         """Query to find key with pattern and retrieve respective record."""
-        query = '{}:*'.format(prefix)
+        query = f'{prefix}:*'
         keys = await self.__instance.keys(query)
         return await self.__instance.mget(keys)
 
@@ -169,7 +167,7 @@ async def unlink_by_key(self, key: str) -> int:
 
     async def mdele_by_prefix(self, prefix: str) -> list:
         """Query to find key with pattern and delete respective record."""
-        query = '{}:*'.format(prefix)
+        query = f'{prefix}:*'
         keys = await self.__instance.keys(query)
         results = []
         for key in keys:

dataops/components/exceptions.py

@@ -6,14 +6,14 @@
 
 from abc import ABCMeta
 from abc import abstractmethod
+from collections.abc import Sequence
 from http.client import BAD_REQUEST
 from http.client import CONFLICT
 from http.client import INTERNAL_SERVER_ERROR
 from http.client import NOT_FOUND
 from http.client import SERVICE_UNAVAILABLE
 from http.client import UNAUTHORIZED
 from http.client import UNPROCESSABLE_ENTITY
-from typing import Sequence
 
 from pydantic.error_wrappers import ErrorList
 

dataops/components/resource_lock/cache.py

@@ -4,8 +4,6 @@
 # Version 3.0 (the "License") available at https://www.gnu.org/licenses/agpl-3.0.en.html.
 # You may not use this file except in compliance with the License.
 
-from typing import List
-
 from dataops.components.cache import Cache
 from dataops.components.resource_lock.schemas import ResourceLockBulkResponseSchema
 from dataops.components.resource_lock.schemas import ResourceLockResponseSchema
@@ -15,15 +13,15 @@
 class ResourceLockerCache(Cache):
     """Manages resource operation locking and unlocking and their respective lock statuses."""
 
-    def str_to_int_list(self, input_: bytes) -> List[int]:
+    def str_to_int_list(self, input_: bytes) -> list[int]:
         """Return decoded strings as a list of integers."""
         return [int(x) for x in input_.decode('utf-8').split(',')]
 
-    def int_list_to_str(self, input_: List[int]) -> str:
+    def int_list_to_str(self, input_: list[int]) -> str:
         """Return joined list as a string with delimiter."""
         return ','.join([str(x) for x in input_])
 
-    async def perform_bulk_lock(self, keys: List[str], operation: str) -> ResourceLockBulkResponseSchema:
+    async def perform_bulk_lock(self, keys: list[str], operation: str) -> ResourceLockBulkResponseSchema:
         """Perform bulk lock for multiple keys.
 
         If one of the lock attempts fails, the locking attempts of the following keys are stopped.
@@ -46,7 +44,7 @@ async def perform_bulk_lock(self, keys: List[str], operation: str) -> ResourceLo
 
         return ResourceLockBulkResponseSchema(keys_status=status)
 
-    async def perform_bulk_unlock(self, keys: List[str], operation: str) -> ResourceLockBulkResponseSchema:
+    async def perform_bulk_unlock(self, keys: list[str], operation: str) -> ResourceLockBulkResponseSchema:
         """Perform bulk unlock for multiple keys."""
 
         keys = sorted(keys)