IEBH-353: Sync with the latest changes

.env.example

@@ -10,7 +10,6 @@ LOGGING_FORMAT=             # example: json
 KAFKA_URL=                  # example: kafka:29099
 KAFKA_TOPIC=                # example: metadata.items
 AUTH_HOST=                  # example: http://auth_service
-CONFIG_CENTER_ENABLED=      # example: false
 OPSDB_UTILITY_USERNAME=     # example: postgres
 OPSDB_UTILITY_HOST=         # example: db
 OPSDB_UTILITY_PORT=         # example: 5432
@@ -25,7 +24,4 @@ CORE_ZONE_VALUE=            # example: 1
 
 # is sensitive/secret
 OPSDB_UTILITY_PASSWORD=     # example: postgres
-VAULT_URL=                  # example: http://vault.vault
-VAULT_CRT=                  # example: /secrets/ca.crt
-VAULT_TOKEN=                # example: vault_token
 RSA_PUBLIC_KEY=             # example: MIGeMA0GCSqGSIb...

.github/workflows/hdc-pipeline.yml

@@ -0,0 +1,36 @@
+name: HDC ci/cd pipeline
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_tests_hdc:
+    uses: PilotDataPlatform/pilot-hdc-ci-tools/.github/workflows/run_tests.yml@main
+    with:
+      min_coverage_percent: 86
+    secrets: inherit
+
+  build_and_publish_hdc:
+    needs: [run_tests_hdc]
+    uses: PilotDataPlatform/pilot-hdc-ci-tools/.github/workflows/build_and_publish.yml@main
+    with:
+      matrix_config: '["alembic","metadata"]'
+      service_name: 'metadata'
+    secrets: inherit
+
+  deploy_hdc:
+    needs: [build_and_publish_hdc]
+    uses: PilotDataPlatform/pilot-hdc-ci-tools/.github/workflows/trigger_deployment.yml@main
+    with:
+      hdc_service_name: 'metadata'
+    secrets: inherit

.pre-commit-config.yaml

@@ -67,15 +67,15 @@ repos:
           '--max-line-length=120',
         ]
 
-#  - repo: https://github.com/PyCQA/docformatter
-#    rev: v1.7.5
-#    hooks:
-#      - id: docformatter
-#        args: [
-#          '--wrap-summaries=120',
-#          '--wrap-descriptions=120',
-#          '--in-place',
-#        ]
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.7
+    hooks:
+      - id: docformatter
+        args: [
+          '--wrap-summaries=120',
+          '--wrap-descriptions=120',
+          '--in-place',
+        ]
 
   - repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.5.5

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker-registry.ebrains.eu/hdc-services-image/base-image:python-3.10.12-v2 AS production-environment
+FROM docker-registry.ebrains.eu/hdc-services-image/base-image:python-3.10.14-v1 AS production-environment
 
 ENV PYTHONDONTWRITEBYTECODE=true \
     PYTHONIOENCODING=UTF-8 \

app/config.py

@@ -7,42 +7,13 @@
 import base64
 import logging
 from functools import lru_cache
-from typing import Any
-from typing import Dict
-from typing import Optional
 
-from common import VaultClient
 from pydantic import BaseSettings
 from pydantic import Extra
 
 
-class VaultConfig(BaseSettings):
-    """Store vault related configuration."""
-
-    APP_NAME: str = 'metadata'
-    CONFIG_CENTER_ENABLED: bool = False
-
-    VAULT_URL: Optional[str]
-    VAULT_CRT: Optional[str]
-    VAULT_TOKEN: Optional[str]
-
-    class Config:
-        env_file = '.env'
-        env_file_encoding = 'utf-8'
-
-
-def load_vault_settings(settings: BaseSettings) -> Dict[str, Any]:
-    config = VaultConfig()
-
-    if not config.CONFIG_CENTER_ENABLED:
-        return {}
-
-    client = VaultClient(config.VAULT_URL, config.VAULT_CRT, config.VAULT_TOKEN)
-    return client.get_from_vault(config.APP_NAME)
-
-
 class Settings(BaseSettings):
-    version = '2.2.0'
+    version = '2.2.10'
     APP_NAME: str = 'metadata_service'
     PORT: int = 5065
     HOST: str = '0.0.0.0'
@@ -78,10 +49,6 @@ class Config:
         env_file_encoding = 'utf-8'
         extra = Extra.allow
 
-        @classmethod
-        def customise_sources(cls, init_settings, env_settings, file_secret_settings):
-            return env_settings, load_vault_settings, init_settings, file_secret_settings
-
     def __init__(self):
         super().__init__()
         self.AUTH_SERVICE = self.AUTH_HOST + '/v1/'

app/models/models_attribute_templates.py

@@ -4,8 +4,6 @@
 # Version 3.0 (the "License") available at https://www.gnu.org/licenses/agpl-3.0.en.html.
 # You may not use this file except in compliance with the License.
 
-from typing import List
-from typing import Optional
 from uuid import UUID
 
 from pydantic import BaseModel
@@ -21,7 +19,7 @@ class GETTemplate(BaseModel):
 
 class GETTemplates(BaseModel):
     project_code: str
-    name: Optional[str]
+    name: str | None
     page_size: int = 10
     page: int = 0
 
@@ -54,7 +52,7 @@ class POSTTemplateAttributes(BaseModel):
     name: str
     optional: bool = True
     type: str = 'text'
-    options: Optional[list[str]]
+    options: list[str] | None
 
     @validator('type')
     def type_validation(cls, v):
@@ -66,7 +64,7 @@ def type_validation(cls, v):
 class POSTTemplate(BaseModel):
     name: str
     project_code: str
-    attributes: List[POSTTemplateAttributes]
+    attributes: list[POSTTemplateAttributes]
 
 
 class POSTTemplateResponse(GETTemplateResponse):

app/models/models_collections.py

@@ -5,7 +5,6 @@
 # You may not use this file except in compliance with the License.
 
 import re
-from typing import Optional
 from uuid import UUID
 
 from pydantic import BaseModel
@@ -87,7 +86,7 @@ class GETCollectionItemsResponse(APIResponse):
 
 
 class POSTCollection(BaseModel):
-    id: Optional[UUID] = Field(example='3fa85f64-5717-4562-b3fc-2c963f66afa6')
+    id: UUID | None = Field(example='3fa85f64-5717-4562-b3fc-2c963f66afa6')
     owner: str
     container_code: str
     name: str

app/models/models_items.py

@@ -6,7 +6,6 @@
 
 from datetime import datetime
 from enum import Enum
-from typing import Optional
 from uuid import UUID
 
 from pydantic import BaseModel
@@ -21,17 +20,19 @@
 
 
 class ItemStatus(str, Enum):
-    """The new enum type for file status
+    """The new enum type for file status.
+
     - REGISTERED means file is created by upload service but not complete yet. either in progress or fail.
     - ACTIVE means file uploading is complete.
-    - ARCHIVED means the file has been deleted."""
+    - ARCHIVED means the file has been deleted.
+    """
 
     REGISTERED = 'REGISTERED'
     ACTIVE = 'ACTIVE'
     ARCHIVED = 'ARCHIVED'
 
     def __str__(self):
-        return '%s' % self.name
+        return f'{self.name}'
 
 
 class ItemType(str, Enum):
@@ -40,15 +41,15 @@ class ItemType(str, Enum):
     NAME_FOLDER = 'name_folder'
 
     def __str__(self):
-        return '%s' % self.name
+        return f'{self.name}'
 
 
 class ContainerType(str, Enum):
     PROJECT = 'project'
     DATASET = 'dataset'
 
     def __str__(self):
-        return '%s' % self.name
+        return f'{self.name}'
 
 
 class GETItem(BaseModel):
@@ -57,7 +58,7 @@ class GETItem(BaseModel):
 
 class GETItemByLocation(BaseModel):
     name: str
-    parent_path: str
+    parent_path: str | None
     container_code: str
     container_type: ContainerType
     zone: int
@@ -70,21 +71,21 @@ class GETItemsByIDs(BaseModel):
 
 
 class GETItemsByLocation(PaginationRequest):
-    container_code: Optional[str]
-    zone: Optional[int]
+    container_code: str | None
+    zone: int | None
     recursive: bool = False
     status: ItemStatus = ItemStatus.ACTIVE
-    parent_path: Optional[str]
-    restore_path: Optional[str]
-    name: Optional[str]
-    owner: Optional[str]
-    type: Optional[str]
-    container_type: Optional[str]
-    auth_user: Optional[str]
-    project_role: Optional[str]
-    fav_user: Optional[str]
-    last_updated_start: Optional[datetime] = None
-    last_updated_end: Optional[datetime] = None
+    parent_path: str | None
+    restore_path: str | None
+    name: str | None
+    owner: str | None
+    type: str | None
+    container_type: str | None
+    auth_user: str | None
+    project_role: str | None
+    fav_user: str | None
+    last_updated_start: datetime | None = None
+    last_updated_end: datetime | None = None
 
     class Config:
         anystr_strip_whitespace = True
@@ -126,9 +127,9 @@ class GETItemResponse(APIResponse):
 
 
 class POSTItem(BaseModel):
-    id: Optional[UUID]
-    parent: Optional[UUID] = Field(example='3fa85f64-5717-4562-b3fc-2c963f66afa6')
-    parent_path: Optional[str] = Field(example='path/to/file')
+    id: UUID | None
+    parent: UUID | None = Field(example='3fa85f64-5717-4562-b3fc-2c963f66afa6')
+    parent_path: str | None = Field(example='path/to/file')
     container_code: str
     container_type: str = 'project'
     type: str = 'file'
@@ -137,15 +138,15 @@ class POSTItem(BaseModel):
     name: str = Field(example='file_name.txt')
     size: int = 0
     owner: str
-    upload_id: Optional[str]
-    location_uri: Optional[str]
-    version: Optional[str]
+    upload_id: str | None
+    location_uri: str | None
+    version: str | None
     tags: list[str] = []
     system_tags: list[str] = []
-    attribute_template_id: Optional[UUID]
-    attributes: Optional[dict] = {}
-    tfrm_source: Optional[UUID] = None
-    tfrm_type: Optional[TransformationType] = None
+    attribute_template_id: UUID | None
+    attributes: dict | None = {}
+    tfrm_source: UUID | None = None
+    tfrm_type: TransformationType | None = None
 
     class Config:
         anystr_strip_whitespace = True
@@ -237,22 +238,22 @@ class POSTItemResponse(GETItemResponse):
 
 
 class PUTItem(POSTItem):
-    parent: Optional[UUID] = Field(example='3fa85f64-5717-4562-b3fc-2c963f66afa6', default='')
-    parent_path: Optional[str] = Field(example='path/to/file', default='')
-    type: Optional[str]
-    status: Optional[ItemStatus]
-    zone: Optional[int]
-    name: Optional[str] = Field(example='file_name.txt')
-    size: Optional[int]
-    owner: Optional[str]
-    container_code: Optional[str]
-    container_type: Optional[str]
-    location_uri: Optional[str]
-    version: Optional[str]
-    tags: Optional[list[str]]
-    system_tags: Optional[list[str]]
-    attribute_template_id: Optional[UUID]
-    attributes: Optional[dict]
+    parent: UUID | None = Field(example='3fa85f64-5717-4562-b3fc-2c963f66afa6', default='')
+    parent_path: str | None = Field(example='path/to/file', default='')
+    type: str | None
+    status: ItemStatus | None
+    zone: int | None
+    name: str | None = Field(example='file_name.txt')
+    size: int | None
+    owner: str | None
+    container_code: str | None
+    container_type: str | None
+    location_uri: str | None
+    version: str | None
+    tags: list[str] | None
+    system_tags: list[str] | None
+    attribute_template_id: UUID | None
+    attributes: dict | None
 
     class Config:
         anystr_strip_whitespace = True
@@ -275,9 +276,9 @@ class DELETEItemResponse(APIResponse):
 
 
 class PUTItemsBequeath(BaseModel):
-    attribute_template_id: Optional[UUID]
-    attributes: Optional[dict]
-    system_tags: Optional[list[str]]
+    attribute_template_id: UUID | None
+    attributes: dict | None
+    system_tags: list[str] | None
 
 
 class PUTItemsBequeathResponse(GETItemResponse):

app/models/models_lineage_provenance.py

@@ -20,7 +20,7 @@ class TransformationType(str, Enum):
     ARCHIVE = 'archive'
 
     def __str__(self):
-        return '%s' % self.name
+        return f'{self.name}'
 
 
 class GETLineageProvenance(BaseModel):

app/routers/router_utils.py

@@ -4,7 +4,7 @@
 # Version 3.0 (the "License") available at https://www.gnu.org/licenses/agpl-3.0.en.html.
 # You may not use this file except in compliance with the License.
 
-from typing import Callable
+from collections.abc import Callable
 
 from pydantic import BaseModel