Dev

.github/workflows/Dev-Container-Publish.yaml

@@ -1,9 +1,7 @@
 name: Dev Container Publish
 
 on:
-  push:
-    branches:
-      - dev
+  workflow_dispatch:
 
 jobs:
   prepare:

.github/workflows/Sync-dev-with-main.yml

@@ -35,14 +35,14 @@ jobs:
       - name: Update the version to <latest>.dev
         id: update_version
         run: |
-          version_file="genpipes/__version__.py"
+          version_file="project_tracking/__version__.py"
           version_number=$(sed -n "s/__version__ = '\([^']*\)'/\1/p" $version_file)
           if [[ "$version_number" != *.dev ]]; then
             echo "__version__ = '${version_number}.dev'" > $version_file
           fi
 
       - name: Commit changes
         run: |
-          git add genpipes/__version__.py
+          git add project_tracking/__version__.py
           git commit -m "Dev Version update"
           git push --force-with-lease

.github/workflows/Tag-and-Release.yml

@@ -1,4 +1,4 @@
-name: Tag and Release Workflow
+name: Tag and Release
 
 on:
   pull_request:

.github/workflows/Update-Version-and-create-Realeases-PR.yml

@@ -1,4 +1,4 @@
-name: Update version and create Release's PR Workflow
+name: Update version and create Release's PR
 
 on:
   workflow_dispatch:

migration/alembic/versions/64a560017524_convert_json_to_jsonb.py

@@ -0,0 +1,172 @@
+"""Convert JSON to JSONB
+
+Revision ID: 64a560017524
+Revises: cacb85b3d114
+Create Date: 2025-10-14 15:55:59.691048
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '64a560017524'
+down_revision: Union[str, None] = 'cacb85b3d114'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('experiment', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('file', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('job', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('location', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('metric', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('operation', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('operation_config', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('project', 'alias',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('project', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('readset', 'alias',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('readset', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('reference', 'alias',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('reference', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('run', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('sample', 'alias',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('sample', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('specimen', 'alias',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    op.alter_column('specimen', 'extra_metadata',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               existing_nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('specimen', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('specimen', 'alias',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('sample', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('sample', 'alias',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('run', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('reference', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('reference', 'alias',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('readset', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('readset', 'alias',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('project', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('project', 'alias',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('operation_config', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('operation', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('metric', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('location', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('job', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('file', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    op.alter_column('experiment', 'extra_metadata',
+               existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    # ### end Alembic commands ###

project_tracking/db_actions/route.py

@@ -96,7 +96,7 @@ def metrics(project_id, session, deliverable=None, specimen_id=None, sample_id=N
             Metric.deleted.is_(deleted),
             Project.id.in_(project_id)
         )
-    )
+    ).distinct()
 
     if metric_id:
         if isinstance(metric_id, int):
@@ -118,7 +118,7 @@ def metrics(project_id, session, deliverable=None, specimen_id=None, sample_id=N
         # Filter metrics based on deliverable status
         stmt = stmt.where(Metric.deliverable.is_(deliverable))
 
-    result = list({metric.id: metric for metric in session.execute(stmt).scalars()}.values())
+    result = session.execute(stmt).scalars().all()
 
     if not result:
         ret["DB_ACTION_WARNING"].append(

project_tracking/model.py

@@ -28,6 +28,8 @@
 
 from sqlalchemy.sql import func
 
+from sqlalchemy.dialects.postgresql import JSONB
+
 from sqlalchemy.ext.mutable import MutableDict, MutableList
 
 from . import database
@@ -216,7 +218,7 @@ class BaseTable(Base):
     creation: Mapped[DateTime] = Column(DateTime(timezone=True), server_default=func.now())
     modification: Mapped[DateTime] = Column(DateTime(timezone=True), onupdate=func.now())
     # extra_metadata: Mapped[dict] = mapped_column(mutable_json_type(dbtype=JSON, nested=True), default=None, nullable=True)
-    extra_metadata: Mapped[dict] = mapped_column(MutableDict.as_mutable(JSON), default=None, nullable=True)
+    extra_metadata: Mapped[dict] = mapped_column(MutableDict.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=dict, nullable=True)
     ext_id: Mapped[int] = mapped_column(default=None, nullable=True)
     ext_src: Mapped[str] = mapped_column(default=None, nullable=True)
 
@@ -316,7 +318,7 @@ class Project(BaseTable):
     )
 
     name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True)
-    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True)
+    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True)
 
     specimens: Mapped[list[Specimen]] = relationship(back_populates="project", cascade="all, delete")
     operations: Mapped[list[Operation]] = relationship(back_populates="project", cascade="all, delete")
@@ -344,7 +346,7 @@ class Specimen(BaseTable):
 
     project_id: Mapped[int] = mapped_column(ForeignKey("project.id"), default=None)
     name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True)
-    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True)
+    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True)
     cohort: Mapped[str] = mapped_column(default=None, nullable=True)
     institution: Mapped[str] = mapped_column(default=None, nullable=True)
 
@@ -413,7 +415,7 @@ class Sample(BaseTable):
 
     specimen_id: Mapped[int] = mapped_column(ForeignKey("specimen.id"), default=None)
     name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True)
-    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True)
+    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True)
     tumour: Mapped[bool] = mapped_column(default=False)
 
     specimen: Mapped[Specimen] = relationship(back_populates="samples")
@@ -566,7 +568,7 @@ def from_attributes(cls, ext_id=None, ext_src=None, name=None, instrument=None,
         ).first()
 
         if run:
-            warning = f"Run with id {run.id} already exists, informations will be attached to this one."
+            warning = f"Run with id {run.id} and name {run.name} already exists, informations will be attached to this one."
 
         if not run:
             run = cls(
@@ -614,7 +616,7 @@ class Readset(BaseTable):
     experiment_id: Mapped[int] = mapped_column(ForeignKey("experiment.id"), default=None)
     run_id: Mapped[int] = mapped_column(ForeignKey("run.id"), default=None)
     name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True)
-    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True)
+    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True)
     lane: Mapped[LaneEnum]  =  mapped_column(default=None, nullable=True)
     adapter1: Mapped[str] = mapped_column(default=None, nullable=True)
     adapter2: Mapped[str] = mapped_column(default=None, nullable=True)
@@ -738,7 +740,7 @@ def from_attributes(
         ).first()
 
         if operation:
-            warning = f"Operation with id {operation.id} already exists, informations will be attached to this one."
+            warning = f"Operation with id {operation.id} and name {operation.name} already exists, informations will be attached to this one."
         else:
             operation = cls(
                 operation_config=operation_config,
@@ -773,7 +775,7 @@ class Reference(BaseTable):
     __tablename__ = "reference"
 
     name: Mapped[str] = mapped_column(default=None, nullable=True)
-    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True)
+    alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True)
     assembly: Mapped[str] = mapped_column(default=None, nullable=True)
     version: Mapped[str] = mapped_column(default=None, nullable=True)
     taxon_id: Mapped[str] = mapped_column(default=None, nullable=True)
@@ -825,7 +827,7 @@ def from_attributes(cls, name=None, version=None, md5sum=None, data=None, sessio
         ).first()
 
         if operation_config:
-            warning = f"OperationConfig with id {operation_config.id} already exists, informations will be attached to this one."
+            warning = f"OperationConfig with id {operation_config.id} and name {operation_config.name} already exists, informations will be attached to this one."
 
         if not operation_config:
             operation_config = cls(
@@ -927,7 +929,7 @@ def from_attributes(
         ).first()
 
         if job:
-            warning = f"Job with id {job.id} already exists, informations will be attached to this one."
+            warning = f"Job with id {job.id} and name {job.name} already exists, informations will be attached to this one."
 
         if not job:
             job = cls(
@@ -1054,7 +1056,7 @@ def get_or_create(
                 cls.name == name,
                 cls.deprecated.is_(deprecated),
                 cls.deleted.is_(deleted),
-                Job.id.is_(job.id)
+                Job.id == job.id
                 # Readset.id.in_([readset.id])
             )
         )

tests/test_ingestion.py

@@ -48,7 +48,7 @@ def test_create_api(client, run_processing_json, transfer_json, genpipes_json, d
     assert json.loads(response.data)["DB_ACTION_OUTPUT"][0]['status'] == "COMPLETED"
     # Test ingesting genpipes a 2nd time to make sure it fetches the already existing entrities and links the new information to them
     response = client.post(f'project/{project_name}/ingest_genpipes', data=json.dumps(genpipes_json))
-    assert json.loads(response.data)["DB_ACTION_WARNING"] == ["OperationConfig with id 1 already exists, informations will be attached to this one.","Operation with id 3 already exists, informations will be attached to this one.","Job with id 3 already exists, informations will be attached to this one.","Job with id 4 already exists, informations will be attached to this one.","Job with id 5 already exists, informations will be attached to this one.","Job with id 4 already exists, informations will be attached to this one.","Job with id 6 already exists, informations will be attached to this one.","Job with id 4 already exists, informations will be attached to this one."]
+    assert json.loads(response.data)["DB_ACTION_WARNING"] == ['OperationConfig with id 1 and name genpipes_ini already exists, informations will be attached to this one.', 'Operation with id 3 and name genpipes already exists, informations will be attached to this one.', 'Job with id 3 and name trimmomatic already exists, informations will be attached to this one.', 'Job with id 4 and name kallisto already exists, informations will be attached to this one.', 'Job with id 5 and name trimmomatic already exists, informations will be attached to this one.', 'Job with id 4 and name kallisto already exists, informations will be attached to this one.', 'Job with id 6 and name trimmomatic already exists, informations will be attached to this one.', 'Job with id 4 and name kallisto already exists, informations will be attached to this one.']
     # Test ingesting delivery
     response = client.post(f'project/{project_name}/ingest_delivery', data=json.dumps(delivery_json))
     assert response.status_code == 200