diff --git a/.github/workflows/Dev-Container-Publish.yaml b/.github/workflows/Dev-Container-Publish.yaml index 732bd30..57954c9 100644 --- a/.github/workflows/Dev-Container-Publish.yaml +++ b/.github/workflows/Dev-Container-Publish.yaml @@ -1,9 +1,7 @@ name: Dev Container Publish on: - push: - branches: - - dev + workflow_dispatch: jobs: prepare: diff --git a/.github/workflows/Sync-dev-with-main.yml b/.github/workflows/Sync-dev-with-main.yml index 84e717b..fc1df13 100644 --- a/.github/workflows/Sync-dev-with-main.yml +++ b/.github/workflows/Sync-dev-with-main.yml @@ -35,7 +35,7 @@ jobs: - name: Update the version to .dev id: update_version run: | - version_file="genpipes/__version__.py" + version_file="project_tracking/__version__.py" version_number=$(sed -n "s/__version__ = '\([^']*\)'/\1/p" $version_file) if [[ "$version_number" != *.dev ]]; then echo "__version__ = '${version_number}.dev'" > $version_file @@ -43,6 +43,6 @@ jobs: - name: Commit changes run: | - git add genpipes/__version__.py + git add project_tracking/__version__.py git commit -m "Dev Version update" git push --force-with-lease diff --git a/.github/workflows/Tag-and-Release.yml b/.github/workflows/Tag-and-Release.yml index 1dc2df3..18f8955 100644 --- a/.github/workflows/Tag-and-Release.yml +++ b/.github/workflows/Tag-and-Release.yml @@ -1,4 +1,4 @@ -name: Tag and Release Workflow +name: Tag and Release on: pull_request: diff --git a/.github/workflows/Update-Version-and-create-Realeases-PR.yml b/.github/workflows/Update-Version-and-create-Realeases-PR.yml index 8e9dc46..410b7e5 100644 --- a/.github/workflows/Update-Version-and-create-Realeases-PR.yml +++ b/.github/workflows/Update-Version-and-create-Realeases-PR.yml @@ -1,4 +1,4 @@ -name: Update version and create Release's PR Workflow +name: Update version and create Release's PR on: workflow_dispatch: diff --git a/migration/alembic/versions/64a560017524_convert_json_to_jsonb.py b/migration/alembic/versions/64a560017524_convert_json_to_jsonb.py new file mode 100644 index 0000000..bdc9376 --- /dev/null +++ b/migration/alembic/versions/64a560017524_convert_json_to_jsonb.py @@ -0,0 +1,172 @@ +"""Convert JSON to JSONB + +Revision ID: 64a560017524 +Revises: cacb85b3d114 +Create Date: 2025-10-14 15:55:59.691048 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '64a560017524' +down_revision: Union[str, None] = 'cacb85b3d114' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('experiment', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('file', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('job', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('location', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('metric', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('operation', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('operation_config', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('project', 'alias', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('project', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('readset', 'alias', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('readset', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('reference', 'alias', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('reference', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('run', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('sample', 'alias', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('sample', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('specimen', 'alias', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + op.alter_column('specimen', 'extra_metadata', + existing_type=postgresql.JSON(astext_type=sa.Text()), + type_=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + existing_nullable=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('specimen', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('specimen', 'alias', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('sample', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('sample', 'alias', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('run', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('reference', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('reference', 'alias', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('readset', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('readset', 'alias', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('project', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('project', 'alias', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('operation_config', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('operation', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('metric', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('location', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('job', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('file', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + op.alter_column('experiment', 'extra_metadata', + existing_type=sa.JSON().with_variant(postgresql.JSONB(astext_type=sa.Text()), 'postgresql'), + type_=postgresql.JSON(astext_type=sa.Text()), + existing_nullable=True) + # ### end Alembic commands ### diff --git a/project_tracking/db_actions/route.py b/project_tracking/db_actions/route.py index 46d540e..75eeccf 100644 --- a/project_tracking/db_actions/route.py +++ b/project_tracking/db_actions/route.py @@ -96,7 +96,7 @@ def metrics(project_id, session, deliverable=None, specimen_id=None, sample_id=N Metric.deleted.is_(deleted), Project.id.in_(project_id) ) - ) + ).distinct() if metric_id: if isinstance(metric_id, int): @@ -118,7 +118,7 @@ def metrics(project_id, session, deliverable=None, specimen_id=None, sample_id=N # Filter metrics based on deliverable status stmt = stmt.where(Metric.deliverable.is_(deliverable)) - result = list({metric.id: metric for metric in session.execute(stmt).scalars()}.values()) + result = session.execute(stmt).scalars().all() if not result: ret["DB_ACTION_WARNING"].append( diff --git a/project_tracking/model.py b/project_tracking/model.py index d2b4f7c..10b1c60 100644 --- a/project_tracking/model.py +++ b/project_tracking/model.py @@ -28,6 +28,8 @@ from sqlalchemy.sql import func +from sqlalchemy.dialects.postgresql import JSONB + from sqlalchemy.ext.mutable import MutableDict, MutableList from . import database @@ -216,7 +218,7 @@ class BaseTable(Base): creation: Mapped[DateTime] = Column(DateTime(timezone=True), server_default=func.now()) modification: Mapped[DateTime] = Column(DateTime(timezone=True), onupdate=func.now()) # extra_metadata: Mapped[dict] = mapped_column(mutable_json_type(dbtype=JSON, nested=True), default=None, nullable=True) - extra_metadata: Mapped[dict] = mapped_column(MutableDict.as_mutable(JSON), default=None, nullable=True) + extra_metadata: Mapped[dict] = mapped_column(MutableDict.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=dict, nullable=True) ext_id: Mapped[int] = mapped_column(default=None, nullable=True) ext_src: Mapped[str] = mapped_column(default=None, nullable=True) @@ -316,7 +318,7 @@ class Project(BaseTable): ) name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True) - alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True) + alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True) specimens: Mapped[list[Specimen]] = relationship(back_populates="project", cascade="all, delete") operations: Mapped[list[Operation]] = relationship(back_populates="project", cascade="all, delete") @@ -344,7 +346,7 @@ class Specimen(BaseTable): project_id: Mapped[int] = mapped_column(ForeignKey("project.id"), default=None) name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True) - alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True) + alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True) cohort: Mapped[str] = mapped_column(default=None, nullable=True) institution: Mapped[str] = mapped_column(default=None, nullable=True) @@ -413,7 +415,7 @@ class Sample(BaseTable): specimen_id: Mapped[int] = mapped_column(ForeignKey("specimen.id"), default=None) name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True) - alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True) + alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True) tumour: Mapped[bool] = mapped_column(default=False) specimen: Mapped[Specimen] = relationship(back_populates="samples") @@ -566,7 +568,7 @@ def from_attributes(cls, ext_id=None, ext_src=None, name=None, instrument=None, ).first() if run: - warning = f"Run with id {run.id} already exists, informations will be attached to this one." + warning = f"Run with id {run.id} and name {run.name} already exists, informations will be attached to this one." if not run: run = cls( @@ -614,7 +616,7 @@ class Readset(BaseTable): experiment_id: Mapped[int] = mapped_column(ForeignKey("experiment.id"), default=None) run_id: Mapped[int] = mapped_column(ForeignKey("run.id"), default=None) name: Mapped[str] = mapped_column(default=None, nullable=False, unique=True) - alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True) + alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True) lane: Mapped[LaneEnum] = mapped_column(default=None, nullable=True) adapter1: Mapped[str] = mapped_column(default=None, nullable=True) adapter2: Mapped[str] = mapped_column(default=None, nullable=True) @@ -738,7 +740,7 @@ def from_attributes( ).first() if operation: - warning = f"Operation with id {operation.id} already exists, informations will be attached to this one." + warning = f"Operation with id {operation.id} and name {operation.name} already exists, informations will be attached to this one." else: operation = cls( operation_config=operation_config, @@ -773,7 +775,7 @@ class Reference(BaseTable): __tablename__ = "reference" name: Mapped[str] = mapped_column(default=None, nullable=True) - alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON), default=list, nullable=True) + alias: Mapped[list] = mapped_column(MutableList.as_mutable(JSON().with_variant(JSONB, 'postgresql')), default=list, nullable=True) assembly: Mapped[str] = mapped_column(default=None, nullable=True) version: Mapped[str] = mapped_column(default=None, nullable=True) taxon_id: Mapped[str] = mapped_column(default=None, nullable=True) @@ -825,7 +827,7 @@ def from_attributes(cls, name=None, version=None, md5sum=None, data=None, sessio ).first() if operation_config: - warning = f"OperationConfig with id {operation_config.id} already exists, informations will be attached to this one." + warning = f"OperationConfig with id {operation_config.id} and name {operation_config.name} already exists, informations will be attached to this one." if not operation_config: operation_config = cls( @@ -927,7 +929,7 @@ def from_attributes( ).first() if job: - warning = f"Job with id {job.id} already exists, informations will be attached to this one." + warning = f"Job with id {job.id} and name {job.name} already exists, informations will be attached to this one." if not job: job = cls( @@ -1054,7 +1056,7 @@ def get_or_create( cls.name == name, cls.deprecated.is_(deprecated), cls.deleted.is_(deleted), - Job.id.is_(job.id) + Job.id == job.id # Readset.id.in_([readset.id]) ) ) diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index 8dc8e50..1a1db2a 100644 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -48,7 +48,7 @@ def test_create_api(client, run_processing_json, transfer_json, genpipes_json, d assert json.loads(response.data)["DB_ACTION_OUTPUT"][0]['status'] == "COMPLETED" # Test ingesting genpipes a 2nd time to make sure it fetches the already existing entrities and links the new information to them response = client.post(f'project/{project_name}/ingest_genpipes', data=json.dumps(genpipes_json)) - assert json.loads(response.data)["DB_ACTION_WARNING"] == ["OperationConfig with id 1 already exists, informations will be attached to this one.","Operation with id 3 already exists, informations will be attached to this one.","Job with id 3 already exists, informations will be attached to this one.","Job with id 4 already exists, informations will be attached to this one.","Job with id 5 already exists, informations will be attached to this one.","Job with id 4 already exists, informations will be attached to this one.","Job with id 6 already exists, informations will be attached to this one.","Job with id 4 already exists, informations will be attached to this one."] + assert json.loads(response.data)["DB_ACTION_WARNING"] == ['OperationConfig with id 1 and name genpipes_ini already exists, informations will be attached to this one.', 'Operation with id 3 and name genpipes already exists, informations will be attached to this one.', 'Job with id 3 and name trimmomatic already exists, informations will be attached to this one.', 'Job with id 4 and name kallisto already exists, informations will be attached to this one.', 'Job with id 5 and name trimmomatic already exists, informations will be attached to this one.', 'Job with id 4 and name kallisto already exists, informations will be attached to this one.', 'Job with id 6 and name trimmomatic already exists, informations will be attached to this one.', 'Job with id 4 and name kallisto already exists, informations will be attached to this one.'] # Test ingesting delivery response = client.post(f'project/{project_name}/ingest_delivery', data=json.dumps(delivery_json)) assert response.status_code == 200