Merge pull request #125 from openzim/rework-components-add-shuttle

Rework backend to isolate the mill and the shuttle

Author: benoit74

.github/workflows/Publish.yml

@@ -42,18 +42,60 @@ jobs:
       - name: Retrieve source code
         uses: actions/checkout@v5
 
-      - name: Build and publish Docker Image
+      - name: Build and publish Docker Image for backend mill
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/cms-mill
+          on-master: latest
+          restrict-to: openzim/cms
+          context: backend
+          dockerfile: Dockerfile-mill
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+
+      - name: Build and publish Docker Image for backend shuttle
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/cms-shuttle
+          on-master: latest
+          restrict-to: openzim/cms
+          context: backend
+          dockerfile: Dockerfile-shuttle
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+
+      - name: Build and publish Docker Image for backend API
         uses: openzim/docker-publish-action@v10
         with:
           image-name: openzim/cms-api
           on-master: latest
           restrict-to: openzim/cms
           context: backend
+          dockerfile: Dockerfile-api
           registries: ghcr.io
           credentials:
             GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
             GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
 
+      # mill should be first since it deploys the alembic migration
+      - name: Deploy CMS Mill changes to cms.openzim.org
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+        with:
+          args: rollout restart deployments mill-deployment -n cms
+
+      - name: Deploy CMS Shuttle changes to cms.openzim.org
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+        with:
+          args: rollout restart deployments shuttle-deployment -n cms
+
       - name: Deploy CMS API changes to cms.openzim.org
         uses: actions-hub/kubectl@master
         env:

.github/workflows/backend-Tests.yml

@@ -55,24 +55,32 @@ jobs:
           psql -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'
           "host=localhost port=5432 dbname=cmstest user=cms password=cmspass"
 
-      - name: Build backend Docker image
+      - name: Build backend mill Docker image
         working-directory: backend
-        run: docker build -t cms-backend:test .
+        run: docker build -t cms-backend-mill:test -f Dockerfile-mill .
 
-      - name: Run backend container
+      - name: Build backend shuttle Docker image
+        working-directory: backend
+        run: docker build -t cms-backend-shuttle:test -f Dockerfile-shuttle .
+
+      - name: Build backend API Docker image
+        working-directory: backend
+        run: docker build -t cms-backend-api:test -f Dockerfile-api .
+
+      - name: Run backend api container
         run: |
           docker run -d --name cms-backend-test \
             -e POSTGRES_URI=$POSTGRES_URI \
             -e JWT_SECRET=DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw \
             -p 8000:80 \
-            cms-backend:test
+            cms-backend-api:test
           # wait for container to be ready
           for i in {1..10}; do
             if curl -s http://localhost:8000/v2/healthcheck > /dev/null; then
-              echo "Backend is up!"
+              echo "Backend API is up!"
               break
             fi
-            echo "Waiting for backend..."
+            echo "Waiting for backend API..."
             sleep 3
           done
 
@@ -81,6 +89,7 @@ jobs:
         env:
           DATABASE_URL: postgresql+psycopg://cms:cmspass@localhost:5432/cmstest
           JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
+          ALEMBIC_UPGRADE_HEAD_ON_START: false
         run: inv coverage --args "-vvv"
 
       - name: Upload coverage report to codecov

README.md

@@ -10,7 +10,10 @@ You are welcome to use it, report bugs and request features!
 
 As an internal tool, we don't provide releases nor changelogs for the CMS. Please check the respective source-code for the two components of the CMS:
 
-- `backend`: the python backend API available via `ghcr.io/openzim/cms-api` docker image.
+- `backend`: the python backend, with 3 sub-components:
+  - the `api`, responsible to provide an API (sic), available via `ghcr.io/openzim/cms-api` docker image
+  - the `mill`, responsible to process background tasks, available via `ghcr.io/openzim/cms-mill` docker image
+  - the `shuttle`, responsible to move files around, available via `ghcr.io/openzim/cms-shuttle` docker image
 - `frontend`: the Vue.js Web UI available via `ghcr.io/openzim/cms-ui` docker image.
 
 ```sh

backend/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.13-slim-bookworm
+LABEL org.opencontainers.image.source https://github.com/openzim/cms
+
+ENV ALEMBIC_UPGRADE_HEAD_ON_START=false
+
+# curl needed for healthcheck
+RUN apt-get update && apt-get install -y curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# install Python dependencies first (since they change more rarely than src code)
+COPY pyproject.toml README.md /app/
+COPY src/cms_backend/__about__.py /app/src/cms_backend/__about__.py
+RUN pip install --no-cache-dir '/app[api]'
+
+COPY src /app/src
+
+RUN pip install --no-cache-dir '/app[api]' \
+    && rm -rf /app/src \
+    && rm /app/pyproject.toml \
+    && rm /app/README.md
+
+EXPOSE 80
+
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
+    CMD curl -f http://localhost/v1/healthcheck || exit 1
+
+CMD ["uvicorn", "cms_backend.api.main:app", "--host", "0.0.0.0", "--port", "80"]

backend/Dockerfile-api

@@ -0,0 +1,24 @@
+FROM python:3.13-slim-bookworm
+LABEL org.opencontainers.image.source https://github.com/openzim/cms
+
+ENV ALEMBIC_UPGRADE_HEAD_ON_START=true
+
+# Install dependencies
+RUN apt-get update && apt-get install -y curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies first (since they change more rarely than src code)
+COPY pyproject.toml README.md /app/
+COPY src/cms_backend/__about__.py /app/src/cms_backend/__about__.py
+RUN pip install --no-cache-dir /app
+
+COPY src /app/src
+
+RUN pip install --no-cache-dir /app \
+    && rm -rf /app/src \
+    && rm /app/pyproject.toml \
+    && rm /app/README.md
+
+WORKDIR /app
+
+CMD ["python", "-m", "cms_backend.mill.main"]

backend/Dockerfile-mill

@@ -0,0 +1,24 @@
+FROM python:3.13-slim-bookworm
+LABEL org.opencontainers.image.source https://github.com/openzim/cms
+
+ENV ALEMBIC_UPGRADE_HEAD_ON_START=false
+
+# Install dependencies
+RUN apt-get update && apt-get install -y curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies first (since they change more rarely than src code)
+COPY pyproject.toml README.md /app/
+COPY src/cms_backend/__about__.py /app/src/cms_backend/__about__.py
+RUN pip install --no-cache-dir /app
+
+COPY src /app/src
+
+RUN pip install --no-cache-dir /app \
+    && rm -rf /app/src \
+    && rm /app/pyproject.toml \
+    && rm /app/README.md
+
+WORKDIR /app
+
+CMD ["python", "-m", "cms_backend.shuttle.main"]

backend/Dockerfile-shuttle

@@ -18,16 +18,19 @@ classifiers = [
 ]
 dependencies = [
     "alembic == 1.15.2",
-    "fastapi[all] == 0.115.2",
     "pydantic == 2.11.4", # this is also a sub-dep of fastapi but we rely a lot on it
     "SQLAlchemy == 2.0.41",
     "pymongo == 4.13.0",
     "psycopg[binary,pool] == 3.2.11",
-    "regex == 2025.10.23"
+    "regex == 2025.10.23",
+    "humanfriendly == 10.0",
 ]
 dynamic = ["version"]
 
 [project.optional-dependencies]
+api = [
+    "fastapi[all] == 0.115.2",
+]
 scripts = [
     "invoke == 2.2.0",
 ]
@@ -37,6 +40,7 @@ lint = [
 ]
 check = [
     "pyright == 1.1.400",
+    "types-humanfriendly == 10.0.0"
 ]
 test = [
     "coverage == 7.8.0",
@@ -46,10 +50,12 @@ test = [
     "pytest-env == 1.1.5",
     "Faker==37.3.0",
     "python-dateutil==2.9.0.post0",
+    "cms_backend[api]",
 ]
 dev = [
     "pre-commit == 4.2.0",
     "ipykernel == 6.29.5",
+    "cms_backend[api]",
     "cms_backend[scripts]",
     "cms_backend[lint]",
     "cms_backend[test]",
@@ -61,6 +67,9 @@ Homepage = "https://github.com/openzim/cms/backend"
 Donate = "https://www.kiwix.org/en/support-us/"
 
 [project.scripts]
+cms-api = "cms_backend.api.main:app"
+cms-mill = "cms_backend.mill.main:main"
+cms-shuttle = "cms_backend.shuttle.main:main"
 create-initial-user = "cms_backend.utils.database:create_initial_user"
 check-db-schema = "cms_backend.utils.database:check_if_schema_is_up_to_date"
 

backend/pyproject.toml

@@ -9,19 +9,19 @@
 from fastapi.responses import JSONResponse
 from pydantic import ValidationError
 
+from cms_backend.api.routes.books import router as books_router
+from cms_backend.api.routes.healthcheck import router as healthcheck_router
+from cms_backend.api.routes.http_errors import BadRequestError
+from cms_backend.api.routes.titles import router as titles_router
+from cms_backend.api.routes.warehouse_paths import router as warehouse_paths_router
+from cms_backend.api.routes.zimfarm_notifications import (
+    router as zimfarm_notification_router,
+)
 from cms_backend.db.exceptions import (
     RecordAlreadyExistsError,
     RecordDisabledError,
     RecordDoesNotExistError,
 )
-from cms_backend.routes.books import router as books_router
-from cms_backend.routes.healthcheck import router as healthcheck_router
-from cms_backend.routes.http_errors import BadRequestError
-from cms_backend.routes.titles import router as titles_router
-from cms_backend.routes.warehouse_paths import router as warehouse_paths_router
-from cms_backend.routes.zimfarm_notifications import (
-    router as zimfarm_notification_router,
-)
 from cms_backend.utils.database import (
     upgrade_db_schema,
 )