Morphisms between diagrammatic instances: data structures and validation

.github/workflows/ci.yml

@@ -75,6 +75,37 @@ jobs:
         run: |
           RUSTDOCFLAGS='--deny warnings' cargo doc --no-deps
 
+  frontend_tests:
+    name: frontend tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "pnpm"
+
+      - name: Install Rust toolchain from file
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          # Don't override flags in cargo config files.
+          rustflags: ""
+
+      - name: Build frontend
+        run: |
+          pnpm install
+          pnpm --filter ./packages/frontend run build -- --mode development
+
+      - name: Run frontend tests
+        run: |
+          pnpm --filter ./packages/frontend run test:no-backend
+
   npm_checks:
     name: npm checks
     runs-on: ubuntu-latest
@@ -98,3 +129,26 @@ jobs:
       - name: Format/linting/import sorting
         run: |
           pnpm --filter "./packages/*" run ci
+
+  build_nixos_system:
+    name: build NixOS system
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install Nix
+      uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:25.05
+
+    - name: Configure Cachix
+      uses: cachix/cachix-action@v14
+      with:
+        name: catcolab-jmoggr
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+    - name: Build the NixOS system for catcolab-next
+      run: nix build .#nixosConfigurations.catcolab-next.config.system.build.toplevel
+

.github/workflows/deploy.yml

@@ -216,21 +216,36 @@ jobs:
     name: Deploy backend to AWS
     runs-on: ubuntu-latest
     if: github.event_name != 'pull_request' && github.ref_name == 'main'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up SSH key
-        run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.CATCOLAB_NEXT_DEPLOYUSER_KEY }}" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan backend-next.catcolab.org >> ~/.ssh/known_hosts
 
-      - name: Rsync the repo to remote host
-        run: |
-           rsync -az --delete ./ catcolab@backend-next.catcolab.org:~/catcolab
-
-      - name: Run nixos-rebuild switch remotely
-        run:
-          ssh catcolab@backend-next.catcolab.org "sudo nixos-rebuild switch --flake ./catcolab#catcolab-next"
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Install Nix
+      uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:25.05
+
+    - name: Configure Cachix
+      uses: cachix/cachix-action@v14
+      with:
+        name: catcolab-jmoggr
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+    - name: Set the SSH key for deployment to catcolab-next
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ secrets.CATCOLAB_NEXT_DEPLOYUSER_KEY }}" > ~/.ssh/id_ed25519
+        chmod 600 ~/.ssh/id_ed25519
+        ssh-keyscan backend-next.catcolab.org >> ~/.ssh/known_hosts
+
+    - name: Deploy to catcolab-next
+      run: |
+       nix run github:serokell/deploy-rs .#catcolab-next
+
+    # Ensure that a copy of the deployed repository is on the machine that it was deployed to. This is
+    # a nice-to-have which enables checking the configuration of the currently deployed system and could
+    # make recovery slightly less aweful in the event nix commands need to be run on the remote.
+    - name: Rsync the repo to remote host
+      run: |
+         rsync -az --delete ./ catcolab@backend-next.catcolab.org:~/catcolab

flake.nix

@@ -320,57 +320,16 @@
         };
       };
 
-      # The backend relies on Firebase, so tests require VM internet access. Enable networking by running
-      # with --no-sandbox.
-      # Docs for nixos tests: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-test-nodes
-      # (google and LLMs are useless)
-      checks.x86_64-linux.integrationTests = nixpkgs.legacyPackages.x86_64-linux.testers.runNixOSTest {
-        name = "Integration Tests";
-
-        skipTypeCheck = true;
-        nodes = {
-          catcolab = import ./infrastructure/hosts/catcolab-vm;
-        };
-
-        node.specialArgs = {
-          inherit inputs self;
-          rustToolchain = rustToolchainLinux;
-        };
-
-        # NOTE: This only checks if the services "start" from systemds perspective, not if they are not
-        # failed immediately after starting...
-        testScript = ''
-          def dump_logs(machine, *units):
-              for u in units:
-                  print(f"\n===== journal for {u} =====")
-                  print(machine.succeed(f"journalctl -u {u} --no-pager"))
-
-          def test_service(machine, service):
-              try:
-                  machine.wait_for_unit(service)
-              except:
-                  dump_logs(machine, service)
-                  raise
-
-          def test_oneshot_service(machine, service):
-              try:
-                  machine.wait_until_succeeds(
-                      f"test $(systemctl is-active {service}) = inactive"
-                  )
-              except:
-                  dump_logs(machine, service)
-                  raise
-
-          test_oneshot_service(catcolab, "database-setup.service")
-          test_oneshot_service(catcolab, "migrations.service")
-
-          test_service(catcolab, "automerge.service");
-          test_service(catcolab, "backend.service");
-          test_service(catcolab, "caddy.service");
-
-          catcolab.start_job("backupdb.service")
-          test_oneshot_service(catcolab, "backupdb.service")
-        '';
-      };
+      # Temporarily disabled until more meaningful tests are developed. Keeping the frontend dependecies
+      # up to date is currently not worth the hassle.
+      # checks.x86_64-linux.integrationTests = import ./infrastructure/tests/integration.nix {
+      #   inherit
+      #     nixpkgs
+      #     inputs
+      #     self
+      #     linuxSystem
+      #     ;
+      #   rustToolchain = rustToolchainLinux;
+      # };
     };
 }

infrastructure/hosts/catcolab-next/default.nix

@@ -25,7 +25,7 @@ in
     };
 
     catcolabSecrets = {
-      file = ../../secrets/.env.next.age;
+      file = ../../secrets/env.next.age;
       mode = "400";
       owner = "catcolab";
     };

infrastructure/hosts/catcolab/default.nix

@@ -22,7 +22,7 @@ in
       owner = "catcolab";
     };
     catcolabSecrets = {
-      file = ../../secrets/.env.prod.age;
+      file = ../../secrets/env.prod.age;
       owner = "catcolab";
     };
   };

infrastructure/secrets/secrets.nix

@@ -7,14 +7,14 @@ let
   catcolab-next-deployuser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM7AYg1fZM0zMxb/BuZTSwK4O3ycUIHruApr1tKoO8nJ deployuser@next.catcolab.org";
 in
 builtins.mapAttrs (_: publicKeys: { inherit publicKeys; }) ({
-  ".env.next.age" = [
+  "env.next.age" = [
     catcolab-next
     owen
     epatters
     jmoggr
     catcolab-next-deployuser
   ];
-  ".env.prod.age" = [
+  "env.prod.age" = [
     catcolab
     owen
     epatters

infrastructure/tests/integration.nix

@@ -0,0 +1,59 @@
+# The backend relies on Firebase, so tests require VM internet access. Enable networking by running
+# with --no-sandbox.
+# Docs for nixos tests: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-test-nodes
+# (google and LLMs are useless)
+{
+  nixpkgs,
+  inputs,
+  self,
+  rustToolchain,
+  linuxSystem,
+}:
+nixpkgs.legacyPackages.${linuxSystem}.testers.runNixOSTest {
+  name = "Integration Tests";
+
+  skipTypeCheck = true;
+
+  nodes = {
+    catcolab = import ../hosts/catcolab-vm;
+  };
+
+  node.specialArgs = {
+    inherit inputs self rustToolchain;
+  };
+
+  # NOTE: This only checks if the services "start" from systemds perspective, not if they are not
+  # failed immediately after starting...
+  testScript = ''
+    def dump_logs(machine, *units):
+        for u in units:
+            print(f"\n===== journal for {u} =====")
+            print(machine.succeed(f"journalctl -u {u} --no-pager"))
+
+    def test_service(machine, service):
+        try:
+            machine.wait_for_unit(service)
+        except:
+            dump_logs(machine, service)
+            raise
+
+    def test_oneshot_service(machine, service):
+        try:
+            machine.wait_until_succeeds(
+                f"test $(systemctl is-active {service}) = inactive"
+            )
+        except:
+            dump_logs(machine, service)
+            raise
+
+    test_oneshot_service(catcolab, "database-setup.service")
+    test_oneshot_service(catcolab, "migrations.service")
+
+    test_service(catcolab, "automerge.service");
+    test_service(catcolab, "backend.service");
+    test_service(catcolab, "caddy.service");
+
+    catcolab.start_job("backupdb.service")
+    test_oneshot_service(catcolab, "backupdb.service")
+  '';
+}

packages/automerge-doc-server/src/server.ts

@@ -15,11 +15,21 @@ import type { Pool as PoolType } from "pg";
 import { PostgresStorageAdapter } from "./postgres_storage_adapter.js";
 import type { NewDocSocketResponse, StartListeningSocketResponse } from "./types.js";
 import type { SocketIOHandlers } from "./socket.js";
+import { serializeError } from "./socket.js";
 import jsonpatch from "fast-json-patch";
 
 // Load environment variables from .env
 dotenv.config();
 
+/** Attempt to migrate a document, returning the migrated document or an error message. */
+function migrateDocument(doc: unknown): { Ok: any } | { Err: string } {
+    try {
+        return { Ok: notbookTypes.migrateDocument(doc) };
+    } catch (e) {
+        return { Err: `Failed to migrate document: ${serializeError(e)}` };
+    }
+}
+
 export class AutomergeServer implements SocketIOHandlers {
     private docMap: Map<string, DocHandle<unknown>>;
 
@@ -66,7 +76,12 @@ export class AutomergeServer implements SocketIOHandlers {
     }
 
     async createDoc(content: unknown): Promise<NewDocSocketResponse> {
-        const handle = this.repo.create(content);
+        const migrateResult = migrateDocument(content);
+        if ("Err" in migrateResult) {
+            return migrateResult;
+        }
+
+        const handle = this.repo.create(migrateResult.Ok);
         if (!handle) {
             return {
                 Err: "Failed to create a new document",
@@ -133,14 +148,18 @@ export class AutomergeServer implements SocketIOHandlers {
         //
         // XXX: frontend/src/api/document.ts needs to be kept up to date with this
         const docBefore = handle.doc();
-        const docAfter = notbookTypes.migrateDocument(docBefore);
+        const migrateResult = migrateDocument(docBefore);
+        if ("Err" in migrateResult) {
+            return migrateResult;
+        }
+        const docAfter = migrateResult.Ok;
+
         if ((docBefore as any).version !== docAfter.version) {
             const patches = jsonpatch.compare(docBefore as any, docAfter);
             handle.change((doc: any) => {
                 jsonpatch.applyPatch(doc, patches);
             });
         }
-
         this.docMap.set(refId, handle);
 
         return { Ok: null };