SurfaceManager

.gitignore

@@ -1,3 +1,6 @@
 docs/javascript/
 docs/react/
 node_modules/
+
+# Incremental compilation that speeds up monorepo builds
+*.tsbuildinfo

package.json

@@ -22,4 +22,4 @@
     "vite": "^7.2.0",
     "ws": "^8.18.3"
   }
-}
+}

packages/javascript/.gitignore

@@ -2,3 +2,4 @@ node_modules
 dist
 yarn-error.log
 docs
+cypress/screenshots

packages/javascript/cypress.config.ts

@@ -0,0 +1,26 @@
+import { defineConfig } from 'cypress';
+
+export default defineConfig({
+  defaultBrowser: 'electron',
+  component: {
+    devServer: {
+      framework: 'react',
+      bundler: 'vite',
+    },
+  },
+
+  viewportHeight: 720,
+  viewportWidth: 1280,
+
+  e2e: {
+    setupNodeEvents(on) {
+      // This is required to log progress to the terminal whilst generating frames
+      on('task', {
+        log(message) {
+          console.log(message);
+          return null;
+        },
+      });
+    },
+  },
+});

packages/javascript/cypress/component/AudioStability.cy.ts

@@ -0,0 +1,156 @@
+import { SurfaceManager } from '../../src/state-based/SurfaceManager';
+
+describe('Audio stability tests', () => {
+  it('can wait without playing', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/sinwave@440hz.wav',
+        type: 'audio',
+        audioOutput: '',
+        keyframes: [
+          [now, { set: { t: 0, rate: 0 } }], // paused at start
+          [now + 60_000, { set: { rate: 1 } }], // play in 1 minute
+        ],
+      },
+    });
+    cy.mount(manager);
+
+    cy.get('audio').should('have.prop', 'paused', true);
+    cy.get('audio').should('have.prop', 'currentTime', 0);
+  });
+
+  it('recovers from a pause', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/metronome@120bpm.wav',
+        type: 'audio',
+        audioOutput: '',
+        keyframes: [[now, { set: { t: 0, rate: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.log('Interfere with audio element');
+    cy.get('audio').should('have.prop', 'paused', false);
+    cy.get('audio').invoke('trigger', 'pause');
+    cy.get('audio').should('have.prop', 'paused', true);
+
+    cy.wait(1000);
+
+    cy.log('audio should have recovered');
+    cy.get('audio').should('have.prop', 'paused', false);
+    cy.get('audio').invoke('prop', 'currentTime').should('be.greaterThan', 1.5);
+  });
+
+  it('recovers from a play', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/metronome@120bpm.wav',
+        type: 'audio',
+        audioOutput: '',
+        keyframes: [[now, { set: { t: 1_500, rate: 0 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    // Wait until audio ready
+    cy.get('audio')
+      .invoke('prop', 'currentTime')
+      .should(($time) => expect(parseFloat($time)).to.be.closeTo(1.5, 0.1));
+
+    cy.log('Interfere with audio element');
+    cy.get('audio').invoke('prop', 'paused').should('be.true');
+    cy.get('audio').invoke('prop', 'playbackRate', 1);
+    cy.get('audio').then(($audio) => $audio.get(0).play().catch(/* do nothing*/));
+    cy.get('audio').invoke('prop', 'paused').should('be.false');
+
+    cy.wait(1000);
+
+    cy.log('audio should have recovered');
+    cy.get('audio')
+      .invoke('prop', 'currentTime')
+      .should(($time) => expect(parseFloat($time)).to.be.closeTo(1.5, 0.1));
+  });
+
+  it('recovers from a seek', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/metronome@120bpm.wav',
+        type: 'audio',
+        audioOutput: '',
+        keyframes: [[now, { set: { t: 0, rate: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.log('Interfere with audio element');
+    cy.get('audio').invoke('prop', 'currentTime', 5);
+
+    cy.wait(500);
+
+    cy.log('audio should have recovered');
+    cy.get('audio').invoke('prop', 'currentTime').should('be.lessThan', 2);
+  });
+
+  it('recovers from volume change', () => {
+    const INITIAL_VOLUME = 0;
+    const CHANGED_VOLUME = 1;
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        type: 'audio',
+        file: 'cypress/fixtures/sinwave@440hz.wav',
+        audioOutput: '',
+        keyframes: [[now, { set: { t: 0, rate: 1, volume: INITIAL_VOLUME } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.get('audio').invoke('prop', 'volume', CHANGED_VOLUME);
+    cy.get('audio').should('have.prop', 'volume', CHANGED_VOLUME);
+
+    cy.wait(1000);
+
+    cy.get('audio').should('have.prop', 'volume', INITIAL_VOLUME);
+  });
+
+  it('recovers from audio element deletion', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        type: 'audio',
+        file: 'cypress/fixtures/sinwave@440hz.wav',
+        audioOutput: '',
+        keyframes: [[now, { set: { t: 0, rate: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.get('audio').should('exist');
+    cy.get('audio').invoke('remove');
+    cy.get('audio').should('not.exist');
+
+    cy.wait(1000);
+
+    cy.get('audio').should('exist');
+  });
+
+  it('smoothly returns to correct time using playbackRate', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        type: 'audio',
+        file: 'cypress/fixtures/metronome@120bpm.wav',
+        audioOutput: '',
+        keyframes: [[now - 500, { set: { t: 0, rate: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.get('audio').invoke('prop', 'currentTime').should('be.lessThan', 2);
+  });
+});

packages/javascript/cypress/component/ImageStability.cy.ts

@@ -0,0 +1,78 @@
+import { SurfaceManager } from '../../src/state-based/SurfaceManager';
+
+describe('Image stability tests', () => {
+  it('can show an image', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/indianred@2560x1440.png',
+        type: 'image',
+        fit: 'cover',
+        keyframes: [[now, { set: { opacity: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+    cy.get('img').should('exist');
+  });
+
+  it("doesn't show a queued image", () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/indianred@2560x1440.png',
+        type: 'image',
+        fit: 'cover',
+        keyframes: [
+          [now + 60_000, { set: { opacity: 1 } }], // show image in 1 minute
+        ],
+      },
+    });
+    cy.mount(manager);
+    cy.get('img').should('not.exist');
+  });
+
+  it('recovers from img element src change', () => {
+    const ORIGINAL_SRC = 'cypress/fixtures/indianred@2560x1440.png';
+    const CHANGED_SRC = '404.png';
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: ORIGINAL_SRC,
+        type: 'image',
+        fit: 'cover',
+        keyframes: [[now, { set: { opacity: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.get('img').should('exist');
+    cy.get('img').invoke('prop', 'src', CHANGED_SRC);
+
+    cy.wait(1_000);
+
+    cy.get('img')
+      .invoke('prop', 'src')
+      .should(($src) => expect($src).to.contain(ORIGINAL_SRC));
+  });
+
+  it('recovers from img element deletion', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({
+      'clip-id': {
+        file: 'cypress/fixtures/indianred@2560x1440.png',
+        type: 'image',
+        fit: 'cover',
+        keyframes: [[now, { set: { opacity: 1 } }]],
+      },
+    });
+    cy.mount(manager);
+
+    cy.get('img').should('exist');
+    cy.get('img').invoke('remove');
+    cy.get('img').should('not.exist');
+
+    cy.wait(1_000);
+
+    cy.get('img').should('exist');
+  });
+});

packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts

@@ -0,0 +1,61 @@
+import { DATA_CLIP_ID, SurfaceManager } from '../../src/state-based/SurfaceManager';
+
+describe('Updating surface state', () => {
+  it('adds and removes a video clip', () => {
+    const manager = new SurfaceManager({});
+    cy.mount(manager);
+
+    cy.get('video')
+      .should('not.exist')
+      .then(() => {
+        const now = Date.now();
+        manager.setState({
+          'clip-id': {
+            file: 'cypress/fixtures/2x2s@2560x1440.mp4',
+            type: 'video',
+            audioOutput: '',
+            fit: 'cover',
+            keyframes: [
+              [now + 1_000, { set: { t: 0, rate: 1 } }], // play in 1s
+            ],
+          },
+        });
+      })
+      .then(() => {
+        // This implicitly waits
+        cy.get('video').should('exist');
+      })
+      .then(() => {
+        manager.setState({});
+      })
+      .then(() => {
+        cy.get('video').should('not.exist');
+      });
+  });
+
+  it('adds multiple media', () => {
+    const now = Date.now();
+    const manager = new SurfaceManager({});
+    cy.mount(manager);
+    expect(manager.element.children.length).to.eq(0);
+
+    manager.setState({
+      'image-background': {
+        type: 'image',
+        file: 'cypress/fixtures/indianred@2560x1440.png',
+        fit: 'cover',
+        keyframes: [[now, {}]],
+      },
+      'video-foreground': {
+        type: 'video',
+        fit: 'contain',
+        audioOutput: '',
+        file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4',
+        keyframes: [[now, {}]],
+      },
+    });
+
+    cy.get(`[${DATA_CLIP_ID}=image-background]`).should('exist');
+    cy.get(`[${DATA_CLIP_ID}=video-foreground]`).should('exist');
+  });
+});