diff --git a/.gitignore b/.gitignore index ea49801..a31c2f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ docs/javascript/ docs/react/ node_modules/ + +# Incremental compilation that speeds up monorepo builds +*.tsbuildinfo \ No newline at end of file diff --git a/package.json b/package.json index 047bcd2..b6933ca 100644 --- a/package.json +++ b/package.json @@ -22,4 +22,4 @@ "vite": "^7.2.0", "ws": "^8.18.3" } -} \ No newline at end of file +} diff --git a/packages/javascript/.gitignore b/packages/javascript/.gitignore index 94b95d1..8e7e28a 100644 --- a/packages/javascript/.gitignore +++ b/packages/javascript/.gitignore @@ -2,3 +2,4 @@ node_modules dist yarn-error.log docs +cypress/screenshots \ No newline at end of file diff --git a/packages/javascript/cypress.config.ts b/packages/javascript/cypress.config.ts new file mode 100644 index 0000000..c8e8d88 --- /dev/null +++ b/packages/javascript/cypress.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from 'cypress'; + +export default defineConfig({ + defaultBrowser: 'electron', + component: { + devServer: { + framework: 'react', + bundler: 'vite', + }, + }, + + viewportHeight: 720, + viewportWidth: 1280, + + e2e: { + setupNodeEvents(on) { + // This is required to log progress to the terminal whilst generating frames + on('task', { + log(message) { + console.log(message); + return null; + }, + }); + }, + }, +}); diff --git a/packages/javascript/cypress/component/AudioStability.cy.ts b/packages/javascript/cypress/component/AudioStability.cy.ts new file mode 100644 index 0000000..da5acb4 --- /dev/null +++ b/packages/javascript/cypress/component/AudioStability.cy.ts @@ -0,0 +1,156 @@ +import { SurfaceManager } from '../../src/state-based/SurfaceManager'; + +describe('Audio stability tests', () => { + it('can wait without playing', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/sinwave@440hz.wav', + type: 'audio', + audioOutput: '', + keyframes: [ + [now, { set: { t: 0, rate: 0 } }], // paused at start + [now + 60_000, { set: { rate: 1 } }], // play in 1 minute + ], + }, + }); + cy.mount(manager); + + cy.get('audio').should('have.prop', 'paused', true); + cy.get('audio').should('have.prop', 'currentTime', 0); + }); + + it('recovers from a pause', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/metronome@120bpm.wav', + type: 'audio', + audioOutput: '', + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.log('Interfere with audio element'); + cy.get('audio').should('have.prop', 'paused', false); + cy.get('audio').invoke('trigger', 'pause'); + cy.get('audio').should('have.prop', 'paused', true); + + cy.wait(1000); + + cy.log('audio should have recovered'); + cy.get('audio').should('have.prop', 'paused', false); + cy.get('audio').invoke('prop', 'currentTime').should('be.greaterThan', 1.5); + }); + + it('recovers from a play', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/metronome@120bpm.wav', + type: 'audio', + audioOutput: '', + keyframes: [[now, { set: { t: 1_500, rate: 0 } }]], + }, + }); + cy.mount(manager); + + // Wait until audio ready + cy.get('audio') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.closeTo(1.5, 0.1)); + + cy.log('Interfere with audio element'); + cy.get('audio').invoke('prop', 'paused').should('be.true'); + cy.get('audio').invoke('prop', 'playbackRate', 1); + cy.get('audio').then(($audio) => $audio.get(0).play().catch(/* do nothing*/)); + cy.get('audio').invoke('prop', 'paused').should('be.false'); + + cy.wait(1000); + + cy.log('audio should have recovered'); + cy.get('audio') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.closeTo(1.5, 0.1)); + }); + + it('recovers from a seek', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/metronome@120bpm.wav', + type: 'audio', + audioOutput: '', + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.log('Interfere with audio element'); + cy.get('audio').invoke('prop', 'currentTime', 5); + + cy.wait(500); + + cy.log('audio should have recovered'); + cy.get('audio').invoke('prop', 'currentTime').should('be.lessThan', 2); + }); + + it('recovers from volume change', () => { + const INITIAL_VOLUME = 0; + const CHANGED_VOLUME = 1; + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + type: 'audio', + file: 'cypress/fixtures/sinwave@440hz.wav', + audioOutput: '', + keyframes: [[now, { set: { t: 0, rate: 1, volume: INITIAL_VOLUME } }]], + }, + }); + cy.mount(manager); + + cy.get('audio').invoke('prop', 'volume', CHANGED_VOLUME); + cy.get('audio').should('have.prop', 'volume', CHANGED_VOLUME); + + cy.wait(1000); + + cy.get('audio').should('have.prop', 'volume', INITIAL_VOLUME); + }); + + it('recovers from audio element deletion', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + type: 'audio', + file: 'cypress/fixtures/sinwave@440hz.wav', + audioOutput: '', + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.get('audio').should('exist'); + cy.get('audio').invoke('remove'); + cy.get('audio').should('not.exist'); + + cy.wait(1000); + + cy.get('audio').should('exist'); + }); + + it('smoothly returns to correct time using playbackRate', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + type: 'audio', + file: 'cypress/fixtures/metronome@120bpm.wav', + audioOutput: '', + keyframes: [[now - 500, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.get('audio').invoke('prop', 'currentTime').should('be.lessThan', 2); + }); +}); diff --git a/packages/javascript/cypress/component/ImageStability.cy.ts b/packages/javascript/cypress/component/ImageStability.cy.ts new file mode 100644 index 0000000..9a7b721 --- /dev/null +++ b/packages/javascript/cypress/component/ImageStability.cy.ts @@ -0,0 +1,78 @@ +import { SurfaceManager } from '../../src/state-based/SurfaceManager'; + +describe('Image stability tests', () => { + it('can show an image', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/indianred@2560x1440.png', + type: 'image', + fit: 'cover', + keyframes: [[now, { set: { opacity: 1 } }]], + }, + }); + cy.mount(manager); + cy.get('img').should('exist'); + }); + + it("doesn't show a queued image", () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/indianred@2560x1440.png', + type: 'image', + fit: 'cover', + keyframes: [ + [now + 60_000, { set: { opacity: 1 } }], // show image in 1 minute + ], + }, + }); + cy.mount(manager); + cy.get('img').should('not.exist'); + }); + + it('recovers from img element src change', () => { + const ORIGINAL_SRC = 'cypress/fixtures/indianred@2560x1440.png'; + const CHANGED_SRC = '404.png'; + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: ORIGINAL_SRC, + type: 'image', + fit: 'cover', + keyframes: [[now, { set: { opacity: 1 } }]], + }, + }); + cy.mount(manager); + + cy.get('img').should('exist'); + cy.get('img').invoke('prop', 'src', CHANGED_SRC); + + cy.wait(1_000); + + cy.get('img') + .invoke('prop', 'src') + .should(($src) => expect($src).to.contain(ORIGINAL_SRC)); + }); + + it('recovers from img element deletion', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/indianred@2560x1440.png', + type: 'image', + fit: 'cover', + keyframes: [[now, { set: { opacity: 1 } }]], + }, + }); + cy.mount(manager); + + cy.get('img').should('exist'); + cy.get('img').invoke('remove'); + cy.get('img').should('not.exist'); + + cy.wait(1_000); + + cy.get('img').should('exist'); + }); +}); diff --git a/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts b/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts new file mode 100644 index 0000000..a796e6d --- /dev/null +++ b/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts @@ -0,0 +1,61 @@ +import { DATA_CLIP_ID, SurfaceManager } from '../../src/state-based/SurfaceManager'; + +describe('Updating surface state', () => { + it('adds and removes a video clip', () => { + const manager = new SurfaceManager({}); + cy.mount(manager); + + cy.get('video') + .should('not.exist') + .then(() => { + const now = Date.now(); + manager.setState({ + 'clip-id': { + file: 'cypress/fixtures/2x2s@2560x1440.mp4', + type: 'video', + audioOutput: '', + fit: 'cover', + keyframes: [ + [now + 1_000, { set: { t: 0, rate: 1 } }], // play in 1s + ], + }, + }); + }) + .then(() => { + // This implicitly waits + cy.get('video').should('exist'); + }) + .then(() => { + manager.setState({}); + }) + .then(() => { + cy.get('video').should('not.exist'); + }); + }); + + it('adds multiple media', () => { + const now = Date.now(); + const manager = new SurfaceManager({}); + cy.mount(manager); + expect(manager.element.children.length).to.eq(0); + + manager.setState({ + 'image-background': { + type: 'image', + file: 'cypress/fixtures/indianred@2560x1440.png', + fit: 'cover', + keyframes: [[now, {}]], + }, + 'video-foreground': { + type: 'video', + fit: 'contain', + audioOutput: '', + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + keyframes: [[now, {}]], + }, + }); + + cy.get(`[${DATA_CLIP_ID}=image-background]`).should('exist'); + cy.get(`[${DATA_CLIP_ID}=video-foreground]`).should('exist'); + }); +}); diff --git a/packages/javascript/cypress/component/VideoStability.cy.ts b/packages/javascript/cypress/component/VideoStability.cy.ts new file mode 100644 index 0000000..2c303c3 --- /dev/null +++ b/packages/javascript/cypress/component/VideoStability.cy.ts @@ -0,0 +1,163 @@ +import { SurfaceManager } from '../../src/state-based/SurfaceManager'; + +describe('Video stability tests', () => { + it('can wait without playing', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + type: 'video', + audioOutput: '', + fit: 'cover', + keyframes: [ + [now, { set: { t: 0, rate: 0 } }], // paused at start + [now + 60_000, { set: { rate: 1 } }], // play in 1 minute + ], + }, + }); + cy.mount(manager); + + cy.get('video').should('have.prop', 'paused', true); + cy.get('video').should('have.prop', 'currentTime', 0); + }); + + it('recovers from a pause', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + type: 'video', + audioOutput: '', + fit: 'cover', + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.log('Interfere with video element'); + cy.get('video').should('have.prop', 'paused', false); + cy.get('video').invoke('trigger', 'pause'); + cy.get('video').should('have.prop', 'paused', true); + + cy.wait(1000); + + cy.log('Video should have recovered'); + cy.get('video').should('have.prop', 'paused', false); + cy.get('video').invoke('prop', 'currentTime').should('be.greaterThan', 1.5); + }); + + it('recovers from a play', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + type: 'video', + audioOutput: '', + fit: 'cover', + keyframes: [[now, { set: { t: 1_500, rate: 0 } }]], + }, + }); + cy.mount(manager); + + // Wait until video ready + cy.get('video') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.closeTo(1.5, 0.1)); + + cy.log('Interfere with video element'); + cy.get('video').invoke('prop', 'paused').should('be.true'); + cy.get('video').invoke('prop', 'playbackRate', 1); + cy.get('video').then(($video) => $video.get(0).play().catch(/* do nothing*/)); + cy.get('video').invoke('prop', 'paused').should('be.false'); + + cy.wait(1000); + + cy.log('Video should have recovered'); + cy.get('video') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.closeTo(1.5, 0.1)); + }); + + it('recovers from a seek', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + type: 'video', + audioOutput: '', + fit: 'cover', + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.log('Interfere with video element'); + cy.get('video').invoke('prop', 'currentTime', 5); + + cy.wait(500); + + cy.log('Video should have recovered'); + cy.get('video').invoke('prop', 'currentTime').should('be.lessThan', 2); + }); + + it('recovers from volume change', () => { + const INITIAL_VOLUME = 0; + const CHANGED_VOLUME = 1; + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + type: 'video', + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + audioOutput: '', + fit: 'cover', + keyframes: [[now, { set: { t: 0, rate: 1, volume: INITIAL_VOLUME } }]], + }, + }); + cy.mount(manager); + + cy.get('video').invoke('prop', 'volume', CHANGED_VOLUME); + cy.get('video').should('have.prop', 'volume', CHANGED_VOLUME); + + cy.wait(1000); + + cy.get('video').should('have.prop', 'volume', INITIAL_VOLUME); + }); + + it('recovers from video element deletion', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + type: 'video', + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + audioOutput: '', + fit: 'cover', + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.get('video').should('exist'); + cy.get('video').invoke('remove'); + cy.get('video').should('not.exist'); + + cy.wait(1000); + + cy.get('video').should('exist'); + }); + + it('smoothly returns to correct time using playbackRate', () => { + const now = Date.now(); + const manager = new SurfaceManager({ + 'clip-id': { + type: 'video', + file: 'cypress/fixtures/yuv444p~5x2s@2560x1440.mp4', + audioOutput: '', + fit: 'cover', + keyframes: [[now - 500, { set: { t: 0, rate: 1 } }]], + }, + }); + cy.mount(manager); + + cy.get('video').invoke('prop', 'currentTime').should('be.lessThan', 2); + }); +}); diff --git a/packages/javascript/cypress/e2e/generate-test-video-frames.cy.ts b/packages/javascript/cypress/e2e/generate-test-video-frames.cy.ts new file mode 100644 index 0000000..e462240 --- /dev/null +++ b/packages/javascript/cypress/e2e/generate-test-video-frames.cy.ts @@ -0,0 +1,24 @@ +const LOOP_DURATION = 2_000; +const FPS = 60; +const TOTAL_LOOPS = 2; +const TOTAL_FRAMES = Math.floor(LOOP_DURATION * (FPS / 1000) * TOTAL_LOOPS); + +describe('template spec', () => { + it('testing', { baseUrl: null }, () => { + const frameDuration = 1000 / FPS; + let frame = 0; + let ms = 0; + + while (frame < TOTAL_FRAMES) { + // Debug log for long running tasks + cy.task('log', `[frame: ${frame + 1}/${TOTAL_FRAMES}] [ms: ${ms.toFixed(2)}/${TOTAL_LOOPS * LOOP_DURATION}]`); + // Go to new frame of the video + cy.visit(`cypress/e2e/test-video.html?loopDurationMs=${LOOP_DURATION}¤tMs=${ms}`); + // Capture screenshot + cy.screenshot(`${frame}`, { capture: 'viewport', overwrite: true }); + // Set up for next iteration + frame++; + ms = frame * frameDuration; + } + }); +}); diff --git a/packages/javascript/cypress/e2e/test-video.html b/packages/javascript/cypress/e2e/test-video.html new file mode 100644 index 0000000..b7c793f --- /dev/null +++ b/packages/javascript/cypress/e2e/test-video.html @@ -0,0 +1,115 @@ + + + + + + + +
+
+
+
+
+
+ + + + \ No newline at end of file diff --git a/packages/javascript/cypress/fixtures/indianred@2560x1440.png b/packages/javascript/cypress/fixtures/indianred@2560x1440.png new file mode 100644 index 0000000..e2f69ac Binary files /dev/null and b/packages/javascript/cypress/fixtures/indianred@2560x1440.png differ diff --git a/packages/javascript/cypress/fixtures/metronome@120bpm.wav b/packages/javascript/cypress/fixtures/metronome@120bpm.wav new file mode 100644 index 0000000..21973e4 Binary files /dev/null and b/packages/javascript/cypress/fixtures/metronome@120bpm.wav differ diff --git a/packages/javascript/cypress/fixtures/sinwave@440hz.wav b/packages/javascript/cypress/fixtures/sinwave@440hz.wav new file mode 100644 index 0000000..d5882d6 Binary files /dev/null and b/packages/javascript/cypress/fixtures/sinwave@440hz.wav differ diff --git a/packages/javascript/cypress/fixtures/yuv420p~2x2s@2560x1440.mp4 b/packages/javascript/cypress/fixtures/yuv420p~2x2s@2560x1440.mp4 new file mode 100644 index 0000000..291227b Binary files /dev/null and b/packages/javascript/cypress/fixtures/yuv420p~2x2s@2560x1440.mp4 differ diff --git a/packages/javascript/cypress/fixtures/yuv420p~5x2s@2560x1440.mp4 b/packages/javascript/cypress/fixtures/yuv420p~5x2s@2560x1440.mp4 new file mode 100644 index 0000000..e044643 Binary files /dev/null and b/packages/javascript/cypress/fixtures/yuv420p~5x2s@2560x1440.mp4 differ diff --git a/packages/javascript/cypress/fixtures/yuv444p~2x2s@2560x1440.mp4 b/packages/javascript/cypress/fixtures/yuv444p~2x2s@2560x1440.mp4 new file mode 100644 index 0000000..0ea41b4 Binary files /dev/null and b/packages/javascript/cypress/fixtures/yuv444p~2x2s@2560x1440.mp4 differ diff --git a/packages/javascript/cypress/fixtures/yuv444p~5x2s@2560x1440.mp4 b/packages/javascript/cypress/fixtures/yuv444p~5x2s@2560x1440.mp4 new file mode 100644 index 0000000..8d0e7a6 Binary files /dev/null and b/packages/javascript/cypress/fixtures/yuv444p~5x2s@2560x1440.mp4 differ diff --git a/packages/javascript/cypress/mount.ts b/packages/javascript/cypress/mount.ts new file mode 100644 index 0000000..ad6400b --- /dev/null +++ b/packages/javascript/cypress/mount.ts @@ -0,0 +1,25 @@ +import { getContainerEl, setupHooks } from '@cypress/mount-utils'; +import { SurfaceManager } from '../src/state-based/SurfaceManager'; + +export function mount(surfaceManager: SurfaceManager): Cypress.Chainable { + const container = getContainerEl(); + + // 100% styling + document.body.style.margin = '0'; + container.style.width = '100vw'; + container.style.height = '100vh'; + + // clean up each time we mount a new component + container.innerHTML = ''; + const prevManager = (window as any).surfaceManager as SurfaceManager; + prevManager?.setState({}); + + // mount component + (window as any).surfaceManager = surfaceManager; + container.append(surfaceManager.element); + + // initialize internal pre/post test hooks + setupHooks(); + + return cy.wrap(surfaceManager.element, { log: false }); +} diff --git a/packages/javascript/cypress/support/commands.ts b/packages/javascript/cypress/support/commands.ts new file mode 100644 index 0000000..95857ae --- /dev/null +++ b/packages/javascript/cypress/support/commands.ts @@ -0,0 +1,37 @@ +/// +// *********************************************** +// This example commands.ts shows you how to +// create various custom commands and overwrite +// existing commands. +// +// For more comprehensive examples of custom +// commands please read more here: +// https://on.cypress.io/custom-commands +// *********************************************** +// +// +// -- This is a parent command -- +// Cypress.Commands.add('login', (email, password) => { ... }) +// +// +// -- This is a child command -- +// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... }) +// +// +// -- This is a dual command -- +// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... }) +// +// +// -- This will overwrite an existing command -- +// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... }) +// +// declare global { +// namespace Cypress { +// interface Chainable { +// login(email: string, password: string): Chainable +// drag(subject: string, options?: Partial): Chainable +// dismiss(subject: string, options?: Partial): Chainable +// visit(originalFn: CommandOriginalFn, url: string, options: Partial): Chainable +// } +// } +// } diff --git a/packages/javascript/cypress/support/component-index.html b/packages/javascript/cypress/support/component-index.html new file mode 100644 index 0000000..ac6e79f --- /dev/null +++ b/packages/javascript/cypress/support/component-index.html @@ -0,0 +1,12 @@ + + + + + + + Components App + + +
+ + \ No newline at end of file diff --git a/packages/javascript/cypress/support/component.ts b/packages/javascript/cypress/support/component.ts new file mode 100644 index 0000000..4ec8287 --- /dev/null +++ b/packages/javascript/cypress/support/component.ts @@ -0,0 +1,36 @@ +/* eslint-disable @typescript-eslint/no-namespace */ +// *********************************************************** +// This example support/component.ts is processed and +// loaded automatically before your test files. +// +// This is a great place to put global configuration and +// behavior that modifies Cypress. +// +// You can change the location of this file or turn off +// automatically serving support files with the +// 'supportFile' configuration option. +// +// You can read more here: +// https://on.cypress.io/configuration +// *********************************************************** + +// Import commands.js using ES2015 syntax: +import { mount } from '../mount'; +import './commands'; + +// Augment the Cypress namespace to include type definitions for +// your custom command. +// Alternatively, can be defined in cypress/support/component.d.ts +// with a at the top of your spec. +declare global { + namespace Cypress { + interface Chainable { + mount: typeof mount; + } + } +} + +Cypress.Commands.add('mount', mount); + +// Example use: +// cy.mount() diff --git a/packages/javascript/cypress/support/e2e.ts b/packages/javascript/cypress/support/e2e.ts new file mode 100644 index 0000000..e66558e --- /dev/null +++ b/packages/javascript/cypress/support/e2e.ts @@ -0,0 +1,17 @@ +// *********************************************************** +// This example support/e2e.ts is processed and +// loaded automatically before your test files. +// +// This is a great place to put global configuration and +// behavior that modifies Cypress. +// +// You can change the location of this file or turn off +// automatically serving support files with the +// 'supportFile' configuration option. +// +// You can read more here: +// https://on.cypress.io/configuration +// *********************************************************** + +// Import commands.js using ES2015 syntax: +import './commands'; diff --git a/packages/javascript/cypress/tsconfig.json b/packages/javascript/cypress/tsconfig.json new file mode 100644 index 0000000..479d839 --- /dev/null +++ b/packages/javascript/cypress/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "es5", + "lib": ["es5", "dom"], + "sourceMap": true, + "types": ["cypress", "node"], + "jsx": "react" + }, + "include": ["**/*.ts"] +} \ No newline at end of file diff --git a/packages/javascript/package.json b/packages/javascript/package.json index 152c641..b74eef1 100644 --- a/packages/javascript/package.json +++ b/packages/javascript/package.json @@ -22,7 +22,7 @@ "default": "./dist/index.js" }, "scripts": { - "test": "yarn types && yarn lint && vitest", + "test": "yarn types && yarn lint && vitest && yarn cy:run", "types": "tsc --noEmit", "lint": "eslint .", "build": "yarn build:ts && yarn build:browser", @@ -31,7 +31,10 @@ "watch-build": "tsc -w", "build-docs": "typedoc --out ../../docs/javascript --name @clockworkdog/cogs-client src/index.ts", "release": "yarn npm publish --access public", - "prerelease": "yarn npm publish --access public --tag=next" + "prerelease": "yarn npm publish --access public --tag=next", + "cy:open": "cypress open", + "cy:run": "cypress run --component", + "cy:generate": "cypress run --e2e" }, "dependencies": { "@clockworkdog/timesync": "workspace:^", @@ -40,15 +43,19 @@ "zod": "^4.1.13" }, "devDependencies": { + "@cypress/mount-utils": "^4.1.2", "@eslint/js": "^9.17.0", "@types/howler": "2.2.12", "@types/jsdom": "^27", "@types/node": "^22.10.2", + "cypress": "^14.5.4", "eslint": "^9.17.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", "jsdom": "^27.1.0", "prettier": "^3.4.2", + "react": "^19.1.1", + "react-dom": "^19.1.1", "typedoc": "^0.27.5", "typescript": "~5.7.2", "typescript-eslint": "^8.18.1", diff --git a/packages/javascript/scripts/generate-video.ts b/packages/javascript/scripts/generate-video.ts new file mode 100644 index 0000000..f6aca1f --- /dev/null +++ b/packages/javascript/scripts/generate-video.ts @@ -0,0 +1,49 @@ +import { spawn } from 'node:child_process'; +const ffmpeg = 'ffmpeg'; + +/** + * - First make sure cypress/screenshots is empty + * This is the directory we'll use to save all video frames + * - Then run `yarn cy:generate` + * This will run the generate procedure in the e2e test + * It will save screenshots to cypres/screenshots + * - Finally run this file to create a test video + */ + +const child = spawn(ffmpeg, [ + // Set the input framerate + '-framerate', + '60', + // Specify the input files + '-i', + 'cypress/screenshots/generate-test-video-frames.cy.ts/%d.png', + // Set the output framerate + '-r', + '60', + // specify the codec + '-c:v', + 'libx264', + // pixel format + '-pix_fmt', + 'yuv420p', + // automatically overwrite + '-y', + // destination + 'out.mp4', +]); + +child.stdout.on('data', (data) => console.log(data.toString())); +child.stderr.on('data', (data) => console.error(data.toString())); + +child.on('error', () => { + console.error('Failed to generate video'); + process.exit(1); +}); + +child.on('close', (code) => { + if (code === 0) { + console.log('Created video'); + } else { + console.error('Failed to create video'); + } +}); diff --git a/packages/javascript/src/index.ts b/packages/javascript/src/index.ts index 30e218b..0936e65 100644 --- a/packages/javascript/src/index.ts +++ b/packages/javascript/src/index.ts @@ -7,6 +7,7 @@ export type { default as MediaObjectFit } from './types/MediaObjectFit'; export * as MediaSchema from './types/MediaSchema'; export { default as CogsAudioPlayer } from './AudioPlayer'; export { default as CogsVideoPlayer } from './VideoPlayer'; +export { SurfaceManager } from './state-based/SurfaceManager'; export * from './types/AudioState'; export { assetUrl, preloadUrl } from './utils/urls'; export { getStateAtTime } from './utils/getStateAtTime'; diff --git a/packages/javascript/src/state-based/AudioManager.ts b/packages/javascript/src/state-based/AudioManager.ts new file mode 100644 index 0000000..999a26e --- /dev/null +++ b/packages/javascript/src/state-based/AudioManager.ts @@ -0,0 +1,125 @@ +import { AudioState, defaultAudioOptions } from '../types/MediaSchema'; +import { getStateAtTime } from '../utils/getStateAtTime'; +import { ClipManager } from './ClipManager'; + +const DEFAULT_AUDIO_POLLING = 1_000; +const TARGET_SYNC_THRESHOLD_MS = 10; // If we're closer than this we're good enough +const MAX_SYNC_THRESHOLD_MS = 1_000; // If we're further away than this, we'll seek instead +const SEEK_LOOKAHEAD_MS = 200; // We won't seek ahead instantly, so lets seek ahead +const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.2; +// We smoothly ramp playbackRate up and down +const PLAYBACK_ADJUSTMENT_SMOOTHING = 0.5; +function playbackSmoothing(deltaTime: number) { + return Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT; +} + +export class AudioManager extends ClipManager { + private audioElement?: HTMLAudioElement; + private isSeeking = false; + + constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: AudioState) { + super(surfaceElement, clipElement, state); + this.clipElement = clipElement; + } + + private updateAudioElement() { + this.destroy(); + this.audioElement = document.createElement('audio'); + this.clipElement.replaceChildren(this.audioElement); + } + + /** + * Helper function to seek to a specified time. + * Works with the update loop to poll until seeked event has fired. + */ + private seekTo(time: number) { + if (!this.audioElement) return; + this.audioElement.addEventListener( + 'seeked', + () => { + this.isSeeking = false; + }, + { once: true, passive: true }, + ); + this.audioElement.currentTime = time / 1_000; + } + + protected update(): void { + // Update loop used to poll until seek finished + if (this.isSeeking) return; + this.delay = DEFAULT_AUDIO_POLLING; + + // Does the