Delta manifest: reduce update response sizes by 75-85%

supabase/functions/_backend/utils/pg.ts

@@ -286,7 +286,7 @@ export function getAlias() {
   return { versionAlias, channelDevicesAlias, channelAlias }
 }
 
-function getSchemaUpdatesAlias(includeMetadata = false) {
+function getSchemaUpdatesAlias(includeMetadata = false, oldVersionId?: number | null) {
   const { versionAlias, channelDevicesAlias, channelAlias } = getAlias()
 
   const versionSelect: any = {
@@ -322,7 +322,24 @@ function getSchemaUpdatesAlias(includeMetadata = false) {
     allow_device_self_set: channelAlias.allow_device_self_set,
     public: channelAlias.public,
   }
-  const manifestSelect = sql<{ file_name: string, file_hash: string, s3_path: string }[]>`COALESCE(json_agg(
+  // Delta manifest: when oldVersionId is provided, exclude files that exist identically in old version
+  const manifestSelect = oldVersionId
+    ? sql<{ file_name: string, file_hash: string, s3_path: string }[]>`COALESCE(json_agg(
+        json_build_object(
+          'file_name', ${schema.manifest.file_name},
+          'file_hash', ${schema.manifest.file_hash},
+          's3_path', ${schema.manifest.s3_path}
+        )
+      ) FILTER (
+        WHERE ${schema.manifest.file_name} IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM ${schema.manifest} AS old_m
+          WHERE old_m.app_version_id = ${oldVersionId}
+          AND old_m.file_name = ${schema.manifest.file_name}
+          AND old_m.file_hash = ${schema.manifest.file_hash}
+        )
+      ), '[]'::json)`
+    : sql<{ file_name: string, file_hash: string, s3_path: string }[]>`COALESCE(json_agg(
         json_build_object(
           'file_name', ${schema.manifest.file_name},
           'file_hash', ${schema.manifest.file_hash},
@@ -339,8 +356,9 @@ export function requestInfosChannelDevicePostgres(
   drizzleClient: ReturnType<typeof getDrizzleClient>,
   includeManifest: boolean,
   includeMetadata = false,
+  oldVersionId?: number | null,
 ) {
-  const { versionSelect, channelDevicesAlias, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata)
+  const { versionSelect, channelDevicesAlias, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata, oldVersionId)
   const baseSelect = {
     channel_devices: {
       device_id: channelDevicesAlias.device_id,
@@ -376,8 +394,9 @@ export function requestInfosChannelPostgres(
   drizzleClient: ReturnType<typeof getDrizzleClient>,
   includeManifest: boolean,
   includeMetadata = false,
+  oldVersionId?: number | null,
 ) {
-  const { versionSelect, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata)
+  const { versionSelect, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata, oldVersionId)
   const platformQuery = platform === 'android' ? channelAlias.android : platform === 'electron' ? channelAlias.electron : channelAlias.ios
   const baseSelect = {
     version: versionSelect,
@@ -422,17 +441,18 @@ export function requestInfosPostgres(
   channelDeviceCount?: number | null,
   manifestBundleCount?: number | null,
   includeMetadata = false,
+  oldVersionId?: number | null,
 ) {
   const shouldQueryChannelOverride = channelDeviceCount === undefined || channelDeviceCount === null ? true : channelDeviceCount > 0
   const shouldFetchManifest = manifestBundleCount === undefined || manifestBundleCount === null ? true : manifestBundleCount > 0
 
   const channelDevice = shouldQueryChannelOverride
-    ? requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata)
+    ? requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata, oldVersionId)
     : Promise.resolve(undefined).then(() => {
         cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override query' })
         return null
       })
-  const channel = requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, shouldFetchManifest, includeMetadata)
+  const channel = requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, shouldFetchManifest, includeMetadata, oldVersionId)
 
   return Promise.all([channelDevice, channel])
     .then(([channelOverride, channelData]) => ({ channelData, channelOverride }))
@@ -509,6 +529,37 @@ export async function getAppVersionPostgres(
   }
 }
 
+export async function getVersionIdByName(
+  c: Context,
+  appId: string,
+  versionName: string,
+  drizzleClient: ReturnType<typeof getDrizzleClient>,
+  allowedDeleted?: boolean,
+): Promise<number | null> {
+  try {
+    // Return null for internal version names
+    if (!versionName || versionName === 'builtin' || versionName === 'unknown')
+      return null
+
+    const result = await drizzleClient
+      .select({ id: schema.app_versions.id })
+      .from(schema.app_versions)
+      .where(and(
+        eq(schema.app_versions.app_id, appId),
+        eq(schema.app_versions.name, versionName),
+        ...(allowedDeleted !== undefined ? [eq(schema.app_versions.deleted, allowedDeleted)] : []),
+      ))
+      .limit(1)
+      .then(data => data[0])
+
+    return result?.id ?? null
+  }
+  catch (e: unknown) {
+    logPgError(c, 'getVersionIdByName', e)
+    return null
+  }
+}
+
 export async function getAppVersionsByAppIdPg(
   c: Context,
   appId: string,

supabase/functions/_backend/utils/update.ts

@@ -15,7 +15,7 @@ import { getBundleUrl, getManifestUrl } from './downloadUrl.ts'
 import { simpleError200 } from './hono.ts'
 import { cloudlog } from './logging.ts'
 import { sendNotifOrg } from './notifications.ts'
-import { closeClient, getAppOwnerPostgres, getDrizzleClient, getPgClient, requestInfosPostgres, setReplicationLagHeader } from './pg.ts'
+import { closeClient, getAppOwnerPostgres, getDrizzleClient, getPgClient, getVersionIdByName, requestInfosPostgres, setReplicationLagHeader } from './pg.ts'
 import { makeDevice } from './plugin_parser.ts'
 import { s3 } from './s3.ts'
 import { createStatsBandwidth, createStatsMau, createStatsVersion, onPremStats, sendStatsAndDevice } from './stats.ts'
@@ -162,7 +162,21 @@ export async function updateWithPG(
   // Only query link/comment if plugin supports it (v5.35.0+, v6.35.0+, v7.35.0+, v8.35.0+) AND app has expose_metadata enabled
   const needsMetadata = appOwner.expose_metadata && !isDeprecatedPluginVersion(pluginVersion, '5.35.0', '6.35.0', '7.35.0', '8.35.0')
 
-  const requestedInto = await requestInfosPostgres(c, platform, app_id, device_id, defaultChannel, drizzleClient, channelDeviceCount, manifestBundleCount, needsMetadata)
+  // Look up old version ID for delta manifest calculation
+  // Only do this if we're fetching manifest entries
+  let oldVersionId: number | null = null
+  if (fetchManifestEntries && version_name && version_name !== 'builtin' && version_name !== 'unknown') {
+    oldVersionId = await getVersionIdByName(c, app_id, version_name, drizzleClient)
+    cloudlog({
+      requestId: c.get('requestId'),
+      message: 'Delta manifest lookup',
+      version_name,
+      oldVersionId,
+      fetchManifestEntries,
+    })
+  }
+
+  const requestedInto = await requestInfosPostgres(c, platform, app_id, device_id, defaultChannel, drizzleClient, channelDeviceCount, manifestBundleCount, needsMetadata, oldVersionId)
   const { channelOverride } = requestedInto
   let { channelData } = requestedInto
   cloudlog({ requestId: c.get('requestId'), message: `channelData exists ? ${channelData !== undefined}, channelOverride exists ? ${channelOverride !== undefined}` })
@@ -185,6 +199,17 @@ export async function updateWithPG(
   const manifestEntries = (channelOverride?.manifestEntries ?? channelData?.manifestEntries ?? []) as Partial<Database['public']['Tables']['manifest']['Row']>[]
   // device.version = versionData ? versionData.id : version.id
 
+  // Check if device is already on the latest version BEFORE checking for missing bundle
+  // This must come first because delta manifest calculation may return empty manifest
+  // when device is already on target version (all files match), which would incorrectly
+  // trigger the no_bundle error for manifest-only bundles
+  if (version_name === version.name) {
+    cloudlog({ requestId: c.get('requestId'), message: 'No new version available', id: device_id, version_name, version: version.name, date: new Date().toISOString() })
+    // TODO: check why this event is send with wrong version_name
+    await sendStatsAndDevice(c, device, [{ action: 'noNew', versionName: version.name }])
+    return simpleError200(c, 'no_new_version_available', 'No new version available')
+  }
+
   // TODO: find better solution to check if device is from apple or google, currently not working in
 
   if (!version.external_url && !version.r2_path && !isInternalVersionName(version.name) && (!manifestEntries || manifestEntries.length === 0)) {
@@ -205,14 +230,6 @@ export async function updateWithPG(
     })
   }
 
-  // cloudlog(c.get('requestId'), 'signedURL', device_id, version_name, version.name)
-  if (version_name === version.name) {
-    cloudlog({ requestId: c.get('requestId'), message: 'No new version available', id: device_id, version_name, version: version.name, date: new Date().toISOString() })
-    // TODO: check why this event is send with wrong version_name
-    await sendStatsAndDevice(c, device, [{ action: 'noNew', versionName: version.name }])
-    return simpleError200(c, 'no_new_version_available', 'No new version available')
-  }
-
   if (channelData) {
     // cloudlog(c.get('requestId'), 'check disableAutoUpdateToMajor', device_id)
     if (!channelData.channels.ios && platform === 'ios') {

tests/updates-manifest.test.ts

@@ -102,6 +102,39 @@
   await resetAppDataStats(APPNAME)
 })
 
+// Delta manifest test constants
+const DELTA_APPNAME = `com.demo.app.delta.${id}`
+let deltaOldVersionId: number | null = null
+let deltaNewVersionId: number | null = null
+
+// Helper to insert multiple manifest entries for a version
+async function insertMultipleManifestEntries(appVersionId: number, entries: { file_name: string, file_hash: string, s3_path: string }[]) {
+  const supabase = getSupabaseClient()
+  // First, delete any existing manifest entries for this version
+  await supabase.from('manifest').delete().eq('app_version_id', appVersionId)
+  // Insert all manifest entries in a single batch
+  const { error } = await supabase.from('manifest').insert(
+    entries.map(entry => ({
+      app_version_id: appVersionId,
+      file_name: entry.file_name,
+      s3_path: entry.s3_path,
+      file_hash: entry.file_hash,
+      file_size: 100,
+    })),
+  )
+  if (error)
+    throw new Error(`Failed to insert manifest entries: ${error.message}`)
+
+  // Update the manifest_count on the version
+  await supabase.from('app_versions').update({ manifest_count: entries.length }).eq('id', appVersionId)
+
+  // Also update manifest_bundle_count on the app to enable manifest fetching
+  const { data: version } = await supabase.from('app_versions').select('app_id').eq('id', appVersionId).single()
+  if (version) {
+    await supabase.from('apps').update({ manifest_bundle_count: entries.length }).eq('app_id', version.app_id)
+  }
+}
+
 describe('update manifest scenarios', () => {
   it('manifest update', async () => {
     // test manifest update working with plugin version >= 6.25.0
@@ -224,3 +257,165 @@
     expect(json.manifest).toBeDefined()
   })
 })
+
+describe('delta manifest scenarios', () => {
+  beforeAll(async () => {
+    // Set up the delta manifest test app with two versions
+    await resetAndSeedAppData(DELTA_APPNAME)
+    const supabase = getSupabaseClient()
+
+    // Get version IDs for old (1.359.0) and new (1.0.0) versions
+    // Note: resetAndSeedAppData creates versions 1.0.0, 1.0.1, 1.359.0, 1.360.0, 1.361.0
+    // The production channel points to version 1.0.0 by default
+    const { data: oldVersion } = await supabase
+      .from('app_versions')
+      .select('id')
+      .eq('name', '1.359.0')
+      .eq('app_id', DELTA_APPNAME)
+      .single()
+
+    const { data: newVersion } = await supabase
+      .from('app_versions')
+      .select('id')
+      .eq('name', '1.0.0')
+      .eq('app_id', DELTA_APPNAME)
+      .single()
+
+    if (oldVersion)
+      deltaOldVersionId = oldVersion.id
+    if (newVersion)
+      deltaNewVersionId = newVersion.id
+
+    // Set up manifest entries for delta testing:
+    // Old version (1.359.0): file_a.js (hash1), file_b.js (hash2)
+    // New version (1.0.0): file_a.js (hash1 - unchanged), file_b.js (hash3 - changed), file_c.js (hash4 - new)
+    if (deltaOldVersionId) {
+      await insertMultipleManifestEntries(deltaOldVersionId, [
+        { file_name: 'file_a.js', file_hash: 'hash_unchanged_1', s3_path: '/file_a.js' },
+        { file_name: 'file_b.js', file_hash: 'hash_old_2', s3_path: '/file_b.js' },
+      ])
+    }
+
+    if (deltaNewVersionId) {
+      await insertMultipleManifestEntries(deltaNewVersionId, [
+        { file_name: 'file_a.js', file_hash: 'hash_unchanged_1', s3_path: '/file_a.js' }, // Same hash - should be excluded
+        { file_name: 'file_b.js', file_hash: 'hash_new_3', s3_path: '/file_b.js' }, // Different hash - should be included
+        { file_name: 'file_c.js', file_hash: 'hash_new_4', s3_path: '/file_c.js' }, // New file - should be included
+      ])
+    }
+  })
+
+  afterAll(async () => {
+    await resetAppData(DELTA_APPNAME)
+    await resetAppDataStats(DELTA_APPNAME)
+  })
+
+  it.concurrent('returns delta manifest with only changed/new files', async () => {
+    // Request update from old version (1.359.0) to new version (1.0.0)
+    // Should only return file_b.js (changed hash) and file_c.js (new file)
+    // file_a.js should be excluded because it has the same hash
+    const baseData = getBaseData(DELTA_APPNAME)
+    baseData.version_name = '1.359.0' // Device is on old version
+    baseData.plugin_version = '7.1.0' // Plugin version that supports manifest
+
+    const response = await postUpdate(baseData)
+    expect(response.status).toBe(200)
+    const json = await response.json<UpdateRes>()
+
+    expect(json.manifest).toBeDefined()
+    expect(json.manifest?.length).toBe(2) // Only file_b and file_c
+
+    // Verify the returned files are the changed/new ones
+    const fileNames = json.manifest?.map(m => m.file_name).sort()
+    expect(fileNames).toEqual(['file_b.js', 'file_c.js'])
+
+    // Verify file_a.js is NOT in the response (it has unchanged hash)
+    expect(json.manifest?.find(m => m.file_name === 'file_a.js')).toBeUndefined()
+
+    // Verify file_b has new hash
+    const fileB = json.manifest?.find(m => m.file_name === 'file_b.js')
+    expect(fileB?.file_hash).toBe('hash_new_3')
+
+    // Verify file_c is the new file
+    const fileC = json.manifest?.find(m => m.file_name === 'file_c.js')
+    expect(fileC?.file_hash).toBe('hash_new_4')
+  })
+
+  it.concurrent('returns full manifest for first install (builtin version)', async () => {
+    // When device is on 'builtin', it should receive full manifest (all 3 files)
+    const baseData = getBaseData(DELTA_APPNAME)
+    baseData.version_name = 'builtin' // First install
+    baseData.plugin_version = '7.1.0'
+
+    const response = await postUpdate(baseData)
+    expect(response.status).toBe(200)
+    const json = await response.json<UpdateRes>()
+
+    expect(json.manifest).toBeDefined()
+    expect(json.manifest?.length).toBe(3) // All files: file_a, file_b, file_c
+
+    const fileNames = json.manifest?.map(m => m.file_name).sort()
+    expect(fileNames).toEqual(['file_a.js', 'file_b.js', 'file_c.js'])
+  })
+
+  it.concurrent('returns full manifest when old version does not exist', async () => {
+    // When device's version doesn't exist in DB, should return full manifest
+    const baseData = getBaseData(DELTA_APPNAME)
+    baseData.version_name = '99.99.99' // Non-existent version
+    baseData.plugin_version = '7.1.0'
+
+    const response = await postUpdate(baseData)
+    expect(response.status).toBe(200)
+    const json = await response.json<UpdateRes>()
+
+    expect(json.manifest).toBeDefined()
+    expect(json.manifest?.length).toBe(3) // All files (graceful fallback)
+
+    const fileNames = json.manifest?.map(m => m.file_name).sort()
+    expect(fileNames).toEqual(['file_a.js', 'file_b.js', 'file_c.js'])
+  })
+
+  // This test modifies data (inserts manifest entries for version 1.0.1), so it runs sequentially
+  it('returns empty manifest when all files are identical', async () => {
+    // Create a scenario where old and new versions have identical manifests
+    const supabase = getSupabaseClient()
+
+    // Get the 1.0.1 version (different from the one on channel)
+    const { data: identicalVersion } = await supabase
+      .from('app_versions')
+      .select('id')
+      .eq('name', '1.0.1')
+      .eq('app_id', DELTA_APPNAME)
+      .single()
+
+    if (identicalVersion) {
+      // Set up identical manifest entries as the new version (1.0.0)
+      await insertMultipleManifestEntries(identicalVersion.id, [
+        { file_name: 'file_a.js', file_hash: 'hash_unchanged_1', s3_path: '/file_a.js' },
+        { file_name: 'file_b.js', file_hash: 'hash_new_3', s3_path: '/file_b.js' },
+        { file_name: 'file_c.js', file_hash: 'hash_new_4', s3_path: '/file_c.js' },
+      ])
+    }
+
+    try {
+      const baseData = getBaseData(DELTA_APPNAME)
+      baseData.version_name = '1.0.1' // Device has identical manifest
+      baseData.plugin_version = '7.1.0'
+
+      const response = await postUpdate(baseData)
+      expect(response.status).toBe(200)
+      const json = await response.json<UpdateRes>()
+
+      // When all files are identical, delta should be empty
+      expect(json.manifest).toBeDefined()
+      expect(json.manifest?.length).toBe(0)
+    }
+    finally {
+      // Clean up: remove the manifest entries we just added
+      if (identicalVersion) {
+        await supabase.from('manifest').delete().eq('app_version_id', identicalVersion.id)
+        await supabase.from('app_versions').update({ manifest_count: 0 }).eq('id', identicalVersion.id)
+      }
+    }
+  })
+})