diff --git a/supabase/functions/_backend/utils/pg.ts b/supabase/functions/_backend/utils/pg.ts index cc4ec94a87..a294c6f519 100644 --- a/supabase/functions/_backend/utils/pg.ts +++ b/supabase/functions/_backend/utils/pg.ts @@ -286,7 +286,7 @@ export function getAlias() { return { versionAlias, channelDevicesAlias, channelAlias } } -function getSchemaUpdatesAlias(includeMetadata = false) { +function getSchemaUpdatesAlias(includeMetadata = false, oldVersionId?: number | null) { const { versionAlias, channelDevicesAlias, channelAlias } = getAlias() const versionSelect: any = { @@ -322,7 +322,24 @@ function getSchemaUpdatesAlias(includeMetadata = false) { allow_device_self_set: channelAlias.allow_device_self_set, public: channelAlias.public, } - const manifestSelect = sql<{ file_name: string, file_hash: string, s3_path: string }[]>`COALESCE(json_agg( + // Delta manifest: when oldVersionId is provided, exclude files that exist identically in old version + const manifestSelect = oldVersionId + ? sql<{ file_name: string, file_hash: string, s3_path: string }[]>`COALESCE(json_agg( + json_build_object( + 'file_name', ${schema.manifest.file_name}, + 'file_hash', ${schema.manifest.file_hash}, + 's3_path', ${schema.manifest.s3_path} + ) + ) FILTER ( + WHERE ${schema.manifest.file_name} IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM ${schema.manifest} AS old_m + WHERE old_m.app_version_id = ${oldVersionId} + AND old_m.file_name = ${schema.manifest.file_name} + AND old_m.file_hash = ${schema.manifest.file_hash} + ) + ), '[]'::json)` + : sql<{ file_name: string, file_hash: string, s3_path: string }[]>`COALESCE(json_agg( json_build_object( 'file_name', ${schema.manifest.file_name}, 'file_hash', ${schema.manifest.file_hash}, @@ -339,8 +356,9 @@ export function requestInfosChannelDevicePostgres( drizzleClient: ReturnType, includeManifest: boolean, includeMetadata = false, + oldVersionId?: number | null, ) { - const { versionSelect, channelDevicesAlias, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata) + const { versionSelect, channelDevicesAlias, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata, oldVersionId) const baseSelect = { channel_devices: { device_id: channelDevicesAlias.device_id, @@ -376,8 +394,9 @@ export function requestInfosChannelPostgres( drizzleClient: ReturnType, includeManifest: boolean, includeMetadata = false, + oldVersionId?: number | null, ) { - const { versionSelect, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata) + const { versionSelect, channelAlias, channelSelect, manifestSelect, versionAlias } = getSchemaUpdatesAlias(includeMetadata, oldVersionId) const platformQuery = platform === 'android' ? channelAlias.android : platform === 'electron' ? channelAlias.electron : channelAlias.ios const baseSelect = { version: versionSelect, @@ -422,17 +441,18 @@ export function requestInfosPostgres( channelDeviceCount?: number | null, manifestBundleCount?: number | null, includeMetadata = false, + oldVersionId?: number | null, ) { const shouldQueryChannelOverride = channelDeviceCount === undefined || channelDeviceCount === null ? true : channelDeviceCount > 0 const shouldFetchManifest = manifestBundleCount === undefined || manifestBundleCount === null ? true : manifestBundleCount > 0 const channelDevice = shouldQueryChannelOverride - ? requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata) + ? requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata, oldVersionId) : Promise.resolve(undefined).then(() => { cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override query' }) return null }) - const channel = requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, shouldFetchManifest, includeMetadata) + const channel = requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, shouldFetchManifest, includeMetadata, oldVersionId) return Promise.all([channelDevice, channel]) .then(([channelOverride, channelData]) => ({ channelData, channelOverride })) @@ -509,6 +529,37 @@ export async function getAppVersionPostgres( } } +export async function getVersionIdByName( + c: Context, + appId: string, + versionName: string, + drizzleClient: ReturnType, + allowedDeleted?: boolean, +): Promise { + try { + // Return null for internal version names + if (!versionName || versionName === 'builtin' || versionName === 'unknown') + return null + + const result = await drizzleClient + .select({ id: schema.app_versions.id }) + .from(schema.app_versions) + .where(and( + eq(schema.app_versions.app_id, appId), + eq(schema.app_versions.name, versionName), + ...(allowedDeleted !== undefined ? [eq(schema.app_versions.deleted, allowedDeleted)] : []), + )) + .limit(1) + .then(data => data[0]) + + return result?.id ?? null + } + catch (e: unknown) { + logPgError(c, 'getVersionIdByName', e) + return null + } +} + export async function getAppVersionsByAppIdPg( c: Context, appId: string, diff --git a/supabase/functions/_backend/utils/update.ts b/supabase/functions/_backend/utils/update.ts index 3f0d5a5f03..d8b5da4dd1 100644 --- a/supabase/functions/_backend/utils/update.ts +++ b/supabase/functions/_backend/utils/update.ts @@ -15,7 +15,7 @@ import { getBundleUrl, getManifestUrl } from './downloadUrl.ts' import { simpleError200 } from './hono.ts' import { cloudlog } from './logging.ts' import { sendNotifOrg } from './notifications.ts' -import { closeClient, getAppOwnerPostgres, getDrizzleClient, getPgClient, requestInfosPostgres, setReplicationLagHeader } from './pg.ts' +import { closeClient, getAppOwnerPostgres, getDrizzleClient, getPgClient, getVersionIdByName, requestInfosPostgres, setReplicationLagHeader } from './pg.ts' import { makeDevice } from './plugin_parser.ts' import { s3 } from './s3.ts' import { createStatsBandwidth, createStatsMau, createStatsVersion, onPremStats, sendStatsAndDevice } from './stats.ts' @@ -162,7 +162,21 @@ export async function updateWithPG( // Only query link/comment if plugin supports it (v5.35.0+, v6.35.0+, v7.35.0+, v8.35.0+) AND app has expose_metadata enabled const needsMetadata = appOwner.expose_metadata && !isDeprecatedPluginVersion(pluginVersion, '5.35.0', '6.35.0', '7.35.0', '8.35.0') - const requestedInto = await requestInfosPostgres(c, platform, app_id, device_id, defaultChannel, drizzleClient, channelDeviceCount, manifestBundleCount, needsMetadata) + // Look up old version ID for delta manifest calculation + // Only do this if we're fetching manifest entries + let oldVersionId: number | null = null + if (fetchManifestEntries && version_name && version_name !== 'builtin' && version_name !== 'unknown') { + oldVersionId = await getVersionIdByName(c, app_id, version_name, drizzleClient) + cloudlog({ + requestId: c.get('requestId'), + message: 'Delta manifest lookup', + version_name, + oldVersionId, + fetchManifestEntries, + }) + } + + const requestedInto = await requestInfosPostgres(c, platform, app_id, device_id, defaultChannel, drizzleClient, channelDeviceCount, manifestBundleCount, needsMetadata, oldVersionId) const { channelOverride } = requestedInto let { channelData } = requestedInto cloudlog({ requestId: c.get('requestId'), message: `channelData exists ? ${channelData !== undefined}, channelOverride exists ? ${channelOverride !== undefined}` }) @@ -185,6 +199,17 @@ export async function updateWithPG( const manifestEntries = (channelOverride?.manifestEntries ?? channelData?.manifestEntries ?? []) as Partial[] // device.version = versionData ? versionData.id : version.id + // Check if device is already on the latest version BEFORE checking for missing bundle + // This must come first because delta manifest calculation may return empty manifest + // when device is already on target version (all files match), which would incorrectly + // trigger the no_bundle error for manifest-only bundles + if (version_name === version.name) { + cloudlog({ requestId: c.get('requestId'), message: 'No new version available', id: device_id, version_name, version: version.name, date: new Date().toISOString() }) + // TODO: check why this event is send with wrong version_name + await sendStatsAndDevice(c, device, [{ action: 'noNew', versionName: version.name }]) + return simpleError200(c, 'no_new_version_available', 'No new version available') + } + // TODO: find better solution to check if device is from apple or google, currently not working in if (!version.external_url && !version.r2_path && !isInternalVersionName(version.name) && (!manifestEntries || manifestEntries.length === 0)) { @@ -205,14 +230,6 @@ export async function updateWithPG( }) } - // cloudlog(c.get('requestId'), 'signedURL', device_id, version_name, version.name) - if (version_name === version.name) { - cloudlog({ requestId: c.get('requestId'), message: 'No new version available', id: device_id, version_name, version: version.name, date: new Date().toISOString() }) - // TODO: check why this event is send with wrong version_name - await sendStatsAndDevice(c, device, [{ action: 'noNew', versionName: version.name }]) - return simpleError200(c, 'no_new_version_available', 'No new version available') - } - if (channelData) { // cloudlog(c.get('requestId'), 'check disableAutoUpdateToMajor', device_id) if (!channelData.channels.ios && platform === 'ios') { diff --git a/tests/updates-manifest.test.ts b/tests/updates-manifest.test.ts index 1897e4853f..2200b481ae 100644 --- a/tests/updates-manifest.test.ts +++ b/tests/updates-manifest.test.ts @@ -102,6 +102,39 @@ afterAll(async () => { await resetAppDataStats(APPNAME) }) +// Delta manifest test constants +const DELTA_APPNAME = `com.demo.app.delta.${id}` +let deltaOldVersionId: number | null = null +let deltaNewVersionId: number | null = null + +// Helper to insert multiple manifest entries for a version +async function insertMultipleManifestEntries(appVersionId: number, entries: { file_name: string, file_hash: string, s3_path: string }[]) { + const supabase = getSupabaseClient() + // First, delete any existing manifest entries for this version + await supabase.from('manifest').delete().eq('app_version_id', appVersionId) + // Insert all manifest entries in a single batch + const { error } = await supabase.from('manifest').insert( + entries.map(entry => ({ + app_version_id: appVersionId, + file_name: entry.file_name, + s3_path: entry.s3_path, + file_hash: entry.file_hash, + file_size: 100, + })), + ) + if (error) + throw new Error(`Failed to insert manifest entries: ${error.message}`) + + // Update the manifest_count on the version + await supabase.from('app_versions').update({ manifest_count: entries.length }).eq('id', appVersionId) + + // Also update manifest_bundle_count on the app to enable manifest fetching + const { data: version } = await supabase.from('app_versions').select('app_id').eq('id', appVersionId).single() + if (version) { + await supabase.from('apps').update({ manifest_bundle_count: entries.length }).eq('app_id', version.app_id) + } +} + describe('update manifest scenarios', () => { it('manifest update', async () => { // test manifest update working with plugin version >= 6.25.0 @@ -224,3 +257,165 @@ describe('update manifest scenarios', () => { expect(json.manifest).toBeDefined() }) }) + +describe('delta manifest scenarios', () => { + beforeAll(async () => { + // Set up the delta manifest test app with two versions + await resetAndSeedAppData(DELTA_APPNAME) + const supabase = getSupabaseClient() + + // Get version IDs for old (1.359.0) and new (1.0.0) versions + // Note: resetAndSeedAppData creates versions 1.0.0, 1.0.1, 1.359.0, 1.360.0, 1.361.0 + // The production channel points to version 1.0.0 by default + const { data: oldVersion } = await supabase + .from('app_versions') + .select('id') + .eq('name', '1.359.0') + .eq('app_id', DELTA_APPNAME) + .single() + + const { data: newVersion } = await supabase + .from('app_versions') + .select('id') + .eq('name', '1.0.0') + .eq('app_id', DELTA_APPNAME) + .single() + + if (oldVersion) + deltaOldVersionId = oldVersion.id + if (newVersion) + deltaNewVersionId = newVersion.id + + // Set up manifest entries for delta testing: + // Old version (1.359.0): file_a.js (hash1), file_b.js (hash2) + // New version (1.0.0): file_a.js (hash1 - unchanged), file_b.js (hash3 - changed), file_c.js (hash4 - new) + if (deltaOldVersionId) { + await insertMultipleManifestEntries(deltaOldVersionId, [ + { file_name: 'file_a.js', file_hash: 'hash_unchanged_1', s3_path: '/file_a.js' }, + { file_name: 'file_b.js', file_hash: 'hash_old_2', s3_path: '/file_b.js' }, + ]) + } + + if (deltaNewVersionId) { + await insertMultipleManifestEntries(deltaNewVersionId, [ + { file_name: 'file_a.js', file_hash: 'hash_unchanged_1', s3_path: '/file_a.js' }, // Same hash - should be excluded + { file_name: 'file_b.js', file_hash: 'hash_new_3', s3_path: '/file_b.js' }, // Different hash - should be included + { file_name: 'file_c.js', file_hash: 'hash_new_4', s3_path: '/file_c.js' }, // New file - should be included + ]) + } + }) + + afterAll(async () => { + await resetAppData(DELTA_APPNAME) + await resetAppDataStats(DELTA_APPNAME) + }) + + it.concurrent('returns delta manifest with only changed/new files', async () => { + // Request update from old version (1.359.0) to new version (1.0.0) + // Should only return file_b.js (changed hash) and file_c.js (new file) + // file_a.js should be excluded because it has the same hash + const baseData = getBaseData(DELTA_APPNAME) + baseData.version_name = '1.359.0' // Device is on old version + baseData.plugin_version = '7.1.0' // Plugin version that supports manifest + + const response = await postUpdate(baseData) + expect(response.status).toBe(200) + const json = await response.json() + + expect(json.manifest).toBeDefined() + expect(json.manifest?.length).toBe(2) // Only file_b and file_c + + // Verify the returned files are the changed/new ones + const fileNames = json.manifest?.map(m => m.file_name).sort() + expect(fileNames).toEqual(['file_b.js', 'file_c.js']) + + // Verify file_a.js is NOT in the response (it has unchanged hash) + expect(json.manifest?.find(m => m.file_name === 'file_a.js')).toBeUndefined() + + // Verify file_b has new hash + const fileB = json.manifest?.find(m => m.file_name === 'file_b.js') + expect(fileB?.file_hash).toBe('hash_new_3') + + // Verify file_c is the new file + const fileC = json.manifest?.find(m => m.file_name === 'file_c.js') + expect(fileC?.file_hash).toBe('hash_new_4') + }) + + it.concurrent('returns full manifest for first install (builtin version)', async () => { + // When device is on 'builtin', it should receive full manifest (all 3 files) + const baseData = getBaseData(DELTA_APPNAME) + baseData.version_name = 'builtin' // First install + baseData.plugin_version = '7.1.0' + + const response = await postUpdate(baseData) + expect(response.status).toBe(200) + const json = await response.json() + + expect(json.manifest).toBeDefined() + expect(json.manifest?.length).toBe(3) // All files: file_a, file_b, file_c + + const fileNames = json.manifest?.map(m => m.file_name).sort() + expect(fileNames).toEqual(['file_a.js', 'file_b.js', 'file_c.js']) + }) + + it.concurrent('returns full manifest when old version does not exist', async () => { + // When device's version doesn't exist in DB, should return full manifest + const baseData = getBaseData(DELTA_APPNAME) + baseData.version_name = '99.99.99' // Non-existent version + baseData.plugin_version = '7.1.0' + + const response = await postUpdate(baseData) + expect(response.status).toBe(200) + const json = await response.json() + + expect(json.manifest).toBeDefined() + expect(json.manifest?.length).toBe(3) // All files (graceful fallback) + + const fileNames = json.manifest?.map(m => m.file_name).sort() + expect(fileNames).toEqual(['file_a.js', 'file_b.js', 'file_c.js']) + }) + + // This test modifies data (inserts manifest entries for version 1.0.1), so it runs sequentially + it('returns empty manifest when all files are identical', async () => { + // Create a scenario where old and new versions have identical manifests + const supabase = getSupabaseClient() + + // Get the 1.0.1 version (different from the one on channel) + const { data: identicalVersion } = await supabase + .from('app_versions') + .select('id') + .eq('name', '1.0.1') + .eq('app_id', DELTA_APPNAME) + .single() + + if (identicalVersion) { + // Set up identical manifest entries as the new version (1.0.0) + await insertMultipleManifestEntries(identicalVersion.id, [ + { file_name: 'file_a.js', file_hash: 'hash_unchanged_1', s3_path: '/file_a.js' }, + { file_name: 'file_b.js', file_hash: 'hash_new_3', s3_path: '/file_b.js' }, + { file_name: 'file_c.js', file_hash: 'hash_new_4', s3_path: '/file_c.js' }, + ]) + } + + try { + const baseData = getBaseData(DELTA_APPNAME) + baseData.version_name = '1.0.1' // Device has identical manifest + baseData.plugin_version = '7.1.0' + + const response = await postUpdate(baseData) + expect(response.status).toBe(200) + const json = await response.json() + + // When all files are identical, delta should be empty + expect(json.manifest).toBeDefined() + expect(json.manifest?.length).toBe(0) + } + finally { + // Clean up: remove the manifest entries we just added + if (identicalVersion) { + await supabase.from('manifest').delete().eq('app_version_id', identicalVersion.id) + await supabase.from('app_versions').update({ manifest_count: 0 }).eq('id', identicalVersion.id) + } + } + }) +})