From 91e8aa590eab5f2aa2e1e66ee0d42b82412d2424 Mon Sep 17 00:00:00 2001 From: Lean Mendoza Date: Wed, 13 Nov 2024 09:56:09 -0300 Subject: [PATCH 1/5] wip add fetch entities ids --- .../handlers/active-entities-ids-handler.ts | 38 +++++++++++++++++++ content/src/controller/routes.ts | 2 + 2 files changed, 40 insertions(+) create mode 100644 content/src/controller/handlers/active-entities-ids-handler.ts diff --git a/content/src/controller/handlers/active-entities-ids-handler.ts b/content/src/controller/handlers/active-entities-ids-handler.ts new file mode 100644 index 000000000..304288717 --- /dev/null +++ b/content/src/controller/handlers/active-entities-ids-handler.ts @@ -0,0 +1,38 @@ +import { Entity } from '@dcl/catalyst-api-specs/lib/client' +import { HandlerContextWithPath, InvalidRequestError } from '../../types' +import Joi from 'joi' + +const schema = Joi.alternatives().try( + Joi.object({ + pointers: Joi.array().items(Joi.string()).min(1).required() + }) +) + +// Method: POST +// Body: { pointers: string[]} +export async function getActiveEntitiesIdsHandler( + context: HandlerContextWithPath<'database' | 'activeEntities' | 'denylist', '/entities/active'> +): Promise<{ status: 200; body: Pick[] }> { + const { database, activeEntities, denylist } = context.components + const { error, value: body } = schema.validate(await context.request.json()) + + if (error) { + throw new InvalidRequestError( + 'pointers must be present. They must be arrays and contain at least one element. None of the elements can be empty.' + ) + } + + const entities: Pick[] = (await activeEntities.withPointers(database, body.pointers)) + .filter((result) => !denylist.isDenylisted(result.id)) + .map((entity) => { + return { + id: entity.id, + pointers: entity.pointers + } + }) + + return { + status: 200, + body: entities + } +} diff --git a/content/src/controller/routes.ts b/content/src/controller/routes.ts index 4831fa6db..9213328a4 100644 --- a/content/src/controller/routes.ts +++ b/content/src/controller/routes.ts @@ -20,6 +20,7 @@ import { getERC721EntityHandler } from './handlers/get-erc721-entity-handler' import { getDeploymentsHandler } from './handlers/get-deployments-handler' import { getChallengeHandler } from './handlers/get-challenge-handler' import { getActiveEntityIdsByDeploymentHashHandler } from './handlers/get-active-entities-by-deployment-hash-handler' +import { getActiveEntitiesIdsHandler } from './handlers/active-entities-ids-handler' // We return the entire router because it will be easier to test than a whole server export async function setupRouter({ components }: GlobalContext): Promise> { @@ -42,6 +43,7 @@ export async function setupRouter({ components }: GlobalContext): Promise Date: Wed, 13 Nov 2024 09:59:48 -0300 Subject: [PATCH 2/5] add timestamp --- .../controller/handlers/active-entities-ids-handler.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/content/src/controller/handlers/active-entities-ids-handler.ts b/content/src/controller/handlers/active-entities-ids-handler.ts index 304288717..e9456c9ed 100644 --- a/content/src/controller/handlers/active-entities-ids-handler.ts +++ b/content/src/controller/handlers/active-entities-ids-handler.ts @@ -12,7 +12,7 @@ const schema = Joi.alternatives().try( // Body: { pointers: string[]} export async function getActiveEntitiesIdsHandler( context: HandlerContextWithPath<'database' | 'activeEntities' | 'denylist', '/entities/active'> -): Promise<{ status: 200; body: Pick[] }> { +): Promise<{ status: 200; body: Pick[] }> { const { database, activeEntities, denylist } = context.components const { error, value: body } = schema.validate(await context.request.json()) @@ -22,12 +22,15 @@ export async function getActiveEntitiesIdsHandler( ) } - const entities: Pick[] = (await activeEntities.withPointers(database, body.pointers)) + const entities: Pick[] = ( + await activeEntities.withPointers(database, body.pointers) + ) .filter((result) => !denylist.isDenylisted(result.id)) .map((entity) => { return { id: entity.id, - pointers: entity.pointers + pointers: entity.pointers, + timestamp: entity.timestamp } }) From cf7e3015a2d9a29539aa2298222f2cf92fe05d2b Mon Sep 17 00:00:00 2001 From: Lean Mendoza Date: Tue, 19 Nov 2024 07:20:31 -0300 Subject: [PATCH 3/5] other approach --- content/src/components.ts | 2 + .../handlers/active-entities-handler.ts | 17 +++ .../handlers/active-entities-ids-handler.ts | 41 ------- content/src/controller/routes.ts | 23 ++-- content/src/logic/deployments.ts | 44 ++++--- content/src/ports/activeEntities.ts | 108 +++++++++++++++--- 6 files changed, 153 insertions(+), 82 deletions(-) delete mode 100644 content/src/controller/handlers/active-entities-ids-handler.ts diff --git a/content/src/components.ts b/content/src/components.ts index bf04f8d55..dd354b925 100644 --- a/content/src/components.ts +++ b/content/src/components.ts @@ -186,6 +186,8 @@ export async function initComponentsWithEnv(env: Environment): Promise +): Promise<{ status: 200; body: Pick[] }> { + const { activeEntities, denylist } = context.components + const entities: Entity[] = activeEntities.getAllCachedScenes().filter((result) => !denylist.isDenylisted(result.id)) + const mapping = entities.map((entity) => ({ + id: entity.id, + pointers: entity.pointers, + timestamp: entity.timestamp + })) + return { + status: 200, + body: mapping + } +} diff --git a/content/src/controller/handlers/active-entities-ids-handler.ts b/content/src/controller/handlers/active-entities-ids-handler.ts deleted file mode 100644 index e9456c9ed..000000000 --- a/content/src/controller/handlers/active-entities-ids-handler.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Entity } from '@dcl/catalyst-api-specs/lib/client' -import { HandlerContextWithPath, InvalidRequestError } from '../../types' -import Joi from 'joi' - -const schema = Joi.alternatives().try( - Joi.object({ - pointers: Joi.array().items(Joi.string()).min(1).required() - }) -) - -// Method: POST -// Body: { pointers: string[]} -export async function getActiveEntitiesIdsHandler( - context: HandlerContextWithPath<'database' | 'activeEntities' | 'denylist', '/entities/active'> -): Promise<{ status: 200; body: Pick[] }> { - const { database, activeEntities, denylist } = context.components - const { error, value: body } = schema.validate(await context.request.json()) - - if (error) { - throw new InvalidRequestError( - 'pointers must be present. They must be arrays and contain at least one element. None of the elements can be empty.' - ) - } - - const entities: Pick[] = ( - await activeEntities.withPointers(database, body.pointers) - ) - .filter((result) => !denylist.isDenylisted(result.id)) - .map((entity) => { - return { - id: entity.id, - pointers: entity.pointers, - timestamp: entity.timestamp - } - }) - - return { - status: 200, - body: entities - } -} diff --git a/content/src/controller/routes.ts b/content/src/controller/routes.ts index 9213328a4..02be443ab 100644 --- a/content/src/controller/routes.ts +++ b/content/src/controller/routes.ts @@ -2,25 +2,24 @@ import { Router } from '@well-known-components/http-server' import { multipartParserWrapper } from '@well-known-components/multipart-wrapper' import { EnvironmentConfig } from '../Environment' import { GlobalContext } from '../types' -import { getActiveEntitiesHandler } from './handlers/active-entities-handler' +import { getActiveEntitiesHandler, getActiveEntitiesScenesHandler } from './handlers/active-entities-handler' import { createEntity } from './handlers/create-entity-handler' -import { createErrorHandler, preventExecutionIfBoostrapping } from './middlewares' import { getFailedDeploymentsHandler } from './handlers/failed-deployments-handler' import { getEntitiesByPointerPrefixHandler } from './handlers/filter-by-urn-handler' +import { getActiveEntityIdsByDeploymentHashHandler } from './handlers/get-active-entities-by-deployment-hash-handler' import { getEntityAuditInformationHandler } from './handlers/get-audit-handler' import { getAvailableContentHandler } from './handlers/get-available-content-handler' -import { getPointerChangesHandler } from './handlers/pointer-changes-handler' -import { getStatusHandler } from './handlers/status-handler' -import { getSnapshotsHandler } from './handlers/get-snapshots-handler' -import { getEntitiesHandler } from './handlers/get-entities-handler' +import { getChallengeHandler } from './handlers/get-challenge-handler' import { getContentHandler } from './handlers/get-content-handler' -import { getEntityThumbnailHandler } from './handlers/get-entity-thumbnail-handler' +import { getDeploymentsHandler } from './handlers/get-deployments-handler' +import { getEntitiesHandler } from './handlers/get-entities-handler' import { getEntityImageHandler } from './handlers/get-entity-image-handler' +import { getEntityThumbnailHandler } from './handlers/get-entity-thumbnail-handler' import { getERC721EntityHandler } from './handlers/get-erc721-entity-handler' -import { getDeploymentsHandler } from './handlers/get-deployments-handler' -import { getChallengeHandler } from './handlers/get-challenge-handler' -import { getActiveEntityIdsByDeploymentHashHandler } from './handlers/get-active-entities-by-deployment-hash-handler' -import { getActiveEntitiesIdsHandler } from './handlers/active-entities-ids-handler' +import { getSnapshotsHandler } from './handlers/get-snapshots-handler' +import { getPointerChangesHandler } from './handlers/pointer-changes-handler' +import { getStatusHandler } from './handlers/status-handler' +import { createErrorHandler, preventExecutionIfBoostrapping } from './middlewares' // We return the entire router because it will be easier to test than a whole server export async function setupRouter({ components }: GlobalContext): Promise> { @@ -43,7 +42,7 @@ export async function setupRouter({ components }: GlobalContext): Promise { - // Generate the select according the info needed - const bothPresent = entityIds && entityIds.length > 0 && pointers && pointers.length > 0 - const nonePresent = !entityIds && !pointers - if (bothPresent || nonePresent) { - throw Error('in getDeploymentsForActiveEntities ids or pointers must be present, but not both') + // Validate that only one parameter is provided + const providedParams = [entityIds && entityIds.length > 0, pointers && pointers.length > 0, entityType].filter( + Boolean + ) + + if (providedParams.length !== 1) { + throw Error('getDeploymentsForActiveEntities requires exactly one of: entityIds, pointers, or entityType') } const query: SQLStatement = SQL` @@ -248,15 +251,23 @@ export async function getDeploymentsForActiveEntities( FROM deployments AS dep1 WHERE dep1.deleter_deployment IS NULL AND `.append( - entityIds + entityIds && entityIds.length > 0 ? SQL`dep1.entity_id = ANY (${entityIds})` - : SQL`dep1.entity_pointers && ${pointers!.map((p) => p.toLowerCase())}` + : pointers && pointers.length > 0 + ? SQL`dep1.entity_pointers && ${pointers.map((p) => p.toLowerCase())}` + : SQL`dep1.entity_type = ${entityType}` ) - const historicalDeploymentsResponse = await database.queryWithValues(query, 'get_active_entities') + const BATCH_SIZE = 1000 + const deploymentsResult: HistoricalDeployment[] = [] + const cursor = await database.streamQuery( + query, + { batchSize: BATCH_SIZE }, + 'get_active_entities' + ) - const deploymentsResult: HistoricalDeployment[] = historicalDeploymentsResponse.rows.map( - (row: HistoricalDeploymentsRow): HistoricalDeployment => ({ + for await (const row of cursor) { + deploymentsResult.push({ deploymentId: row.id, entityType: row.entity_type, entityId: row.entity_id, @@ -269,12 +280,15 @@ export async function getDeploymentsForActiveEntities( localTimestamp: row.local_timestamp, overwrittenBy: row.overwritten_by ?? undefined }) - ) - - const deploymentIds = deploymentsResult.map(({ deploymentId }) => deploymentId) + } - const content = await getContentFiles(database, deploymentIds) + // Batch fetch all content files at once + const content = await getContentFiles( + database, + deploymentsResult.map((d) => d.deploymentId) + ) + // Map results to final format return deploymentsResult.map((result) => ({ entityVersion: result.version as EntityVersion, entityType: result.entityType as EntityType, diff --git a/content/src/ports/activeEntities.ts b/content/src/ports/activeEntities.ts index 3297a6d0d..ee5a243d7 100644 --- a/content/src/ports/activeEntities.ts +++ b/content/src/ports/activeEntities.ts @@ -69,6 +69,16 @@ export type ActiveEntities = IBaseComponent & { * Note: only used in stale profiles GC */ clearPointers(pointers: string[]): Promise + + /** + * Initialize the cache with the active entities for the cached entity types + */ + initialize(database: DatabaseClient): Promise + + /** + * Get all cached scenes + */ + getAllCachedScenes(): Entity[] } /** @@ -81,9 +91,6 @@ export function createActiveEntitiesComponent( components: Pick ): ActiveEntities { const logger = components.logs.getLogger('ActiveEntities') - const cache = new LRU({ - max: components.env.getConfig(EnvironmentConfig.ENTITIES_CACHE_SIZE) - }) const collectionUrnsByPrefixCache = new LRU({ ttl: 1000 * 60 * 60 * 24, // 24 hours @@ -94,6 +101,47 @@ export function createActiveEntitiesComponent( const normalizePointerCacheKey = (pointer: string) => pointer.toLowerCase() + const cache = new LRU({ + max: components.env.getConfig(EnvironmentConfig.ENTITIES_CACHE_SIZE) + }) + const fixedCache = new Map() + + const createLRUandFixedCache = (maxLRU: number, fixedTypes: EntityType[]) => { + return { + get(entityId: string) { + return cache.get(entityId) || fixedCache.get(entityId) + }, + set(entityId: string, entity: Entity | NotActiveEntity) { + const isFixed = fixedCache.has(entityId) || (typeof entity !== 'string' && fixedTypes.includes(entity.type)) + if (isFixed) { + return fixedCache.set(entityId, entity) + } + return cache.set(entityId, entity) + }, + setFixed(entityId: string, entity: Entity | NotActiveEntity) { + return fixedCache.set(entityId, entity) + }, + get max() { + return cache.max + }, + clear() { + cache.clear() + fixedCache.clear() + }, + has(entityId: string) { + return cache.has(entityId) || fixedCache.has(entityId) + } + } + } + + const cachedEntityntityTypes = [EntityType.SCENE] + + // Entities cache by key=entityId + const entityCache = createLRUandFixedCache( + components.env.getConfig(EnvironmentConfig.ENTITIES_CACHE_SIZE), + cachedEntityntityTypes + ) + const createEntityByPointersCache = (): Map => { const entityIdByPointers = new Map() return { @@ -116,7 +164,7 @@ export function createActiveEntitiesComponent( const entityIdByPointers = createEntityByPointersCache() // init gauge metrics - components.metrics.observe('dcl_entities_cache_storage_max_size', {}, cache.max) + components.metrics.observe('dcl_entities_cache_storage_max_size', {}, entityCache.max) Object.values(EntityType).forEach((entityType) => { components.metrics.observe('dcl_entities_cache_storage_size', { entity_type: entityType }, 0) }) @@ -133,9 +181,9 @@ export function createActiveEntitiesComponent( // pointer now have a different active entity, let's update the old one const entityId = entityIdByPointers.get(pointer) if (isPointingToEntity(entityId)) { - const entity = cache.get(entityId) // it should be present + const entity = entityCache.get(entityId) // it should be present if (isEntityPresent(entity)) { - cache.set(entityId, 'NOT_ACTIVE_ENTITY') + entityCache.set(entityId, 'NOT_ACTIVE_ENTITY') for (const pointer of entity.pointers) { entityIdByPointers.set(pointer, 'NOT_ACTIVE_ENTITY') } @@ -158,7 +206,7 @@ export function createActiveEntitiesComponent( entityIdByPointers.set(pointer, isEntityPresent(entity) ? entity.id : entity) } if (isEntityPresent(entity)) { - cache.set(entity.id, entity) + entityCache.set(entity.id, entity) components.metrics.increment('dcl_entities_cache_storage_size', { entity_type: entity.type }) // Store in the db the new entity pointed by pointers await updateActiveDeployments(database, pointers, entity.id) @@ -196,7 +244,7 @@ export function createActiveEntitiesComponent( ) for (const entityId of entityIdsWithoutActiveEntity) { - cache.set(entityId, 'NOT_ACTIVE_ENTITY') + entityCache.set(entityId, 'NOT_ACTIVE_ENTITY') logger.debug('entityId has no active entity', { entityId }) } } @@ -227,6 +275,19 @@ export function createActiveEntitiesComponent( return entities } + async function populateEntityType(database: DatabaseClient, entityType: EntityType): Promise { + const deployments = await getDeploymentsForActiveEntities(database, undefined, undefined, entityType) + + logger.info('Populating cache for entity type', { entityType, deployments: deployments.length }) + + for (const deployment of deployments) { + reportCacheAccess(deployment.entityType, 'miss') + } + + const entities = mapDeploymentsToEntities(deployments) + await updateCache(database, entities, {}) + } + /** * Retrieve active entities by their ids */ @@ -236,7 +297,7 @@ export function createActiveEntitiesComponent( const onCache: (Entity | NotActiveEntity)[] = [] const remaining: string[] = [] for (const entityId of uniqueEntityIds) { - const entity = cache.get(entityId) + const entity = entityCache.get(entityId) if (entity) { onCache.push(entity) if (isEntityPresent(entity)) { @@ -278,6 +339,8 @@ export function createActiveEntitiesComponent( } } + logger.info('Retrieving entities by pointers', { pointers: remaining.length }) + // once we get the ids, retrieve from cache or find const entityIds = Array.from(uniqueEntityIds.values()) const entitiesById = await withIds(database, entityIds) @@ -313,7 +376,7 @@ export function createActiveEntitiesComponent( for (const pointer of pointers) { if (entityIdByPointers.has(pointer)) { const entityId = entityIdByPointers.get(pointer)! - cache.set(entityId, 'NOT_ACTIVE_ENTITY') + entityCache.set(entityId, 'NOT_ACTIVE_ENTITY') entityIdByPointers.set(pointer, 'NOT_ACTIVE_ENTITY') } } @@ -322,7 +385,23 @@ export function createActiveEntitiesComponent( function reset() { entityIdByPointers.clear() collectionUrnsByPrefixCache.clear() - cache.clear() + entityCache.clear() + } + + async function initialize(database: DatabaseClient) { + logger.info('Initializing active entities cache', { + entityTypes: cachedEntityntityTypes.map((t) => t.toString()).toString() + }) + for (const entityType of cachedEntityntityTypes) { + await populateEntityType(database, entityType) + } + logger.info('Active entities cache initialized') + } + + function getAllCachedScenes() { + return Array.from(fixedCache.values()).filter( + (entity): entity is Entity => typeof entity !== 'string' && entity.type === EntityType.SCENE + ) } return { @@ -333,10 +412,11 @@ export function createActiveEntitiesComponent( update, clear, clearPointers, - + initialize, + getAllCachedScenes, getCachedEntity(idOrPointer) { - if (cache.has(idOrPointer)) { - const cachedEntity = cache.get(idOrPointer) + if (entityCache.has(idOrPointer)) { + const cachedEntity = entityCache.get(idOrPointer) return isEntityPresent(cachedEntity) ? cachedEntity.id : cachedEntity } return entityIdByPointers.get(idOrPointer) From cbe0c52db033b2e440bef350f07ea01583e76dd8 Mon Sep 17 00:00:00 2001 From: Lean Mendoza Date: Tue, 19 Nov 2024 07:58:34 -0300 Subject: [PATCH 4/5] optional initialization --- content/src/ports/activeEntities.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/content/src/ports/activeEntities.ts b/content/src/ports/activeEntities.ts index ee5a243d7..22099a629 100644 --- a/content/src/ports/activeEntities.ts +++ b/content/src/ports/activeEntities.ts @@ -392,10 +392,14 @@ export function createActiveEntitiesComponent( logger.info('Initializing active entities cache', { entityTypes: cachedEntityntityTypes.map((t) => t.toString()).toString() }) - for (const entityType of cachedEntityntityTypes) { - await populateEntityType(database, entityType) + try { + for (const entityType of cachedEntityntityTypes) { + await populateEntityType(database, entityType) + } + logger.info('Active entities cache initialized') + } catch (error) { + logger.error('Error initializing active entities cache', { error }) } - logger.info('Active entities cache initialized') } function getAllCachedScenes() { From e1d3d01ae5276a9a04fb32e24d527ce5f32a9816 Mon Sep 17 00:00:00 2001 From: Lean Mendoza Date: Tue, 19 Nov 2024 08:49:13 -0300 Subject: [PATCH 5/5] add tests Signed-off-by: Lean Mendoza --- .../test/unit/ports/activeEntities.spec.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/content/test/unit/ports/activeEntities.spec.ts b/content/test/unit/ports/activeEntities.spec.ts index 7d18ab8fc..40d6a3465 100644 --- a/content/test/unit/ports/activeEntities.spec.ts +++ b/content/test/unit/ports/activeEntities.spec.ts @@ -332,6 +332,62 @@ describe('activeEntities', () => { expect(firstResult).toMatchObject(fourthResult) }) }) + + describe('after initialization', () => { + it('should populate cache with scenes during initialization', async () => { + const components = await buildComponents() + const sceneDeployment = { + ...fakeDeployment, + entityType: EntityType.SCENE, + entityId: 'scene1' + } + + sut.mockImplementation(() => Promise.resolve([sceneDeployment])) + + await components.activeEntities.initialize(components.database) + + const cachedScenes = components.activeEntities.getAllCachedScenes() + expect(cachedScenes).toHaveLength(1) + expect(cachedScenes[0].id).toBe('scene1') + expect(cachedScenes[0].type).toBe(EntityType.SCENE) + }) + + it('should only cache scene type entities', async () => { + const components = await buildComponents() + const mixedDeployments = [ + { + ...fakeDeployment, + entityType: EntityType.SCENE, + entityId: 'scene1', + pointers: ['bafkscene'] + }, + { + ...fakeDeployment, + entityType: EntityType.PROFILE, + entityId: 'profile1', + pointers: ['0xprofile'] + } + ] + + sut.mockImplementation(() => Promise.resolve(mixedDeployments)) + + await components.activeEntities.initialize(components.database) + + const cachedScenes = components.activeEntities.getAllCachedScenes() + expect(cachedScenes).toHaveLength(1) + expect(cachedScenes[0].id).toBe('scene1') + }) + + it('should return empty array when no scenes are cached', async () => { + const components = await buildComponents() + sut.mockImplementation(() => Promise.resolve([])) + + await components.activeEntities.initialize(components.database) + + const cachedScenes = components.activeEntities.getAllCachedScenes() + expect(cachedScenes).toHaveLength(0) + }) + }) }) async function buildComponents() {