diff --git a/deploy/api-gateway.tf b/deploy/api-gateway.tf index 5f13fa1..239b40d 100644 --- a/deploy/api-gateway.tf +++ b/deploy/api-gateway.tf @@ -100,6 +100,7 @@ resource "aws_api_gateway_integration" "artists_tracks_integration" { integration_http_method = "POST" type = "AWS_PROXY" uri = aws_lambda_function.lambda.invoke_arn + timeout_milliseconds = 60000 } @@ -187,6 +188,7 @@ resource "aws_api_gateway_deployment" "deployment" { aws_api_gateway_resource.artists_tracks_resource.path_part, aws_api_gateway_method.artists_tracks_method.id, aws_api_gateway_integration.artists_tracks_integration.id, + aws_api_gateway_integration.artists_tracks_integration.timeout_milliseconds, aws_api_gateway_resource.me_resource.id, aws_api_gateway_resource.me_resource.path_part, aws_api_gateway_method.me_method.id, diff --git a/deploy/lambda.tf b/deploy/lambda.tf index 1b75463..2bb01a6 100644 --- a/deploy/lambda.tf +++ b/deploy/lambda.tf @@ -5,7 +5,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda_role.arn handler = "index.handler" runtime = "nodejs18.x" - timeout = 30 + timeout = 61 source_code_hash = data.archive_file.lambda_zip.output_base64sha256 memory_size = 1024 diff --git a/package.json b/package.json index dbfcfa1..d409b8a 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "dependencies": { "aws-lambda": "^1.0.7", "cookie": "^1.0.2", - "express": "^4.18.2" + "express": "^4.18.2", + "p-limit": "^6.2.0" }, "devDependencies": { "@types/aws-lambda": "^8.10.150", diff --git a/src/spotify/getArtistsAlbums.ts b/src/spotify/getArtistsAlbums.ts index 7ec1ada..d8b3c00 100644 --- a/src/spotify/getArtistsAlbums.ts +++ b/src/spotify/getArtistsAlbums.ts @@ -1,4 +1,5 @@ import { getArtistsAlbumsForAlbumGroup } from './getArtistsAlbumsForAlbumGroup'; +import pLimit from 'p-limit'; export const getArtistsAlbums = async ({ accessToken, @@ -8,12 +9,16 @@ export const getArtistsAlbums = async ({ artistId: string; }): Promise => { const groups = ['album', 'single', 'compilation', 'appears_on']; + + const limit = pLimit(1); const getAlbumsForAlbumGroupPromises = groups.map((group) => - getArtistsAlbumsForAlbumGroup({ - accessToken, - artistId, - group - }) + limit(() => + getArtistsAlbumsForAlbumGroup({ + accessToken, + artistId, + group + }) + ) ); const promiseResults = await Promise.all(getAlbumsForAlbumGroupPromises); diff --git a/src/spotify/getArtistsAlbumsForAlbumGroup.ts b/src/spotify/getArtistsAlbumsForAlbumGroup.ts index dea3cd5..88e5a15 100644 --- a/src/spotify/getArtistsAlbumsForAlbumGroup.ts +++ b/src/spotify/getArtistsAlbumsForAlbumGroup.ts @@ -1,5 +1,6 @@ import { fetchSpotifyData } from './fetchSpotifyData'; import { Album } from './getArtistsAlbums'; +import pLimit from 'p-limit'; const limit = 50; @@ -84,13 +85,15 @@ const getAdditionalAlbums = async ({ urlsForOtherPages.push(initialUrl + `&offset=${i}`); } + const concurrencyLimit = pLimit(1); const otherPagesPromises = urlsForOtherPages.map((url) => - fetchSpotifyData({ - accessToken, - url - }) + concurrencyLimit(() => + fetchSpotifyData({ + accessToken, + url + }) + ) ); - const otherPagesData = await Promise.all(otherPagesPromises); const albums: Album[] = []; otherPagesData.forEach(({ items }) => { diff --git a/src/spotify/getArtistsTracks.ts b/src/spotify/getArtistsTracks.ts index 9d06270..6e053e7 100644 --- a/src/spotify/getArtistsTracks.ts +++ b/src/spotify/getArtistsTracks.ts @@ -1,6 +1,7 @@ import { Album } from './getArtistsAlbums'; import { fetchSpotifyData } from './fetchSpotifyData'; import { getArtistsAlbums } from './getArtistsAlbums'; +import pLimit from 'p-limit'; export const getArtistsTracks = async ({ accessToken, @@ -11,8 +12,6 @@ export const getArtistsTracks = async ({ }): Promise => { const albums = await getArtistsAlbums({ accessToken, artistId }); - const tracks: Track[] = []; - // Process albums in batches (Spotify API limit) const maxAlbumsPerRequest = 20; const albumBatches = splitIntoChunks({ @@ -20,13 +19,21 @@ export const getArtistsTracks = async ({ chunkSize: maxAlbumsPerRequest }); - for (const batch of albumBatches) { - const tracksThisBatch = await getTracksForAlbumBatch({ - albumsBatch: batch, - accessToken - }); + const limit = pLimit(1); + const getTracksPromises = albumBatches.map((batch) => + limit(() => + getTracksForAlbumBatch({ + albumsBatch: batch, + accessToken + }) + ) + ); + const getTracksResults = await Promise.all(getTracksPromises); + + const tracks: Track[] = []; + getTracksResults.forEach((tracksThisBatch) => { tracks.push(...tracksThisBatch); - } + }); // All tracks for all the albums the artist features on have been obtained // Some albums may have tracks that don't feature the given artist diff --git a/yarn.lock b/yarn.lock index 4ccb937..25d24d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4775,6 +4775,15 @@ __metadata: languageName: node linkType: hard +"p-limit@npm:^6.2.0": + version: 6.2.0 + resolution: "p-limit@npm:6.2.0" + dependencies: + yocto-queue: "npm:^1.1.1" + checksum: 10c0/448bf55a1776ca1444594d53b3c731e68cdca00d44a6c8df06a2f6e506d5bbd540ebb57b05280f8c8bff992a630ed782a69612473f769a7473495d19e2270166 + languageName: node + linkType: hard + "p-locate@npm:^4.1.0": version: 4.1.0 resolution: "p-locate@npm:4.1.0" @@ -5160,6 +5169,7 @@ __metadata: jest-util: "npm:^30.0.2" jest-when: "npm:^3.7.0" nock: "npm:^14.0.5" + p-limit: "npm:^6.2.0" prettier: "npm:^3.6.2" ts-jest: "npm:^29.4.0" typescript: "npm:^5.8.3" @@ -6232,3 +6242,10 @@ __metadata: checksum: 10c0/dceb44c28578b31641e13695d200d34ec4ab3966a5729814d5445b194933c096b7ced71494ce53a0e8820685d1d010df8b2422e5bf2cdea7e469d97ffbea306f languageName: node linkType: hard + +"yocto-queue@npm:^1.1.1": + version: 1.2.1 + resolution: "yocto-queue@npm:1.2.1" + checksum: 10c0/5762caa3d0b421f4bdb7a1926b2ae2189fc6e4a14469258f183600028eb16db3e9e0306f46e8ebf5a52ff4b81a881f22637afefbef5399d6ad440824e9b27f9f + languageName: node + linkType: hard