From 820d605066e2e749103be05fe722fe60afeca18a Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Sun, 27 Jul 2025 12:58:54 +0530 Subject: [PATCH 1/9] deploy: increase timeout of gateway and lambda --- deploy/api-gateway.tf | 1 + deploy/lambda.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/api-gateway.tf b/deploy/api-gateway.tf index 5f13fa1..9910eee 100644 --- a/deploy/api-gateway.tf +++ b/deploy/api-gateway.tf @@ -100,6 +100,7 @@ resource "aws_api_gateway_integration" "artists_tracks_integration" { integration_http_method = "POST" type = "AWS_PROXY" uri = aws_lambda_function.lambda.invoke_arn + timeout_milliseconds = 60000 } diff --git a/deploy/lambda.tf b/deploy/lambda.tf index 1b75463..2bb01a6 100644 --- a/deploy/lambda.tf +++ b/deploy/lambda.tf @@ -5,7 +5,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda_role.arn handler = "index.handler" runtime = "nodejs18.x" - timeout = 30 + timeout = 61 source_code_hash = data.archive_file.lambda_zip.output_base64sha256 memory_size = 1024 From c4d7defc96ba3012b88e9e5a683dc2e0830a23a6 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Sun, 27 Jul 2025 13:08:26 +0530 Subject: [PATCH 2/9] Revert "deploy: increase timeout of gateway and lambda" This reverts commit 820d605066e2e749103be05fe722fe60afeca18a. --- deploy/api-gateway.tf | 1 - deploy/lambda.tf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/api-gateway.tf b/deploy/api-gateway.tf index 9910eee..5f13fa1 100644 --- a/deploy/api-gateway.tf +++ b/deploy/api-gateway.tf @@ -100,7 +100,6 @@ resource "aws_api_gateway_integration" "artists_tracks_integration" { integration_http_method = "POST" type = "AWS_PROXY" uri = aws_lambda_function.lambda.invoke_arn - timeout_milliseconds = 60000 } diff --git a/deploy/lambda.tf b/deploy/lambda.tf index 2bb01a6..1b75463 100644 --- a/deploy/lambda.tf +++ b/deploy/lambda.tf @@ -5,7 +5,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda_role.arn handler = "index.handler" runtime = "nodejs18.x" - timeout = 61 + timeout = 30 source_code_hash = data.archive_file.lambda_zip.output_base64sha256 memory_size = 1024 From 23bc70e7e1040e8f328e00d270811c8e899b47b0 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Sun, 27 Jul 2025 13:13:31 +0530 Subject: [PATCH 3/9] deploy: increase timeout of gateway and lambda --- deploy/api-gateway.tf | 2 ++ deploy/lambda.tf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/api-gateway.tf b/deploy/api-gateway.tf index 5f13fa1..239b40d 100644 --- a/deploy/api-gateway.tf +++ b/deploy/api-gateway.tf @@ -100,6 +100,7 @@ resource "aws_api_gateway_integration" "artists_tracks_integration" { integration_http_method = "POST" type = "AWS_PROXY" uri = aws_lambda_function.lambda.invoke_arn + timeout_milliseconds = 60000 } @@ -187,6 +188,7 @@ resource "aws_api_gateway_deployment" "deployment" { aws_api_gateway_resource.artists_tracks_resource.path_part, aws_api_gateway_method.artists_tracks_method.id, aws_api_gateway_integration.artists_tracks_integration.id, + aws_api_gateway_integration.artists_tracks_integration.timeout_milliseconds, aws_api_gateway_resource.me_resource.id, aws_api_gateway_resource.me_resource.path_part, aws_api_gateway_method.me_method.id, diff --git a/deploy/lambda.tf b/deploy/lambda.tf index 1b75463..2bb01a6 100644 --- a/deploy/lambda.tf +++ b/deploy/lambda.tf @@ -5,7 +5,7 @@ resource "aws_lambda_function" "lambda" { role = aws_iam_role.lambda_role.arn handler = "index.handler" runtime = "nodejs18.x" - timeout = 30 + timeout = 61 source_code_hash = data.archive_file.lambda_zip.output_base64sha256 memory_size = 1024 From 6e1da6742decfd03a39d36348629aefbe5c6f552 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Sun, 27 Jul 2025 13:29:11 +0530 Subject: [PATCH 4/9] feat: add retries on 429 error response --- src/spotify/fetchSpotifyData.ts | 44 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/spotify/fetchSpotifyData.ts b/src/spotify/fetchSpotifyData.ts index 63849bd..ddb6a11 100644 --- a/src/spotify/fetchSpotifyData.ts +++ b/src/spotify/fetchSpotifyData.ts @@ -2,25 +2,43 @@ export const fetchSpotifyData = async ({ url, accessToken, body, - method = 'GET' + method = 'GET', + maxRetries = 3 }: { url: string; accessToken: string; body?: any; method?: string; + maxRetries?: number; }): Promise => { - const response = await fetch(url, { - method, - headers: { - Authorization: `Bearer ${accessToken}` - }, - body: JSON.stringify(body) - }); + let retries = 0; - if (!response.ok) { - console.log('Error text:', await response.text()); - throw new Error('Request to spotify was unsuccessful'); - } + while (true) { + const response = await fetch(url, { + method, + headers: { + Authorization: `Bearer ${accessToken}` + }, + body: JSON.stringify(body) + }); + + if (response.status === 429 && retries < maxRetries) { + // Get retry delay from header or default to 1 second + const retryAfter = response.headers.get('Retry-After'); + const delaySeconds = retryAfter ? parseInt(retryAfter, 10) : 1; + console.log(`Rate limited by Spotify. Retrying after ${delaySeconds} seconds. Retry ${retries + 1}/${maxRetries}`); + + // Wait for the specified delay + await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000)); + retries++; + continue; + } - return (await response.json()) as T; + if (!response.ok) { + console.log('Error text:', await response.text()); + throw new Error('Request to spotify was unsuccessful'); + } + + return (await response.json()) as T; + } }; From 8e7bca5b4c2a1734f98349721ace38f9f72ecad5 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Tue, 29 Jul 2025 12:13:12 +0530 Subject: [PATCH 5/9] Revert "feat: add retries on 429 error response" This reverts commit 6e1da6742decfd03a39d36348629aefbe5c6f552. --- src/spotify/fetchSpotifyData.ts | 44 ++++++++++----------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/src/spotify/fetchSpotifyData.ts b/src/spotify/fetchSpotifyData.ts index ddb6a11..63849bd 100644 --- a/src/spotify/fetchSpotifyData.ts +++ b/src/spotify/fetchSpotifyData.ts @@ -2,43 +2,25 @@ export const fetchSpotifyData = async ({ url, accessToken, body, - method = 'GET', - maxRetries = 3 + method = 'GET' }: { url: string; accessToken: string; body?: any; method?: string; - maxRetries?: number; }): Promise => { - let retries = 0; + const response = await fetch(url, { + method, + headers: { + Authorization: `Bearer ${accessToken}` + }, + body: JSON.stringify(body) + }); - while (true) { - const response = await fetch(url, { - method, - headers: { - Authorization: `Bearer ${accessToken}` - }, - body: JSON.stringify(body) - }); - - if (response.status === 429 && retries < maxRetries) { - // Get retry delay from header or default to 1 second - const retryAfter = response.headers.get('Retry-After'); - const delaySeconds = retryAfter ? parseInt(retryAfter, 10) : 1; - console.log(`Rate limited by Spotify. Retrying after ${delaySeconds} seconds. Retry ${retries + 1}/${maxRetries}`); - - // Wait for the specified delay - await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000)); - retries++; - continue; - } - - if (!response.ok) { - console.log('Error text:', await response.text()); - throw new Error('Request to spotify was unsuccessful'); - } - - return (await response.json()) as T; + if (!response.ok) { + console.log('Error text:', await response.text()); + throw new Error('Request to spotify was unsuccessful'); } + + return (await response.json()) as T; }; From 7e53a568fc62c0c7136ca87bfb68498be8c1aec1 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Tue, 29 Jul 2025 13:16:54 +0530 Subject: [PATCH 6/9] feat: add 5 requests per sec limit --- src/spotify/fetchSpotifyData.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/spotify/fetchSpotifyData.ts b/src/spotify/fetchSpotifyData.ts index 63849bd..eb5d81a 100644 --- a/src/spotify/fetchSpotifyData.ts +++ b/src/spotify/fetchSpotifyData.ts @@ -1,3 +1,8 @@ +// Keep track of recent API calls +const recentApiCalls: number[] = []; +const MAX_CALLS_PER_SECOND = 5; // 5 requests per second +const TIME_WINDOW_MS = 1000; // 1 second window + export const fetchSpotifyData = async ({ url, accessToken, @@ -9,6 +14,29 @@ export const fetchSpotifyData = async ({ body?: any; method?: string; }): Promise => { + // Rate limiting logic + const now = Date.now(); + + // Remove timestamps older than our time window + while (recentApiCalls.length > 0 && recentApiCalls[0] < now - TIME_WINDOW_MS) { + recentApiCalls.shift(); + } + + // If we've made too many calls recently, wait until we can make another + if (recentApiCalls.length >= MAX_CALLS_PER_SECOND) { + const oldestCall = recentApiCalls[0]; + const timeToWait = oldestCall + TIME_WINDOW_MS - now; + + if (timeToWait > 0) { + console.log(`Rate limit reached. Waiting ${timeToWait}ms before next request.`); + await new Promise(resolve => setTimeout(resolve, timeToWait)); + } + } + + // Add current timestamp to our tracking array + recentApiCalls.push(Date.now()); + + // Make the actual API call const response = await fetch(url, { method, headers: { From 510ddb10bbdb8c5f53c92e89c1bf8f368efc1576 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Tue, 29 Jul 2025 13:19:11 +0530 Subject: [PATCH 7/9] feat: get tracks in parallel --- src/spotify/getArtistsTracks.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/spotify/getArtistsTracks.ts b/src/spotify/getArtistsTracks.ts index 9d06270..509afb7 100644 --- a/src/spotify/getArtistsTracks.ts +++ b/src/spotify/getArtistsTracks.ts @@ -11,8 +11,6 @@ export const getArtistsTracks = async ({ }): Promise => { const albums = await getArtistsAlbums({ accessToken, artistId }); - const tracks: Track[] = []; - // Process albums in batches (Spotify API limit) const maxAlbumsPerRequest = 20; const albumBatches = splitIntoChunks({ @@ -20,13 +18,18 @@ export const getArtistsTracks = async ({ chunkSize: maxAlbumsPerRequest }); - for (const batch of albumBatches) { - const tracksThisBatch = await getTracksForAlbumBatch({ + const getTracksPromises = albumBatches.map((batch) => + getTracksForAlbumBatch({ albumsBatch: batch, accessToken - }); + }) + ); + const getTracksResults = await Promise.all(getTracksPromises); + + const tracks: Track[] = []; + getTracksResults.forEach((tracksThisBatch) => { tracks.push(...tracksThisBatch); - } + }); // All tracks for all the albums the artist features on have been obtained // Some albums may have tracks that don't feature the given artist From 2f14403ee51110ebe22ee829f663a27cf4e0d131 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Thu, 31 Jul 2025 23:12:29 +0530 Subject: [PATCH 8/9] Revert "feat: add 5 requests per sec limit" This reverts commit 7e53a568fc62c0c7136ca87bfb68498be8c1aec1. --- src/spotify/fetchSpotifyData.ts | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/spotify/fetchSpotifyData.ts b/src/spotify/fetchSpotifyData.ts index eb5d81a..63849bd 100644 --- a/src/spotify/fetchSpotifyData.ts +++ b/src/spotify/fetchSpotifyData.ts @@ -1,8 +1,3 @@ -// Keep track of recent API calls -const recentApiCalls: number[] = []; -const MAX_CALLS_PER_SECOND = 5; // 5 requests per second -const TIME_WINDOW_MS = 1000; // 1 second window - export const fetchSpotifyData = async ({ url, accessToken, @@ -14,29 +9,6 @@ export const fetchSpotifyData = async ({ body?: any; method?: string; }): Promise => { - // Rate limiting logic - const now = Date.now(); - - // Remove timestamps older than our time window - while (recentApiCalls.length > 0 && recentApiCalls[0] < now - TIME_WINDOW_MS) { - recentApiCalls.shift(); - } - - // If we've made too many calls recently, wait until we can make another - if (recentApiCalls.length >= MAX_CALLS_PER_SECOND) { - const oldestCall = recentApiCalls[0]; - const timeToWait = oldestCall + TIME_WINDOW_MS - now; - - if (timeToWait > 0) { - console.log(`Rate limit reached. Waiting ${timeToWait}ms before next request.`); - await new Promise(resolve => setTimeout(resolve, timeToWait)); - } - } - - // Add current timestamp to our tracking array - recentApiCalls.push(Date.now()); - - // Make the actual API call const response = await fetch(url, { method, headers: { From 97febd74e5335dd80323d61c733c7e1eebf4af37 Mon Sep 17 00:00:00 2001 From: Henry Davies Date: Thu, 31 Jul 2025 23:26:49 +0530 Subject: [PATCH 9/9] feat: use p-limit for promise alls --- package.json | 3 ++- src/spotify/getArtistsAlbums.ts | 15 ++++++++++----- src/spotify/getArtistsAlbumsForAlbumGroup.ts | 13 ++++++++----- src/spotify/getArtistsTracks.ts | 12 ++++++++---- yarn.lock | 17 +++++++++++++++++ 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index dbfcfa1..d409b8a 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "dependencies": { "aws-lambda": "^1.0.7", "cookie": "^1.0.2", - "express": "^4.18.2" + "express": "^4.18.2", + "p-limit": "^6.2.0" }, "devDependencies": { "@types/aws-lambda": "^8.10.150", diff --git a/src/spotify/getArtistsAlbums.ts b/src/spotify/getArtistsAlbums.ts index 7ec1ada..d8b3c00 100644 --- a/src/spotify/getArtistsAlbums.ts +++ b/src/spotify/getArtistsAlbums.ts @@ -1,4 +1,5 @@ import { getArtistsAlbumsForAlbumGroup } from './getArtistsAlbumsForAlbumGroup'; +import pLimit from 'p-limit'; export const getArtistsAlbums = async ({ accessToken, @@ -8,12 +9,16 @@ export const getArtistsAlbums = async ({ artistId: string; }): Promise => { const groups = ['album', 'single', 'compilation', 'appears_on']; + + const limit = pLimit(1); const getAlbumsForAlbumGroupPromises = groups.map((group) => - getArtistsAlbumsForAlbumGroup({ - accessToken, - artistId, - group - }) + limit(() => + getArtistsAlbumsForAlbumGroup({ + accessToken, + artistId, + group + }) + ) ); const promiseResults = await Promise.all(getAlbumsForAlbumGroupPromises); diff --git a/src/spotify/getArtistsAlbumsForAlbumGroup.ts b/src/spotify/getArtistsAlbumsForAlbumGroup.ts index dea3cd5..88e5a15 100644 --- a/src/spotify/getArtistsAlbumsForAlbumGroup.ts +++ b/src/spotify/getArtistsAlbumsForAlbumGroup.ts @@ -1,5 +1,6 @@ import { fetchSpotifyData } from './fetchSpotifyData'; import { Album } from './getArtistsAlbums'; +import pLimit from 'p-limit'; const limit = 50; @@ -84,13 +85,15 @@ const getAdditionalAlbums = async ({ urlsForOtherPages.push(initialUrl + `&offset=${i}`); } + const concurrencyLimit = pLimit(1); const otherPagesPromises = urlsForOtherPages.map((url) => - fetchSpotifyData({ - accessToken, - url - }) + concurrencyLimit(() => + fetchSpotifyData({ + accessToken, + url + }) + ) ); - const otherPagesData = await Promise.all(otherPagesPromises); const albums: Album[] = []; otherPagesData.forEach(({ items }) => { diff --git a/src/spotify/getArtistsTracks.ts b/src/spotify/getArtistsTracks.ts index 509afb7..6e053e7 100644 --- a/src/spotify/getArtistsTracks.ts +++ b/src/spotify/getArtistsTracks.ts @@ -1,6 +1,7 @@ import { Album } from './getArtistsAlbums'; import { fetchSpotifyData } from './fetchSpotifyData'; import { getArtistsAlbums } from './getArtistsAlbums'; +import pLimit from 'p-limit'; export const getArtistsTracks = async ({ accessToken, @@ -18,11 +19,14 @@ export const getArtistsTracks = async ({ chunkSize: maxAlbumsPerRequest }); + const limit = pLimit(1); const getTracksPromises = albumBatches.map((batch) => - getTracksForAlbumBatch({ - albumsBatch: batch, - accessToken - }) + limit(() => + getTracksForAlbumBatch({ + albumsBatch: batch, + accessToken + }) + ) ); const getTracksResults = await Promise.all(getTracksPromises); diff --git a/yarn.lock b/yarn.lock index 4ccb937..25d24d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4775,6 +4775,15 @@ __metadata: languageName: node linkType: hard +"p-limit@npm:^6.2.0": + version: 6.2.0 + resolution: "p-limit@npm:6.2.0" + dependencies: + yocto-queue: "npm:^1.1.1" + checksum: 10c0/448bf55a1776ca1444594d53b3c731e68cdca00d44a6c8df06a2f6e506d5bbd540ebb57b05280f8c8bff992a630ed782a69612473f769a7473495d19e2270166 + languageName: node + linkType: hard + "p-locate@npm:^4.1.0": version: 4.1.0 resolution: "p-locate@npm:4.1.0" @@ -5160,6 +5169,7 @@ __metadata: jest-util: "npm:^30.0.2" jest-when: "npm:^3.7.0" nock: "npm:^14.0.5" + p-limit: "npm:^6.2.0" prettier: "npm:^3.6.2" ts-jest: "npm:^29.4.0" typescript: "npm:^5.8.3" @@ -6232,3 +6242,10 @@ __metadata: checksum: 10c0/dceb44c28578b31641e13695d200d34ec4ab3966a5729814d5445b194933c096b7ced71494ce53a0e8820685d1d010df8b2422e5bf2cdea7e469d97ffbea306f languageName: node linkType: hard + +"yocto-queue@npm:^1.1.1": + version: 1.2.1 + resolution: "yocto-queue@npm:1.2.1" + checksum: 10c0/5762caa3d0b421f4bdb7a1926b2ae2189fc6e4a14469258f183600028eb16db3e9e0306f46e8ebf5a52ff4b81a881f22637afefbef5399d6ad440824e9b27f9f + languageName: node + linkType: hard