From 6a3b0bbb16222272741fad80888e359ddb85e2f0 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Thu, 4 Dec 2025 18:27:21 -0500 Subject: [PATCH 1/5] fix(fetch): fixup generators and add error handling --- .../next-data/generators/supportersData.mjs | 42 +++--- .../next-data/generators/vulnerabilities.mjs | 134 ++++++++---------- apps/site/next.constants.mjs | 6 + apps/site/types/index.ts | 1 + apps/site/types/supporters.ts | 9 ++ 5 files changed, 97 insertions(+), 95 deletions(-) create mode 100644 apps/site/types/supporters.ts diff --git a/apps/site/next-data/generators/supportersData.mjs b/apps/site/next-data/generators/supportersData.mjs index 1c1c19f15b77b..6b2947ea4b2d0 100644 --- a/apps/site/next-data/generators/supportersData.mjs +++ b/apps/site/next-data/generators/supportersData.mjs @@ -1,28 +1,24 @@ +import { OPENCOLLECTIVE_MEMBERS_URL } from '#site/next.constants.mjs'; + /** * Fetches supporters data from Open Collective API, filters active backers, * and maps it to the Supporters type. * - * @returns {Promise>} Array of supporters + * @returns {Promise>} Array of supporters */ -async function fetchOpenCollectiveData() { - const endpoint = 'https://opencollective.com/nodejs/members/all.json'; - - const response = await fetch(endpoint); - - const payload = await response.json(); - - const members = payload - .filter(({ role, isActive }) => role === 'BACKER' && isActive) - .sort((a, b) => b.totalAmountDonated - a.totalAmountDonated) - .map(({ name, website, image, profile }) => ({ - name, - image, - url: website, - profile, - source: 'opencollective', - })); - - return members; -} - -export default fetchOpenCollectiveData; +export default () => + fetch(OPENCOLLECTIVE_MEMBERS_URL) + .then(response => response.json()) + .then(payload => + payload + .filter(({ role, isActive }) => role === 'BACKER' && isActive) + .sort((a, b) => b.totalAmountDonated - a.totalAmountDonated) + .map(({ name, website, image, profile }) => ({ + name, + image, + url: website, + profile, + source: 'opencollective', + })) + ) + .catch(() => []); diff --git a/apps/site/next-data/generators/vulnerabilities.mjs b/apps/site/next-data/generators/vulnerabilities.mjs index 82378a71d3577..e86ceee24e4f1 100644 --- a/apps/site/next-data/generators/vulnerabilities.mjs +++ b/apps/site/next-data/generators/vulnerabilities.mjs @@ -8,86 +8,76 @@ const RANGE_REGEX = /([<>]=?)\s*(\d+)(?:\.(\d+))?/; * * @returns {Promise} Grouped vulnerabilities */ -export default async function generateVulnerabilityData() { - const response = await fetch(VULNERABILITIES_URL); +export default () => + fetch(VULNERABILITIES_URL) + .then(response => response.json()) + .then(payload => { + /** @type {Array} */ + const data = Object.values(payload); + + /** @type {Promise */ + const grouped = {}; + + // Helper function to add vulnerability to a major version group + const addToGroup = (majorVersion, vulnerability) => { + grouped[majorVersion] ??= []; + grouped[majorVersion].push(vulnerability); + }; + + // Helper function to process version patterns + const processVersion = (version, vulnerability) => { + // Handle 0.X versions (pre-semver) + if (/^0\.\d+(\.x)?$/.test(version)) { + addToGroup('0', vulnerability); + + return; + } + + // Handle simple major.x patterns (e.g., 12.x) + if (/^\d+\.x$/.test(version)) { + const majorVersion = version.split('.')[0]; - /** @type {Array} */ - const data = Object.values(await response.json()); - - /** @type {Promise */ - const grouped = {}; - - // Helper function to add vulnerability to a major version group - const addToGroup = (majorVersion, vulnerability) => { - grouped[majorVersion] ??= []; - grouped[majorVersion].push(vulnerability); - }; - - // Helper function to process version patterns - const processVersion = (version, vulnerability) => { - // Handle 0.X versions (pre-semver) - if (/^0\.\d+(\.x)?$/.test(version)) { - addToGroup('0', vulnerability); - - return; - } - - // Handle simple major.x patterns (e.g., 12.x) - if (/^\d+\.x$/.test(version)) { - const majorVersion = version.split('.')[0]; + addToGroup(majorVersion, vulnerability); - addToGroup(majorVersion, vulnerability); + return; + } - return; - } + // Handle version ranges (>, >=, <, <=) + const rangeMatch = RANGE_REGEX.exec(version); - // Handle version ranges (>, >=, <, <=) - const rangeMatch = RANGE_REGEX.exec(version); + if (rangeMatch) { + const [, operator, majorVersion] = rangeMatch; - if (rangeMatch) { - const [, operator, majorVersion] = rangeMatch; + const majorNum = parseInt(majorVersion, 10); - const majorNum = parseInt(majorVersion, 10); + switch (operator) { + case '>=': + case '>': + case '<=': + addToGroup(majorVersion, vulnerability); - switch (operator) { - case '>=': - case '>': - case '<=': - addToGroup(majorVersion, vulnerability); + break; + case '<': + // Add to all major versions below the specified version + for (let i = majorNum - 1; i >= 0; i--) { + addToGroup(i.toString(), vulnerability); + } - break; - case '<': - // Add to all major versions below the specified version - for (let i = majorNum - 1; i >= 0; i--) { - addToGroup(i.toString(), vulnerability); + break; } + } + }; - break; + for (const { ref, ...vulnerability } of Object.values(data)) { + vulnerability.url = ref; + // Process all potential versions from the vulnerable field + const versions = vulnerability.vulnerable.split(' || ').filter(Boolean); + + for (const version of versions) { + processVersion(version, vulnerability); + } } - } - }; - - for (const vulnerability of Object.values(data)) { - const parsedVulnerability = { - cve: vulnerability.cve, - url: vulnerability.ref, - vulnerable: vulnerability.vulnerable, - patched: vulnerability.patched, - description: vulnerability.description, - overview: vulnerability.overview, - affectedEnvironments: vulnerability.affectedEnvironments, - severity: vulnerability.severity, - }; - - // Process all potential versions from the vulnerable field - const versions = parsedVulnerability.vulnerable - .split(' || ') - .filter(Boolean); - - for (const version of versions) { - processVersion(version, parsedVulnerability); - } - } - - return grouped; -} + + return grouped; + }) + .catch(() => ({})); diff --git a/apps/site/next.constants.mjs b/apps/site/next.constants.mjs index 8a8e9ec4196cb..c90c61711b7c6 100644 --- a/apps/site/next.constants.mjs +++ b/apps/site/next.constants.mjs @@ -213,3 +213,9 @@ export const EOL_VERSION_IDENTIFIER = 'End-of-life'; */ export const VULNERABILITIES_URL = 'https://raw.githubusercontent.com/nodejs/security-wg/main/vuln/core/index.json'; + +/** + * The location of the OpenCollective data + */ +export const OPENCOLLECTIVE_MEMBERS_URL = + 'https://opencollective.com/nodejs/members/all.json'; diff --git a/apps/site/types/index.ts b/apps/site/types/index.ts index 35643a647dbbe..3e0fd77eb4e11 100644 --- a/apps/site/types/index.ts +++ b/apps/site/types/index.ts @@ -16,3 +16,4 @@ export * from './download'; export * from './userAgent'; export * from './vulnerabilities'; export * from './page'; +export * from './supporters'; diff --git a/apps/site/types/supporters.ts b/apps/site/types/supporters.ts new file mode 100644 index 0000000000000..1078201c77cef --- /dev/null +++ b/apps/site/types/supporters.ts @@ -0,0 +1,9 @@ +export type Supporter = { + name: string; + image: string; + url: string; + profile: string; + source: string; +}; + +export type OpenCollectiveSupporter = Supporter & { source: 'opencollective' }; From d26505268a618ad38b3dd4bf6dc3ac23c3de5168 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Sun, 7 Dec 2025 15:45:27 -0500 Subject: [PATCH 2/5] fixup! --- .../next-data/generators/supportersData.mjs | 6 ++--- .../next-data/generators/vulnerabilities.mjs | 12 ++++++---- apps/site/next.calendar.mjs | 3 ++- apps/site/types/supporters.ts | 6 ++--- apps/site/util/fetch.ts | 24 +++++++++++++++++++ 5 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 apps/site/util/fetch.ts diff --git a/apps/site/next-data/generators/supportersData.mjs b/apps/site/next-data/generators/supportersData.mjs index 6b2947ea4b2d0..4dbb09c2979b3 100644 --- a/apps/site/next-data/generators/supportersData.mjs +++ b/apps/site/next-data/generators/supportersData.mjs @@ -1,4 +1,5 @@ import { OPENCOLLECTIVE_MEMBERS_URL } from '#site/next.constants.mjs'; +import { fetchWithRetry } from '#site/util/fetch'; /** * Fetches supporters data from Open Collective API, filters active backers, @@ -7,7 +8,7 @@ import { OPENCOLLECTIVE_MEMBERS_URL } from '#site/next.constants.mjs'; * @returns {Promise>} Array of supporters */ export default () => - fetch(OPENCOLLECTIVE_MEMBERS_URL) + fetchWithRetry(OPENCOLLECTIVE_MEMBERS_URL) .then(response => response.json()) .then(payload => payload @@ -20,5 +21,4 @@ export default () => profile, source: 'opencollective', })) - ) - .catch(() => []); + ); diff --git a/apps/site/next-data/generators/vulnerabilities.mjs b/apps/site/next-data/generators/vulnerabilities.mjs index e86ceee24e4f1..5d2dfaa1aded8 100644 --- a/apps/site/next-data/generators/vulnerabilities.mjs +++ b/apps/site/next-data/generators/vulnerabilities.mjs @@ -1,6 +1,9 @@ import { VULNERABILITIES_URL } from '#site/next.constants.mjs'; +import { fetchWithRetry } from '#site/util/fetch'; const RANGE_REGEX = /([<>]=?)\s*(\d+)(?:\.(\d+))?/; +const V0_REGEX = /^0\.\d+(\.x)?$/; +const VER_REGEX = /^\d+\.x$/; /** * Fetches vulnerability data from the Node.js Security Working Group repository, @@ -9,7 +12,7 @@ const RANGE_REGEX = /([<>]=?)\s*(\d+)(?:\.(\d+))?/; * @returns {Promise} Grouped vulnerabilities */ export default () => - fetch(VULNERABILITIES_URL) + fetchWithRetry(VULNERABILITIES_URL) .then(response => response.json()) .then(payload => { /** @type {Array} */ @@ -27,14 +30,14 @@ export default () => // Helper function to process version patterns const processVersion = (version, vulnerability) => { // Handle 0.X versions (pre-semver) - if (/^0\.\d+(\.x)?$/.test(version)) { + if (V0_REGEX.test(version)) { addToGroup('0', vulnerability); return; } // Handle simple major.x patterns (e.g., 12.x) - if (/^\d+\.x$/.test(version)) { + if (VER_REGEX.test(version)) { const majorVersion = version.split('.')[0]; addToGroup(majorVersion, vulnerability); @@ -79,5 +82,4 @@ export default () => } return grouped; - }) - .catch(() => ({})); + }); diff --git a/apps/site/next.calendar.mjs b/apps/site/next.calendar.mjs index d32b4f5af27c3..b7cdb666fcd73 100644 --- a/apps/site/next.calendar.mjs +++ b/apps/site/next.calendar.mjs @@ -4,6 +4,7 @@ import { BASE_CALENDAR_URL, SHARED_CALENDAR_KEY, } from './next.calendar.constants.mjs'; +import { fetchWithRetry } from './util/fetch'; /** * @@ -33,7 +34,7 @@ export const getCalendarEvents = async (calendarId = '', maxResults = 20) => { calendarQueryUrl.searchParams.append(key, value) ); - return fetch(calendarQueryUrl) + return fetchWithRetry(calendarQueryUrl) .then(response => response.json()) .then(calendar => calendar.items ?? []); }; diff --git a/apps/site/types/supporters.ts b/apps/site/types/supporters.ts index 1078201c77cef..5da04e07c50ca 100644 --- a/apps/site/types/supporters.ts +++ b/apps/site/types/supporters.ts @@ -1,9 +1,9 @@ -export type Supporter = { +export type Supporter = { name: string; image: string; url: string; profile: string; - source: string; + source: T; }; -export type OpenCollectiveSupporter = Supporter & { source: 'opencollective' }; +export type OpenCollectiveSupporter = Supporter<'opencollective'>; diff --git a/apps/site/util/fetch.ts b/apps/site/util/fetch.ts new file mode 100644 index 0000000000000..239403cbb81bf --- /dev/null +++ b/apps/site/util/fetch.ts @@ -0,0 +1,24 @@ +import { setTimeout } from 'node:timers/promises'; + +type RetryOptions = RequestInit & { + maxRetry: number; + delay: number; +}; + +export const fetchWithRetry = async ( + url: string, + { maxRetry = 3, delay = 100, ...options }: RetryOptions +) => { + for (let i = 1; i <= maxRetry; i++) { + try { + return fetch(url, options); + } catch (e) { + if (i === maxRetry) { + throw e; + } + + await setTimeout(delay); + continue; + } + } +}; From 23ef0464874e40d53cd222e54b81b920abce8f45 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Sun, 7 Dec 2025 15:49:27 -0500 Subject: [PATCH 3/5] fixup! --- apps/site/util/fetch.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/site/util/fetch.ts b/apps/site/util/fetch.ts index 239403cbb81bf..cc4b7b3786311 100644 --- a/apps/site/util/fetch.ts +++ b/apps/site/util/fetch.ts @@ -1,13 +1,13 @@ import { setTimeout } from 'node:timers/promises'; type RetryOptions = RequestInit & { - maxRetry: number; - delay: number; + maxRetry?: number; + delay?: number; }; export const fetchWithRetry = async ( url: string, - { maxRetry = 3, delay = 100, ...options }: RetryOptions + { maxRetry = 3, delay = 100, ...options }: RetryOptions = {} ) => { for (let i = 1; i <= maxRetry; i++) { try { From a7d80d1eed1f7b01030ab3d11aecec5dc7c4554b Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Thu, 18 Dec 2025 08:56:43 -0500 Subject: [PATCH 4/5] fixup! --- .../next-data/generators/supportersData.mjs | 35 ++--- .../next-data/generators/vulnerabilities.mjs | 120 +++++++++--------- apps/site/util/fetch.ts | 13 +- 3 files changed, 92 insertions(+), 76 deletions(-) diff --git a/apps/site/next-data/generators/supportersData.mjs b/apps/site/next-data/generators/supportersData.mjs index 4dbb09c2979b3..fc77c4b383e8b 100644 --- a/apps/site/next-data/generators/supportersData.mjs +++ b/apps/site/next-data/generators/supportersData.mjs @@ -7,18 +7,23 @@ import { fetchWithRetry } from '#site/util/fetch'; * * @returns {Promise>} Array of supporters */ -export default () => - fetchWithRetry(OPENCOLLECTIVE_MEMBERS_URL) - .then(response => response.json()) - .then(payload => - payload - .filter(({ role, isActive }) => role === 'BACKER' && isActive) - .sort((a, b) => b.totalAmountDonated - a.totalAmountDonated) - .map(({ name, website, image, profile }) => ({ - name, - image, - url: website, - profile, - source: 'opencollective', - })) - ); +async function fetchOpenCollectiveData() { + const response = await fetchWithRetry(OPENCOLLECTIVE_MEMBERS_URL); + + const payload = await response.json(); + + const members = payload + .filter(({ role, isActive }) => role === 'BACKER' && isActive) + .sort((a, b) => b.totalAmountDonated - a.totalAmountDonated) + .map(({ name, website, image, profile }) => ({ + name, + image, + url: website, + profile, + source: 'opencollective', + })); + + return members; +} + +export default fetchOpenCollectiveData; diff --git a/apps/site/next-data/generators/vulnerabilities.mjs b/apps/site/next-data/generators/vulnerabilities.mjs index 5d2dfaa1aded8..d461d0f50d15a 100644 --- a/apps/site/next-data/generators/vulnerabilities.mjs +++ b/apps/site/next-data/generators/vulnerabilities.mjs @@ -11,75 +11,75 @@ const VER_REGEX = /^\d+\.x$/; * * @returns {Promise} Grouped vulnerabilities */ -export default () => - fetchWithRetry(VULNERABILITIES_URL) - .then(response => response.json()) - .then(payload => { - /** @type {Array} */ - const data = Object.values(payload); - - /** @type {Promise */ - const grouped = {}; - - // Helper function to add vulnerability to a major version group - const addToGroup = (majorVersion, vulnerability) => { - grouped[majorVersion] ??= []; - grouped[majorVersion].push(vulnerability); - }; - - // Helper function to process version patterns - const processVersion = (version, vulnerability) => { - // Handle 0.X versions (pre-semver) - if (V0_REGEX.test(version)) { - addToGroup('0', vulnerability); - - return; - } - - // Handle simple major.x patterns (e.g., 12.x) - if (VER_REGEX.test(version)) { - const majorVersion = version.split('.')[0]; +export default async function generateVulnerabilityData() { + const response = await fetchWithRetry(VULNERABILITIES_URL); - addToGroup(majorVersion, vulnerability); + /** @type {Array} */ + const data = Object.values(await response.json()); - return; - } + /** @type {Promise */ + const grouped = {}; - // Handle version ranges (>, >=, <, <=) - const rangeMatch = RANGE_REGEX.exec(version); + // Helper function to add vulnerability to a major version group + const addToGroup = (majorVersion, vulnerability) => { + grouped[majorVersion] ??= []; + grouped[majorVersion].push(vulnerability); + }; - if (rangeMatch) { - const [, operator, majorVersion] = rangeMatch; + // Helper function to process version patterns + const processVersion = (version, vulnerability) => { + // Handle 0.X versions (pre-semver) + if (V0_REGEX.test(version)) { + addToGroup('0', vulnerability); - const majorNum = parseInt(majorVersion, 10); + return; + } - switch (operator) { - case '>=': - case '>': - case '<=': - addToGroup(majorVersion, vulnerability); + // Handle simple major.x patterns (e.g., 12.x) + if (VER_REGEX.test(version)) { + const majorVersion = version.split('.')[0]; - break; - case '<': - // Add to all major versions below the specified version - for (let i = majorNum - 1; i >= 0; i--) { - addToGroup(i.toString(), vulnerability); - } + addToGroup(majorVersion, vulnerability); - break; - } - } - }; + return; + } + + // Handle version ranges (>, >=, <, <=) + const rangeMatch = RANGE_REGEX.exec(version); - for (const { ref, ...vulnerability } of Object.values(data)) { - vulnerability.url = ref; - // Process all potential versions from the vulnerable field - const versions = vulnerability.vulnerable.split(' || ').filter(Boolean); + if (rangeMatch) { + const [, operator, majorVersion] = rangeMatch; + + const majorNum = parseInt(majorVersion, 10); + + switch (operator) { + case '>=': + case '>': + case '<=': + addToGroup(majorVersion, vulnerability); - for (const version of versions) { - processVersion(version, vulnerability); - } + break; + case '<': + // Add to all major versions below the specified version + for (let i = majorNum - 1; i >= 0; i--) { + addToGroup(i.toString(), vulnerability); + } + + break; } + } + }; + + for (const { ref, ...vulnerability } of Object.values(data)) { + vulnerability.url = ref; + + // Process all potential versions from the vulnerable field + const versions = vulnerability.vulnerable.split(' || ').filter(Boolean); + + for (const version of versions) { + processVersion(version, vulnerability); + } + } - return grouped; - }); + return grouped; +} diff --git a/apps/site/util/fetch.ts b/apps/site/util/fetch.ts index cc4b7b3786311..032866ba558bb 100644 --- a/apps/site/util/fetch.ts +++ b/apps/site/util/fetch.ts @@ -5,6 +5,12 @@ type RetryOptions = RequestInit & { delay?: number; }; +type FetchError = { + cause: { + code: string; + }; +}; + export const fetchWithRetry = async ( url: string, { maxRetry = 3, delay = 100, ...options }: RetryOptions = {} @@ -13,7 +19,12 @@ export const fetchWithRetry = async ( try { return fetch(url, options); } catch (e) { - if (i === maxRetry) { + console.debug( + `fetch of ${url} failed at ${Date.now()}, attempt ${i}/${maxRetry}`, + e + ); + + if (i === maxRetry || (e as FetchError).cause.code !== 'ETIMEDOUT') { throw e; } From b6805861b0ce4b0a22fcffb810c2a8e359fd152a Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Thu, 18 Dec 2025 09:11:07 -0500 Subject: [PATCH 5/5] fixup! --- apps/site/util/fetch.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/site/util/fetch.ts b/apps/site/util/fetch.ts index 032866ba558bb..61378982c6616 100644 --- a/apps/site/util/fetch.ts +++ b/apps/site/util/fetch.ts @@ -28,8 +28,7 @@ export const fetchWithRetry = async ( throw e; } - await setTimeout(delay); - continue; + await setTimeout(delay * i); } } };