From 1728bf79b81b56f9e575cf4d305edefa302f4c0d Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 3 Aug 2026 17:18:53 +0200 Subject: [PATCH 1/3] fix: npm 429 retries (CM-XXXX) Signed-off-by: anilb --- .../packages_worker/src/npm/activities.ts | 39 ++++++++++++++++--- .../packages_worker/src/npm/fetchPackument.ts | 10 ++++- .../apps/packages_worker/src/npm/types.ts | 3 ++ .../src/packages/npmPackageState.ts | 18 +++++++++ 4 files changed, 63 insertions(+), 7 deletions(-) diff --git a/services/apps/packages_worker/src/npm/activities.ts b/services/apps/packages_worker/src/npm/activities.ts index 4cdb71566d..e54877e781 100644 --- a/services/apps/packages_worker/src/npm/activities.ts +++ b/services/apps/packages_worker/src/npm/activities.ts @@ -1,4 +1,4 @@ -import { Context } from '@temporalio/activity' +import { ApplicationFailure, Context } from '@temporalio/activity' import { type Dispatcher, ProxyAgent } from 'undici' import { @@ -11,6 +11,7 @@ import { getNpmPurlsDueForLast30dHistory, getNpmPurlsDueForLatest30d, getNpmPurlsForChangedNames, + getNpmPurlsScannedSince, getUnscannedNpmPurls, insertDailyDownloads, logAuditFieldChanges, @@ -97,8 +98,9 @@ export async function commitNpmChangesSeq(lastSeq: string): Promise { } // 4xx errors get a few quick in-lane retries with a small linear backoff (1s, 2s), -// then the package is given up on and marked scanned. 429/5xx/network errors are NOT -// handled here — they throw and ride Temporal's exponential activity-retry instead. +// then the package is given up on and marked scanned. 5xx/network errors throw and ride +// Temporal's exponential activity-retry; 429s throw with the retry scheduled past the +// server-stated Retry-After window instead. const INGEST_4XX_ATTEMPTS = 3 const INGEST_4XX_BACKOFF_MS = 1000 @@ -118,7 +120,17 @@ async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatche return } - // 429 / 5xx / network → bubble up so Temporal retries the activity with exponential backoff. + // 429 → fail the attempt, but schedule the retry past the server-stated penalty window + // (npm blocks for ~300s; the default 30/60/120s ladder always lands back inside it). + if (packumentResult.kind === 'RATE_LIMIT') { + const delaySec = Math.min(Math.max(packumentResult.retryAfterSec ?? 300, 30), 900) + 5 + throw ApplicationFailure.create({ + message: `Failed to fetch packument for ${name}: ${packumentResult.message}`, + nextRetryDelay: `${delaySec}s`, + }) + } + + // 5xx / network → bubble up so Temporal retries the activity with exponential backoff. // MALFORMED is permanent (a 200 body that isn't a packument — retrying won't change it), // so it takes the quick-retry-then-skip path below instead of poisoning the lane forever. if ( @@ -168,11 +180,21 @@ export async function ingestNpmPackageBatch(purls: string[], laneIndex: number): if (purls.length === 0) return const qx = await getPackagesDb() + + // A retry replays the same purls array — drop what an earlier attempt already settled + // (scanned since first schedule), so the replay doesn't re-spend rate budget on re-downloads. + let pending = purls + const { attempt, scheduledTimestampMs } = Context.current().info + if (attempt > 1) { + const done = new Set(await getNpmPurlsScannedSince(qx, purls, new Date(scheduledTimestampMs))) + if (done.size > 0) pending = purls.filter((p) => !done.has(p)) + } + const proxy = proxyForLane(laneIndex) const dispatcher = proxy ? new ProxyAgent(proxyUrl(proxy)) : undefined try { - for (const purl of purls) { + for (const purl of pending) { await sleep(ingestSleepMs()) await ingestOne(qx, purl, dispatcher) } @@ -181,7 +203,12 @@ export async function ingestNpmPackageBatch(purls: string[], laneIndex: number): } log.info( - { laneIndex, count: purls.length, exit: proxy?.host ?? 'direct' }, + { + laneIndex, + count: pending.length, + skipped: purls.length - pending.length, + exit: proxy?.host ?? 'direct', + }, 'Ingested npm package batch', ) } diff --git a/services/apps/packages_worker/src/npm/fetchPackument.ts b/services/apps/packages_worker/src/npm/fetchPackument.ts index a19741d8d9..3548811cd0 100644 --- a/services/apps/packages_worker/src/npm/fetchPackument.ts +++ b/services/apps/packages_worker/src/npm/fetchPackument.ts @@ -43,7 +43,15 @@ export async function fetchPackument( if (res.status === 404) return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } - if (res.status === 429) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + if (res.status === 429) { + const retryAfter = parseInt(res.headers.get('retry-after') ?? '', 10) + return { + kind: 'RATE_LIMIT', + message: 'rate limited', + statusCode: 429, + retryAfterSec: Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : undefined, + } + } if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } let json: unknown diff --git a/services/apps/packages_worker/src/npm/types.ts b/services/apps/packages_worker/src/npm/types.ts index e46276ec7b..d18dc5d83d 100644 --- a/services/apps/packages_worker/src/npm/types.ts +++ b/services/apps/packages_worker/src/npm/types.ts @@ -30,6 +30,9 @@ export interface FetchError { kind: FetchErrorKind message: string statusCode?: number + // Server-stated wait (Retry-After) on RATE_LIMIT, so retries can honor the real + // penalty window instead of guessing with exponential backoff. + retryAfterSec?: number } export function isFetchError(v: unknown): v is FetchError { diff --git a/services/libs/data-access-layer/src/packages/npmPackageState.ts b/services/libs/data-access-layer/src/packages/npmPackageState.ts index 3b359bfdb6..85e4ac2f8c 100644 --- a/services/libs/data-access-layer/src/packages/npmPackageState.ts +++ b/services/libs/data-access-layer/src/packages/npmPackageState.ts @@ -28,6 +28,24 @@ export async function markNpmPackageScanned( ) } +// Of the given purls, the ones whose metadata scan ran at/after `since`. Used by activity +// retries to skip purls an earlier attempt of the same activity already settled. +export async function getNpmPurlsScannedSince( + qx: QueryExecutor, + purls: string[], + since: Date, +): Promise { + if (purls.length === 0) return [] + const rows: Array<{ purl: string }> = await qx.select( + `SELECT purl + FROM npm_package_state + WHERE purl = ANY($(purls)::text[]) + AND metadata_last_run_at >= $(since)`, + { purls, since }, + ) + return rows.map((r) => r.purl) +} + // Critical npm packages in the `packages` table whose metadata has never been scanned. // Only is_critical packages are enriched — metadata is deep, per-package work, so it is // scoped to the critical set (matching the daily-downloads pass). From ea31d903eb0b45ff5f2834009601d3033720406d Mon Sep 17 00:00:00 2001 From: anilb Date: Tue, 4 Aug 2026 09:30:05 +0200 Subject: [PATCH 2/3] fix: address pr review comments Signed-off-by: anilb --- .../apps/packages_worker/src/npm/activities.ts | 2 +- .../packages_worker/src/npm/fetchPackument.ts | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/npm/activities.ts b/services/apps/packages_worker/src/npm/activities.ts index e54877e781..1c6f2e591d 100644 --- a/services/apps/packages_worker/src/npm/activities.ts +++ b/services/apps/packages_worker/src/npm/activities.ts @@ -123,7 +123,7 @@ async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatche // 429 → fail the attempt, but schedule the retry past the server-stated penalty window // (npm blocks for ~300s; the default 30/60/120s ladder always lands back inside it). if (packumentResult.kind === 'RATE_LIMIT') { - const delaySec = Math.min(Math.max(packumentResult.retryAfterSec ?? 300, 30), 900) + 5 + const delaySec = Math.min(Math.max(packumentResult.retryAfterSec ?? 300, 30), 3600) + 5 throw ApplicationFailure.create({ message: `Failed to fetch packument for ${name}: ${packumentResult.message}`, nextRetryDelay: `${delaySec}s`, diff --git a/services/apps/packages_worker/src/npm/fetchPackument.ts b/services/apps/packages_worker/src/npm/fetchPackument.ts index 3548811cd0..1dcdaa55da 100644 --- a/services/apps/packages_worker/src/npm/fetchPackument.ts +++ b/services/apps/packages_worker/src/npm/fetchPackument.ts @@ -44,12 +44,11 @@ export async function fetchPackument( if (res.status === 404) return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } if (res.status === 429) { - const retryAfter = parseInt(res.headers.get('retry-after') ?? '', 10) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429, - retryAfterSec: Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : undefined, + retryAfterSec: parseRetryAfterSec(res.headers.get('retry-after')), } } if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } @@ -70,6 +69,18 @@ export async function fetchPackument( return json } +// Retry-After is either delta-seconds or an HTTP-date (RFC 9110). Returns whole seconds, +// or undefined when absent/unparseable/expired so the caller applies its own default. +function parseRetryAfterSec(header: string | null): number | undefined { + if (!header) return undefined + const seconds = Number(header) + if (Number.isFinite(seconds)) return seconds > 0 ? Math.ceil(seconds) : undefined + const date = new Date(header) + if (Number.isNaN(date.getTime())) return undefined + const untilSec = Math.ceil((date.getTime() - Date.now()) / 1000) + return untilSec > 0 ? untilSec : undefined +} + function isPackument(v: unknown): v is Packument { return typeof v === 'object' && v !== null && 'name' in v && 'versions' in v && 'dist-tags' in v } From 8f95cd81b4d0c01069d6694048441d0034544f2d Mon Sep 17 00:00:00 2001 From: anilb Date: Tue, 4 Aug 2026 09:52:47 +0200 Subject: [PATCH 3/3] refactor: npm fetch error kind enum Signed-off-by: anilb --- .../blast-radius/clients/npmAbbreviated.ts | 16 ++++---- .../src/blast-radius/dependentsScan.ts | 4 +- .../packages_worker/src/npm/activities.ts | 6 +-- .../packages_worker/src/npm/fetchChanges.ts | 21 ++++++---- .../packages_worker/src/npm/fetchDownloads.ts | 41 ++++++++++--------- .../packages_worker/src/npm/fetchPackument.ts | 15 +++---- .../apps/packages_worker/src/npm/types.ts | 7 +++- 7 files changed, 62 insertions(+), 48 deletions(-) diff --git a/services/apps/packages_worker/src/blast-radius/clients/npmAbbreviated.ts b/services/apps/packages_worker/src/blast-radius/clients/npmAbbreviated.ts index debe1b42da..33677fe461 100644 --- a/services/apps/packages_worker/src/blast-radius/clients/npmAbbreviated.ts +++ b/services/apps/packages_worker/src/blast-radius/clients/npmAbbreviated.ts @@ -1,5 +1,5 @@ import { combineSignals } from '../../npm/signals' -import type { FetchError } from '../../npm/types' +import { type FetchError, FetchErrorKind } from '../../npm/types' const REGISTRY = 'https://registry.npmjs.org' const USER_AGENT = 'lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org)' @@ -43,21 +43,23 @@ export async function fetchAbbreviatedPackument( signal: combinedSignal, }) } catch (err) { - return { kind: 'TRANSIENT', message: String(err) } + return { kind: FetchErrorKind.TRANSIENT, message: String(err) } } finally { clearTimeout(timer) } if (res.status === 404) - return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } - if (res.status === 429) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + return { kind: FetchErrorKind.NOT_FOUND, message: `${name} not found`, statusCode: 404 } + if (res.status === 429) + return { kind: FetchErrorKind.RATE_LIMIT, message: 'rate limited', statusCode: 429 } + if (!res.ok) + return { kind: FetchErrorKind.TRANSIENT, message: `HTTP ${res.status}`, statusCode: res.status } let json: unknown try { json = await res.json() } catch { - return { kind: 'MALFORMED', message: 'invalid JSON' } + return { kind: FetchErrorKind.MALFORMED, message: 'invalid JSON' } } if ( @@ -66,7 +68,7 @@ export async function fetchAbbreviatedPackument( !('versions' in json) || !('dist-tags' in json) ) { - return { kind: 'MALFORMED', message: 'unexpected shape' } + return { kind: FetchErrorKind.MALFORMED, message: 'unexpected shape' } } return json as AbbreviatedPackument diff --git a/services/apps/packages_worker/src/blast-radius/dependentsScan.ts b/services/apps/packages_worker/src/blast-radius/dependentsScan.ts index 844dcdca4c..d3ed7ecaac 100644 --- a/services/apps/packages_worker/src/blast-radius/dependentsScan.ts +++ b/services/apps/packages_worker/src/blast-radius/dependentsScan.ts @@ -4,7 +4,7 @@ import * as path from 'path' import { fetchBulkPointRange, fetchPointRange } from '../npm/fetchDownloads' import { fetchPackument } from '../npm/fetchPackument' -import { FetchError, isFetchError } from '../npm/types' +import { FetchError, FetchErrorKind, isFetchError } from '../npm/types' import { fetchAbbreviatedPackument } from './clients/npmAbbreviated' import { downloadAndExtractTarball } from './clients/npmTarball' @@ -120,7 +120,7 @@ async function withRateLimitRetry( ): Promise { let result = await fetchFn() for (const delayMs of RATE_LIMIT_RETRY_DELAYS_MS) { - if (!isFetchError(result) || result.kind !== 'RATE_LIMIT' || signal?.aborted) break + if (!isFetchError(result) || result.kind !== FetchErrorKind.RATE_LIMIT || signal?.aborted) break await new Promise((resolve) => setTimeout(resolve, delayMs)) if (signal?.aborted) break result = await fetchFn() diff --git a/services/apps/packages_worker/src/npm/activities.ts b/services/apps/packages_worker/src/npm/activities.ts index 1c6f2e591d..66fcb6054b 100644 --- a/services/apps/packages_worker/src/npm/activities.ts +++ b/services/apps/packages_worker/src/npm/activities.ts @@ -39,7 +39,7 @@ import { import { fetchPackument } from './fetchPackument' import { Last30dWindow, computeMissingLast30dWindows } from './last30dGaps' import { laneCount, proxyForLane } from './proxies' -import { isFetchError } from './types' +import { FetchErrorKind, isFetchError } from './types' import { upsertPackage } from './upsertPackage' const log = getServiceChildLogger('npm') @@ -122,7 +122,7 @@ async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatche // 429 → fail the attempt, but schedule the retry past the server-stated penalty window // (npm blocks for ~300s; the default 30/60/120s ladder always lands back inside it). - if (packumentResult.kind === 'RATE_LIMIT') { + if (packumentResult.kind === FetchErrorKind.RATE_LIMIT) { const delaySec = Math.min(Math.max(packumentResult.retryAfterSec ?? 300, 30), 3600) + 5 throw ApplicationFailure.create({ message: `Failed to fetch packument for ${name}: ${packumentResult.message}`, @@ -135,7 +135,7 @@ async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatche // so it takes the quick-retry-then-skip path below instead of poisoning the lane forever. if ( !isClientError(packumentResult.statusCode, packumentResult.kind) && - packumentResult.kind !== 'MALFORMED' + packumentResult.kind !== FetchErrorKind.MALFORMED ) { throw new Error(`Failed to fetch packument for ${name}: ${packumentResult.message}`) } diff --git a/services/apps/packages_worker/src/npm/fetchChanges.ts b/services/apps/packages_worker/src/npm/fetchChanges.ts index 4ee5aa1b94..e500d24036 100644 --- a/services/apps/packages_worker/src/npm/fetchChanges.ts +++ b/services/apps/packages_worker/src/npm/fetchChanges.ts @@ -1,4 +1,4 @@ -import type { FetchError } from './types' +import { type FetchError, FetchErrorKind } from './types' const USER_AGENT = 'lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org)' @@ -21,21 +21,23 @@ export async function fetchChangesSince(since: string): Promise() for (const row of body.results as Array<{ id?: unknown }>) { @@ -58,20 +60,21 @@ export async function fetchCurrentSeq(): Promise { signal: abort.signal, }) } catch (err) { - return { kind: 'TRANSIENT', message: String(err) } + return { kind: FetchErrorKind.TRANSIENT, message: String(err) } } finally { clearTimeout(timer) } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}` } + if (!res.ok) return { kind: FetchErrorKind.TRANSIENT, message: `HTTP ${res.status}` } let body: { update_seq?: unknown } try { body = (await res.json()) as { update_seq?: unknown } } catch { - return { kind: 'MALFORMED', message: 'invalid JSON' } + return { kind: FetchErrorKind.MALFORMED, message: 'invalid JSON' } } - if (body.update_seq === undefined) return { kind: 'MALFORMED', message: 'missing update_seq' } + if (body.update_seq === undefined) + return { kind: FetchErrorKind.MALFORMED, message: 'missing update_seq' } return String(body.update_seq) } diff --git a/services/apps/packages_worker/src/npm/fetchDownloads.ts b/services/apps/packages_worker/src/npm/fetchDownloads.ts index 89494da7aa..aa455e9c42 100644 --- a/services/apps/packages_worker/src/npm/fetchDownloads.ts +++ b/services/apps/packages_worker/src/npm/fetchDownloads.ts @@ -1,7 +1,7 @@ import type { Dispatcher } from 'undici' import { combineSignals } from './signals' -import type { FetchError } from './types' +import { type FetchError, FetchErrorKind } from './types' const USER_AGENT = 'lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org)' @@ -52,34 +52,35 @@ export async function fetchBulkPointRange( try { res = await fetch(url, downloadInit(dispatcher, combineSignals(abort.signal, signal))) } catch (err) { - return { kind: 'TRANSIENT', message: String(err) } + return { kind: FetchErrorKind.TRANSIENT, message: String(err) } } finally { clearTimeout(timer) } if (res.status === 404) - return { kind: 'NOT_FOUND', message: `bulk request not found`, statusCode: 404 } + return { kind: FetchErrorKind.NOT_FOUND, message: `bulk request not found`, statusCode: 404 } if (res.status === 429) { const headers: Record = {} for (const [k, v] of res.headers.entries()) headers[k] = v const body = await res.text().catch(() => '') return { - kind: 'RATE_LIMIT', + kind: FetchErrorKind.RATE_LIMIT, message: `rate limited by npm downloads API — headers: ${JSON.stringify(headers)} body: ${body}`, statusCode: 429, } } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + if (!res.ok) + return { kind: FetchErrorKind.TRANSIENT, message: `HTTP ${res.status}`, statusCode: res.status } let json: unknown try { json = await res.json() } catch { - return { kind: 'MALFORMED', message: 'invalid JSON' } + return { kind: FetchErrorKind.MALFORMED, message: 'invalid JSON' } } if (typeof json !== 'object' || json === null) - return { kind: 'MALFORMED', message: 'expected object response' } + return { kind: FetchErrorKind.MALFORMED, message: 'expected object response' } const raw = json as Record const counts = new Map() @@ -111,35 +112,36 @@ export async function fetchPointRange( try { res = await fetch(url, downloadInit(dispatcher, combineSignals(abort.signal, signal))) } catch (err) { - return { kind: 'TRANSIENT', message: String(err) } + return { kind: FetchErrorKind.TRANSIENT, message: String(err) } } finally { clearTimeout(timer) } if (res.status === 404) - return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + return { kind: FetchErrorKind.NOT_FOUND, message: `${name} not found`, statusCode: 404 } if (res.status === 429) { const headers: Record = {} for (const [k, v] of res.headers.entries()) headers[k] = v const body = await res.text().catch(() => '') return { - kind: 'RATE_LIMIT', + kind: FetchErrorKind.RATE_LIMIT, message: `rate limited by npm downloads API — headers: ${JSON.stringify(headers)} body: ${body}`, statusCode: 429, } } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + if (!res.ok) + return { kind: FetchErrorKind.TRANSIENT, message: `HTTP ${res.status}`, statusCode: res.status } let json: unknown try { json = await res.json() } catch { - return { kind: 'MALFORMED', message: 'invalid JSON' } + return { kind: FetchErrorKind.MALFORMED, message: 'invalid JSON' } } const data = json as { downloads?: unknown; start?: unknown; end?: unknown } if (typeof data.downloads !== 'number') - return { kind: 'MALFORMED', message: 'missing downloads field' } + return { kind: FetchErrorKind.MALFORMED, message: 'missing downloads field' } return { count: data.downloads, @@ -168,35 +170,36 @@ export async function fetchDailyRange( try { res = await fetch(url, downloadInit(dispatcher, abort.signal)) } catch (err) { - return { kind: 'TRANSIENT', message: String(err) } + return { kind: FetchErrorKind.TRANSIENT, message: String(err) } } finally { clearTimeout(timer) } if (res.status === 404) - return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + return { kind: FetchErrorKind.NOT_FOUND, message: `${name} not found`, statusCode: 404 } if (res.status === 429) { const headers: Record = {} for (const [k, v] of res.headers.entries()) headers[k] = v const body = await res.text().catch(() => '') return { - kind: 'RATE_LIMIT', + kind: FetchErrorKind.RATE_LIMIT, message: `rate limited by npm downloads API — headers: ${JSON.stringify(headers)} body: ${body}`, statusCode: 429, } } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + if (!res.ok) + return { kind: FetchErrorKind.TRANSIENT, message: `HTTP ${res.status}`, statusCode: res.status } let json: unknown try { json = await res.json() } catch { - return { kind: 'MALFORMED', message: 'invalid JSON' } + return { kind: FetchErrorKind.MALFORMED, message: 'invalid JSON' } } const data = json as { start?: unknown; end?: unknown; downloads?: unknown } if (!Array.isArray(data.downloads)) - return { kind: 'MALFORMED', message: 'missing downloads array' } + return { kind: FetchErrorKind.MALFORMED, message: 'missing downloads array' } return { start: typeof data.start === 'string' ? data.start : start, diff --git a/services/apps/packages_worker/src/npm/fetchPackument.ts b/services/apps/packages_worker/src/npm/fetchPackument.ts index 1dcdaa55da..684452cbcb 100644 --- a/services/apps/packages_worker/src/npm/fetchPackument.ts +++ b/services/apps/packages_worker/src/npm/fetchPackument.ts @@ -1,7 +1,7 @@ import type { Dispatcher } from 'undici' import { combineSignals } from './signals' -import type { FetchError, Packument } from './types' +import { type FetchError, FetchErrorKind, type Packument } from './types' const REGISTRY = 'https://registry.npmjs.org' const USER_AGENT = 'lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org)' @@ -36,34 +36,35 @@ export async function fetchPackument( if (dispatcher) init.dispatcher = dispatcher res = await fetch(url, init as RequestInit) } catch (err) { - return { kind: 'TRANSIENT', message: String(err) } + return { kind: FetchErrorKind.TRANSIENT, message: String(err) } } finally { clearTimeout(timer) } if (res.status === 404) - return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + return { kind: FetchErrorKind.NOT_FOUND, message: `${name} not found`, statusCode: 404 } if (res.status === 429) { return { - kind: 'RATE_LIMIT', + kind: FetchErrorKind.RATE_LIMIT, message: 'rate limited', statusCode: 429, retryAfterSec: parseRetryAfterSec(res.headers.get('retry-after')), } } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + if (!res.ok) + return { kind: FetchErrorKind.TRANSIENT, message: `HTTP ${res.status}`, statusCode: res.status } let json: unknown try { json = await res.json() } catch { - return { kind: 'MALFORMED', message: 'invalid JSON' } + return { kind: FetchErrorKind.MALFORMED, message: 'invalid JSON' } } if (!isPackument(json)) { const stub = asUnpublishedStub(json) if (stub) return stub - return { kind: 'MALFORMED', message: 'unexpected shape' } + return { kind: FetchErrorKind.MALFORMED, message: 'unexpected shape' } } delete (json as unknown as Record).readme return json diff --git a/services/apps/packages_worker/src/npm/types.ts b/services/apps/packages_worker/src/npm/types.ts index d18dc5d83d..4c4de10694 100644 --- a/services/apps/packages_worker/src/npm/types.ts +++ b/services/apps/packages_worker/src/npm/types.ts @@ -24,7 +24,12 @@ export interface Packument { unpublished?: unknown } -export type FetchErrorKind = 'RATE_LIMIT' | 'TRANSIENT' | 'NOT_FOUND' | 'MALFORMED' +export enum FetchErrorKind { + RATE_LIMIT = 'RATE_LIMIT', + TRANSIENT = 'TRANSIENT', + NOT_FOUND = 'NOT_FOUND', + MALFORMED = 'MALFORMED', +} export interface FetchError { kind: FetchErrorKind