fix(readme): fetch from jsDelivr when packument readme is likely truncated

The npm registry truncates the packument readme field at 65,536
characters. When the readme length exceeds 64,000 characters, fetch
the full file from jsDelivr CDN instead.

Closes #1458

Author: lukeocodes

server/utils/readme-loaders.ts

@@ -1,6 +1,10 @@
 import * as v from 'valibot'
 import { PackageRouteParamsSchema } from '#shared/schemas/package'
-import { CACHE_MAX_AGE_ONE_HOUR, NPM_MISSING_README_SENTINEL } from '#shared/utils/constants'
+import {
+  CACHE_MAX_AGE_ONE_HOUR,
+  NPM_MISSING_README_SENTINEL,
+  NPM_README_TRUNCATION_THRESHOLD,
+} from '#shared/utils/constants'
 
 /** Standard README filenames to try when fetching from jsdelivr (case-sensitive CDN) */
 const standardReadmeFilenames = [
@@ -58,34 +62,39 @@ export const resolvePackageReadmeSource = defineCachedFunction(
     })
 
     const packageData = await fetchNpmPackage(packageName)
-    const resolvedVersion = version ?? packageData['dist-tags']?.latest
 
-    // Prefer jsDelivr (actual file from npm tarball) because the npm registry
-    // truncates the packument readme field at 65,536 characters.
-    let readmeContent = await fetchReadmeFromJsdelivr(
-      packageName,
-      standardReadmeFilenames,
-      resolvedVersion,
-    )
-
-    // Fall back to packument readme if jsDelivr didn't have a standard README.
-    // This covers packages with non-standard readme filenames (e.g. README.zh-TW.md)
-    // or packages that don't include a README in the tarball.
-    if (!readmeContent) {
-      let packumentReadme: string | undefined
-
-      if (version) {
-        packumentReadme = packageData.versions?.[version]?.readme
-      } else {
-        packumentReadme = packageData.readme
+    let readmeContent: string | undefined
+    let readmeFilename: string | undefined
+
+    if (version) {
+      const versionData = packageData.versions[version]
+      if (versionData) {
+        readmeContent = versionData.readme
+        readmeFilename = versionData.readmeFilename
       }
+    } else {
+      readmeContent = packageData.readme
+      readmeFilename = packageData.readmeFilename
+    }
 
-      if (packumentReadme && packumentReadme !== NPM_MISSING_README_SENTINEL) {
-        readmeContent = packumentReadme
+    const hasValidNpmReadme = readmeContent && readmeContent !== NPM_MISSING_README_SENTINEL
+
+    const isLikelyTruncated =
+      hasValidNpmReadme && readmeContent!.length >= NPM_README_TRUNCATION_THRESHOLD
+
+    if (!hasValidNpmReadme || !isStandardReadme(readmeFilename) || isLikelyTruncated) {
+      const resolvedVersion = version ?? packageData['dist-tags']?.latest
+      const jsdelivrReadme = await fetchReadmeFromJsdelivr(
+        packageName,
+        standardReadmeFilenames,
+        resolvedVersion,
+      )
+      if (jsdelivrReadme) {
+        readmeContent = jsdelivrReadme
       }
     }
 
-    if (!readmeContent) {
+    if (!readmeContent || readmeContent === NPM_MISSING_README_SENTINEL) {
       return {
         packageName,
         version,

shared/utils/constants.ts

@@ -20,6 +20,8 @@ export const ERROR_PACKAGE_REQUIREMENTS_FAILED =
 export const ERROR_FILE_LIST_FETCH_FAILED = 'Failed to fetch file list.'
 export const ERROR_CALC_INSTALL_SIZE_FAILED = 'Failed to calculate install size.'
 export const NPM_MISSING_README_SENTINEL = 'ERROR: No README data found!'
+/** The npm registry truncates the packument readme field at 65,536 characters (2^16) */
+export const NPM_README_TRUNCATION_THRESHOLD = 64_000
 export const ERROR_JSR_FETCH_FAILED = 'Failed to fetch package from JSR registry.'
 export const ERROR_NPM_FETCH_FAILED = 'Failed to fetch package from npm registry.'
 export const ERROR_PROVENANCE_FETCH_FAILED = 'Failed to fetch provenance.'

test/unit/server/utils/readme-loaders.spec.ts

@@ -81,14 +81,13 @@ describe('resolvePackageReadmeSource', () => {
     parseRepositoryInfoMock.mockReset()
   })
 
-  it('prefers jsDelivr readme over packument readme (latest)', async () => {
-    const jsdelivrContent = '# Full README from CDN'
+  it('returns markdown and repoInfo when package has valid npm readme (latest)', async () => {
+    const markdown = '# Hello'
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': '# Truncated',
-      'readmeFilename': 'README.md',
-      'repository': { url: 'https://github.com/u/r' },
-      'versions': {},
-      'dist-tags': { latest: '2.0.0' },
+      readme: markdown,
+      readmeFilename: 'README.md',
+      repository: { url: 'https://github.com/u/r' },
+      versions: {},
     })
     parseRepositoryInfoMock.mockReturnValue({
       provider: 'github',
@@ -97,53 +96,46 @@ describe('resolvePackageReadmeSource', () => {
       rawBaseUrl: 'https://raw.githubusercontent.com/u/r/HEAD',
       blobBaseUrl: 'https://github.com/u/r/blob/HEAD',
     })
-    const fetchMock = vi.fn().mockResolvedValue({
-      ok: true,
-      text: async () => jsdelivrContent,
-    })
-    vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('some-pkg')
 
     expect(result).toMatchObject({
       packageName: 'some-pkg',
       version: undefined,
-      markdown: jsdelivrContent,
+      markdown,
       repoInfo: { provider: 'github', owner: 'u', repo: 'r' },
     })
     expect(fetchNpmPackageMock).toHaveBeenCalledWith('some-pkg')
   })
 
-  it('uses resolved latest version for jsDelivr when no version specified', async () => {
+  it('returns markdown from version when packagePath includes version', async () => {
+    const markdown = '# Version readme'
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': '# Packument',
-      'readmeFilename': 'README.md',
-      'repository': undefined,
-      'versions': {},
-      'dist-tags': { latest: '3.1.0' },
+      readme: 'latest readme',
+      readmeFilename: 'README.md',
+      repository: undefined,
+      versions: {
+        '1.0.0': { readme: markdown, readmeFilename: 'README.md' },
+      },
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
-    const fetchMock = vi.fn().mockResolvedValue({
-      ok: true,
-      text: async () => '# CDN',
-    })
-    vi.stubGlobal('fetch', fetchMock)
 
-    await resolvePackageReadmeSource('pkg')
+    const result = await resolvePackageReadmeSource('some-pkg/v/1.0.0')
 
-    expect(fetchMock).toHaveBeenCalledWith(expect.stringContaining('pkg@3.1.0'))
+    expect(result).toMatchObject({
+      packageName: 'some-pkg',
+      version: '1.0.0',
+      markdown,
+    })
   })
 
-  it('returns markdown from specific version jsDelivr when packagePath includes version', async () => {
-    const jsdelivrContent = '# Version readme from CDN'
+  it('falls back to jsdelivr when npm readme is missing sentinel', async () => {
+    const jsdelivrContent = '# From CDN'
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': 'latest readme',
-      'readmeFilename': 'README.md',
-      'repository': undefined,
-      'versions': {
-        '1.0.0': { readme: 'version readme from packument', readmeFilename: 'README.md' },
-      },
-      'dist-tags': { latest: '2.0.0' },
+      readme: NPM_MISSING_README_SENTINEL,
+      readmeFilename: 'README.md',
+      repository: undefined,
+      versions: {},
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
     const fetchMock = vi.fn().mockResolvedValue({
@@ -152,69 +144,64 @@ describe('resolvePackageReadmeSource', () => {
     })
     vi.stubGlobal('fetch', fetchMock)
 
-    const result = await resolvePackageReadmeSource('some-pkg/v/1.0.0')
+    const result = await resolvePackageReadmeSource('pkg')
 
     expect(result).toMatchObject({
-      packageName: 'some-pkg',
-      version: '1.0.0',
+      packageName: 'pkg',
       markdown: jsdelivrContent,
+      repoInfo: undefined,
     })
-    expect(fetchMock).toHaveBeenCalledWith(expect.stringContaining('some-pkg@1.0.0'))
+    expect(fetchMock).toHaveBeenCalled()
   })
 
-  it('falls back to packument readme when jsDelivr fails', async () => {
-    const packumentReadme = '# From packument'
+  it('falls back to jsdelivr when readmeFilename is not standard', async () => {
+    const jsdelivrContent = '# From CDN'
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': packumentReadme,
-      'readmeFilename': 'README.md',
-      'repository': undefined,
-      'versions': {},
-      'dist-tags': { latest: '1.0.0' },
+      readme: 'content',
+      readmeFilename: 'DOCS.md',
+      repository: undefined,
+      versions: {},
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
-    const fetchMock = vi.fn().mockResolvedValue({ ok: false })
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: async () => jsdelivrContent,
+    })
     vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('pkg')
 
-    expect(result).toMatchObject({
-      packageName: 'pkg',
-      markdown: packumentReadme,
-    })
+    expect(result).toMatchObject({ markdown: jsdelivrContent })
   })
 
-  it('falls back to version packument readme when jsDelivr fails', async () => {
-    const versionReadme = '# Version readme'
+  it('returns undefined markdown when no content and jsdelivr fails', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': 'latest readme',
-      'repository': undefined,
-      'versions': {
-        '1.0.0': { readme: versionReadme },
-      },
-      'dist-tags': { latest: '1.0.0' },
+      readme: undefined,
+      readmeFilename: undefined,
+      repository: undefined,
+      versions: {},
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
     const fetchMock = vi.fn().mockResolvedValue({ ok: false })
     vi.stubGlobal('fetch', fetchMock)
 
-    const result = await resolvePackageReadmeSource('pkg/v/1.0.0')
+    const result = await resolvePackageReadmeSource('pkg')
 
     expect(result).toMatchObject({
       packageName: 'pkg',
-      version: '1.0.0',
-      markdown: versionReadme,
+      version: undefined,
+      markdown: undefined,
+      repoInfo: undefined,
     })
   })
 
-  it('skips packument readme with missing sentinel in fallback', async () => {
+  it('returns undefined markdown when content is NPM_MISSING_README_SENTINEL and jsdelivr fails', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': NPM_MISSING_README_SENTINEL,
-      'readmeFilename': 'README.md',
-      'repository': undefined,
-      'versions': {},
-      'dist-tags': { latest: '1.0.0' },
+      readme: NPM_MISSING_README_SENTINEL,
+      readmeFilename: 'README.md',
+      repository: undefined,
+      versions: {},
     })
-    parseRepositoryInfoMock.mockReturnValue(undefined)
     const fetchMock = vi.fn().mockResolvedValue({ ok: false })
     vi.stubGlobal('fetch', fetchMock)
 
@@ -227,35 +214,35 @@ describe('resolvePackageReadmeSource', () => {
     })
   })
 
-  it('returns undefined markdown when no content anywhere', async () => {
+  it('fetches from jsdelivr when packument readme exceeds truncation threshold', async () => {
+    const truncatedReadme = 'x'.repeat(64_000)
+    const fullReadme = 'x'.repeat(80_000)
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': undefined,
-      'readmeFilename': undefined,
+      'readme': truncatedReadme,
+      'readmeFilename': 'README.md',
       'repository': undefined,
       'versions': {},
       'dist-tags': { latest: '1.0.0' },
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
-    const fetchMock = vi.fn().mockResolvedValue({ ok: false })
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: async () => fullReadme,
+    })
     vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('pkg')
 
-    expect(result).toMatchObject({
-      packageName: 'pkg',
-      version: undefined,
-      markdown: undefined,
-      repoInfo: undefined,
-    })
+    expect(result).toMatchObject({ markdown: fullReadme })
+    expect(fetchMock).toHaveBeenCalled()
   })
 
   it('uses package repository for repoInfo when markdown is present', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      'readme': '# Hi',
-      'readmeFilename': 'README.md',
-      'repository': { url: 'https://github.com/a/b' },
-      'versions': {},
-      'dist-tags': { latest: '1.0.0' },
+      readme: '# Hi',
+      readmeFilename: 'README.md',
+      repository: { url: 'https://github.com/a/b' },
+      versions: {},
     })
     const repoInfo = {
       provider: 'github' as const,
@@ -265,11 +252,6 @@ describe('resolvePackageReadmeSource', () => {
       blobBaseUrl: 'https://github.com/a/b/blob/HEAD',
     }
     parseRepositoryInfoMock.mockReturnValue(repoInfo)
-    const fetchMock = vi.fn().mockResolvedValue({
-      ok: true,
-      text: async () => '# CDN',
-    })
-    vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('pkg')