fix(readme): fetch README from jsDelivr to avoid npm registry truncation

The npm registry truncates the packument readme field at exactly
65,536 characters, causing large READMEs to render incomplete.

Fetch the actual README file from jsDelivr CDN (npm tarball) as the
primary source, falling back to the packument field when jsDelivr
doesn't have it.

Closes #1458

Author: lukeocodes

server/utils/readme-loaders.ts

@@ -13,13 +13,6 @@ const standardReadmeFilenames = [
   'readme.markdown',
 ]
 
-/** Matches standard README filenames (case-insensitive, for checking registry metadata) */
-const standardReadmePattern = /^readme(?:\.md|\.markdown)?$/i
-
-export function isStandardReadme(filename: string | undefined): boolean {
-  return !!filename && standardReadmePattern.test(filename)
-}
-
 /**
  * Fetch README from jsdelivr CDN for a specific package version.
  * Falls back through common README filenames.
@@ -58,35 +51,25 @@ export const resolvePackageReadmeSource = defineCachedFunction(
     })
 
     const packageData = await fetchNpmPackage(packageName)
+    const resolvedVersion = version ?? packageData['dist-tags']?.latest
 
-    let readmeContent: string | undefined
-    let readmeFilename: string | undefined
-
-    if (version) {
-      const versionData = packageData.versions[version]
-      if (versionData) {
-        readmeContent = versionData.readme
-        readmeFilename = versionData.readmeFilename
-      }
-    } else {
-      readmeContent = packageData.readme
-      readmeFilename = packageData.readmeFilename
-    }
-
-    const hasValidNpmReadme = readmeContent && readmeContent !== NPM_MISSING_README_SENTINEL
-
-    if (!hasValidNpmReadme || !isStandardReadme(readmeFilename)) {
-      const jsdelivrReadme = await fetchReadmeFromJsdelivr(
-        packageName,
-        standardReadmeFilenames,
-        version,
-      )
-      if (jsdelivrReadme) {
-        readmeContent = jsdelivrReadme
+    // Prefer jsDelivr (actual file from npm tarball) over the packument readme field,
+    // because the npm registry truncates readme content at 65,536 characters.
+    let readmeContent = await fetchReadmeFromJsdelivr(
+      packageName,
+      standardReadmeFilenames,
+      resolvedVersion,
+    )
+
+    // Fall back to packument readme if jsDelivr didn't have it
+    if (!readmeContent) {
+      const packumentReadme = version ? packageData.versions[version]?.readme : packageData.readme
+      if (packumentReadme && packumentReadme !== NPM_MISSING_README_SENTINEL) {
+        readmeContent = packumentReadme
       }
     }
 
-    if (!readmeContent || readmeContent === NPM_MISSING_README_SENTINEL) {
+    if (!readmeContent) {
       return {
         packageName,
         version,

test/unit/server/utils/readme-loaders.spec.ts

@@ -14,29 +14,9 @@ vi.stubGlobal('fetchNpmPackage', fetchNpmPackageMock)
 const parseRepositoryInfoMock = vi.fn()
 vi.stubGlobal('parseRepositoryInfo', parseRepositoryInfoMock)
 
-const { fetchReadmeFromJsdelivr, isStandardReadme, resolvePackageReadmeSource } =
+const { fetchReadmeFromJsdelivr, resolvePackageReadmeSource } =
   await import('../../../../server/utils/readme-loaders')
 
-describe('isStandardReadme', () => {
-  it('returns true for standard README filenames', () => {
-    expect(isStandardReadme('README.md')).toBe(true)
-    expect(isStandardReadme('readme.md')).toBe(true)
-    expect(isStandardReadme('Readme.md')).toBe(true)
-    expect(isStandardReadme('README')).toBe(true)
-    expect(isStandardReadme('readme')).toBe(true)
-    expect(isStandardReadme('README.markdown')).toBe(true)
-    expect(isStandardReadme('readme.markdown')).toBe(true)
-  })
-
-  it('returns false for non-standard filenames', () => {
-    expect(isStandardReadme('CONTRIBUTING.md')).toBe(false)
-    expect(isStandardReadme('README.txt')).toBe(false)
-    expect(isStandardReadme('readme.rst')).toBe(false)
-    expect(isStandardReadme(undefined)).toBe(false)
-    expect(isStandardReadme('')).toBe(false)
-  })
-})
-
 describe('fetchReadmeFromJsdelivr', () => {
   it('returns content when first filename succeeds', async () => {
     const content = '# Package'
@@ -81,13 +61,14 @@ describe('resolvePackageReadmeSource', () => {
     parseRepositoryInfoMock.mockReset()
   })
 
-  it('returns markdown and repoInfo when package has valid npm readme (latest)', async () => {
-    const markdown = '# Hello'
+  it('prefers jsDelivr readme over packument readme (latest)', async () => {
+    const jsdelivrContent = '# Full README from CDN'
     fetchNpmPackageMock.mockResolvedValue({
-      readme: markdown,
-      readmeFilename: 'README.md',
-      repository: { url: 'https://github.com/u/r' },
-      versions: {},
+      'readme': '# Truncated',
+      'readmeFilename': 'README.md',
+      'repository': { url: 'https://github.com/u/r' },
+      'versions': {},
+      'dist-tags': { latest: '2.0.0' },
     })
     parseRepositoryInfoMock.mockReturnValue({
       provider: 'github',
@@ -96,90 +77,122 @@ describe('resolvePackageReadmeSource', () => {
       rawBaseUrl: 'https://raw.githubusercontent.com/u/r/HEAD',
       blobBaseUrl: 'https://github.com/u/r/blob/HEAD',
     })
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: async () => jsdelivrContent,
+    })
+    vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('some-pkg')
 
     expect(result).toMatchObject({
       packageName: 'some-pkg',
       version: undefined,
-      markdown,
+      markdown: jsdelivrContent,
       repoInfo: { provider: 'github', owner: 'u', repo: 'r' },
     })
     expect(fetchNpmPackageMock).toHaveBeenCalledWith('some-pkg')
   })
 
-  it('returns markdown from version when packagePath includes version', async () => {
-    const markdown = '# Version readme'
+  it('uses resolved latest version for jsDelivr when no version specified', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      readme: 'latest readme',
-      readmeFilename: 'README.md',
-      repository: undefined,
-      versions: {
-        '1.0.0': { readme: markdown, readmeFilename: 'README.md' },
+      'readme': '# Packument',
+      'readmeFilename': 'README.md',
+      'repository': undefined,
+      'versions': {},
+      'dist-tags': { latest: '3.1.0' },
+    })
+    parseRepositoryInfoMock.mockReturnValue(undefined)
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: async () => '# CDN',
+    })
+    vi.stubGlobal('fetch', fetchMock)
+
+    await resolvePackageReadmeSource('pkg')
+
+    expect(fetchMock).toHaveBeenCalledWith(expect.stringContaining('pkg@3.1.0'))
+  })
+
+  it('returns markdown from specific version jsDelivr when packagePath includes version', async () => {
+    const jsdelivrContent = '# Version readme from CDN'
+    fetchNpmPackageMock.mockResolvedValue({
+      'readme': 'latest readme',
+      'readmeFilename': 'README.md',
+      'repository': undefined,
+      'versions': {
+        '1.0.0': { readme: 'version readme from packument', readmeFilename: 'README.md' },
       },
+      'dist-tags': { latest: '2.0.0' },
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: async () => jsdelivrContent,
+    })
+    vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('some-pkg/v/1.0.0')
 
     expect(result).toMatchObject({
       packageName: 'some-pkg',
       version: '1.0.0',
-      markdown,
+      markdown: jsdelivrContent,
     })
+    expect(fetchMock).toHaveBeenCalledWith(expect.stringContaining('some-pkg@1.0.0'))
   })
 
-  it('falls back to jsdelivr when npm readme is missing sentinel', async () => {
-    const jsdelivrContent = '# From CDN'
+  it('falls back to packument readme when jsDelivr fails', async () => {
+    const packumentReadme = '# From packument'
     fetchNpmPackageMock.mockResolvedValue({
-      readme: NPM_MISSING_README_SENTINEL,
-      readmeFilename: 'README.md',
-      repository: undefined,
-      versions: {},
+      'readme': packumentReadme,
+      'readmeFilename': 'README.md',
+      'repository': undefined,
+      'versions': {},
+      'dist-tags': { latest: '1.0.0' },
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
-    const fetchMock = vi.fn().mockResolvedValue({
-      ok: true,
-      text: async () => jsdelivrContent,
-    })
+    const fetchMock = vi.fn().mockResolvedValue({ ok: false })
     vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('pkg')
 
     expect(result).toMatchObject({
       packageName: 'pkg',
-      markdown: jsdelivrContent,
-      repoInfo: undefined,
+      markdown: packumentReadme,
     })
-    expect(fetchMock).toHaveBeenCalled()
   })
 
-  it('falls back to jsdelivr when readmeFilename is not standard', async () => {
-    const jsdelivrContent = '# From CDN'
+  it('falls back to version packument readme when jsDelivr fails', async () => {
+    const versionReadme = '# Version readme'
     fetchNpmPackageMock.mockResolvedValue({
-      readme: 'content',
-      readmeFilename: 'DOCS.md',
-      repository: undefined,
-      versions: {},
+      'readme': 'latest readme',
+      'repository': undefined,
+      'versions': {
+        '1.0.0': { readme: versionReadme },
+      },
+      'dist-tags': { latest: '1.0.0' },
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
-    const fetchMock = vi.fn().mockResolvedValue({
-      ok: true,
-      text: async () => jsdelivrContent,
-    })
+    const fetchMock = vi.fn().mockResolvedValue({ ok: false })
     vi.stubGlobal('fetch', fetchMock)
 
-    const result = await resolvePackageReadmeSource('pkg')
+    const result = await resolvePackageReadmeSource('pkg/v/1.0.0')
 
-    expect(result).toMatchObject({ markdown: jsdelivrContent })
+    expect(result).toMatchObject({
+      packageName: 'pkg',
+      version: '1.0.0',
+      markdown: versionReadme,
+    })
   })
 
-  it('returns undefined markdown when no content and jsdelivr fails', async () => {
+  it('skips packument readme with missing sentinel in fallback', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      readme: undefined,
-      readmeFilename: undefined,
-      repository: undefined,
-      versions: {},
+      'readme': NPM_MISSING_README_SENTINEL,
+      'readmeFilename': 'README.md',
+      'repository': undefined,
+      'versions': {},
+      'dist-tags': { latest: '1.0.0' },
     })
     parseRepositoryInfoMock.mockReturnValue(undefined)
     const fetchMock = vi.fn().mockResolvedValue({ ok: false })
@@ -189,37 +202,40 @@ describe('resolvePackageReadmeSource', () => {
 
     expect(result).toMatchObject({
       packageName: 'pkg',
-      version: undefined,
       markdown: undefined,
       repoInfo: undefined,
     })
   })
 
-  it('returns undefined markdown when content is NPM_MISSING_README_SENTINEL and jsdelivr fails', async () => {
+  it('returns undefined markdown when no content anywhere', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      readme: NPM_MISSING_README_SENTINEL,
-      readmeFilename: 'README.md',
-      repository: undefined,
-      versions: {},
+      'readme': undefined,
+      'readmeFilename': undefined,
+      'repository': undefined,
+      'versions': {},
+      'dist-tags': { latest: '1.0.0' },
     })
+    parseRepositoryInfoMock.mockReturnValue(undefined)
     const fetchMock = vi.fn().mockResolvedValue({ ok: false })
     vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('pkg')
 
     expect(result).toMatchObject({
       packageName: 'pkg',
+      version: undefined,
       markdown: undefined,
       repoInfo: undefined,
     })
   })
 
   it('uses package repository for repoInfo when markdown is present', async () => {
     fetchNpmPackageMock.mockResolvedValue({
-      readme: '# Hi',
-      readmeFilename: 'README.md',
-      repository: { url: 'https://github.com/a/b' },
-      versions: {},
+      'readme': '# Hi',
+      'readmeFilename': 'README.md',
+      'repository': { url: 'https://github.com/a/b' },
+      'versions': {},
+      'dist-tags': { latest: '1.0.0' },
     })
     const repoInfo = {
       provider: 'github' as const,
@@ -229,6 +245,11 @@ describe('resolvePackageReadmeSource', () => {
       blobBaseUrl: 'https://github.com/a/b/blob/HEAD',
     }
     parseRepositoryInfoMock.mockReturnValue(repoInfo)
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: async () => '# CDN',
+    })
+    vi.stubGlobal('fetch', fetchMock)
 
     const result = await resolvePackageReadmeSource('pkg')