Skip to content

Commit ed04df7

Browse files
lukeocodeslukeocodes
committed
fix(readme): fetch README from jsDelivr to avoid npm registry truncation
The npm registry truncates the packument readme field at exactly 65,536 characters, causing large READMEs to render incomplete. Fetch the actual README file from jsDelivr CDN (npm tarball) as the primary source, falling back to the packument field when jsDelivr doesn't have it. Closes #1458
1 parent a5b2e9c commit ed04df7

2 files changed

Lines changed: 118 additions & 78 deletions

File tree

‎server/utils/readme-loaders.ts‎

Lines changed: 22 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -58,35 +58,34 @@ export const resolvePackageReadmeSource = defineCachedFunction(
5858
})
5959

6060
const packageData = await fetchNpmPackage(packageName)
61+
const resolvedVersion = version ?? packageData['dist-tags']?.latest
6162

62-
let readmeContent: string | undefined
63-
let readmeFilename: string | undefined
64-
65-
if (version) {
66-
const versionData = packageData.versions[version]
67-
if (versionData) {
68-
readmeContent = versionData.readme
69-
readmeFilename = versionData.readmeFilename
63+
// Prefer jsDelivr (actual file from npm tarball) because the npm registry
64+
// truncates the packument readme field at 65,536 characters.
65+
let readmeContent = await fetchReadmeFromJsdelivr(
66+
packageName,
67+
standardReadmeFilenames,
68+
resolvedVersion,
69+
)
70+
71+
// Fall back to packument readme if jsDelivr didn't have a standard README.
72+
// This covers packages with non-standard readme filenames (e.g. README.zh-TW.md)
73+
// or packages that don't include a README in the tarball.
74+
if (!readmeContent) {
75+
let packumentReadme: string | undefined
76+
77+
if (version) {
78+
packumentReadme = packageData.versions[version]?.readme
79+
} else {
80+
packumentReadme = packageData.readme
7081
}
71-
} else {
72-
readmeContent = packageData.readme
73-
readmeFilename = packageData.readmeFilename
74-
}
7582

76-
const hasValidNpmReadme = readmeContent && readmeContent !== NPM_MISSING_README_SENTINEL
77-
78-
if (!hasValidNpmReadme || !isStandardReadme(readmeFilename)) {
79-
const jsdelivrReadme = await fetchReadmeFromJsdelivr(
80-
packageName,
81-
standardReadmeFilenames,
82-
version,
83-
)
84-
if (jsdelivrReadme) {
85-
readmeContent = jsdelivrReadme
83+
if (packumentReadme && packumentReadme !== NPM_MISSING_README_SENTINEL) {
84+
readmeContent = packumentReadme
8685
}
8786
}
8887

89-
if (!readmeContent || readmeContent === NPM_MISSING_README_SENTINEL) {
88+
if (!readmeContent) {
9089
return {
9190
packageName,
9291
version,

‎test/unit/server/utils/readme-loaders.spec.ts‎

Lines changed: 96 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,14 @@ describe('resolvePackageReadmeSource', () => {
8181
parseRepositoryInfoMock.mockReset()
8282
})
8383

84-
it('returns markdown and repoInfo when package has valid npm readme (latest)', async () => {
85-
const markdown = '# Hello'
84+
it('prefers jsDelivr readme over packument readme (latest)', async () => {
85+
const jsdelivrContent = '# Full README from CDN'
8686
fetchNpmPackageMock.mockResolvedValue({
87-
readme: markdown,
88-
readmeFilename: 'README.md',
89-
repository: { url: 'https://github.com/u/r' },
90-
versions: {},
87+
'readme': '# Truncated',
88+
'readmeFilename': 'README.md',
89+
'repository': { url: 'https://github.com/u/r' },
90+
'versions': {},
91+
'dist-tags': { latest: '2.0.0' },
9192
})
9293
parseRepositoryInfoMock.mockReturnValue({
9394
provider: 'github',
@@ -96,90 +97,122 @@ describe('resolvePackageReadmeSource', () => {
9697
rawBaseUrl: 'https://raw.githubusercontent.com/u/r/HEAD',
9798
blobBaseUrl: 'https://github.com/u/r/blob/HEAD',
9899
})
100+
const fetchMock = vi.fn().mockResolvedValue({
101+
ok: true,
102+
text: async () => jsdelivrContent,
103+
})
104+
vi.stubGlobal('fetch', fetchMock)
99105

100106
const result = await resolvePackageReadmeSource('some-pkg')
101107

102108
expect(result).toMatchObject({
103109
packageName: 'some-pkg',
104110
version: undefined,
105-
markdown,
111+
markdown: jsdelivrContent,
106112
repoInfo: { provider: 'github', owner: 'u', repo: 'r' },
107113
})
108114
expect(fetchNpmPackageMock).toHaveBeenCalledWith('some-pkg')
109115
})
110116

111-
it('returns markdown from version when packagePath includes version', async () => {
112-
const markdown = '# Version readme'
117+
it('uses resolved latest version for jsDelivr when no version specified', async () => {
118+
fetchNpmPackageMock.mockResolvedValue({
119+
'readme': '# Packument',
120+
'readmeFilename': 'README.md',
121+
'repository': undefined,
122+
'versions': {},
123+
'dist-tags': { latest: '3.1.0' },
124+
})
125+
parseRepositoryInfoMock.mockReturnValue(undefined)
126+
const fetchMock = vi.fn().mockResolvedValue({
127+
ok: true,
128+
text: async () => '# CDN',
129+
})
130+
vi.stubGlobal('fetch', fetchMock)
131+
132+
await resolvePackageReadmeSource('pkg')
133+
134+
expect(fetchMock).toHaveBeenCalledWith(expect.stringContaining('pkg@3.1.0'))
135+
})
136+
137+
it('returns markdown from specific version jsDelivr when packagePath includes version', async () => {
138+
const jsdelivrContent = '# Version readme from CDN'
113139
fetchNpmPackageMock.mockResolvedValue({
114-
readme: 'latest readme',
115-
readmeFilename: 'README.md',
116-
repository: undefined,
117-
versions: {
118-
'1.0.0': { readme: markdown, readmeFilename: 'README.md' },
140+
'readme': 'latest readme',
141+
'readmeFilename': 'README.md',
142+
'repository': undefined,
143+
'versions': {
144+
'1.0.0': { readme: 'version readme from packument', readmeFilename: 'README.md' },
119145
},
146+
'dist-tags': { latest: '2.0.0' },
120147
})
121148
parseRepositoryInfoMock.mockReturnValue(undefined)
149+
const fetchMock = vi.fn().mockResolvedValue({
150+
ok: true,
151+
text: async () => jsdelivrContent,
152+
})
153+
vi.stubGlobal('fetch', fetchMock)
122154

123155
const result = await resolvePackageReadmeSource('some-pkg/v/1.0.0')
124156

125157
expect(result).toMatchObject({
126158
packageName: 'some-pkg',
127159
version: '1.0.0',
128-
markdown,
160+
markdown: jsdelivrContent,
129161
})
162+
expect(fetchMock).toHaveBeenCalledWith(expect.stringContaining('some-pkg@1.0.0'))
130163
})
131164

132-
it('falls back to jsdelivr when npm readme is missing sentinel', async () => {
133-
const jsdelivrContent = '# From CDN'
165+
it('falls back to packument readme when jsDelivr fails', async () => {
166+
const packumentReadme = '# From packument'
134167
fetchNpmPackageMock.mockResolvedValue({
135-
readme: NPM_MISSING_README_SENTINEL,
136-
readmeFilename: 'README.md',
137-
repository: undefined,
138-
versions: {},
168+
'readme': packumentReadme,
169+
'readmeFilename': 'README.md',
170+
'repository': undefined,
171+
'versions': {},
172+
'dist-tags': { latest: '1.0.0' },
139173
})
140174
parseRepositoryInfoMock.mockReturnValue(undefined)
141-
const fetchMock = vi.fn().mockResolvedValue({
142-
ok: true,
143-
text: async () => jsdelivrContent,
144-
})
175+
const fetchMock = vi.fn().mockResolvedValue({ ok: false })
145176
vi.stubGlobal('fetch', fetchMock)
146177

147178
const result = await resolvePackageReadmeSource('pkg')
148179

149180
expect(result).toMatchObject({
150181
packageName: 'pkg',
151-
markdown: jsdelivrContent,
152-
repoInfo: undefined,
182+
markdown: packumentReadme,
153183
})
154-
expect(fetchMock).toHaveBeenCalled()
155184
})
156185

157-
it('falls back to jsdelivr when readmeFilename is not standard', async () => {
158-
const jsdelivrContent = '# From CDN'
186+
it('falls back to version packument readme when jsDelivr fails', async () => {
187+
const versionReadme = '# Version readme'
159188
fetchNpmPackageMock.mockResolvedValue({
160-
readme: 'content',
161-
readmeFilename: 'DOCS.md',
162-
repository: undefined,
163-
versions: {},
189+
'readme': 'latest readme',
190+
'repository': undefined,
191+
'versions': {
192+
'1.0.0': { readme: versionReadme },
193+
},
194+
'dist-tags': { latest: '1.0.0' },
164195
})
165196
parseRepositoryInfoMock.mockReturnValue(undefined)
166-
const fetchMock = vi.fn().mockResolvedValue({
167-
ok: true,
168-
text: async () => jsdelivrContent,
169-
})
197+
const fetchMock = vi.fn().mockResolvedValue({ ok: false })
170198
vi.stubGlobal('fetch', fetchMock)
171199

172-
const result = await resolvePackageReadmeSource('pkg')
200+
const result = await resolvePackageReadmeSource('pkg/v/1.0.0')
173201

174-
expect(result).toMatchObject({ markdown: jsdelivrContent })
202+
expect(result).toMatchObject({
203+
packageName: 'pkg',
204+
version: '1.0.0',
205+
markdown: versionReadme,
206+
})
175207
})
176208

177-
it('returns undefined markdown when no content and jsdelivr fails', async () => {
209+
it('skips packument readme with missing sentinel in fallback', async () => {
178210
fetchNpmPackageMock.mockResolvedValue({
179-
readme: undefined,
180-
readmeFilename: undefined,
181-
repository: undefined,
182-
versions: {},
211+
'readme': NPM_MISSING_README_SENTINEL,
212+
'readmeFilename': 'README.md',
213+
'repository': undefined,
214+
'versions': {},
215+
'dist-tags': { latest: '1.0.0' },
183216
})
184217
parseRepositoryInfoMock.mockReturnValue(undefined)
185218
const fetchMock = vi.fn().mockResolvedValue({ ok: false })
@@ -189,37 +222,40 @@ describe('resolvePackageReadmeSource', () => {
189222

190223
expect(result).toMatchObject({
191224
packageName: 'pkg',
192-
version: undefined,
193225
markdown: undefined,
194226
repoInfo: undefined,
195227
})
196228
})
197229

198-
it('returns undefined markdown when content is NPM_MISSING_README_SENTINEL and jsdelivr fails', async () => {
230+
it('returns undefined markdown when no content anywhere', async () => {
199231
fetchNpmPackageMock.mockResolvedValue({
200-
readme: NPM_MISSING_README_SENTINEL,
201-
readmeFilename: 'README.md',
202-
repository: undefined,
203-
versions: {},
232+
'readme': undefined,
233+
'readmeFilename': undefined,
234+
'repository': undefined,
235+
'versions': {},
236+
'dist-tags': { latest: '1.0.0' },
204237
})
238+
parseRepositoryInfoMock.mockReturnValue(undefined)
205239
const fetchMock = vi.fn().mockResolvedValue({ ok: false })
206240
vi.stubGlobal('fetch', fetchMock)
207241

208242
const result = await resolvePackageReadmeSource('pkg')
209243

210244
expect(result).toMatchObject({
211245
packageName: 'pkg',
246+
version: undefined,
212247
markdown: undefined,
213248
repoInfo: undefined,
214249
})
215250
})
216251

217252
it('uses package repository for repoInfo when markdown is present', async () => {
218253
fetchNpmPackageMock.mockResolvedValue({
219-
readme: '# Hi',
220-
readmeFilename: 'README.md',
221-
repository: { url: 'https://github.com/a/b' },
222-
versions: {},
254+
'readme': '# Hi',
255+
'readmeFilename': 'README.md',
256+
'repository': { url: 'https://github.com/a/b' },
257+
'versions': {},
258+
'dist-tags': { latest: '1.0.0' },
223259
})
224260
const repoInfo = {
225261
provider: 'github' as const,
@@ -229,6 +265,11 @@ describe('resolvePackageReadmeSource', () => {
229265
blobBaseUrl: 'https://github.com/a/b/blob/HEAD',
230266
}
231267
parseRepositoryInfoMock.mockReturnValue(repoInfo)
268+
const fetchMock = vi.fn().mockResolvedValue({
269+
ok: true,
270+
text: async () => '# CDN',
271+
})
272+
vi.stubGlobal('fetch', fetchMock)
232273

233274
const result = await resolvePackageReadmeSource('pkg')
234275

0 commit comments

Comments
 (0)