Skip to content

Commit d2e0a30

Browse files
committed
refactor(llms-txt): add handler factory and split llms.txt/llms_full.txt content
Add createPackageLlmsTxtHandler factory for DRY route creation. handleLlmsTxt now accepts includeAgentFiles option to control whether agent instruction files are included (llms_full.txt) or omitted (llms.txt). Add handleOrgLlmsTxt for org-level package listings and generateRootLlmsTxt for the root /llms.txt discovery page. Simplify route handlers to single-line factory calls.
1 parent 594b42d commit d2e0a30

3 files changed

Lines changed: 193 additions & 66 deletions

File tree

Lines changed: 2 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,3 @@
1-
import * as v from 'valibot'
2-
import { PackageRouteParamsSchema } from '#shared/schemas/package'
3-
import { CACHE_MAX_AGE_ONE_HOUR } from '#shared/utils/constants'
4-
import { handleApiError } from '#server/utils/error-handler'
5-
import { handleLlmsTxt } from '#server/utils/llms-txt'
1+
import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt'
62

7-
/**
8-
* Serves llms.txt for an npm package.
9-
*
10-
* Handles all URL shapes via re-exports:
11-
* - /package/:name/llms.txt
12-
* - /package/:org/:name/llms.txt
13-
* - /package/:name/v/:version/llms.txt
14-
* - /package/:org/:name/v/:version/llms.txt
15-
*/
16-
export default defineCachedEventHandler(
17-
async event => {
18-
const org = getRouterParam(event, 'org')
19-
const name = getRouterParam(event, 'name')
20-
const rawVersion = getRouterParam(event, 'version')
21-
if (!name) {
22-
throw createError({ statusCode: 404, message: 'Package name is required.' })
23-
}
24-
25-
const rawPackageName = org ? `${org}/${name}` : name
26-
27-
try {
28-
const { packageName, version } = v.parse(PackageRouteParamsSchema, {
29-
packageName: rawPackageName,
30-
version: rawVersion,
31-
})
32-
33-
const content = await handleLlmsTxt(packageName, version)
34-
setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
35-
return content
36-
} catch (error: unknown) {
37-
handleApiError(error, {
38-
statusCode: 502,
39-
message: 'Failed to generate llms.txt.',
40-
})
41-
}
42-
},
43-
{
44-
maxAge: CACHE_MAX_AGE_ONE_HOUR,
45-
swr: true,
46-
getKey: event => {
47-
const org = getRouterParam(event, 'org')
48-
const name = getRouterParam(event, 'name')
49-
const version = getRouterParam(event, 'version')
50-
const pkg = org ? `${org}/${name}` : name
51-
return version ? `llms-txt:${pkg}@${version}` : `llms-txt:${pkg}`
52-
},
53-
},
54-
)
3+
export default createPackageLlmsTxtHandler()
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
export { default } from '../../[name]/llms.txt.get'
1+
import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt'
2+
3+
export default createPackageLlmsTxtHandler()

server/utils/llms-txt.ts

Lines changed: 188 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
1+
import * as v from 'valibot'
12
import type { Packument } from '@npm/types'
23
import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types'
3-
import { NPM_MISSING_README_SENTINEL } from '#shared/utils/constants'
4+
import { PackageRouteParamsSchema } from '#shared/schemas/package'
5+
import {
6+
NPM_MISSING_README_SENTINEL,
7+
NPM_REGISTRY,
8+
CACHE_MAX_AGE_ONE_HOUR,
9+
} from '#shared/utils/constants'
10+
import { handleApiError } from '#server/utils/error-handler'
411

512
/** Well-known agent instruction files at the package root */
613
const ROOT_AGENT_FILES: Record<string, string> = {
@@ -116,7 +123,7 @@ export async function fetchAgentFiles(
116123
* - Blockquote description (if available)
117124
* - Metadata list (homepage, repository, npm)
118125
* - README section
119-
* - Agent Instructions section (one sub-heading per file)
126+
* - Agent Instructions section (one sub-heading per file, full mode only)
120127
*/
121128
export function generateLlmsTxt(result: LlmsTxtResult): string {
122129
const lines: string[] = []
@@ -204,12 +211,18 @@ function parseRepoUrl(
204211

205212
/**
206213
* Orchestrates fetching all data and generating llms.txt for a package.
207-
* Shared by both versioned and unversioned route handlers.
214+
*
215+
* When `includeAgentFiles` is false (default, for llms.txt), skips the file tree
216+
* fetch and agent file discovery entirely — only returns README + metadata.
217+
* When true (for llms_full.txt), includes agent instruction files.
208218
*/
209219
export async function handleLlmsTxt(
210220
packageName: string,
211221
requestedVersion?: string,
222+
options?: { includeAgentFiles?: boolean },
212223
): Promise<string> {
224+
const includeAgentFiles = options?.includeAgentFiles ?? false
225+
213226
const packageData = await fetchNpmPackage(packageName)
214227
const resolvedVersion = requestedVersion ?? packageData['dist-tags']?.latest
215228

@@ -220,18 +233,25 @@ export async function handleLlmsTxt(
220233
// Extract README from packument (sync)
221234
const readmeFromPackument = getReadmeFromPackument(packageData, requestedVersion)
222235

223-
// Fetch file tree (and README from CDN if packument didn't have one)
224-
const [fileTreeData, cdnReadme] = await Promise.all([
225-
fetchFileTree(packageName, resolvedVersion),
226-
readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion),
227-
])
236+
let agentFiles: AgentFile[] = []
237+
let cdnReadme: string | null = null
238+
239+
if (includeAgentFiles) {
240+
// Full mode: fetch file tree for agent discovery + README fallback in parallel
241+
const [fileTreeData, readme] = await Promise.all([
242+
fetchFileTree(packageName, resolvedVersion),
243+
readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion),
244+
])
245+
cdnReadme = readme
246+
const agentFilePaths = discoverAgentFiles(fileTreeData.files)
247+
agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths)
248+
} else if (!readmeFromPackument) {
249+
// Standard mode: only fetch README from CDN if packument lacks it
250+
cdnReadme = await fetchReadmeFromCdn(packageName, resolvedVersion)
251+
}
228252

229253
const readme = readmeFromPackument ?? cdnReadme ?? undefined
230254

231-
// Discover and fetch agent files
232-
const agentFilePaths = discoverAgentFiles(fileTreeData.files)
233-
const agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths)
234-
235255
const result: LlmsTxtResult = {
236256
packageName,
237257
version: resolvedVersion,
@@ -244,3 +264,159 @@ export async function handleLlmsTxt(
244264

245265
return generateLlmsTxt(result)
246266
}
267+
268+
// Validation for org names (matches server/api/registry/org/[org]/packages.get.ts)
269+
const NPM_ORG_NAME_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i
270+
271+
/**
272+
* Generate llms.txt for an npm organization/scope.
273+
* Lists all packages in the org with links to their llms.txt pages.
274+
*/
275+
export async function handleOrgLlmsTxt(orgName: string, baseUrl: string): Promise<string> {
276+
if (!orgName || orgName.length > 50 || !NPM_ORG_NAME_RE.test(orgName)) {
277+
throw createError({ statusCode: 404, message: `Invalid org name: ${orgName}` })
278+
}
279+
280+
const data = await $fetch<Record<string, string>>(
281+
`${NPM_REGISTRY}/-/org/${encodeURIComponent(orgName)}/package`,
282+
)
283+
284+
const packages = Object.keys(data).sort()
285+
286+
if (packages.length === 0) {
287+
throw createError({ statusCode: 404, message: `No packages found for @${orgName}` })
288+
}
289+
290+
const lines: string[] = []
291+
292+
lines.push(`# @${orgName}`)
293+
lines.push('')
294+
lines.push(`> npm packages published under the @${orgName} scope`)
295+
lines.push('')
296+
lines.push(`- npm: https://www.npmjs.com/org/${orgName}`)
297+
lines.push('')
298+
lines.push('## Packages')
299+
lines.push('')
300+
301+
for (const pkg of packages) {
302+
const encodedPkg = pkg.replace('/', '/')
303+
lines.push(`- [${pkg}](${baseUrl}/package/${encodedPkg}/llms.txt)`)
304+
}
305+
306+
lines.push('')
307+
308+
return lines.join('\n').trimEnd() + '\n'
309+
}
310+
311+
/**
312+
* Generate the root /llms.txt explaining available routes.
313+
*/
314+
export function generateRootLlmsTxt(baseUrl: string): string {
315+
const lines: string[] = []
316+
317+
lines.push('# npmx.dev')
318+
lines.push('')
319+
lines.push('> A fast, modern browser for the npm registry')
320+
lines.push('')
321+
lines.push('This site provides LLM-friendly documentation for npm packages.')
322+
lines.push('')
323+
lines.push('## Available Routes')
324+
lines.push('')
325+
lines.push('### Package Documentation (llms.txt)')
326+
lines.push('')
327+
lines.push('README and package metadata in markdown format.')
328+
lines.push('')
329+
lines.push(`- \`${baseUrl}/package/<name>/llms.txt\` — unscoped package (latest version)`)
330+
lines.push(
331+
`- \`${baseUrl}/package/<name>/v/<version>/llms.txt\` — unscoped package (specific version)`,
332+
)
333+
lines.push(`- \`${baseUrl}/package/@<org>/<name>/llms.txt\` — scoped package (latest version)`)
334+
lines.push(
335+
`- \`${baseUrl}/package/@<org>/<name>/v/<version>/llms.txt\` — scoped package (specific version)`,
336+
)
337+
lines.push('')
338+
lines.push('### Full Package Documentation (llms_full.txt)')
339+
lines.push('')
340+
lines.push(
341+
'README, package metadata, and agent instruction files (CLAUDE.md, .cursorrules, etc.).',
342+
)
343+
lines.push('')
344+
lines.push(`- \`${baseUrl}/package/<name>/llms_full.txt\` — unscoped package (latest version)`)
345+
lines.push(
346+
`- \`${baseUrl}/package/<name>/v/<version>/llms_full.txt\` — unscoped package (specific version)`,
347+
)
348+
lines.push(
349+
`- \`${baseUrl}/package/@<org>/<name>/llms_full.txt\` — scoped package (latest version)`,
350+
)
351+
lines.push(
352+
`- \`${baseUrl}/package/@<org>/<name>/v/<version>/llms_full.txt\` — scoped package (specific version)`,
353+
)
354+
lines.push('')
355+
lines.push('### Organization Packages (llms.txt)')
356+
lines.push('')
357+
lines.push('List of all packages under an npm scope with links to their documentation.')
358+
lines.push('')
359+
lines.push(`- \`${baseUrl}/package/@<org>/llms.txt\` — organization package listing`)
360+
lines.push('')
361+
lines.push('## Examples')
362+
lines.push('')
363+
lines.push(`- [nuxt llms.txt](${baseUrl}/package/nuxt/llms.txt)`)
364+
lines.push(`- [nuxt llms_full.txt](${baseUrl}/package/nuxt/llms_full.txt)`)
365+
lines.push(`- [@nuxt/kit llms.txt](${baseUrl}/package/@nuxt/kit/llms.txt)`)
366+
lines.push(`- [@nuxt org packages](${baseUrl}/package/@nuxt/llms.txt)`)
367+
lines.push('')
368+
369+
return lines.join('\n').trimEnd() + '\n'
370+
}
371+
372+
/**
373+
* Create a cached event handler for package-level llms.txt or llms_full.txt.
374+
*
375+
* Each route file should call this factory and `export default` the result.
376+
* This avoids the re-export pattern that Nitro doesn't register as routes.
377+
*/
378+
export function createPackageLlmsTxtHandler(options?: { full?: boolean }) {
379+
const full = options?.full ?? false
380+
381+
return defineCachedEventHandler(
382+
async event => {
383+
const org = getRouterParam(event, 'org')
384+
const name = getRouterParam(event, 'name')
385+
const rawVersion = getRouterParam(event, 'version')
386+
387+
if (!name) {
388+
throw createError({ statusCode: 404, message: 'Package name is required.' })
389+
}
390+
391+
const rawPackageName = org ? `${org}/${name}` : name
392+
393+
try {
394+
const { packageName, version } = v.parse(PackageRouteParamsSchema, {
395+
packageName: rawPackageName,
396+
version: rawVersion,
397+
})
398+
399+
const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full })
400+
setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
401+
return content
402+
} catch (error: unknown) {
403+
handleApiError(error, {
404+
statusCode: 502,
405+
message: `Failed to generate ${full ? 'llms_full.txt' : 'llms.txt'}.`,
406+
})
407+
}
408+
},
409+
{
410+
maxAge: CACHE_MAX_AGE_ONE_HOUR,
411+
swr: true,
412+
getKey: event => {
413+
const org = getRouterParam(event, 'org')
414+
const name = getRouterParam(event, 'name')
415+
const version = getRouterParam(event, 'version')
416+
const pkg = org ? `${org}/${name}` : name
417+
const prefix = full ? 'llms-full-txt' : 'llms-txt'
418+
return version ? `${prefix}:${pkg}@${version}` : `${prefix}:${pkg}`
419+
},
420+
},
421+
)
422+
}

0 commit comments

Comments
 (0)