perf: use osv batch api for dep analysis

Author: danielroe

server/utils/dependency-analysis.ts

@@ -1,5 +1,6 @@
 import type {
   OsvQueryResponse,
+  OsvBatchResponse,
   OsvVulnerability,
   OsvSeverityLevel,
   VulnerabilitySummary,
@@ -10,29 +11,68 @@ import type {
 } from '#shared/types/dependency-analysis'
 import { resolveDependencyTree } from './dependency-resolver'
 
-/** Result of a single OSV query */
-type OsvQueryResult = { status: 'ok'; data: PackageVulnerabilityInfo | null } | { status: 'error' }
+/** Package info needed for OSV queries */
+interface PackageQueryInfo {
+  name: string
+  version: string
+  depth: DependencyDepth
+  path: string[]
+}
 
 /**
- * Query OSV for vulnerabilities in a package
+ * Query OSV batch API to find which packages have vulnerabilities.
+ * Returns indices of packages that have vulnerabilities (for follow-up detailed queries).
+ * @see https://google.github.io/osv.dev/post-v1-querybatch/
  */
-async function queryOsv(
-  name: string,
-  version: string,
-  depth: DependencyDepth,
-  path: string[],
-): Promise<OsvQueryResult> {
+async function queryOsvBatch(
+  packages: PackageQueryInfo[],
+): Promise<{ vulnerableIndices: number[]; failed: boolean }> {
+  if (packages.length === 0) return { vulnerableIndices: [], failed: false }
+
+  try {
+    const response = await $fetch<OsvBatchResponse>('https://api.osv.dev/v1/querybatch', {
+      method: 'POST',
+      body: {
+        queries: packages.map(pkg => ({
+          package: { name: pkg.name, ecosystem: 'npm' },
+          version: pkg.version,
+        })),
+      },
+    })
+
+    // Find indices of packages that have vulnerabilities
+    const vulnerableIndices: number[] = []
+    for (let i = 0; i < response.results.length; i++) {
+      const result = response.results[i]
+      if (result?.vulns && result.vulns.length > 0) {
+        vulnerableIndices.push(i)
+      }
+    }
+
+    return { vulnerableIndices, failed: false }
+  } catch (error) {
+    // oxlint-disable-next-line no-console -- log OSV API failures for debugging
+    console.warn(`[dep-analysis] OSV batch query failed:`, error)
+    return { vulnerableIndices: [], failed: true }
+  }
+}
+
+/**
+ * Query OSV for full vulnerability details for a single package.
+ * Only called for packages known to have vulnerabilities.
+ */
+async function queryOsvDetails(pkg: PackageQueryInfo): Promise<PackageVulnerabilityInfo | null> {
   try {
     const response = await $fetch<OsvQueryResponse>('https://api.osv.dev/v1/query', {
       method: 'POST',
       body: {
-        package: { name, ecosystem: 'npm' },
-        version,
+        package: { name: pkg.name, ecosystem: 'npm' },
+        version: pkg.version,
       },
     })
 
     const vulns = response.vulns || []
-    if (vulns.length === 0) return { status: 'ok', data: null }
+    if (vulns.length === 0) return null
 
     const counts = { total: vulns.length, critical: 0, high: 0, moderate: 0, low: 0 }
     const vulnerabilities: VulnerabilitySummary[] = []
@@ -65,11 +105,18 @@ async function queryOsv(
       })
     }
 
-    return { status: 'ok', data: { name, version, depth, path, vulnerabilities, counts } }
+    return {
+      name: pkg.name,
+      version: pkg.version,
+      depth: pkg.depth,
+      path: pkg.path,
+      vulnerabilities,
+      counts,
+    }
   } catch (error) {
     // oxlint-disable-next-line no-console -- log OSV API failures for debugging
-    console.warn(`[dep-analysis] OSV query failed for ${name}@${version}:`, error)
-    return { status: 'error' }
+    console.warn(`[dep-analysis] OSV detail query failed for ${pkg.name}@${pkg.version}:`, error)
+    return null
   }
 }
 
@@ -110,17 +157,24 @@ function getSeverityLevel(vuln: OsvVulnerability): OsvSeverityLevel {
 
 /**
  * Analyze entire dependency tree for vulnerabilities and deprecated packages.
+ * Uses OSV batch API for efficient vulnerability discovery, then fetches
+ * full details only for packages with known vulnerabilities.
  */
 export const analyzeDependencyTree = defineCachedFunction(
   async (name: string, version: string): Promise<VulnerabilityTreeResult> => {
     // Resolve all packages in the tree with depth tracking
     const resolved = await resolveDependencyTree(name, version, { trackDepth: true })
 
-    // Convert to array for OSV querying
-    const packages = [...resolved.values()]
+    // Convert to array with query info
+    const packages: PackageQueryInfo[] = [...resolved.values()].map(pkg => ({
+      name: pkg.name,
+      version: pkg.version,
+      depth: pkg.depth!,
+      path: pkg.path || [],
+    }))
 
     // Collect deprecated packages (no API call needed - already in packument data)
-    const deprecatedPackages: DeprecatedPackageInfo[] = packages
+    const deprecatedPackages: DeprecatedPackageInfo[] = [...resolved.values()]
       .filter(pkg => pkg.deprecated)
       .map(pkg => ({
         name: pkg.name,
@@ -135,22 +189,27 @@ export const analyzeDependencyTree = defineCachedFunction(
         return depthOrder[a.depth] - depthOrder[b.depth]
       })
 
-    // Query OSV for all packages in parallel batches
-    const vulnerablePackages: PackageVulnerabilityInfo[] = []
-    let failedQueries = 0
-    const batchSize = 10
+    // Step 1: Use batch API to find which packages have vulnerabilities
+    // This is much faster than individual queries - one request for all packages
+    const { vulnerableIndices, failed: batchFailed } = await queryOsvBatch(packages)
+
+    let vulnerablePackages: PackageVulnerabilityInfo[] = []
+    let failedQueries = batchFailed ? packages.length : 0
+
+    if (!batchFailed && vulnerableIndices.length > 0) {
+      // Step 2: Fetch full vulnerability details only for packages with vulns
+      // This is typically a small fraction of total packages
+      const vulnerablePackageInfos = vulnerableIndices.map(i => packages[i]!)
 
-    for (let i = 0; i < packages.length; i += batchSize) {
-      const batch = packages.slice(i, i + batchSize)
-      const results = await Promise.all(
-        batch.map(pkg => queryOsv(pkg.name, pkg.version, pkg.depth!, pkg.path || [])),
+      const detailResults = await Promise.all(
+        vulnerablePackageInfos.map(pkg => queryOsvDetails(pkg)),
       )
 
-      for (const result of results) {
-        if (result.status === 'error') {
+      for (const result of detailResults) {
+        if (result) {
+          vulnerablePackages.push(result)
+        } else {
           failedQueries++
-        } else if (result.data) {
-          vulnerablePackages.push(result.data)
         }
       }
     }
@@ -175,11 +234,11 @@ export const analyzeDependencyTree = defineCachedFunction(
       totalCounts.low += pkg.counts.low
     }
 
-    // Log critical failures (>50% of queries failed)
-    if (failedQueries > 0 && failedQueries > packages.length / 2) {
+    // Log if batch query failed entirely
+    if (batchFailed) {
       // oxlint-disable-next-line no-console -- critical error logging
       console.error(
-        `[dep-analysis] Critical: ${failedQueries}/${packages.length} OSV queries failed for ${name}@${version}`,
+        `[dep-analysis] Critical: OSV batch query failed for ${name}@${version} (${packages.length} packages)`,
       )
     }
 
@@ -197,6 +256,6 @@ export const analyzeDependencyTree = defineCachedFunction(
     maxAge: 60 * 60,
     swr: true,
     name: 'dependency-analysis',
-    getKey: (name: string, version: string) => `v1:${name}@${version}`,
+    getKey: (name: string, version: string) => `v2:${name}@${version}`,
   },
 )

shared/types/dependency-analysis.ts

@@ -64,6 +64,30 @@ export interface OsvQueryResponse {
   next_page_token?: string
 }
 
+/**
+ * Single result from OSV batch query (minimal info - just ID and modified)
+ */
+export interface OsvBatchVulnRef {
+  id: string
+  modified: string
+}
+
+/**
+ * Single result in OSV batch response
+ */
+export interface OsvBatchResult {
+  vulns?: OsvBatchVulnRef[]
+  next_page_token?: string
+}
+
+/**
+ * OSV batch query response
+ * @see https://google.github.io/osv.dev/post-v1-querybatch/
+ */
+export interface OsvBatchResponse {
+  results: OsvBatchResult[]
+}
+
 /**
  * Simplified vulnerability info for display
  */