feat: progressive loading for org packages

app/composables/npm/useAlgoliaSearch.ts

@@ -214,11 +214,9 @@ export function useAlgoliaSearch() {
     }
   }
 
-  /** Fetch metadata for specific packages by exact name using Algolia's getObjects API. */
-  async function getPackagesByName(packageNames: string[]): Promise<NpmSearchResponse> {
-    if (packageNames.length === 0) {
-      return { isStale: false, objects: [], total: 0, time: new Date().toISOString() }
-    }
+  /** Fetch metadata for a single batch of packages (max 1000) by exact name. */
+  async function getPackagesByNameSlice(names: string[]): Promise<NpmSearchResult[]> {
+    if (names.length === 0) return []
 
     const response = await $fetch<{ results: (AlgoliaHit | null)[] }>(
       `https://${algolia.appId}-dsn.algolia.net/1/indexes/*/objects`,
@@ -229,7 +227,7 @@ export function useAlgoliaSearch() {
           'x-algolia-application-id': algolia.appId,
         },
         body: {
-          requests: packageNames.map(name => ({
+          requests: names.map(name => ({
             indexName,
             objectID: name,
             attributesToRetrieve: ATTRIBUTES_TO_RETRIEVE,
@@ -238,11 +236,31 @@ export function useAlgoliaSearch() {
       },
     )
 
-    const hits = response.results.filter((r): r is AlgoliaHit => r !== null && 'name' in r)
+    return response.results
+      .filter((r): r is AlgoliaHit => r !== null && 'name' in r)
+      .map(hitToSearchResult)
+  }
+
+  /** Fetch metadata for specific packages by exact name using Algolia's getObjects API. */
+  async function getPackagesByName(packageNames: string[]): Promise<NpmSearchResponse> {
+    if (packageNames.length === 0) {
+      return { isStale: false, objects: [], total: 0, time: new Date().toISOString() }
+    }
+
+    // Algolia getObjects has a limit of 1000 objects per request, so batch if needed
+    const BATCH_SIZE = 1000
+    const batches: string[][] = []
+    for (let i = 0; i < packageNames.length; i += BATCH_SIZE) {
+      batches.push(packageNames.slice(i, i + BATCH_SIZE))
+    }
+
+    const results = await Promise.all(batches.map(batch => getPackagesByNameSlice(batch)))
+    const allObjects = results.flat()
+
     return {
       isStale: false,
-      objects: hits.map(hitToSearchResult),
-      total: hits.length,
+      objects: allObjects,
+      total: allObjects.length,
       time: new Date().toISOString(),
     }
   }
@@ -349,5 +367,6 @@ export function useAlgoliaSearch() {
     searchWithSuggestions,
     searchByOwner,
     getPackagesByName,
+    getPackagesByNameSlice,
   }
 }

app/composables/npm/useOrgPackages.ts

@@ -1,9 +1,34 @@
+import type { NpmSearchResponse, NpmSearchResult, PackageMetaResponse } from '#shared/types'
+import { emptySearchResponse, metaToSearchResult } from './search-utils'
+import { mapWithConcurrency } from '#shared/utils/async'
+
+/** Number of packages to fetch metadata for in the initial load */
+const INITIAL_BATCH_SIZE = 50
+
+/** Max names per Algolia getObjects request */
+const ALGOLIA_BATCH_SIZE = 1000
+
+export interface OrgPackagesResponse extends NpmSearchResponse {
+  /** Total number of packages in the org (may exceed objects.length before loadAll) */
+  totalPackages: number
+  /** All package names in the org (used by loadMore to know what to fetch next) */
+  allPackageNames: string[]
+}
+
+function emptyOrgResponse(): OrgPackagesResponse {
+  return {
+    ...emptySearchResponse(),
+    totalPackages: 0,
+    allPackageNames: [],
+  }
+}
+
 /**
- * Fetch all packages for an npm organization.
+ * Fetch packages for an npm organization with progressive loading.
  *
  * 1. Gets the authoritative package list from the npm registry (single request)
- * 2. Fetches metadata from Algolia by exact name (single request)
- * 3. Falls back to lightweight server-side package-meta lookups
+ * 2. Fetches metadata for the first batch immediately (fast SSR)
+ * 3. Remaining packages are loaded on-demand via `loadAll()`
  */
 export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
   const route = useRoute()
@@ -13,17 +38,22 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
     if (p === 'npm' || searchProvider.value === 'npm') return 'npm'
     return 'algolia'
   })
-  const { getPackagesByName } = useAlgoliaSearch()
+  const { getPackagesByNameSlice } = useAlgoliaSearch()
+
+  const loadedObjects = shallowRef<NpmSearchResult[]>([])
+
+  // Promise lock — scoped inside the composable to avoid cross-instance sharing
+  let loadAllPromise: Promise<void> | null = null
 
   const asyncData = useLazyAsyncData(
     () => `org-packages:${searchProviderValue.value}:${toValue(orgName)}`,
     async ({ ssrContext }, { signal }) => {
       const org = toValue(orgName)
       if (!org) {
-        return emptySearchResponse()
+        return emptyOrgResponse()
       }
 
-      // Get the authoritative package list from the npm registry (single request)
+      // Get the authoritative package list from the npm registry
       let packageNames: string[]
       try {
         const { packages } = await $fetch<{ packages: string[]; count: number }>(
@@ -32,7 +62,6 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
         )
         packageNames = packages
       } catch (err) {
-        // Check if this is a 404 (org not found)
         if (err && typeof err === 'object' && 'statusCode' in err && err.statusCode === 404) {
           const error = createError({
             statusCode: 404,
@@ -44,34 +73,131 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
           }
           throw error
         }
-        // For other errors (network, etc.), return empty array to be safe
         packageNames = []
       }
 
       if (packageNames.length === 0) {
-        return emptySearchResponse()
+        loadedObjects.value = []
+        return emptyOrgResponse()
       }
 
-      // Fetch metadata + downloads from Algolia (single request via getObjects)
+      const initialNames = packageNames.slice(0, INITIAL_BATCH_SIZE)
+
+      // Fetch metadata for first batch
+      let initialObjects: NpmSearchResult[] = []
+
       if (searchProviderValue.value === 'algolia') {
         try {
-          const response = await getPackagesByName(packageNames)
-          if (response.objects.length > 0) {
-            return response
-          }
+          initialObjects = await getPackagesByNameSlice(initialNames)
         } catch {
-          // Fall through to npm registry path
+          // Fall through to npm fallback
         }
       }
 
-      // npm fallback: fetch lightweight metadata via server proxy
+      // Staleness guard
+      if (toValue(orgName) !== org) return emptyOrgResponse()
+
+      // npm fallback for initial batch
+      if (initialObjects.length === 0) {
+        const metaResults = await mapWithConcurrency(
+          initialNames,
+          async name => {
+            try {
+              return await $fetch<PackageMetaResponse>(
+                `/api/registry/package-meta/${encodePackageName(name)}`,
+                { signal },
+              )
+            } catch {
+              return null
+            }
+          },
+          10,
+        )
+
+        if (toValue(orgName) !== org) return emptyOrgResponse()
+
+        initialObjects = metaResults
+          .filter((meta): meta is PackageMetaResponse => meta !== null)
+          .map(metaToSearchResult)
+      }
+
+      loadedObjects.value = initialObjects
+
+      return {
+        isStale: false,
+        objects: initialObjects,
+        total: initialObjects.length,
+        totalPackages: packageNames.length,
+        allPackageNames: packageNames,
+        time: new Date().toISOString(),
+      } satisfies OrgPackagesResponse
+    },
+    { default: emptyOrgResponse },
+  )
+
+  /** Read allPackageNames from async data (survives SSR→client hydration via Nuxt payload). */
+  function allPackageNames(): string[] {
+    return asyncData.data.value?.allPackageNames ?? []
+  }
+
+  /** Load the next batch of packages (default: 1 Algolia batch of 1000). */
+  async function loadMore(count: number = ALGOLIA_BATCH_SIZE): Promise<void> {
+    const loadedSet = new Set(loadedObjects.value.map(o => o.package.name))
+    if (loadedSet.size >= allPackageNames().length) return
+
+    // Reuse in-flight promise to prevent duplicate fetches
+    if (loadAllPromise) {
+      await loadAllPromise
+      return
+    }
+
+    loadAllPromise = _doLoadMore(count)
+    try {
+      await loadAllPromise
+    } finally {
+      loadAllPromise = null
+    }
+  }
+
+  /** Load ALL remaining packages (used when filters need the full dataset). */
+  async function loadAll(): Promise<void> {
+    const remaining = allPackageNames().length - loadedObjects.value.length
+    if (remaining <= 0) return
+    await loadMore(remaining)
+  }
+
+  async function _doLoadMore(count: number): Promise<void> {
+    const names = allPackageNames()
+    const current = loadedObjects.value
+    const loadedSet = new Set(current.map(o => o.package.name))
+    const remainingNames = names.filter(n => !loadedSet.has(n)).slice(0, count)
+    if (remainingNames.length === 0) return
+
+    const org = toValue(orgName)
+    let newObjects: NpmSearchResult[] = []
+
+    if (searchProviderValue.value === 'algolia') {
+      const batches: string[][] = []
+      for (let i = 0; i < remainingNames.length; i += ALGOLIA_BATCH_SIZE) {
+        batches.push(remainingNames.slice(i, i + ALGOLIA_BATCH_SIZE))
+      }
+
+      const results = await Promise.allSettled(batches.map(batch => getPackagesByNameSlice(batch)))
+
+      if (toValue(orgName) !== org) return
+
+      for (const result of results) {
+        if (result.status === 'fulfilled') {
+          newObjects.push(...result.value)
+        }
+      }
+    } else {
       const metaResults = await mapWithConcurrency(
-        packageNames,
+        remainingNames,
         async name => {
           try {
             return await $fetch<PackageMetaResponse>(
               `/api/registry/package-meta/${encodePackageName(name)}`,
-              { signal },
             )
           } catch {
             return null
@@ -80,19 +206,33 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
         10,
       )
 
-      const results: NpmSearchResult[] = metaResults
+      if (toValue(orgName) !== org) return
+
+      newObjects = metaResults
         .filter((meta): meta is PackageMetaResponse => meta !== null)
         .map(metaToSearchResult)
+    }
 
-      return {
+    if (newObjects.length > 0) {
+      const deduped = newObjects.filter(o => !loadedSet.has(o.package.name))
+      const all = [...current, ...deduped]
+      loadedObjects.value = all
+
+      // Update asyncData so the page sees the new objects
+      asyncData.data.value = {
         isStale: false,
-        objects: results,
-        total: results.length,
+        objects: all,
+        total: all.length,
+        totalPackages: names.length,
+        allPackageNames: names,
         time: new Date().toISOString(),
-      } satisfies NpmSearchResponse
-    },
-    { default: emptySearchResponse },
-  )
+      }
+    }
+  }
 
-  return asyncData
+  return {
+    ...asyncData,
+    loadMore,
+    loadAll,
+  }
 }

app/composables/useVisibleItems.ts

@@ -1,8 +1,23 @@
 import { computed, shallowRef, toValue } from 'vue'
 import type { MaybeRefOrGetter } from 'vue'
 
-export function useVisibleItems<T>(items: MaybeRefOrGetter<T[]>, limit: number) {
+export interface UseVisibleItemsOptions {
+  /**
+   * Called when expanding. Useful for loading remaining data on demand.
+   * If it returns a promise, `isExpanding` will be `true` until it resolves.
+   * Return `false` to signal a partial load — `showAll` stays false so
+   * `hasMore` remains true and the user can retry.
+   */
+  onExpand?: () => void | boolean | Promise<void | boolean>
+}
+
+export function useVisibleItems<T>(
+  items: MaybeRefOrGetter<T[]>,
+  limit: number,
+  options?: UseVisibleItemsOptions,
+) {
   const showAll = shallowRef(false)
+  const isExpanding = shallowRef(false)
 
   const visibleItems = computed(() => {
     const list = toValue(items)
@@ -15,15 +30,30 @@ export function useVisibleItems<T>(items: MaybeRefOrGetter<T[]>, limit: number)
 
   const hasMore = computed(() => !showAll.value && toValue(items).length > limit)
 
-  const expand = () => {
-    showAll.value = true
+  const expand = async () => {
+    if (showAll.value) return
+    let fullyLoaded = true
+    if (options?.onExpand) {
+      isExpanding.value = true
+      try {
+        const result = await options.onExpand()
+        if (result === false) fullyLoaded = false
+      } finally {
+        isExpanding.value = false
+      }
+    }
+    if (fullyLoaded) showAll.value = true
   }
   const collapse = () => {
     showAll.value = false
   }
-  const toggle = () => {
-    showAll.value = !showAll.value
+  const toggle = async () => {
+    if (showAll.value) {
+      collapse()
+    } else {
+      await expand()
+    }
   }
 
-  return { visibleItems, hiddenCount, hasMore, showAll, expand, collapse, toggle }
+  return { visibleItems, hiddenCount, hasMore, isExpanding, showAll, expand, collapse, toggle }
 }