fix: batch Algolia getObjects requests for large orgs

The root cause of #1946 and #2507 is that Algolia's getObjects API
has a 1000-item limit. Sending >1000 names returns a 400, which
triggers the slow npm fallback path (1 request per package).

Fix: batch getPackagesByName into chunks of 1000 with a concurrency
limit of 3. The org page and useOrgPackages stay unchanged — all
packages are fetched upfront and sorted client-side.

Client-side paging is tracked separately in #2529.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Adebesin-Cell

app/composables/npm/useAlgoliaSearch.ts

@@ -254,8 +254,14 @@ export function useAlgoliaSearch() {
       batches.push(packageNames.slice(i, i + BATCH_SIZE))
     }
 
-    const results = await Promise.all(batches.map(batch => getPackagesByNameSlice(batch)))
-    const allObjects = results.flat()
+    // Fetch batches with concurrency limit to avoid overwhelming the API
+    const CONCURRENCY = 3
+    const allObjects: NpmSearchResult[] = []
+    for (let i = 0; i < batches.length; i += CONCURRENCY) {
+      const chunk = batches.slice(i, i + CONCURRENCY)
+      const results = await Promise.all(chunk.map(batch => getPackagesByNameSlice(batch)))
+      allObjects.push(...results.flat())
+    }
 
     return {
       isStale: false,
@@ -367,6 +373,5 @@ export function useAlgoliaSearch() {
     searchWithSuggestions,
     searchByOwner,
     getPackagesByName,
-    getPackagesByNameSlice,
   }
 }

app/composables/npm/useOrgPackages.ts

@@ -2,33 +2,12 @@ import type { NpmSearchResponse, NpmSearchResult, PackageMetaResponse } from '#s
 import { emptySearchResponse, metaToSearchResult } from './search-utils'
 import { mapWithConcurrency } from '#shared/utils/async'
 
-/** Number of packages to fetch metadata for in the initial load */
-const INITIAL_BATCH_SIZE = 50
-
-/** Max names per Algolia getObjects request */
-const ALGOLIA_BATCH_SIZE = 1000
-
-export interface OrgPackagesResponse extends NpmSearchResponse {
-  /** Total number of packages in the org (may exceed objects.length before loadAll) */
-  totalPackages: number
-  /** All package names in the org (used by loadMore to know what to fetch next) */
-  allPackageNames: string[]
-}
-
-function emptyOrgResponse(): OrgPackagesResponse {
-  return {
-    ...emptySearchResponse(),
-    totalPackages: 0,
-    allPackageNames: [],
-  }
-}
-
 /**
- * Fetch packages for an npm organization with progressive loading.
+ * Fetch all packages for an npm organization.
  *
  * 1. Gets the authoritative package list from the npm registry (single request)
- * 2. Fetches metadata for the first batch immediately (fast SSR)
- * 3. Remaining packages are loaded on-demand via `loadAll()`
+ * 2. Fetches metadata from Algolia by exact name (batched, max 1000 per request)
+ * 3. Falls back to lightweight server-side package-meta lookups
  */
 export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
   const route = useRoute()
@@ -38,22 +17,17 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
     if (p === 'npm' || searchProvider.value === 'npm') return 'npm'
     return 'algolia'
   })
-  const { getPackagesByNameSlice } = useAlgoliaSearch()
-
-  const loadedObjects = shallowRef<NpmSearchResult[]>([])
-
-  // Promise lock — scoped inside the composable to avoid cross-instance sharing
-  let loadAllPromise: Promise<void> | null = null
+  const { getPackagesByName } = useAlgoliaSearch()
 
   const asyncData = useLazyAsyncData(
     () => `org-packages:${searchProviderValue.value}:${toValue(orgName)}`,
     async ({ ssrContext }, { signal }) => {
       const org = toValue(orgName)
       if (!org) {
-        return emptyOrgResponse()
+        return emptySearchResponse()
       }
 
-      // Get the authoritative package list from the npm registry
+      // Get the authoritative package list from the npm registry (single request)
       let packageNames: string[]
       try {
         const { packages } = await $fetch<{ packages: string[]; count: number }>(
@@ -77,127 +51,32 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
       }
 
       if (packageNames.length === 0) {
-        loadedObjects.value = []
-        return emptyOrgResponse()
+        return emptySearchResponse()
       }
 
-      const initialNames = packageNames.slice(0, INITIAL_BATCH_SIZE)
-
-      // Fetch metadata for first batch
-      let initialObjects: NpmSearchResult[] = []
-
+      // Fetch metadata from Algolia (batched in chunks of 1000, parallel)
       if (searchProviderValue.value === 'algolia') {
         try {
-          initialObjects = await getPackagesByNameSlice(initialNames)
+          const response = await getPackagesByName(packageNames)
+          if (response.objects.length > 0) {
+            return response
+          }
         } catch {
-          // Fall through to npm fallback
+          // Fall through to npm registry path
         }
       }
 
       // Staleness guard
-      if (toValue(orgName) !== org) return emptyOrgResponse()
-
-      // npm fallback for initial batch
-      if (initialObjects.length === 0) {
-        const metaResults = await mapWithConcurrency(
-          initialNames,
-          async name => {
-            try {
-              return await $fetch<PackageMetaResponse>(
-                `/api/registry/package-meta/${encodePackageName(name)}`,
-                { signal },
-              )
-            } catch {
-              return null
-            }
-          },
-          10,
-        )
-
-        if (toValue(orgName) !== org) return emptyOrgResponse()
-
-        initialObjects = metaResults
-          .filter((meta): meta is PackageMetaResponse => meta !== null)
-          .map(metaToSearchResult)
-      }
-
-      loadedObjects.value = initialObjects
-
-      return {
-        isStale: false,
-        objects: initialObjects,
-        total: initialObjects.length,
-        totalPackages: packageNames.length,
-        allPackageNames: packageNames,
-        time: new Date().toISOString(),
-      } satisfies OrgPackagesResponse
-    },
-    { default: emptyOrgResponse },
-  )
-
-  /** Read allPackageNames from async data (survives SSR→client hydration via Nuxt payload). */
-  function allPackageNames(): string[] {
-    return asyncData.data.value?.allPackageNames ?? []
-  }
-
-  /** Load the next batch of packages (default: 1 Algolia batch of 1000). */
-  async function loadMore(count: number = ALGOLIA_BATCH_SIZE): Promise<void> {
-    const loadedSet = new Set(loadedObjects.value.map(o => o.package.name))
-    if (loadedSet.size >= allPackageNames().length) return
-
-    // Reuse in-flight promise to prevent duplicate fetches
-    if (loadAllPromise) {
-      await loadAllPromise
-      return
-    }
-
-    loadAllPromise = _doLoadMore(count)
-    try {
-      await loadAllPromise
-    } finally {
-      loadAllPromise = null
-    }
-  }
-
-  /** Load ALL remaining packages (used when filters need the full dataset). */
-  async function loadAll(): Promise<void> {
-    const remaining = allPackageNames().length - loadedObjects.value.length
-    if (remaining <= 0) return
-    await loadMore(remaining)
-  }
-
-  async function _doLoadMore(count: number): Promise<void> {
-    const names = allPackageNames()
-    const current = loadedObjects.value
-    const loadedSet = new Set(current.map(o => o.package.name))
-    const remainingNames = names.filter(n => !loadedSet.has(n)).slice(0, count)
-    if (remainingNames.length === 0) return
-
-    const org = toValue(orgName)
-    let newObjects: NpmSearchResult[] = []
-
-    if (searchProviderValue.value === 'algolia') {
-      const batches: string[][] = []
-      for (let i = 0; i < remainingNames.length; i += ALGOLIA_BATCH_SIZE) {
-        batches.push(remainingNames.slice(i, i + ALGOLIA_BATCH_SIZE))
-      }
+      if (toValue(orgName) !== org) return emptySearchResponse()
 
-      const results = await Promise.allSettled(batches.map(batch => getPackagesByNameSlice(batch)))
-
-      if (toValue(orgName) !== org) return
-
-      for (const result of results) {
-        if (result.status === 'fulfilled') {
-          newObjects.push(...result.value)
-        }
-      }
-    } else {
+      // npm fallback: fetch lightweight metadata via server proxy
       const metaResults = await mapWithConcurrency(
-        remainingNames,
+        packageNames,
         async name => {
           try {
             return await $fetch<PackageMetaResponse>(
               `/api/registry/package-meta/${encodePackageName(name)}`,
+              { signal },
             )
           } catch {
             return null
@@ -206,33 +85,19 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
         10,
       )
 
-      if (toValue(orgName) !== org) return
-
-      newObjects = metaResults
+      const results: NpmSearchResult[] = metaResults
         .filter((meta): meta is PackageMetaResponse => meta !== null)
         .map(metaToSearchResult)
-    }
 
-    if (newObjects.length > 0) {
-      const deduped = newObjects.filter(o => !loadedSet.has(o.package.name))
-      const all = [...current, ...deduped]
-      loadedObjects.value = all
-
-      // Update asyncData so the page sees the new objects
-      asyncData.data.value = {
+      return {
         isStale: false,
-        objects: all,
-        total: all.length,
-        totalPackages: names.length,
-        allPackageNames: names,
+        objects: results,
+        total: results.length,
         time: new Date().toISOString(),
-      }
-    }
-  }
+      } satisfies NpmSearchResponse
+    },
+    { default: emptySearchResponse },
+  )
 
-  return {
-    ...asyncData,
-    loadMore,
-    loadAll,
-  }
+  return asyncData
 }

app/composables/useVisibleItems.ts

@@ -1,23 +1,8 @@
 import { computed, shallowRef, toValue } from 'vue'
 import type { MaybeRefOrGetter } from 'vue'
 
-export interface UseVisibleItemsOptions {
-  /**
-   * Called when expanding. Useful for loading remaining data on demand.
-   * If it returns a promise, `isExpanding` will be `true` until it resolves.
-   * Return `false` to signal a partial load — `showAll` stays false so
-   * `hasMore` remains true and the user can retry.
-   */
-  onExpand?: () => void | boolean | Promise<void | boolean>
-}
-
-export function useVisibleItems<T>(
-  items: MaybeRefOrGetter<T[]>,
-  limit: number,
-  options?: UseVisibleItemsOptions,
-) {
+export function useVisibleItems<T>(items: MaybeRefOrGetter<T[]>, limit: number) {
   const showAll = shallowRef(false)
-  const isExpanding = shallowRef(false)
 
   const visibleItems = computed(() => {
     const list = toValue(items)
@@ -30,30 +15,15 @@ export function useVisibleItems<T>(
 
   const hasMore = computed(() => !showAll.value && toValue(items).length > limit)
 
-  const expand = async () => {
-    if (showAll.value) return
-    let fullyLoaded = true
-    if (options?.onExpand) {
-      isExpanding.value = true
-      try {
-        const result = await options.onExpand()
-        if (result === false) fullyLoaded = false
-      } finally {
-        isExpanding.value = false
-      }
-    }
-    if (fullyLoaded) showAll.value = true
+  const expand = () => {
+    showAll.value = true
   }
   const collapse = () => {
     showAll.value = false
   }
-  const toggle = async () => {
-    if (showAll.value) {
-      collapse()
-    } else {
-      await expand()
-    }
+  const toggle = () => {
+    showAll.value = !showAll.value
   }
 
-  return { visibleItems, hiddenCount, hasMore, isExpanding, showAll, expand, collapse, toggle }
+  return { visibleItems, hiddenCount, hasMore, showAll, expand, collapse, toggle }
 }