refactor: progressive org loading with incremental batches

- Extend useVisibleItems with onExpand callback + partial load support
- useOrgPackages: fetch first 50 on SSR, loadMore(1000) per page,
  loadAll() only when filters need the full dataset
- Track remaining by name (not position) for retry safety
- Pagination shows real total (11,397) while loading incrementally

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Adebesin-Cell

app/composables/npm/useOrgPackages.ts

@@ -3,7 +3,7 @@ import { emptySearchResponse, metaToSearchResult } from './search-utils'
 import { mapWithConcurrency } from '#shared/utils/async'
 
 /** Number of packages to fetch metadata for in the initial load */
-const INITIAL_BATCH_SIZE = 250
+const INITIAL_BATCH_SIZE = 50
 
 /** Max names per Algolia getObjects request */
 const ALGOLIA_BATCH_SIZE = 1000
@@ -135,8 +135,8 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
     { default: emptyOrgResponse },
   )
 
-  /** Load all remaining packages that weren't fetched in the initial batch. */
-  async function loadAll(): Promise<void> {
+  /** Load the next batch of packages (default: 1 Algolia batch of 1000). */
+  async function loadMore(count: number = ALGOLIA_BATCH_SIZE): Promise<void> {
     const loadedSet = new Set(loadedObjects.value.map(o => o.package.name))
     if (loadedSet.size >= allNames.value.length) return
 
@@ -146,23 +146,29 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
       return
     }
 
-    loadAllPromise = _doLoadAll()
+    loadAllPromise = _doLoadMore(count)
     try {
       await loadAllPromise
     } finally {
       loadAllPromise = null
     }
   }
 
-  async function _doLoadAll(): Promise<void> {
+  /** Load ALL remaining packages (used when filters need the full dataset). */
+  async function loadAll(): Promise<void> {
+    const remaining = allNames.value.length - loadedObjects.value.length
+    if (remaining <= 0) return
+    await loadMore(remaining)
+  }
+
+  async function _doLoadMore(count: number): Promise<void> {
     const names = allNames.value
     const current = loadedObjects.value
     const loadedSet = new Set(current.map(o => o.package.name))
-    const remainingNames = names.filter(n => !loadedSet.has(n))
+    const remainingNames = names.filter(n => !loadedSet.has(n)).slice(0, count)
     if (remainingNames.length === 0) return
 
     const org = toValue(orgName)
-
     let newObjects: NpmSearchResult[] = []
 
     if (searchProviderValue.value === 'algolia') {
@@ -203,8 +209,7 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
     }
 
     if (newObjects.length > 0) {
-      const existingNames = new Set(current.map(o => o.package.name))
-      const deduped = newObjects.filter(o => !existingNames.has(o.package.name))
+      const deduped = newObjects.filter(o => !loadedSet.has(o.package.name))
       const all = [...current, ...deduped]
       loadedObjects.value = all
 
@@ -221,6 +226,7 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
 
   return {
     ...asyncData,
+    loadMore,
     loadAll,
   }
 }

app/pages/org/[org].vue

@@ -15,7 +15,7 @@ const orgName = computed(() => route.params.org.toLowerCase())
 const { isConnected } = useConnector()
 
 // Fetch packages progressively (first 250 on SSR, rest on demand via loadAll)
-const { data: results, status, error, loadAll } = useOrgPackages(orgName)
+const { data: results, status, error, loadMore, loadAll } = useOrgPackages(orgName)
 
 // Handle 404 errors reactively (since we're not awaiting)
 watch(
@@ -32,28 +32,41 @@ watch(
   { immediate: true },
 )
 
-const allPackages = computed(() => results.value?.objects ?? [])
+const packages = computed(() => results.value?.objects ?? [])
 const totalPackages = computed(() => results.value?.totalPackages ?? 0)
-
-// Show first 250 packages initially; expanding triggers loadAll() to fetch remaining data
-const {
-  visibleItems: packages,
-  hasMore: hasMoreVisible,
-  isExpanding,
-  expand,
-} = useVisibleItems(allPackages, 250, { onExpand: loadAll })
 const packageCount = computed(() => packages.value.length)
 
-// hasMore combines both: hidden in-memory items AND unfetched server data
-const hasMore = computed(
-  () => hasMoreVisible.value || allPackages.value.length < totalPackages.value,
-)
+// Progressive loading: first 50 on SSR, rest fetched on demand via loadAll()
+const hasMore = computed(() => packages.value.length < totalPackages.value)
+const isLoadingMore = shallowRef(false)
+
+/** Fetch the next batch of packages (1000 at a time). */
+async function fetchNextBatch() {
+  if (!hasMore.value || isLoadingMore.value) return
+  isLoadingMore.value = true
+  try {
+    await loadMore()
+  } finally {
+    isLoadingMore.value = false
+  }
+}
+
+/** Fetch ALL remaining packages (needed for client-side filtering). */
+async function fetchAllRemaining() {
+  if (!hasMore.value || isLoadingMore.value) return
+  isLoadingMore.value = true
+  try {
+    await loadAll()
+  } finally {
+    isLoadingMore.value = false
+  }
+}
 
 // Preferences (persisted to localStorage)
 const { viewMode, paginationMode, pageSize, columns, toggleColumn, resetColumns } =
   usePackageListPreferences()
 
-// Structured filters and sorting (operates on visible set; expand() widens it)
+// Structured filters and sorting (operates on all loaded packages)
 const {
   filters,
   sortOption,
@@ -77,20 +90,25 @@ const {
 // Pagination state
 const currentPage = shallowRef(1)
 
+// Total items accounts for unfetched packages so pagination shows the real count
+const paginationTotal = computed(() =>
+  hasMore.value ? totalPackages.value : sortedPackages.value.length,
+)
+
 // Calculate total pages
 const totalPages = computed(() => {
-  return Math.ceil(sortedPackages.value.length / pageSize.value)
+  return Math.ceil(paginationTotal.value / pageSize.value)
 })
 
-// Reset to page 1 when filters change; expand so filtering works across all packages
+// Reset to page 1 when filters change; load all packages so client-side filtering works
 watch([filters, sortOption], () => {
   currentPage.value = 1
   if (
     filters.value.text ||
     filters.value.keywords.length > 0 ||
     sortOption.value !== 'updated-desc'
   ) {
-    expand()
+    fetchAllRemaining()
   }
 })
 
@@ -101,6 +119,14 @@ watch(totalPages, newTotal => {
   }
 })
 
+// Load next batch when user navigates beyond what's loaded
+watch(currentPage, page => {
+  const loadedPages = Math.ceil(sortedPackages.value.length / pageSize.value)
+  if (page > loadedPages && hasMore.value) {
+    fetchNextBatch()
+  }
+})
+
 // Debounced URL update for filter/sort
 const updateUrl = debounce((updates: { filter?: string; sort?: string }) => {
   router.replace({
@@ -273,7 +299,7 @@ defineOgImageComponent('Default', {
 
     <!-- Loading state (only when no packages loaded yet) -->
     <LoadingSpinner
-      v-if="status === 'pending' && allPackages.length === 0"
+      v-if="status === 'pending' && packages.length === 0"
       :text="$t('common.loading_packages')"
     />
 
@@ -288,7 +314,7 @@ defineOgImageComponent('Default', {
     </div>
 
     <!-- Empty state -->
-    <div v-else-if="allPackages.length === 0" class="py-12 text-center">
+    <div v-else-if="packageCount === 0 && totalPackages === 0" class="py-12 text-center">
       <p class="text-fg-muted font-mono">
         {{ $t('org.page.no_packages') }} <span class="text-fg">@{{ orgName }}</span>
       </p>
@@ -337,15 +363,15 @@ defineOgImageComponent('Default', {
         <PackageList
           :results="sortedPackages"
           :has-more="hasMore"
-          :is-loading="isExpanding"
+          :is-loading="isLoadingMore"
           :view-mode="viewMode"
           :columns="columns"
           :filters="filters"
           v-model:sort-option="sortOption"
           :pagination-mode="paginationMode"
           :page-size="pageSize"
           :current-page="currentPage"
-          @load-more="expand"
+          @load-more="fetchNextBatch"
           @click-keyword="toggleKeyword"
         />
 
@@ -354,7 +380,7 @@ defineOgImageComponent('Default', {
           v-model:mode="paginationMode"
           v-model:page-size="pageSize"
           v-model:current-page="currentPage"
-          :total-items="sortedPackages.length"
+          :total-items="paginationTotal"
           :view-mode="viewMode"
         />
       </template>