Surface truncated repos when a list hits 1000 items

Author: norascheuch

extensions/ql-vscode/src/databases/config/db-config-store.ts

@@ -148,7 +148,7 @@ export class DbConfigStore extends DisposableObject {
   public async addRemoteReposToList(
     repoNwoList: string[],
     parentList: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repos if config is not loaded");
     }
@@ -161,23 +161,22 @@ export class DbConfigStore extends DisposableObject {
       throw Error(`Cannot find parent list '${parentList}'`);
     }
 
-    const newRepositoriesList = new Set([
-      ...new Set(parent.repositories),
-      ...new Set(repoNwoList),
-    ]);
+    // Remove duplicates from the list of repositories.
+    const newRepositoriesList = [
+      ...new Set([...new Set(parent.repositories), ...new Set(repoNwoList)]),
+    ];
+
+    parent.repositories = newRepositoriesList.slice(0, 1000);
+    const truncatedRepositories = newRepositoriesList.slice(1000);
 
-    if (newRepositoriesList.size > 1000) {
-      parent.repositories = [...Array.from(newRepositoriesList).slice(0, 1000)];
-    } else {
-      parent.repositories = [...newRepositoriesList];
-    }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repo if config is not loaded");
     }
@@ -192,6 +191,7 @@ export class DbConfigStore extends DisposableObject {
       );
     }
 
+    const truncatedRepositories = [];
     const config = cloneDbConfig(this.config);
     if (parentList) {
       const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -200,12 +200,15 @@ export class DbConfigStore extends DisposableObject {
       if (!parent) {
         throw Error(`Cannot find parent list '${parentList}'`);
       } else {
-        parent.repositories.push(repoNwo);
+        const newRepositories = [...parent.repositories, repoNwo];
+        parent.repositories = newRepositories.slice(0, 1000);
+        truncatedRepositories.push(...newRepositories.slice(1000));
       }
     } else {
       config.databases.variantAnalysis.repositories.push(repoNwo);
     }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/db-manager.ts

@@ -96,15 +96,15 @@ export class DbManager {
   public async addNewRemoteRepo(
     nwo: string,
     parentList?: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
   }
 
   public async addNewRemoteReposToList(
     nwoList: string[],
     parentList: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
   }
 
   public async addNewRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -179,7 +179,14 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await this.dbManager.addNewRemoteRepo(nwo, parentList);
+    const truncatedRepositories = await this.dbManager.addNewRemoteRepo(
+      nwo,
+      parentList,
+    );
+
+    if (parentList) {
+      this.truncatedReposNote(truncatedRepositories, parentList);
+    }
   }
 
   private async addNewRemoteOwner(): Promise<void> {
@@ -373,10 +380,27 @@ export class DbPanel extends DisposableObject {
       `${codeSearchQuery} language:${codeSearchLanguage.language}`,
     );
 
-    await this.dbManager.addNewRemoteReposToList(
+    const truncatedRepositories = await this.dbManager.addNewRemoteReposToList(
       repositories,
       treeViewItem.dbItem.listName,
     );
+    this.truncatedReposNote(
+      truncatedRepositories,
+      treeViewItem.dbItem.listName,
+    );
+  }
+
+  private truncatedReposNote(
+    truncatedRepositories: string[],
+    listName: string,
+  ) {
+    if (truncatedRepositories.length > 0) {
+      void showAndLogErrorMessage(
+        `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join(
+          ", ",
+        )}`,
+      );
+    }
   }
 
   private async onDidCollapseElement(

extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts

@@ -282,7 +282,7 @@ describe("db config store", () => {
       configStore.dispose();
     });
 
-    it("should add no more than 1000 repositories to a list", async () => {
+    it("should add no more than 1000 repositories to a remote list using #addRemoteReposToList", async () => {
       // Initial set up
       const dbConfig = createDbConfig({
         remoteLists: [
@@ -296,7 +296,7 @@ describe("db config store", () => {
       const configStore = await initializeConfig(dbConfig, configPath, app);
 
       // Add
-      await configStore.addRemoteReposToList(
+      const reponse = await configStore.addRemoteReposToList(
         [...Array(1001).keys()].map((i) => `owner/db${i}`),
         "list1",
       );
@@ -311,6 +311,38 @@ describe("db config store", () => {
       expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
         1000,
       );
+      expect(reponse).toEqual(["owner/db1000"]);
+
+      configStore.dispose();
+    });
+
+    it("should add no more than 1000 repositories to a remote list using #addRemoteRepo", async () => {
+      // Initial set up
+      const dbConfig = createDbConfig({
+        remoteLists: [
+          {
+            name: "list1",
+            repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
+          },
+        ],
+      });
+
+      const configStore = await initializeConfig(dbConfig, configPath, app);
+
+      // Add
+      const reponse = await configStore.addRemoteRepo("owner/db1000", "list1");
+
+      // Read the config file
+      const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
+
+      // Check that the config file has been updated
+      const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
+      expect(updatedRemoteDbs.repositories).toHaveLength(0);
+      expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
+      expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
+        1000,
+      );
+      expect(reponse).toEqual(["owner/db1000"]);
 
       configStore.dispose();
     });

extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts

@@ -115,6 +115,46 @@ describe("db manager", () => {
         });
       });
 
+      it("should return truncated repos when adding to a user defined list using #addNewRemoteReposToList", async () => {
+        const dbConfig: DbConfig = createDbConfig({
+          remoteLists: [
+            {
+              name: "my-list-1",
+              repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
+            },
+          ],
+        });
+
+        await saveDbConfig(dbConfig);
+
+        const response = await dbManager.addNewRemoteReposToList(
+          ["owner2/repo2"],
+          "my-list-1",
+        );
+
+        expect(response).toEqual(["owner2/repo2"]);
+      });
+
+      it("should return truncated repos when adding to a user defined list using #addNewRemoteRepo", async () => {
+        const dbConfig: DbConfig = createDbConfig({
+          remoteLists: [
+            {
+              name: "my-list-1",
+              repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
+            },
+          ],
+        });
+
+        await saveDbConfig(dbConfig);
+
+        const response = await dbManager.addNewRemoteRepo(
+          "owner2/repo2",
+          "my-list-1",
+        );
+
+        expect(response).toEqual(["owner2/repo2"]);
+      });
+
       it("should add a new remote repo to a user defined list", async () => {
         const dbConfig: DbConfig = createDbConfig({
           remoteLists: [