Merge pull request #2476 from github/nora/use-code-search-api

Code search: use code search api

Author: norascheuch

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -0,0 +1,70 @@
+import { retry } from "@octokit/plugin-retry";
+import { throttling } from "@octokit/plugin-throttling";
+import { Octokit } from "@octokit/rest";
+import { Progress, CancellationToken } from "vscode";
+import { showAndLogWarningMessage } from "../helpers";
+import { Credentials } from "../common/authentication";
+
+export async function getCodeSearchRepositories(
+  query: string,
+  progress: Progress<{
+    message?: string | undefined;
+    increment?: number | undefined;
+  }>,
+  token: CancellationToken,
+  credentials: Credentials,
+): Promise<string[]> {
+  let nwos: string[] = [];
+  const octokit = await provideOctokitWithThrottling(credentials);
+
+  for await (const response of octokit.paginate.iterator(
+    octokit.rest.search.code,
+    {
+      q: query,
+      per_page: 100,
+    },
+  )) {
+    nwos.push(...response.data.map((item) => item.repository.full_name));
+    // calculate progress bar: 80% of the progress bar is used for the code search
+    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
+    // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10
+    const increment =
+      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
+    progress.report({ increment });
+
+    if (token.isCancellationRequested) {
+      nwos = [];
+      break;
+    }
+  }
+
+  return [...new Set(nwos)];
+}
+
+async function provideOctokitWithThrottling(
+  credentials: Credentials,
+): Promise<Octokit> {
+  const MyOctokit = Octokit.plugin(throttling);
+  const auth = await credentials.getAccessToken();
+
+  const octokit = new MyOctokit({
+    auth,
+    retry,
+    throttle: {
+      onRateLimit: (retryAfter: number, options: any): boolean => {
+        void showAndLogWarningMessage(
+          `Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`,
+        );
+
+        return true;
+      },
+      onSecondaryRateLimit: (_retryAfter: number, options: any): void => {
+        void showAndLogWarningMessage(
+          `Secondary Rate Limit detected for request ${options.method} ${options.url}`,
+        );
+      },
+    },
+  });
+
+  return octokit;
+}

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -36,8 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query";
 import { getErrorMessage } from "../../pure/helpers-pure";
 import { DatabasePanelCommands } from "../../common/commands";
 import { App } from "../../common/app";
-import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client";
 import { QueryLanguage } from "../../common/query-language";
+import { getCodeSearchRepositories } from "../code-search-api";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
@@ -351,13 +351,19 @@ export class DbPanel extends DisposableObject {
 
     const listName = treeViewItem.dbItem.listName;
 
-    const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values(
-      QueryLanguage,
-    ).map((language) => ({
-      label: language.toString(),
-      alwaysShow: true,
-      language: language.toString(),
-    }));
+    const languageQuickPickItems: CodeSearchQuickPickItem[] = [
+      {
+        label: "No specific language",
+        alwaysShow: true,
+        language: "",
+      },
+    ].concat(
+      Object.values(QueryLanguage).map((language) => ({
+        label: language.toString(),
+        alwaysShow: true,
+        language: language.toString(),
+      })),
+    );
 
     const codeSearchLanguage =
       await window.showQuickPick<CodeSearchQuickPickItem>(
@@ -372,6 +378,10 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
+    const languagePrompt = codeSearchLanguage.language
+      ? `language:${codeSearchLanguage.language}`
+      : "";
+
     const codeSearchQuery = await window.showInputBox({
       title: "GitHub Code Search",
       prompt:
@@ -392,10 +402,10 @@ export class DbPanel extends DisposableObject {
         progress.report({ increment: 10 });
 
         const repositories = await getCodeSearchRepositories(
-          this.app.credentials,
-          `${codeSearchQuery} language:${codeSearchLanguage.language}`,
+          `${codeSearchQuery} ${languagePrompt}`,
           progress,
           token,
+          this.app.credentials,
         );
 
         token.onCancellationRequested(() => {

extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts

@@ -7,43 +7,6 @@ import {
   VariantAnalysisSubmissionRequest,
 } from "./variant-analysis";
 import { Repository } from "./repository";
-import { Progress } from "vscode";
-import { CancellationToken } from "vscode-jsonrpc";
-
-export async function getCodeSearchRepositories(
-  credentials: Credentials,
-  query: string,
-  progress: Progress<{
-    message?: string | undefined;
-    increment?: number | undefined;
-  }>,
-  token: CancellationToken,
-): Promise<string[]> {
-  let nwos: string[] = [];
-  const octokit = await credentials.getOctokit();
-  for await (const response of octokit.paginate.iterator(
-    octokit.rest.search.repos,
-    {
-      q: query,
-      per_page: 100,
-    },
-  )) {
-    nwos.push(...response.data.map((item) => item.full_name));
-    // calculate progress bar: 80% of the progress bar is used for the code search
-    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
-    // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10
-    const increment =
-      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
-    progress.report({ increment });
-
-    if (token.isCancellationRequested) {
-      nwos = [];
-      break;
-    }
-  }
-
-  return [...new Set(nwos)];
-}
 
 export async function submitVariantAnalysis(
   credentials: Credentials,