Merge pull request #1692 from github/koesie10/add-metadata-to-repo-row

Add repository metadata to row

Author: koesie10

extensions/ql-vscode/package-lock.json

@@ -1314,6 +1314,7 @@
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
     "@faker-js/faker": "^7.5.0",
+    "@octokit/plugin-throttling": "^4.3.2",
     "@storybook/addon-actions": "^6.5.10",
     "@storybook/addon-essentials": "^6.5.10",
     "@storybook/addon-interactions": "^6.5.10",

extensions/ql-vscode/package.json

@@ -0,0 +1,128 @@
+/**
+ * This scripts helps after adding a new field in the GitHub API. You will
+ * need to modify this script to add the new field to the scenarios. This
+ * is just a template and should not be used as-is since it has already been
+ * applied.
+ *
+ * Depending on the actual implementation of the script, you might run into
+ * rate limits. If that happens, you can set a `GITHUB_TOKEN` environment
+ * variable. For example, use: ``export GITHUB_TOKEN=`gh auth token```.
+ *
+ * Usage: npx ts-node scripts/add-fields-to-scenarios.ts
+ */
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { Octokit, type RestEndpointMethodTypes } from '@octokit/rest';
+import { throttling } from '@octokit/plugin-throttling';
+
+import { getFiles } from './util/files';
+import type { GitHubApiRequest } from '../src/mocks/gh-api-request';
+import { isGetVariantAnalysisRequest } from '../src/mocks/gh-api-request';
+import { VariantAnalysis } from '../src/remote-queries/gh-api/variant-analysis';
+import { RepositoryWithMetadata } from '../src/remote-queries/gh-api/repository';
+
+const extensionDirectory = path.resolve(__dirname, '..');
+const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
+
+// Make sure we don't run into rate limits by automatically waiting until we can
+// make another request.
+const MyOctokit = Octokit.plugin(throttling);
+
+const auth = process.env.GITHUB_TOKEN;
+
+const octokit = new MyOctokit({
+  auth,
+  throttle: {
+    onRateLimit: (retryAfter: number, options: any, octokit: Octokit): boolean => {
+      octokit.log.warn(
+        `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`
+      );
+
+      return true;
+    },
+    onSecondaryRateLimit: (_retryAfter: number, options: any, octokit: Octokit): void => {
+      octokit.log.warn(
+        `SecondaryRateLimit detected for request ${options.method} ${options.url}`
+      );
+    },
+  }
+});
+const repositories = new Map<number, RestEndpointMethodTypes['repos']['get']['response']['data']>();
+
+async function addFieldsToRepository(repository: RepositoryWithMetadata) {
+  if (!repositories.has(repository.id)) {
+    const [owner, repo] = repository.full_name.split('/');
+
+    const apiRepository = await octokit.repos.get({
+      owner,
+      repo,
+    });
+
+    repositories.set(repository.id, apiRepository.data);
+  }
+
+  const apiRepository = repositories.get(repository.id)!;
+
+  repository.stargazers_count = apiRepository.stargazers_count;
+  repository.updated_at = apiRepository.updated_at;
+}
+
+async function addFieldsToScenarios() {
+  if (!(await fs.pathExists(scenariosDirectory))) {
+    console.error('Scenarios directory does not exist: ' + scenariosDirectory);
+    return;
+  }
+
+  for await (const file of getFiles(scenariosDirectory)) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
+    const data: GitHubApiRequest = await fs.readJson(file);
+
+    if (!isGetVariantAnalysisRequest(data)) {
+      continue;
+    }
+
+    if (!data.response.body || !('controller_repo' in data.response.body)) {
+      continue;
+    }
+
+    console.log(`Adding fields to '${path.relative(scenariosDirectory, file)}'`);
+
+    const variantAnalysis = data.response.body as VariantAnalysis;
+
+    if (variantAnalysis.scanned_repositories) {
+      for (const item of variantAnalysis.scanned_repositories) {
+        await addFieldsToRepository(item.repository);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.access_mismatch_repos) {
+      for (const item of variantAnalysis.skipped_repositories.access_mismatch_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.no_codeql_db_repos) {
+      for (const item of variantAnalysis.skipped_repositories.no_codeql_db_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.over_limit_repos) {
+      for (const item of variantAnalysis.skipped_repositories.over_limit_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    await fs.writeJson(file, data, { spaces: 2 });
+  }
+}
+
+addFieldsToScenarios().catch(e => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -4,6 +4,8 @@ import * as path from 'path';
 import Ajv from 'ajv';
 import * as tsj from 'ts-json-schema-generator';
 
+import { getFiles } from './util/files';
+
 const extensionDirectory = path.resolve(__dirname, '..');
 const rootDirectory = path.resolve(extensionDirectory, '../..');
 const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
@@ -60,19 +62,6 @@ async function lintScenarios() {
   }
 }
 
-// https://stackoverflow.com/a/45130990
-async function* getFiles(dir: string): AsyncGenerator<string> {
-  const dirents = await fs.readdir(dir, { withFileTypes: true });
-  for (const dirent of dirents) {
-    const res = path.resolve(dir, dirent.name);
-    if (dirent.isDirectory()) {
-      yield* getFiles(res);
-    } else {
-      yield res;
-    }
-  }
-}
-
 lintScenarios().catch(e => {
   console.error(e);
   process.exit(2);

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -2,5 +2,9 @@
   "$schema": "https://json.schemastore.org/tsconfig",
   "extends": "../tsconfig.json",
   "include": ["**/*.ts"],
-  "exclude": []
+  "exclude": [],
+  "compilerOptions": {
+    "rootDir": "..",
+    "noEmit": true
+  }
 }

extensions/ql-vscode/scripts/tsconfig.json

@@ -0,0 +1,15 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+// https://stackoverflow.com/a/45130990
+export async function* getFiles(dir: string): AsyncGenerator<string> {
+  const dirents = await fs.readdir(dir, { withFileTypes: true });
+  for (const dirent of dirents) {
+    const res = path.resolve(dir, dirent.name);
+    if (dirent.isDirectory()) {
+      yield* getFiles(res);
+    } else {
+      yield res;
+    }
+  }
+}

extensions/ql-vscode/scripts/util/files.ts

@@ -105,7 +105,9 @@
             "id": 206444,
             "name": "hive",
             "full_name": "apache/hive",
-            "private": false
+            "private": false,
+            "stargazers_count": 4523,
+            "updated_at": "2022-11-02T10:04:02Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 81841,
@@ -116,7 +118,9 @@
             "id": 20753500,
             "name": "ng-nice",
             "full_name": "angular-cn/ng-nice",
-            "private": false
+            "private": false,
+            "stargazers_count": 192,
+            "updated_at": "2022-03-17T08:34:30Z"
           },
           "analysis_status": "in_progress"
         },
@@ -125,7 +129,9 @@
             "id": 23418517,
             "name": "hadoop",
             "full_name": "apache/hadoop",
-            "private": false
+            "private": false,
+            "stargazers_count": 13030,
+            "updated_at": "2022-11-02T08:23:58Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 66895,
@@ -136,7 +142,9 @@
             "id": 236095576,
             "name": "backstage",
             "full_name": "backstage/backstage",
-            "private": false
+            "private": false,
+            "stargazers_count": 19033,
+            "updated_at": "2022-11-02T10:25:24Z"
           },
           "analysis_status": "in_progress"
         },
@@ -145,7 +153,9 @@
             "id": 257485422,
             "name": "vite",
             "full_name": "vitejs/vite",
-            "private": false
+            "private": false,
+            "stargazers_count": 49064,
+            "updated_at": "2022-11-02T11:29:22Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 702,

extensions/ql-vscode/src/mocks/scenarios/problem-query-success/11-getVariantAnalysis.json

@@ -105,7 +105,9 @@
             "id": 206444,
             "name": "hive",
             "full_name": "apache/hive",
-            "private": false
+            "private": false,
+            "stargazers_count": 4523,
+            "updated_at": "2022-11-02T10:04:02Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 81841,
@@ -116,7 +118,9 @@
             "id": 20753500,
             "name": "ng-nice",
             "full_name": "angular-cn/ng-nice",
-            "private": false
+            "private": false,
+            "stargazers_count": 192,
+            "updated_at": "2022-03-17T08:34:30Z"
           },
           "analysis_status": "in_progress"
         },
@@ -125,7 +129,9 @@
             "id": 23418517,
             "name": "hadoop",
             "full_name": "apache/hadoop",
-            "private": false
+            "private": false,
+            "stargazers_count": 13030,
+            "updated_at": "2022-11-02T08:23:58Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 66895,
@@ -136,7 +142,9 @@
             "id": 236095576,
             "name": "backstage",
             "full_name": "backstage/backstage",
-            "private": false
+            "private": false,
+            "stargazers_count": 19033,
+            "updated_at": "2022-11-02T10:25:24Z"
           },
           "analysis_status": "in_progress"
         },
@@ -145,7 +153,9 @@
             "id": 257485422,
             "name": "vite",
             "full_name": "vitejs/vite",
-            "private": false
+            "private": false,
+            "stargazers_count": 49064,
+            "updated_at": "2022-11-02T11:29:22Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 702,