Merge branch 'main' into robertbrignull/ResultTables-Header

Author: robertbrignull

extensions/ql-vscode/CHANGELOG.md

@@ -7,6 +7,7 @@
 - Remove "last updated" information and sorting from variant analysis results view. [#2637](https://github.com/github/vscode-codeql/pull/2637)
 - Links to code on GitHub now include column numbers as well as line numbers. [#2406](https://github.com/github/vscode-codeql/pull/2406)
 - No longer highlight trailing commas for jump to definition.  [#2615](https://github.com/github/vscode-codeql/pull/2615)
+- Fix a bug where the QHelp preview page was not being refreshed after changes to the underlying `.qhelp` file. [#2660](https://github.com/github/vscode-codeql/pull/2660)
 
 ## 1.8.8 - 17 July 2023
 

extensions/ql-vscode/package-lock.json

@@ -1655,6 +1655,13 @@
           "title": "CodeQL",
           "icon": "media/logo.svg"
         }
+      ],
+      "panel": [
+        {
+          "id": "codeql-model-details",
+          "title": "CodeQL Model Details",
+          "icon": "media/logo.svg"
+        }
       ]
     },
     "views": {
@@ -1685,9 +1692,20 @@
           "name": "Evaluator Log Viewer",
           "when": "config.codeQL.canary"
         }
+      ],
+      "codeql-model-details": [
+        {
+          "id": "codeQLModelDetails",
+          "name": "CodeQL Model Details",
+          "when": "config.codeQL.canary && config.codeQL.dataExtensions.modelDetailsView"
+        }
       ]
     },
     "viewsWelcome": [
+      {
+        "view": "codeQLModelDetails",
+        "contents": "Loading..."
+      },
       {
         "view": "codeQLAstViewer",
         "contents": "Run the 'CodeQL: View AST' command on an open source file from a CodeQL database.\n[View AST](command:codeQL.viewAst)"
@@ -1756,8 +1774,6 @@
     "fs-extra": "^11.1.1",
     "immutable": "^4.0.0",
     "js-yaml": "^4.1.0",
-    "minimatch": "^9.0.0",
-    "minimist": "^1.2.6",
     "msw": "^1.2.0",
     "nanoid": "^3.2.0",
     "node-fetch": "^2.6.7",
@@ -1767,10 +1783,8 @@
     "semver": "^7.5.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
-    "stream": "^0.0.2",
-    "stream-chain": "^2.2.4",
     "stream-json": "^1.7.3",
-    "styled-components": "^5.3.3",
+    "styled-components": "^6.0.2",
     "tmp": "^0.1.0",
     "tmp-promise": "^3.0.2",
     "tree-kill": "^1.2.2",
@@ -1811,21 +1825,17 @@
     "@types/d3-graphviz": "^2.6.6",
     "@types/del": "^4.0.0",
     "@types/fs-extra": "^11.0.1",
-    "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "^1.1.0",
-    "@types/gulp-sourcemaps": "0.0.32",
     "@types/jest": "^29.0.2",
     "@types/js-yaml": "^3.12.5",
-    "@types/jszip": "^3.1.6",
     "@types/nanoid": "^3.0.0",
     "@types/node": "^16.11.25",
     "@types/node-fetch": "^2.5.2",
     "@types/react": "^18.0.28",
     "@types/react-dom": "^18.0.11",
     "@types/sarif": "^2.1.2",
     "@types/semver": "^7.2.0",
-    "@types/stream-chain": "^2.0.1",
     "@types/stream-json": "^1.7.1",
     "@types/styled-components": "^5.1.11",
     "@types/tar-stream": "^2.2.2",
@@ -1835,19 +1845,19 @@
     "@types/vscode": "^1.67.0",
     "@types/webpack": "^5.28.0",
     "@types/webpack-env": "^1.18.0",
-    "@typescript-eslint/eslint-plugin": "^5.38.0",
-    "@typescript-eslint/parser": "^5.38.0",
+    "@typescript-eslint/eslint-plugin": "^6.2.1",
+    "@typescript-eslint/parser": "^6.2.1",
     "@vscode/test-electron": "^2.2.0",
     "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
-    "cosmiconfig": "^7.1.0",
+    "cosmiconfig": "^8.2.0",
     "cross-env": "^7.0.3",
     "css-loader": "^6.8.1",
     "del": "^6.0.0",
     "esbuild": "^0.15.15",
     "eslint": "^8.23.1",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-etc": "^2.0.2",
     "eslint-plugin-github": "^4.4.1",
     "eslint-plugin-jest-dom": "^5.0.1",
@@ -1871,7 +1881,7 @@
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "patch-package": "^7.0.0",
+    "patch-package": "^8.0.0",
     "prettier": "^3.0.0",
     "storybook": "^7.1.0",
     "tar-stream": "^3.0.0",
@@ -1880,8 +1890,7 @@
     "ts-json-schema-generator": "^1.1.2",
     "ts-loader": "^9.4.2",
     "ts-node": "^10.7.0",
-    "ts-protoc-gen": "^0.9.0",
-    "ts-unused-exports": "^9.0.5",
+    "ts-unused-exports": "^10.0.0",
     "typescript": "^5.0.2",
     "webpack": "^5.76.0",
     "webpack-cli": "^5.0.1"

extensions/ql-vscode/package.json

@@ -12,6 +12,7 @@ import type {
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import type { Usage } from "../data-extensions-editor/external-api-usage";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -58,7 +59,9 @@ export type ExplorerSelectionCommandFunction<Item> = (
 type BuiltInVsCodeCommands = {
   // The codeQLDatabases.focus command is provided by VS Code because we've registered the custom view
   "codeQLDatabases.focus": () => Promise<void>;
+  "codeQLModelDetails.focus": () => Promise<void>;
   "markdown.showPreviewToSide": (uri: Uri) => Promise<void>;
+  "workbench.action.closeActiveEditor": () => Promise<void>;
   revealFileInOS: (uri: Uri) => Promise<void>;
   setContext: (
     key: `${"codeql" | "codeQL"}${string}`,
@@ -302,6 +305,10 @@ export type PackagingCommands = {
 
 export type DataExtensionsEditorCommands = {
   "codeQL.openDataExtensionsEditor": () => Promise<void>;
+  "codeQLDataExtensionsEditor.jumpToUsageLocation": (
+    usage: Usage,
+    databaseItem: DatabaseItem,
+  ) => Promise<void>;
 };
 
 export type EvalLogViewerCommands = {

extensions/ql-vscode/src/common/commands.ts

@@ -510,6 +510,12 @@ interface AddModeledMethodsMessage {
   modeledMethods: Record<string, ModeledMethod>;
 }
 
+interface SetInProgressMethodsMessage {
+  t: "setInProgressMethods";
+  packageName: string;
+  inProgressMethods: string[];
+}
+
 interface SwitchModeMessage {
   t: "switchMode";
   mode: Mode;
@@ -544,10 +550,16 @@ interface GenerateExternalApiMessage {
 
 interface GenerateExternalApiFromLlmMessage {
   t: "generateExternalApiFromLlm";
+  packageName: string;
   externalApiUsages: ExternalApiUsage[];
   modeledMethods: Record<string, ModeledMethod>;
 }
 
+interface StopGeneratingExternalApiFromLlmMessage {
+  t: "stopGeneratingExternalApiFromLlm";
+  packageName: string;
+}
+
 interface ModelDependencyMessage {
   t: "modelDependency";
 }
@@ -556,7 +568,8 @@ export type ToDataExtensionsEditorMessage =
   | SetExtensionPackStateMessage
   | SetExternalApiUsagesMessage
   | LoadModeledMethodsMessage
-  | AddModeledMethodsMessage;
+  | AddModeledMethodsMessage
+  | SetInProgressMethodsMessage;
 
 export type FromDataExtensionsEditorMessage =
   | ViewLoadedMsg
@@ -568,4 +581,5 @@ export type FromDataExtensionsEditorMessage =
   | SaveModeledMethods
   | GenerateExternalApiMessage
   | GenerateExternalApiFromLlmMessage
+  | StopGeneratingExternalApiFromLlmMessage
   | ModelDependencyMessage;

extensions/ql-vscode/src/common/interface-types.ts

@@ -2,7 +2,7 @@ import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
 import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 
-interface SarifLink {
+export interface SarifLink {
   dest: number;
   text: string;
 }

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -714,6 +714,11 @@ const EXTENSIONS_DIRECTORY = new Setting(
   "extensionsDirectory",
   DATA_EXTENSIONS,
 );
+const MODEL_DETAILS_VIEW = new Setting("modelDetailsView", DATA_EXTENSIONS);
+
+export function showModelDetailsView(): boolean {
+  return !!MODEL_DETAILS_VIEW.getValue<boolean>();
+}
 
 export function showLlmGeneration(): boolean {
   return !!LLM_GENERATION.getValue<boolean>();

extensions/ql-vscode/src/config.ts

@@ -17,6 +17,10 @@ import { redactableError } from "../common/errors";
 import { interpretResultsSarif } from "../query-results";
 import { join } from "path";
 import { assertNever } from "../common/helpers-pure";
+import { dir } from "tmp-promise";
+import { writeFile, outputFile } from "fs-extra";
+import { dump as dumpYaml } from "js-yaml";
+import { MethodSignature } from "./external-api-usage";
 
 type AutoModelQueryOptions = {
   queryTag: string;
@@ -26,6 +30,7 @@ type AutoModelQueryOptions = {
   databaseItem: DatabaseItem;
   qlpack: QlPacksForLanguage;
   sourceInfo: SourceInfo | undefined;
+  additionalPacks: string[];
   extensionPacks: string[];
   queryStorageDir: string;
 
@@ -52,6 +57,7 @@ async function runAutoModelQuery({
   databaseItem,
   qlpack,
   sourceInfo,
+  additionalPacks,
   extensionPacks,
   queryStorageDir,
   progress,
@@ -99,7 +105,7 @@ async function runAutoModelQuery({
       quickEvalCountOnly: false,
     },
     false,
-    getOnDiskWorkspaceFolders(),
+    additionalPacks,
     extensionPacks,
     queryStorageDir,
     undefined,
@@ -147,12 +153,14 @@ async function runAutoModelQuery({
 
 type AutoModelQueriesOptions = {
   mode: Mode;
+  candidateMethods: MethodSignature[];
   cliServer: CodeQLCliServer;
   queryRunner: QueryRunner;
   databaseItem: DatabaseItem;
   queryStorageDir: string;
 
   progress: ProgressCallback;
+  cancellationTokenSource: CancellationTokenSource;
 };
 
 export type AutoModelQueriesResult = {
@@ -161,17 +169,14 @@ export type AutoModelQueriesResult = {
 
 export async function runAutoModelQueries({
   mode,
+  candidateMethods,
   cliServer,
   queryRunner,
   databaseItem,
   queryStorageDir,
   progress,
+  cancellationTokenSource,
 }: AutoModelQueriesOptions): Promise<AutoModelQueriesResult | undefined> {
-  // maxStep for this part is 1500
-  const maxStep = 1500;
-
-  const cancellationTokenSource = new CancellationTokenSource();
-
   const qlpack = await qlpackOfDatabase(cliServer, databaseItem);
 
   // CodeQL needs to have access to the database to be able to retrieve the
@@ -189,17 +194,17 @@ export async function runAutoModelQueries({
           sourceLocationPrefix,
         };
 
-  const additionalPacks = getOnDiskWorkspaceFolders();
+  // Generate a pack containing the candidate filters
+  const filterPackDir = await generateCandidateFilterPack(
+    databaseItem.language,
+    candidateMethods,
+  );
+
+  const additionalPacks = [...getOnDiskWorkspaceFolders(), filterPackDir];
   const extensionPacks = Object.keys(
     await cliServer.resolveQlpacks(additionalPacks, true),
   );
 
-  progress({
-    step: 0,
-    maxStep,
-    message: "Finding candidates and examples",
-  });
-
   const candidates = await runAutoModelQuery({
     mode,
     queryTag: "candidates",
@@ -208,15 +213,10 @@ export async function runAutoModelQueries({
     databaseItem,
     qlpack,
     sourceInfo,
+    additionalPacks,
     extensionPacks,
     queryStorageDir,
-    progress: (update) => {
-      progress({
-        step: update.step,
-        maxStep,
-        message: "Finding candidates and examples",
-      });
-    },
+    progress,
     token: cancellationTokenSource.token,
   });
 
@@ -228,3 +228,59 @@ export async function runAutoModelQueries({
     candidates,
   };
 }
+
+/**
+ * generateCandidateFilterPack will create a temporary extension pack.
+ * This pack will contain a filter that will restrict the automodel queries
+ * to the specified candidate methods only.
+ * This is done using the `extensible` predicate "automodelCandidateFilter".
+ * @param language
+ * @param candidateMethods
+ * @returns
+ */
+export async function generateCandidateFilterPack(
+  language: string,
+  candidateMethods: MethodSignature[],
+): Promise<string> {
+  // Pack resides in a temporary directory, to not pollute the workspace.
+  const packDir = (await dir({ unsafeCleanup: true })).path;
+
+  const syntheticConfigPack = {
+    name: "codeql/automodel-filter",
+    version: "0.0.0",
+    library: true,
+    extensionTargets: {
+      [`codeql/${language}-queries`]: "*",
+    },
+    dataExtensions: ["filter.yml"],
+  };
+
+  const qlpackFile = join(packDir, "codeql-pack.yml");
+  await outputFile(qlpackFile, dumpYaml(syntheticConfigPack), "utf8");
+
+  // The predicate has the following defintion:
+  // extensible predicate automodelCandidateFilter(string package, string type, string name, string signature)
+  const dataRows = candidateMethods.map((method) => [
+    method.packageName,
+    method.typeName,
+    method.methodName,
+    method.methodParameters,
+  ]);
+
+  const filter = {
+    extensions: [
+      {
+        addsTo: {
+          pack: `codeql/${language}-queries`,
+          extensible: "automodelCandidateFilter",
+        },
+        data: dataRows,
+      },
+    ],
+  };
+
+  const filterFile = join(packDir, "filter.yml");
+  await writeFile(filterFile, dumpYaml(filter), "utf8");
+
+  return packDir;
+}