Merge branch 'main' into robertbrignull/data-modeled-methods-tests

Author: robertbrignull

extensions/ql-vscode/media/dark/symbol-misc.svg

@@ -19,6 +19,7 @@ export type WebviewPanelConfig = {
   viewColumn: ViewColumn;
   view: WebviewView;
   preserveFocus?: boolean;
+  iconPath?: Uri | { dark: Uri; light: Uri };
   additionalOptions?: WebviewPanelOptions & WebviewOptions;
   allowWasmEval?: boolean;
 };
@@ -86,6 +87,8 @@ export abstract class AbstractWebview<
       );
       this.panel = panel;
 
+      this.panel.iconPath = config.iconPath;
+
       this.setupPanel(panel, config);
 
       this.panelResolves.forEach((resolve) => resolve(panel));

extensions/ql-vscode/media/light/symbol-misc.svg

@@ -140,6 +140,12 @@ export function parsePredictedClassifications(
         input: "",
         output: "",
         provenance: "ai-generated",
+        signature,
+        // predictedBySignature[signature] always has at least element
+        packageName: predictedMethods[0].package,
+        typeName: predictedMethods[0].type,
+        methodName: predictedMethods[0].name,
+        methodParameters: predictedMethods[0].signature,
       };
       continue;
     }
@@ -157,6 +163,11 @@ export function parsePredictedClassifications(
       input: sink.input ?? "",
       output: sink.output ?? "",
       provenance: "ai-generated",
+      signature,
+      packageName: sink.package,
+      typeName: sink.type,
+      methodName: sink.name,
+      methodParameters: sink.signature,
     };
   }
 

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -1,5 +1,11 @@
 import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
-import { Call, ExternalApiUsage } from "./external-api-usage";
+import {
+  Call,
+  CallClassification,
+  ExternalApiUsage,
+} from "./external-api-usage";
+import { ModeledMethodType } from "./modeled-method";
+import { parseLibraryFilename } from "./library";
 
 export function decodeBqrsToExternalApiUsages(
   chunk: DecodedBqrsChunk,
@@ -10,7 +16,10 @@ export function decodeBqrsToExternalApiUsages(
     const usage = tuple[0] as Call;
     const signature = tuple[1] as string;
     const supported = (tuple[2] as string) === "true";
-    const library = tuple[4] as string;
+    let library = tuple[4] as string;
+    let libraryVersion: string | undefined = tuple[5] as string;
+    const type = tuple[6] as ModeledMethodType;
+    const classification = tuple[8] as CallClassification;
 
     const [packageWithType, methodDeclaration] = signature.split("#");
 
@@ -30,21 +39,41 @@ export function decodeBqrsToExternalApiUsages(
       methodDeclaration.indexOf("("),
     );
 
+    // For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
+    // properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
+    // for Java.
+    if (library.endsWith(".jar") || libraryVersion === "") {
+      const { name, version } = parseLibraryFilename(library);
+      library = name;
+      if (version) {
+        libraryVersion = version;
+      }
+    }
+
+    if (libraryVersion === "") {
+      libraryVersion = undefined;
+    }
+
     if (!methodsByApiName.has(signature)) {
       methodsByApiName.set(signature, {
         library,
+        libraryVersion,
         signature,
         packageName,
         typeName,
         methodName,
         methodParameters,
         supported,
+        supportedType: type,
         usages: [],
       });
     }
 
     const method = methodsByApiName.get(signature)!;
-    method.usages.push(usage);
+    method.usages.push({
+      ...usage,
+      classification,
+    });
   });
 
   return Array.from(methodsByApiName.values());

extensions/ql-vscode/src/data-extensions-editor/auto-model.ts

@@ -76,6 +76,14 @@ export class DataExtensionsEditorView extends AbstractWebview<
       viewColumn: ViewColumn.Active,
       preserveFocus: true,
       view: "data-extensions-editor",
+      iconPath: {
+        dark: Uri.file(
+          join(this.ctx.extensionPath, "media/dark/symbol-misc.svg"),
+        ),
+        light: Uri.file(
+          join(this.ctx.extensionPath, "media/light/symbol-misc.svg"),
+        ),
+      },
     };
   }
 
@@ -311,11 +319,11 @@ export class DataExtensionsEditorView extends AbstractWebview<
         queryRunner: this.queryRunner,
         queryStorageDir: this.queryStorageDir,
         databaseItem: addedDatabase ?? this.databaseItem,
-        onResults: async (results) => {
+        onResults: async (modeledMethods) => {
           const modeledMethodsByName: Record<string, ModeledMethod> = {};
 
-          for (const result of results) {
-            modeledMethodsByName[result.signature] = result.modeledMethod;
+          for (const modeledMethod of modeledMethods) {
+            modeledMethodsByName[modeledMethod.signature] = modeledMethod;
           }
 
           await this.postMessage({

extensions/ql-vscode/src/data-extensions-editor/bqrs.ts

@@ -1,15 +1,27 @@
 import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { ModeledMethodType } from "./modeled-method";
 
 export type Call = {
   label: string;
   url: ResolvableLocationValue;
 };
 
-export type ExternalApiUsage = {
+export enum CallClassification {
+  Unknown = "unknown",
+  Source = "source",
+  Test = "test",
+  Generated = "generated",
+}
+
+export type Usage = Call & {
+  classification: CallClassification;
+};
+
+export interface MethodSignature {
   /**
-   * Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
+   * Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
    */
-  library: string;
+  libraryVersion?: string;
   /**
    * A unique signature that can be used to identify this external API usage.
    *
@@ -25,10 +37,18 @@ export type ExternalApiUsage = {
    * The method parameters, including enclosing parentheses, e.g. `(String, String)`
    */
   methodParameters: string;
+}
+
+export interface ExternalApiUsage extends MethodSignature {
+  /**
+   * Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
+   */
+  library: string;
   /**
    * Is this method already supported by CodeQL standard libraries.
    * If so, there is no need for the user to model it themselves.
    */
   supported: boolean;
-  usages: Call[];
-};
+  supportedType: ModeledMethodType;
+  usages: Usage[];
+}

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -8,10 +8,7 @@ import { extLogger } from "../common/logging/vscode";
 import { extensiblePredicateDefinitions } from "./predicates";
 import { ProgressCallback } from "../common/vscode/progress";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import {
-  ModeledMethodType,
-  ModeledMethodWithSignature,
-} from "./modeled-method";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
 import { redactableError } from "../common/errors";
 import { QueryResultType } from "../query-server/new-messages";
 import { file } from "tmp-promise";
@@ -27,7 +24,7 @@ type FlowModelOptions = {
   databaseItem: DatabaseItem;
   progress: ProgressCallback;
   token: CancellationToken;
-  onResults: (results: ModeledMethodWithSignature[]) => void | Promise<void>;
+  onResults: (results: ModeledMethod[]) => void | Promise<void>;
 };
 
 async function resolveQueries(
@@ -79,7 +76,7 @@ async function getModeledMethodsFromFlow(
     progress,
     token,
   }: Omit<FlowModelOptions, "onResults">,
-): Promise<ModeledMethodWithSignature[]> {
+): Promise<ModeledMethod[]> {
   if (queryPath === undefined) {
     void showAndLogExceptionWithTelemetry(
       extLogger,

extensions/ql-vscode/src/data-extensions-editor/external-api-usage.ts

@@ -0,0 +1,58 @@
+import { basename, extname } from "../common/path";
+
+// From the semver package using
+// const { re, t } = require("semver/internal/re");
+// console.log(re[t.LOOSE]);
+// Modifications:
+// - Added version named group which does not capture the v prefix
+// - Removed the ^ and $ anchors
+// - Made the minor and patch versions optional
+// - Added a hyphen to the start of the version
+// - Added a dot as a valid separator between the version and the label
+// - Made the patch version optional even if a label is given
+// This will match any semver string at the end of a larger string
+const semverRegex =
+  /-[v=\s]*(?<version>([0-9]+)(\.([0-9]+)(?:(\.([0-9]+))?(?:[-.]?((?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*)(?:\.(?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?)?)?)/g;
+
+export interface Library {
+  name: string;
+  version?: string;
+}
+
+export function parseLibraryFilename(filename: string): Library {
+  let libraryName = basename(filename);
+  const extension = extname(libraryName);
+  libraryName = libraryName.slice(0, -extension.length);
+
+  let libraryVersion: string | undefined;
+
+  let match: RegExpMatchArray | null = null;
+
+  // Reset the regex
+  semverRegex.lastIndex = 0;
+
+  // Find the last occurence of the regex within the library name
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const currentMatch = semverRegex.exec(libraryName);
+    if (currentMatch === null) {
+      break;
+    }
+
+    match = currentMatch;
+  }
+
+  if (match?.groups) {
+    libraryVersion = match.groups?.version;
+    // Remove everything after the start of the match
+    libraryName = libraryName.slice(0, match.index);
+  }
+
+  // Remove any leading or trailing hyphens or dots
+  libraryName = libraryName.replaceAll(/^[.-]+|[.-]+$/g, "");
+
+  return {
+    name: libraryName,
+    version: libraryVersion,
+  };
+}

extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts

@@ -1,3 +1,5 @@
+import { MethodSignature } from "./external-api-usage";
+
 export type ModeledMethodType =
   | "none"
   | "source"
@@ -17,15 +19,10 @@ export type Provenance =
   // Entered by the user in the editor manually
   | "manual";
 
-export type ModeledMethod = {
+export interface ModeledMethod extends MethodSignature {
   type: ModeledMethodType;
   input: string;
   output: string;
   kind: string;
   provenance: Provenance;
-};
-
-export type ModeledMethodWithSignature = {
-  signature: string;
-  modeledMethod: ModeledMethod;
-};
+}