Merge pull request #2731 from github/starcke/no-submodule

Make automodel and flow queries work without submodule.

Author: starcke

extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts

@@ -3,7 +3,6 @@ import { CoreCompletedQuery, QueryRunner } from "../query-server";
 import { DatabaseItem } from "../databases/local-databases";
 import { ProgressCallback } from "../common/vscode/progress";
 import * as Sarif from "sarif";
-import { qlpackOfDatabase, resolveQueries } from "../local-queries";
 import { Mode } from "./shared/mode";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { interpretResultsSarif } from "../query-results";
@@ -16,6 +15,7 @@ import { MethodSignature } from "./external-api-usage";
 import { runQuery } from "../local-queries/run-query";
 import { QueryMetadata } from "../common/interface-types";
 import { CancellationTokenSource } from "vscode";
+import { resolveQueries } from "../local-queries";
 
 function modeTag(mode: Mode): string {
   switch (mode) {
@@ -84,6 +84,7 @@ export async function runAutoModelQueries({
     extensionPacks,
     progress,
     token: cancellationTokenSource.token,
+    createLockFile: false,
   });
 
   if (!completedQuery) {
@@ -127,15 +128,15 @@ async function resolveAutomodelQuery(
   queryTag: string,
   mode: Mode,
 ): Promise<string> {
-  const qlpack = await qlpackOfDatabase(cliServer, databaseItem);
+  const packsToSearch = [`codeql/${databaseItem.language}-queries`];
 
   // First, resolve the query that we want to run.
   // All queries are tagged like this:
   // internal extract automodel <mode> <queryTag>
   // Example: internal extract automodel framework-mode candidates
   const queries = await resolveQueries(
     cliServer,
-    qlpack,
+    packsToSearch,
     `Extract automodel ${queryTag}`,
     {
       kind: "problem",

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -136,11 +136,10 @@ export class DataExtensionsEditorModule extends DisposableObject {
 
             // Create new temporary directory for query files and pack dependencies
             const queryDir = (await dir({ unsafeCleanup: true })).path;
-            const success = await setUpPack(queryDir, language);
+            const success = await setUpPack(this.cliServer, queryDir, language);
             if (!success) {
               return;
             }
-            await this.cliServer.packInstall(queryDir);
 
             const view = new DataExtensionsEditorView(
               this.ctx,

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-queries.ts

@@ -3,6 +3,7 @@ import { QueryLanguage } from "../common/query-language";
 import { writeFile } from "fs-extra";
 import { dump } from "js-yaml";
 import { prepareExternalApiQuery } from "./external-api-usage-queries";
+import { CodeQLCliServer } from "../codeql-cli/cli";
 
 /**
  * setUpPack sets up a directory to use for the data extension editor queries.
@@ -11,6 +12,7 @@ import { prepareExternalApiQuery } from "./external-api-usage-queries";
  * @returns true if the setup was successful, false otherwise.
  */
 export async function setUpPack(
+  cliServer: CodeQLCliServer,
   queryDir: string,
   language: QueryLanguage,
 ): Promise<boolean> {
@@ -23,7 +25,7 @@ export async function setUpPack(
     return false;
   }
 
-  // Set up a synthetic query pack to resolve dependencies.
+  // Set up a synthetic pack so that the query can be resolved later.
   const syntheticQueryPack = {
     name: "codeql/external-api-usage",
     version: "0.0.0",
@@ -34,5 +36,10 @@ export async function setUpPack(
 
   const qlpackFile = join(queryDir, "codeql-pack.yml");
   await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
+  await cliServer.packInstall(queryDir);
+
+  // Install the other needed query packs
+  await cliServer.packDownload([`codeql/${language}-queries`]);
+
   return true;
 }

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-queries.ts

@@ -94,6 +94,8 @@ export async function runExternalApiQueries(
     extensionPacks,
     progress,
     token,
+    // We need to create a lock file, because the query is inside our own pack
+    createLockFile: true,
   });
 
   if (!completedQuery) {

extensions/ql-vscode/src/data-extensions-editor/flow-model-queries.ts

@@ -10,9 +10,9 @@ import { ProgressCallback } from "../common/vscode/progress";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { ModeledMethod, ModeledMethodType } from "./modeled-method";
 import { redactableError } from "../common/errors";
-import { qlpackOfDatabase, resolveQueries } from "../local-queries";
 import { telemetryListener } from "../common/vscode/telemetry";
 import { runQuery } from "../local-queries/run-query";
+import { resolveQueries } from "../local-queries";
 
 type FlowModelOptions = {
   cliServer: CodeQLCliServer;
@@ -83,11 +83,16 @@ async function resolveFlowQueries(
   cliServer: CodeQLCliServer,
   databaseItem: DatabaseItem,
 ): Promise<string[]> {
-  const qlpacks = await qlpackOfDatabase(cliServer, databaseItem);
+  const packsToSearch = [`codeql/${databaseItem.language}-queries`];
 
-  return await resolveQueries(cliServer, qlpacks, "flow model generator", {
-    "tags contain": ["modelgenerator"],
-  });
+  return await resolveQueries(
+    cliServer,
+    packsToSearch,
+    "flow model generator",
+    {
+      "tags contain": ["modelgenerator"],
+    },
+  );
 }
 
 async function runSingleFlowQuery(
@@ -129,6 +134,7 @@ async function runSingleFlowQuery(
         maxStep: 4000,
       }),
     token,
+    createLockFile: false,
   });
 
   if (!completedQuery) {

extensions/ql-vscode/src/language-support/contextual/query-resolver.ts

@@ -8,7 +8,7 @@ import {
 } from "./key-type";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { DatabaseItem } from "../../databases/local-databases";
-import { resolveQueries as resolveLocalQueries } from "../../local-queries/query-resolver";
+import { resolveQueriesByLanguagePack as resolveLocalQueries } from "../../local-queries/query-resolver";
 import { extLogger } from "../../common/logging/vscode";
 import { TeeLogger } from "../../common/logging";
 import { CancellationToken } from "vscode";

extensions/ql-vscode/src/local-queries/query-resolver.ts

@@ -72,16 +72,7 @@ async function resolveQueriesFromPacks(
   );
 }
 
-/**
- * Finds the queries with the specified kind and tags in a QLPack.
- *
- * @param cli The CLI instance to use.
- * @param qlpacks The list of packs to search.
- * @param name The name of the query to use in error messages.
- * @param constraints Constraints on the queries to search for.
- * @returns The found queries from the first pack in which any matching queries were found.
- */
-export async function resolveQueries(
+export async function resolveQueriesByLanguagePack(
   cli: CodeQLCliServer,
   qlpacks: QlPacksForLanguage,
   name: string,
@@ -95,6 +86,24 @@ export async function resolveQueries(
     packsToSearch.push(qlpacks.queryPack);
   }
 
+  return resolveQueries(cli, packsToSearch, name, constraints);
+}
+
+/**
+ * Finds the queries with the specified kind and tags in a QLPack.
+ *
+ * @param cli The CLI instance to use.
+ * @param packsToSearch The list of packs to search.
+ * @param name The name of the query to use in error messages.
+ * @param constraints Constraints on the queries to search for.
+ * @returns The found queries from the first pack in which any matching queries were found.
+ */
+export async function resolveQueries(
+  cli: CodeQLCliServer,
+  packsToSearch: string[],
+  name: string,
+  constraints: QueryConstraints,
+): Promise<string[]> {
   const queries = await resolveQueriesFromPacks(
     cli,
     packsToSearch,

extensions/ql-vscode/src/local-queries/run-query.ts

@@ -21,6 +21,7 @@ type RunQueryOptions = {
   extensionPacks: string[] | undefined;
   progress: ProgressCallback;
   token: CancellationToken;
+  createLockFile: boolean;
 };
 
 export async function runQuery({
@@ -33,12 +34,17 @@ export async function runQuery({
   extensionPacks,
   progress,
   token,
+  createLockFile,
 }: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
-  // Create a lock file for the query. This is required to resolve dependencies and library path for the query.
-  const { cleanup: cleanupLockFile } = await createLockFileForStandardQuery(
-    cliServer,
-    queryPath,
-  );
+  let cleanupLockFile: (() => Promise<void>) | undefined = undefined;
+  if (createLockFile) {
+    // Create a lock file for the query. This is required to resolve dependencies and library path for the query.
+    const { cleanup } = await createLockFileForStandardQuery(
+      cliServer,
+      queryPath,
+    );
+    cleanupLockFile = cleanup;
+  }
 
   // Create a query run to execute
   const queryRun = queryRunner.createQueryRun(

extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/auto-model-codeml-queries.test.ts

@@ -14,7 +14,6 @@ import { mockedObject, mockedUri } from "../../utils/mocking.helpers";
 import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
 import { QueryRunner } from "../../../../src/query-server";
 import * as queryResolver from "../../../../src/local-queries/query-resolver";
-import * as standardQueries from "../../../../src/local-queries/standard-queries";
 import { MethodSignature } from "../../../../src/data-extensions-editor/external-api-usage";
 import { join } from "path";
 import { exists, readFile } from "fs-extra";
@@ -23,40 +22,27 @@ import { CancellationTokenSource } from "vscode-jsonrpc";
 import { QueryOutputDir } from "../../../../src/run-queries-shared";
 
 describe("runAutoModelQueries", () => {
-  const qlpack = {
-    dbschemePack: "dbschemePack",
-    dbschemePackIsLibraryPack: false,
-  };
-
   let resolveQueriesSpy: jest.SpiedFunction<
     typeof queryResolver.resolveQueries
   >;
-  let createLockFileForStandardQuerySpy: jest.SpiedFunction<
-    typeof standardQueries.createLockFileForStandardQuery
-  >;
-
   beforeEach(() => {
-    jest.spyOn(queryResolver, "qlpackOfDatabase").mockResolvedValue(qlpack);
-
     resolveQueriesSpy = jest
       .spyOn(queryResolver, "resolveQueries")
-      .mockImplementation(async (_cliServer, _qlPack, _name, constraints) => {
-        if (constraints["tags contain all"]?.includes("candidates")) {
-          return ["/a/b/c/ql/candidates.ql"];
-        }
-        if (constraints["tags contain all"]?.includes("positive")) {
-          return ["/a/b/c/ql/positive-examples.ql"];
-        }
-        if (constraints["tags contain all"]?.includes("negative")) {
-          return ["/a/b/c/ql/negative-examples.ql"];
-        }
+      .mockImplementation(
+        async (_cliServer, _packsToSearch, _name, constraints) => {
+          if (constraints["tags contain all"]?.includes("candidates")) {
+            return ["/a/b/c/ql/candidates.ql"];
+          }
+          if (constraints["tags contain all"]?.includes("positive")) {
+            return ["/a/b/c/ql/positive-examples.ql"];
+          }
+          if (constraints["tags contain all"]?.includes("negative")) {
+            return ["/a/b/c/ql/negative-examples.ql"];
+          }
 
-        return [];
-      });
-
-    createLockFileForStandardQuerySpy = jest
-      .spyOn(standardQueries, "createLockFileForStandardQuery")
-      .mockResolvedValue({});
+          return [];
+        },
+      );
   });
 
   it("should run the query and return the results", async () => {
@@ -154,18 +140,13 @@ describe("runAutoModelQueries", () => {
     expect(resolveQueriesSpy).toHaveBeenCalledTimes(1);
     expect(resolveQueriesSpy).toHaveBeenCalledWith(
       options.cliServer,
-      qlpack,
+      ["codeql/java-queries"],
       "Extract automodel candidates",
       {
         kind: "problem",
         "tags contain all": ["automodel", "application-mode", "candidates"],
       },
     );
-    expect(createLockFileForStandardQuerySpy).toHaveBeenCalledTimes(1);
-    expect(createLockFileForStandardQuerySpy).toHaveBeenCalledWith(
-      options.cliServer,
-      "/a/b/c/ql/candidates.ql",
-    );
     expect(options.queryRunner.createQueryRun).toHaveBeenCalledTimes(1);
     expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith(
       "/a/b/c/src.zip",

extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/data-extensions-editor-queries.test.ts

@@ -6,8 +6,15 @@ import { dirSync } from "tmp-promise";
 import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries";
 import { QueryLanguage } from "../../../../src/common/query-language";
 import { Mode } from "../../../../src/data-extensions-editor/shared/mode";
+import { mockedObject } from "../../utils/mocking.helpers";
+import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
 
 describe("setUpPack", () => {
+  const cliServer = mockedObject<CodeQLCliServer>({
+    packDownload: jest.fn(),
+    packInstall: jest.fn(),
+  });
+
   const languages = Object.keys(fetchExternalApiQueries).flatMap((lang) => {
     const queryDir = dirSync({ unsafeCleanup: true }).name;
     const query = fetchExternalApiQueries[lang as QueryLanguage];
@@ -21,7 +28,7 @@ describe("setUpPack", () => {
   test.each(languages)(
     "should create files for $language",
     async ({ language, queryDir, query }) => {
-      await setUpPack(queryDir, language);
+      await setUpPack(cliServer, queryDir, language);
 
       const queryFiles = await readdir(queryDir);
       expect(queryFiles.sort()).toEqual(