Mock config instead of writing settings files

In our tests, we were writing settings files to disk because we were
using the VSCode configuration API which writes settings to files. This
results in flaky tests because concurrency can cause the VSCode API to
misbehave.

This will switch the tests to use a mocked API by default. For some
tests the real implementation is used, but the large majority of tests
is now using a mocked version which only keeps track of the
configuration in memory. This makes it easier to reset the state between
tests since we can just empty out the in-memory configuration.

Author: koesie10

extensions/ql-vscode/src/config.ts

@@ -56,15 +56,6 @@ export class Setting {
       .getConfiguration(this.parent.qualifiedName)
       .update(this.name, value, target);
   }
-
-  inspect<T>(): InspectionResult<T> | undefined {
-    if (this.parent === undefined) {
-      throw new Error("Cannot update the value of a root setting.");
-    }
-    return workspace
-      .getConfiguration(this.parent.qualifiedName)
-      .inspect(this.name);
-  }
 }
 
 export interface InspectionResult<T> {

extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts

@@ -8,23 +8,24 @@ import { dirname } from "path";
 import fetch from "node-fetch";
 import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper";
 import * as tmp from "tmp";
-import { getTestSetting } from "../test-config";
 import { CUSTOM_CODEQL_PATH_SETTING } from "../../../src/config";
-import { extensions, workspace } from "vscode";
+import { ConfigurationTarget, env, extensions, workspace } from "vscode";
+import { beforeEachAction } from "../test-config";
 
-import baseJestSetup from "../jest.setup";
-
-export default baseJestSetup;
+(env as any).openExternal = () => {
+  /**/
+};
 
 // create an extension storage location
 let removeStorage: tmp.DirResult["removeCallback"] | undefined;
 
 beforeAll(async () => {
   // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
-  await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setInitialTestValue(
+  await beforeEachAction();
+  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
     process.env.CLI_PATH,
+    ConfigurationTarget.Workspace,
   );
-  await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setup();
 
   // ensure the test database is downloaded
   mkdirpSync(dirname(dbLoc));
@@ -78,6 +79,15 @@ beforeAll(async () => {
   await extensions.getExtension("GitHub.vscode-codeql")?.activate();
 });
 
+beforeEach(async () => {
+  await beforeEachAction();
+
+  await CUSTOM_CODEQL_PATH_SETTING.updateValue(
+    process.env.CLI_PATH,
+    ConfigurationTarget.Global,
+  );
+});
+
 // ensure extension is cleaned up.
 afterAll(async () => {
   // ensure temp directory is cleaned up.

extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-submission-integration.test.ts

@@ -3,6 +3,7 @@ import { resolve } from "path";
 import {
   authentication,
   commands,
+  ConfigurationTarget,
   extensions,
   QuickPickItem,
   TextDocument,
@@ -12,7 +13,10 @@ import {
 
 import { CodeQLExtensionInterface } from "../../../../src/extension";
 import { MockGitHubApiServer } from "../../../../src/mocks/mock-gh-api-server";
-import { mockConfiguration } from "../../utils/configuration-helpers";
+import {
+  CANARY_FEATURES,
+  setRemoteControllerRepo,
+} from "../../../../src/config";
 
 jest.setTimeout(30_000);
 
@@ -36,17 +40,8 @@ describe("Variant Analysis Submission Integration", () => {
   let showErrorMessageSpy: jest.SpiedFunction<typeof window.showErrorMessage>;
 
   beforeEach(async () => {
-    mockConfiguration({
-      values: {
-        codeQL: {
-          canary: true,
-        },
-        "codeQL.variantAnalysis": {
-          liveResults: true,
-          controllerRepo: "github/vscode-codeql",
-        },
-      },
-    });
+    await CANARY_FEATURES.updateValue(true, ConfigurationTarget.Global);
+    await setRemoteControllerRepo("github/vscode-codeql");
 
     jest.spyOn(authentication, "getSession").mockResolvedValue({
       id: "test",

extensions/ql-vscode/test/vscode-tests/jest.setup.ts

@@ -1,10 +1,10 @@
 import { env } from "vscode";
-import { jestTestConfigHelper } from "./test-config";
+import { beforeEachAction } from "./test-config";
 
 (env as any).openExternal = () => {
   /**/
 };
 
-export default async function setupEnv() {
-  await jestTestConfigHelper();
-}
+beforeEach(async () => {
+  await beforeEachAction();
+});

extensions/ql-vscode/test/vscode-tests/minimal-workspace/config.test.ts

@@ -6,8 +6,13 @@ import {
   QueryHistoryConfigListener,
   QueryServerConfigListener,
 } from "../../../src/config";
+import { vscodeGetConfigurationMock } from "../test-config";
 
 describe("config listeners", () => {
+  beforeEach(() => {
+    vscodeGetConfigurationMock.mockRestore();
+  });
+
   interface TestConfig<T> {
     clazz: new () => ConfigListener;
     settings: Array<{

extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts

@@ -13,7 +13,7 @@ import { DbTreeViewItem } from "../../../../src/databases/ui/db-tree-view-item";
 import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
 import { createMockExtensionContext } from "../../../factories/extension-context";
 import { createDbConfig } from "../../../factories/db-config-factories";
-import { mockConfiguration } from "../../utils/configuration-helpers";
+import { setRemoteControllerRepo } from "../../../../src/config";
 
 describe("db panel rendering nodes", () => {
   const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -50,12 +50,8 @@ describe("db panel rendering nodes", () => {
   });
 
   describe("when controller repo is not set", () => {
-    mockConfiguration({
-      values: {
-        "codeQL.variantAnalysis": {
-          controllerRepo: undefined,
-        },
-      },
+    beforeEach(async () => {
+      await setRemoteControllerRepo(undefined);
     });
 
     it("should not have any items", async () => {
@@ -81,14 +77,8 @@ describe("db panel rendering nodes", () => {
   });
 
   describe("when controller repo is set", () => {
-    beforeEach(() => {
-      mockConfiguration({
-        values: {
-          "codeQL.variantAnalysis": {
-            controllerRepo: "github/codeql",
-          },
-        },
-      });
+    beforeEach(async () => {
+      await setRemoteControllerRepo("github/codeql");
     });
 
     it("should render default remote nodes when the config is empty", async () => {

extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel.test.ts

@@ -9,7 +9,7 @@ import { DbTreeViewItem } from "../../../../src/databases/ui/db-tree-view-item";
 import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
 import { createMockExtensionContext } from "../../../factories/extension-context";
 import { createDbConfig } from "../../../factories/db-config-factories";
-import { mockConfiguration } from "../../utils/configuration-helpers";
+import { setRemoteControllerRepo } from "../../../../src/config";
 
 describe("db panel", () => {
   const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -40,13 +40,7 @@ describe("db panel", () => {
   beforeEach(async () => {
     await ensureDir(workspaceStoragePath);
 
-    mockConfiguration({
-      values: {
-        "codeQL.variantAnalysis": {
-          controllerRepo: "github/codeql",
-        },
-      },
-    });
+    await setRemoteControllerRepo("github/codeql");
   });
 
   afterEach(async () => {

extensions/ql-vscode/test/vscode-tests/no-workspace/databases/db-panel-selection.test.ts

@@ -15,7 +15,7 @@ import {
 import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
 import { createMockExtensionContext } from "../../../factories/extension-context";
 import { createDbConfig } from "../../../factories/db-config-factories";
-import { mockConfiguration } from "../../utils/configuration-helpers";
+import { setRemoteControllerRepo } from "../../../../src/config";
 
 describe("db panel selection", () => {
   const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -46,13 +46,7 @@ describe("db panel selection", () => {
   beforeEach(async () => {
     await ensureDir(workspaceStoragePath);
 
-    mockConfiguration({
-      values: {
-        "codeQL.variantAnalysis": {
-          controllerRepo: "github/codeql",
-        },
-      },
-    });
+    await setRemoteControllerRepo("github/codeql");
   });
 
   afterEach(async () => {

extensions/ql-vscode/test/vscode-tests/no-workspace/telemetry.test.ts

@@ -13,6 +13,7 @@ import { UserCancellationException } from "../../../src/commandRunner";
 import { ENABLE_TELEMETRY } from "../../../src/config";
 import * as Config from "../../../src/config";
 import { createMockExtensionContext } from "./index";
+import { vscodeGetConfigurationMock } from "../test-config";
 
 // setting preferences can trigger lots of background activity
 // so need to bump up the timeout of this test.
@@ -40,6 +41,8 @@ describe("telemetry reporting", () => {
   >;
 
   beforeEach(async () => {
+    vscodeGetConfigurationMock.mockRestore();
+
     try {
       // in case a previous test has accidentally activated this extension,
       // need to disable it first.