Merge branch 'main' into aeisenberg/multi-token

Author: aeisenberg

.github/workflows/cli-test.yml

@@ -62,7 +62,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -64,7 +64,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -81,6 +81,8 @@ jobs:
 
       - name: Lint
         working-directory: extensions/ql-vscode
+        env:
+          NODE_OPTIONS: '--max-old-space-size=4096'
         run: |
           npm run lint
 
@@ -110,7 +112,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -149,7 +151,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -183,7 +185,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -251,7 +253,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '18.15.0'
+          node-version-file: extensions/ql-vscode/.nvmrc
 
       - name: Install dependencies
         run: |

docs/node-version.md

@@ -15,14 +15,14 @@ You can find this info by seleting "About Visual Studio Code" from the top menu.
 
 The following files will need to be updated:
 
-- `.github/workflows/cli-test.yml` - the "node-version: '[VERSION]'" setting
-- `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
-- `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting
-- `extensions/ql-vscode/.nvmrc` - this will enable nvm to automatically switch to the correct node version when you're in the project folder
-- `extensions/ql-vscode/package-lock.json` - the "engines.node: '[VERSION]'" setting
+- `extensions/ql-vscode/.nvmrc` - this will enable nvm to automatically switch to the correct Node
+   version when you're in the project folder. It will also change the Node version the GitHub Actions
+   workflows use.
 - `extensions/ql-vscode/package.json` - the "engines.node: '[VERSION]'" setting
 - `extensions/ql-vscode/package.json` - the "@types/node: '[VERSION]'" dependency
 
+Then run `npm install` to update the `extensions/ql-vscode/package-lock.json` file.
+
 ## Node.js version used in tests
 
 Unit tests will use whatever version of Node.js is installed locally. In CI this will be the version specified in the workflow.

extensions/ql-vscode/src/config.ts

@@ -709,6 +709,7 @@ const MODEL_SETTING = new Setting("model", ROOT_SETTING);
 const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING);
 const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
+const SHOW_MULTIPLE_MODELS = new Setting("showMultipleModels", MODEL_SETTING);
 
 export function showFlowGeneration(): boolean {
   return !!FLOW_GENERATION.getValue<boolean>();
@@ -723,3 +724,7 @@ export function getExtensionsDirectory(languageId: string): string | undefined {
     languageId,
   });
 }
+
+export function showMultipleModels(): boolean {
+  return !!SHOW_MULTIPLE_MODELS.getValue<boolean>();
+}

extensions/ql-vscode/src/extension.ts

@@ -871,6 +871,7 @@ async function activateWithInstalledDistribution(
     ctx,
     queryHistoryConfigurationListener,
     labelProvider,
+    languageContext,
     async (
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo,

extensions/ql-vscode/src/model-editor/auto-modeler.ts

@@ -16,6 +16,7 @@ import { QueryRunner } from "../query-server";
 import { DatabaseItem } from "../databases/local-databases";
 import { Mode } from "./shared/mode";
 import { CancellationTokenSource } from "vscode";
+import { convertToLegacyModeledMethods } from "./modeled-methods-legacy";
 
 // Limit the number of candidates we send to the model in each request
 // to avoid long requests.
@@ -192,11 +193,13 @@ export class AutoModeler {
       filename: "auto-model.yml",
     });
 
-    const loadedMethods = loadDataExtensionYaml(models);
-    if (!loadedMethods) {
+    const rawLoadedMethods = loadDataExtensionYaml(models);
+    if (!rawLoadedMethods) {
       return;
     }
 
+    const loadedMethods = convertToLegacyModeledMethods(rawLoadedMethods);
+
     // Any candidate that was part of the response is a negative result
     // meaning that the canidate is not a sink for the kinds that the LLM is checking for.
     // For now we model this as a sink neutral method, however this is subject

extensions/ql-vscode/src/model-editor/model-editor-view.ts

@@ -34,7 +34,11 @@ import {
 import { Method, Usage } from "./method";
 import { ModeledMethod } from "./modeled-method";
 import { ExtensionPack } from "./shared/extension-pack";
-import { showFlowGeneration, showLlmGeneration } from "../config";
+import {
+  showFlowGeneration,
+  showLlmGeneration,
+  showMultipleModels,
+} from "../config";
 import { Mode } from "./shared/mode";
 import { loadModeledMethods, saveModeledMethods } from "./modeled-method-fs";
 import { pickExtensionPack } from "./extension-pack-picker";
@@ -365,6 +369,7 @@ export class ModelEditorView extends AbstractWebview<
         extensionPack: this.extensionPack,
         showFlowGeneration: showFlowGeneration(),
         showLlmButton,
+        showMultipleModels: showMultipleModels(),
         mode: this.mode,
       },
     });

extensions/ql-vscode/src/model-editor/modeled-method-fs.ts

@@ -10,6 +10,11 @@ import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { load as loadYaml } from "js-yaml";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { pathsEqual } from "../common/files";
+import {
+  convertFromLegacyModeledMethods,
+  convertFromLegacyModeledMethodsFiles,
+  convertToLegacyModeledMethods,
+} from "./modeled-methods-legacy";
 
 export async function saveModeledMethods(
   extensionPack: ExtensionPack,
@@ -29,8 +34,8 @@ export async function saveModeledMethods(
   const yamls = createDataExtensionYamls(
     language,
     methods,
-    modeledMethods,
-    existingModeledMethods,
+    convertFromLegacyModeledMethods(modeledMethods),
+    convertFromLegacyModeledMethodsFiles(existingModeledMethods),
     mode,
   );
 
@@ -68,7 +73,8 @@ async function loadModeledMethodFiles(
       );
       continue;
     }
-    modeledMethodsByFile[modelFile] = modeledMethods;
+    modeledMethodsByFile[modelFile] =
+      convertToLegacyModeledMethods(modeledMethods);
   }
 
   return modeledMethodsByFile;

extensions/ql-vscode/src/model-editor/modeled-methods-legacy.ts

@@ -0,0 +1,33 @@
+import { ModeledMethod } from "./modeled-method";
+
+export function convertFromLegacyModeledMethods(
+  modeledMethods: Record<string, ModeledMethod>,
+): Record<string, ModeledMethod[]> {
+  // Convert a single ModeledMethod to an array of ModeledMethods
+  return Object.fromEntries(
+    Object.entries(modeledMethods).map(([signature, modeledMethod]) => {
+      return [signature, [modeledMethod]];
+    }),
+  );
+}
+
+export function convertToLegacyModeledMethods(
+  modeledMethods: Record<string, ModeledMethod[]>,
+): Record<string, ModeledMethod> {
+  // Always take the first modeled method in the array
+  return Object.fromEntries(
+    Object.entries(modeledMethods).map(([signature, modeledMethods]) => {
+      return [signature, modeledMethods[0]];
+    }),
+  );
+}
+
+export function convertFromLegacyModeledMethodsFiles(
+  modeledMethods: Record<string, Record<string, ModeledMethod>>,
+): Record<string, Record<string, ModeledMethod[]>> {
+  return Object.fromEntries(
+    Object.entries(modeledMethods).map(([filename, modeledMethods]) => {
+      return [filename, convertFromLegacyModeledMethods(modeledMethods)];
+    }),
+  );
+}