Merge remote-tracking branch 'origin/main' into koesie10/type-model

Author: koesie10

docs/releasing.md

@@ -1,80 +1,62 @@
 # Releasing (write access required)
 
-1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
+1. Determine the new version number. We default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
+    * Making substantial new features available to all users. This can include lifting a feature flag.
+    * Breakage in compatibility with recent versions of the CLI.
+    * Minimum required version of VS Code is increased.
+    * New telemetry events are added.
+    * Deprecation or removal of commands.
+    * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
+1. Create a release branch named after the new version (e.g. `v1.3.6`):
+    * For a regular scheduled release this branch will be based on latest `main`.
+        * Make sure your local copy of `main` is up to date so you are including all changes.
+    * To do a minimal bug-fix release, base the release branch on the tag from the most recent release and then add only the changes you want to release.
+        * Choose this option if you want to release a specific set of changes (e.g. a bug fix) and don't want to incur extra risk by including other changes that have been merged to the `main` branch.
+
+        ```bash
+        git checkout -b <new_release_branch> <previous_release_tag>
+        ```
+
+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green.
+    * You can skip this step if you are releasing from `main` and there were no merges since the most recent daily scheduled run of this workflow.
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
-    * Go through all recent PRs and make sure they are properly accounted for.
+    * Go through PRs that have been merged since the previous release and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
-    * For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
-      * Making substantial new features available to all users. This can include lifting a feature flag.
-      * Breakage in compatibility with recent versions of the CLI.
-      * Minimum required version of VS Code is increased.
-      * New telemetry events are added.
-      * Deprecation or removal of commands.
-      * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. See the [Node.js version instructions](./node-version.md) for more information.
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
-1. Create a PR for this release:
-    * This PR will contain any missing bits from steps 1, 2 and 3. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
-    * Create a new branch for the release named after the new version. For example: `v1.3.6`
-    * Create a new commit with a message the same as the branch name.
-    * Create a PR for this branch.
-    * Wait for the PR to be merged into `main`
-1. Switch to `main` branch and pull latest changes
-1. Lock the `main` branch.
-    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-    * Select "Lock branch"
-    * Click "Save changes"
-1. Ensure that no PRs have been merged since the release PR that you merged. If there were, you might need to unlock `main` temporarily and update the CHANGELOG again.
-1. Build the extension `npm run build` and install it on your VS Code using "Install from VSIX".
+1. Commit any changes made during steps 4 and 5 with a commit message the same as the branch name (e.g. `v1.3.6`).
+1. Open a PR for this release.
+    * The PR diff should contain:
+        * Any missing bits from steps 4 and 5. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
+        * If releasing from a branch other than `main`, this PR will also contain the extension changes being released.
+1. Build the extension using `npm run build` and install it on your VS Code using "Install from VSIX".
 1. Go through [our test plan](./test-plan.md) to ensure that the extension is working as expected.
-1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+1. Create a new tag on the release branch with your new version (named after the release), e.g.
 
     ```bash
-    git checkout main
     git tag v1.3.6
     ```
 
-   If you've accidentally created a badly named tag, you can delete it via
-
-   ```bash
-   git tag -d badly-named-tag
-   ```
-
-1. Unlock the main branch
-    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-    * Deselect "Lock branch"
-    * Click "Save changes"
+1. Merge the release PR into `main`.
+    * If there are conflicts in the changelog, make sure to place any new changelog entries at the top, above the section for the current release, as these new entries are not part of the current release and should be placed in the "unreleased" section.
+    * The release PR must be merged before pushing the tag to ensure that we always release a commit that is present on the `main` branch. It's not required that the commit is the head of the `main` branch, but there should be no chance of a future release accidentally not including changes from this release.
 1. Push the new tag up:
 
-   a. If you're using a fork of the repo:
-
-    ```bash
-    git push upstream refs/tags/v1.3.6
-    ```
-
-   b. If you're working straight in this repo:
-
     ```bash
     git push origin refs/tags/v1.3.6
     ```
 
-   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
-
-    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
-    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
-1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
+1. Find the [Release](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease) workflow run that was just triggered by pushing the tag, and monitor the status of the release build.
     * DO NOT approve the "publish" stages of the workflow yet.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
 1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
-1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+1. Approve the deployments of the [Release](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease) workflow run. This will automatically publish to Open VSX and VS Code Marketplace.
     * If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
-1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
-1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
-1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
+1. Go to the draft GitHub release in [the releases page](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release.
 1. If documentation changes need to be published, notify documentation team that release has been made.
-1. Review and merge the version bump PR that is automatically created by Actions.
+1. Review and merge the version bump PR that is automatically created by the Release workflow.
 
 ## Secrets and authentication for publishing
 

extensions/ql-vscode/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [UNRELEASED]
 
+- The "Sort by Language" action in the databases view now sorts by name within each language. [#3055](https://github.com/github/vscode-codeql/pull/3055)
+
+## 1.9.4 - 6 November 2023
+
+No user facing changes.
+
 ## 1.9.3 - 26 October 2023
 
 - Sorted result set filenames now include a hash of the result set name instead of the full name. [#2955](https://github.com/github/vscode-codeql/pull/2955)

extensions/ql-vscode/package-lock.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.9.4",
+  "version": "1.9.5",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",

extensions/ql-vscode/package.json

@@ -158,9 +158,17 @@ class DatabaseTreeDataProvider
           case SortOrder.NameDesc:
             return db2.name.localeCompare(db1.name, env.language);
           case SortOrder.LanguageAsc:
-            return db1.language.localeCompare(db2.language, env.language);
+            return (
+              db1.language.localeCompare(db2.language, env.language) ||
+              // If the languages are the same, sort by name
+              db1.name.localeCompare(db2.name, env.language)
+            );
           case SortOrder.LanguageDesc:
-            return db2.language.localeCompare(db1.language, env.language);
+            return (
+              db2.language.localeCompare(db1.language, env.language) ||
+              // If the languages are the same, sort by name
+              db2.name.localeCompare(db1.name, env.language)
+            );
           case SortOrder.DateAddedAsc:
             return (db1.dateAdded || 0) - (db2.dateAdded || 0);
           case SortOrder.DateAddedDesc:

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -40,17 +40,19 @@ function makeKey(
 const DEPENDENT_PREDICATES_REGEXP = (() => {
   const regexps = [
     // SCAN id
-    String.raw`SCAN\s+([0-9a-zA-Z:#_]+)\s`,
+    String.raw`SCAN\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s`,
     // JOIN id WITH id
-    String.raw`JOIN\s+([0-9a-zA-Z:#_]+)\s+WITH\s+([0-9a-zA-Z:#_]+)\s`,
+    String.raw`JOIN\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s+WITH\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s`,
     // AGGREGATE id, id
-    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+)\s*,\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s*,\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)`,
     // id AND NOT id
-    String.raw`([0-9a-zA-Z:#_]+)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)`,
     // INVOKE HIGHER-ORDER RELATION rel ON <id, ..., id>
-    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+)((?:,[0-9a-zA-Z:#_<>]+)*)>`,
+    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+|\`[^\`\r\n]*\`)((?:,[0-9a-zA-Z:#_<>]+|,\`[^\`\r\n]*\`)*)>`,
     // SELECT id
-    String.raw`SELECT\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`SELECT\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)`,
+    // REWRITE id WITH
+    String.raw`REWRITE\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s+WITH\s`,
   ];
   return new RegExp(
     `${String.raw`\{[0-9]+\}\s+[0-9a-zA-Z]+\s=\s(?:` + regexps.join("|")})`,
@@ -65,7 +67,12 @@ function getDependentPredicates(operations: string[]): I.List<string> {
         .rest() // Skip the first group as it's just the entire string
         .filter((x) => !!x && !x.match("r[0-9]+|PRIMITIVE")) // Only keep the references to predicates.
         .flatMap((x) => x.split(",")) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
-        .filter((x) => !!x); // Remove empty strings
+        .filter((x) => !!x) // Remove empty strings
+        .map((x) =>
+          x.startsWith("`") && x.endsWith("`")
+            ? x.substring(1, x.length - 1)
+            : x,
+        ); // Remove quotes from quoted identifiers
     } else {
       return I.List();
     }

extensions/ql-vscode/src/log-insights/join-order.ts

@@ -208,27 +208,6 @@ export class AutoModeler {
       return;
     }
 
-    // Any candidate that was part of the response is a negative result
-    // meaning that the canidate is not a sink for the kinds that the LLM is checking for.
-    // For now we model this as a sink neutral method, however this is subject
-    // to discussion.
-    for (const candidate of candidateMethods) {
-      if (!(candidate.signature in loadedMethods)) {
-        loadedMethods[candidate.signature] = [
-          {
-            type: "neutral",
-            kind: "sink",
-            provenance: "ai-generated",
-            signature: candidate.signature,
-            packageName: candidate.packageName,
-            typeName: candidate.typeName,
-            methodName: candidate.methodName,
-            methodParameters: candidate.methodParameters,
-          },
-        ];
-      }
-    }
-
     await this.addModeledMethods(loadedMethods);
   }
 

extensions/ql-vscode/src/model-editor/auto-modeler.ts

@@ -1,4 +1,4 @@
-import { MethodDefinition } from "../method";
+import { MethodArgument, MethodDefinition } from "../method";
 import {
   ModeledMethod,
   NeutralModeledMethod,
@@ -47,6 +47,11 @@ export type ModelsAsDataLanguagePredicates = {
   type?: ModelsAsDataLanguagePredicate<TypeModeledMethod>;
 };
 
+export type MethodArgumentOptions = {
+  options: MethodArgument[];
+  defaultArgumentPath: string;
+};
+
 export type ModelsAsDataLanguage = {
   /**
    * The modes that are available for this language. If not specified, all
@@ -56,4 +61,9 @@ export type ModelsAsDataLanguage = {
   createMethodSignature: (method: MethodDefinition) => string;
   predicates: ModelsAsDataLanguagePredicates;
   modelGeneration?: ModelsAsDataLanguageModelGeneration;
+  /**
+   * Returns the list of valid arguments that can be selected for the given method.
+   * @param method The method to get the valid arguments for.
+   */
+  getArgumentOptions: (method: MethodDefinition) => MethodArgumentOptions;
 };

extensions/ql-vscode/src/model-editor/languages/models-as-data.ts

@@ -2,6 +2,7 @@ import { ModelsAsDataLanguage } from "../models-as-data";
 import { sharedExtensiblePredicates, sharedKinds } from "../shared";
 import { Mode } from "../../shared/mode";
 import { parseGenerateModelResults } from "./generate";
+import { getArgumentsList, MethodArgument } from "../../method";
 
 function parseRubyMethodFromPath(path: string): string {
   const match = path.match(/Method\[([^\]]+)].*/);
@@ -181,4 +182,34 @@ export const ruby: ModelsAsDataLanguage = {
     },
     parseResults: parseGenerateModelResults,
   },
+  getArgumentOptions: (method) => {
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => {
+        if (argument.endsWith(":")) {
+          return {
+            path: `Argument[${argument}]`,
+            label: `Argument[${argument}]`,
+          };
+        }
+
+        return {
+          path: `Argument[${index}]`,
+          label: `Argument[${index}]: ${argument}`,
+        };
+      },
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[self]",
+          label: "Argument[self]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[self]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[self]",
+    };
+  },
 };

extensions/ql-vscode/src/model-editor/languages/ruby/index.ts

@@ -3,6 +3,7 @@ import { Provenance } from "../../modeled-method";
 import { DataTuple } from "../../model-extension-file";
 import { sharedExtensiblePredicates, sharedKinds } from "../shared";
 import { filterFlowModelQueries, parseFlowModelResults } from "./generate";
+import { getArgumentsList, MethodArgument } from "../../method";
 
 function readRowToMethod(row: DataTuple[]): string {
   return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
@@ -145,4 +146,25 @@ export const staticLanguage: ModelsAsDataLanguage = {
     filterQueries: filterFlowModelQueries,
     parseResults: parseFlowModelResults,
   },
+  getArgumentOptions: (method) => {
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => ({
+        path: `Argument[${index}]`,
+        label: `Argument[${index}]: ${argument}`,
+      }),
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[this]",
+          label: "Argument[this]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[this]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[this]",
+    };
+  },
 };