Merge branch 'main' into nsstring

Author: geoffw0

.github/actions/cache-query-compilation/action.yml

@@ -43,12 +43,83 @@ runs:
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
     - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      shell: bash
-      run: |
-        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
-
-        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env:
+      uses: actions/github-script@v6
+      env: 
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
+      with:
+        script: |
+          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+          // mkdir -p ${COMBINED_CACHE_DIR}
+          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+          // # copy the contents of the .cache folders into the combined cache folder.
+          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+          // # clean up the .cache folders
+          // rm -rf **/.cache/*
+
+          const fs = require("fs");
+          const path = require("path");
+
+          // the first argv is the cache folder to create.
+          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
+
+          function* walkCaches(dir) {
+            const files = fs.readdirSync(dir, { withFileTypes: true });
+            for (const file of files) {
+              if (file.isDirectory()) {
+                const filePath = path.join(dir, file.name);
+                yield* walkCaches(filePath);
+                if (file.name === ".cache") {
+                  yield filePath;
+                }
+              }
+            }
+          }
+
+          async function copyDir(src, dest) {
+            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+              const srcPath = path.join(src, file.name);
+              const destPath = path.join(dest, file.name);
+              if (file.isDirectory()) {
+                if (!fs.existsSync(destPath)) {
+                  fs.mkdirSync(destPath);
+                }
+                await copyDir(srcPath, destPath);
+              } else {
+                await fs.promises.copyFile(srcPath, destPath);
+              }
+            }
+          }
+
+          async function main() {
+            const cacheDirs = [...walkCaches(".")];
+
+            for (const dir of cacheDirs) {
+              console.log(`Found .cache dir at ${dir}`);
+            }
+
+            // mkdir -p ${COMBINED_CACHE_DIR}
+            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+            await Promise.all(
+              cacheDirs.map((cacheDir) =>
+                (async function () {
+                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+                })()
+              )
+            );
+
+            // # copy the contents of the .cache folders into the combined cache folder.
+            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+            await Promise.all(
+              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+            );
+
+            // # clean up the .cache folders
+            // rm -rf **/.cache/*
+            await Promise.all(
+              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+            );
+          }
+          main();

.github/actions/cache-query-compilation/move-caches.js

@@ -12,10 +12,10 @@ jobs:
     name: Test MacOS
     runs-on: macos-latest
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code
@@ -47,10 +47,10 @@ jobs:
     name: Test Windows
     runs-on: windows-latest-xl
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code

.github/workflows/go-tests-other-os.yml

@@ -20,10 +20,10 @@ jobs:
     name: Test Linux (Ubuntu)
     runs-on: ubuntu-latest-xl
     steps:
-      - name: Set up Go 1.19
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.0
         id: go
 
       - name: Check out code

.github/workflows/go-tests.yml

@@ -5,13 +5,6 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
-    paths:
-      - "ql/**"
-      - "**.qll"
-      - "**.ql"
-      - "**.dbscheme"
-      - "**/qlpack.yml"
-      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/ql-for-ql-build.yml

@@ -131,6 +131,14 @@ IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -667,23 +667,78 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(NodeEx arg |
-      fwdFlow(arg, _, config) and
-      viableParamArgEx(_, node, arg) and
-      cc = true and
-      not fullBarrier(node, config)
-    )
+    fwdFlowIn(_, _, _, node, config) and
+    cc = true
     or
     // flow out of a callable
+    fwdFlowOut(_, node, false, config) and
+    cc = false
+    or
+    // flow through a callable
     exists(DataFlowCall call |
-      fwdFlowOut(call, node, false, config) and
-      cc = false
-      or
       fwdFlowOutFromArg(call, node, config) and
       fwdFlowIsEntered(call, cc, config)
     )
   }
 
+  // inline to reduce the number of iterations
+  pragma[inline]
+  private predicate fwdFlowIn(
+    DataFlowCall call, NodeEx arg, Cc cc, ParamNodeEx p, Configuration config
+  ) {
+    // call context cannot help reduce virtual dispatch
+    fwdFlow(arg, cc, config) and
+    viableParamArgEx(call, p, arg) and
+    not fullBarrier(p, config) and
+    (
+      cc = false
+      or
+      cc = true and
+      not reducedViableImplInCallContext(call, _, _)
+    )
+    or
+    // call context may help reduce virtual dispatch
+    exists(DataFlowCallable target |
+      fwdFlowInReducedViableImplInSomeCallContext(call, arg, p, target, config) and
+      target = viableImplInSomeFwdFlowCallContextExt(call, config) and
+      cc = true
+    )
+  }
+
+  /**
+   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
+   */
+  pragma[nomagic]
+  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
+    fwdFlowIn(call, _, cc, _, config)
+  }
+
+  pragma[nomagic]
+  private predicate fwdFlowInReducedViableImplInSomeCallContext(
+    DataFlowCall call, NodeEx arg, ParamNodeEx p, DataFlowCallable target, Configuration config
+  ) {
+    fwdFlow(arg, true, config) and
+    viableParamArgEx(call, p, arg) and
+    reducedViableImplInCallContext(call, _, _) and
+    target = p.getEnclosingCallable() and
+    not fullBarrier(p, config)
+  }
+
+  /**
+   * Gets a viable dispatch target of `call` in the context `ctx`. This is
+   * restricted to those `call`s for which a context might make a difference,
+   * and to `ctx`s that are reachable in `fwdFlow`.
+   */
+  pragma[nomagic]
+  private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(
+    DataFlowCall call, Configuration config
+  ) {
+    exists(DataFlowCall ctx |
+      fwdFlowIsEntered(ctx, _, config) and
+      result = viableImplInCallContextExt(call, ctx)
+    )
+  }
+
   private predicate fwdFlow(NodeEx node, Configuration config) { fwdFlow(node, _, config) }
 
   pragma[nomagic]
@@ -726,7 +781,8 @@ private module Stage1 implements StageSig {
     )
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc, Configuration config) {
     exists(ReturnPosition pos |
       fwdFlowReturnPosition(pos, cc, config) and
@@ -740,17 +796,6 @@ private module Stage1 implements StageSig {
     fwdFlowOut(call, out, true, config)
   }
 
-  /**
-   * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
-   */
-  pragma[nomagic]
-  private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc, Configuration config) {
-    exists(ArgNodeEx arg |
-      fwdFlow(arg, cc, config) and
-      viableParamArgEx(call, _, arg)
-    )
-  }
-
   private predicate stateStepFwd(FlowState state1, FlowState state2, Configuration config) {
     exists(NodeEx node1 |
       additionalLocalStateStep(node1, state1, _, state2, config) or
@@ -817,20 +862,21 @@ private module Stage1 implements StageSig {
     )
     or
     // flow into a callable
-    exists(DataFlowCall call |
-      revFlowIn(call, node, false, config) and
-      toReturn = false
-      or
-      revFlowInToReturn(call, node, config) and
-      revFlowIsReturned(call, toReturn, config)
-    )
+    revFlowIn(_, node, false, config) and
+    toReturn = false
     or
     // flow out of a callable
     exists(ReturnPosition pos |
       revFlowOut(pos, config) and
       node.(RetNodeEx).getReturnPosition() = pos and
       toReturn = true
     )
+    or
+    // flow through a callable
+    exists(DataFlowCall call |
+      revFlowInToReturn(call, node, config) and
+      revFlowIsReturned(call, toReturn, config)
+    )
   }
 
   /**
@@ -886,11 +932,11 @@ private module Stage1 implements StageSig {
   additional predicate viableParamArgNodeCandFwd1(
     DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, Configuration config
   ) {
-    viableParamArgEx(call, p, arg) and
-    fwdFlow(arg, config)
+    fwdFlowIn(call, arg, _, p, config)
   }
 
-  pragma[nomagic]
+  // inline to reduce the number of iterations
+  pragma[inline]
   private predicate revFlowIn(
     DataFlowCall call, ArgNodeEx arg, boolean toReturn, Configuration config
   ) {