Merge branch 'main' into amammad-go-fastHttp

Author: am0o0

.github/labeler.yml

@@ -45,11 +45,7 @@ documentation:
 
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
+  - "shared/dataflow/**/*"
 
 "ATM":
   - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/csharp-qltest.yml

@@ -53,7 +53,6 @@ jobs:
         slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/fetch-codeql
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -62,12 +61,7 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
@@ -80,7 +74,24 @@ jobs:
           dotnet-version: 7.0.102
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
           dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+  stubgentest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Run stub generator tests
+        run: |
+          # Generate (Asp)NetCore stubs
+          STUBS_PATH=stubs_output
+          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger latest "$STUBS_PATH"
+          rm -rf ql/test/resources/stubs/_frameworks
+          # Update existing stubs in the repo with the freshly generated ones
+          mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
+          git status
+          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

config/identical-files.json

@@ -28,8 +28,6 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl1.qll"
   ],
   "TaintTracking Legacy Configuration Java/C++/C#/Go/Python/Ruby/Swift": [
@@ -552,4 +550,4 @@
     "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
     "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
-}
+}

cpp/ql/lib/semmle/code/cpp/controlflow/IRGuards.qll

@@ -240,7 +240,7 @@ private class GuardConditionFromIR extends GuardCondition {
    */
   private predicate controlsBlock(BasicBlock controlled, boolean testIsTrue) {
     exists(IRBlock irb |
-      forex(IRGuardCondition inst | inst = ir | inst.controls(irb, testIsTrue)) and
+      ir.controls(irb, testIsTrue) and
       irb.getAnInstruction().getAst().(ControlFlowNode).getBasicBlock() = controlled and
       not isUnreachedBlock(irb)
     )

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowDispatch.qll

@@ -79,13 +79,3 @@ class ArgumentPosition extends int {
 /** Holds if arguments at position `apos` match parameters at position `ppos`. */
 pragma[inline]
 predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
-
-/**
- * Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
- *
- * This is a temporary hook to support technical debt in the Go language; do not use.
- */
-pragma[inline]
-predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
-  any()
-}

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowPrivate.qll

@@ -208,6 +208,8 @@ predicate expectsContent(Node n, ContentSet c) { none() }
 
 predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) { none() }
 
+predicate localMustFlowStep(Node node1, Node node2) { none() }
+
 /** Gets the type of `n` used for type pruning. */
 Type getNodeType(Node n) {
   suppressUnusedNode(n) and
@@ -295,12 +297,3 @@ class ContentApprox = Unit;
 /** Gets an approximated value for content `c`. */
 pragma[inline]
 ContentApprox getContentApprox(Content c) { any() }
-
-/**
- * Gets an additional term that is added to the `join` and `branch` computations to reflect
- * an additional forward or backwards branching factor that is not taken into account
- * when calculating the (virtual) dispatch cost.
- *
- * Argument `arg` is part of a path from a source to a sink, and `p` is the target parameter.
- */
-int getAdditionalFlowIntoCallNodeTerm(ArgumentNode arg, ParameterNode p) { none() }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll

@@ -271,13 +271,3 @@ DataFlowCallable viableImplInCallContext(DataFlowCall call, DataFlowCall ctx) {
 /** Holds if arguments at position `apos` match parameters at position `ppos`. */
 pragma[inline]
 predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) { ppos = apos }
-
-/**
- * Holds if flow from `call`'s argument `arg` to parameter `p` is permissible.
- *
- * This is a temporary hook to support technical debt in the Go language; do not use.
- */
-pragma[inline]
-predicate golangSpecificParamArgFilter(DataFlowCall call, ParameterNode p, ArgumentNode arg) {
-  any()
-}

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplSpecific.qll

@@ -18,4 +18,6 @@ module CppDataFlow implements InputSig {
   import Public
 
   Node exprNode(DataFlowExpr e) { result = Public::exprNode(e) }
+
+  predicate getAdditionalFlowIntoCallNodeTerm = Private::getAdditionalFlowIntoCallNodeTerm/2;
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -804,6 +804,8 @@ predicate expectsContent(Node n, ContentSet c) { none() }
 
 predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) { none() }
 
+predicate localMustFlowStep(Node node1, Node node2) { none() }
+
 /** Gets the type of `n` used for type pruning. */
 DataFlowType getNodeType(Node n) {
   suppressUnusedNode(n) and

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -1066,15 +1066,28 @@ private module GetConvertedResultExpression {
   private import semmle.code.cpp.ir.implementation.raw.internal.TranslatedExpr
   private import semmle.code.cpp.ir.implementation.raw.internal.InstructionTag
 
+  private Operand getAnInitializeDynamicAllocationInstructionAddress() {
+    result = any(InitializeDynamicAllocationInstruction init).getAllocationAddressOperand()
+  }
+
   /**
    * Gets the expression that should be returned as the result expression from `instr`.
    *
-   * Note that this predicate may return multiple results in cases where a conversion belond to a
+   * Note that this predicate may return multiple results in cases where a conversion belongs to a
    * different AST element than its operand.
    */
   Expr getConvertedResultExpression(Instruction instr, int n) {
-    // Only fully converted instructions has a result for `asConvertedExpr`
-    not conversionFlow(unique( | | getAUse(instr)), _, false, false) and
+    // Only fully converted instructions have a result for `asConvertedExpr`
+    not conversionFlow(unique(Operand op |
+        // The address operand of a `InitializeDynamicAllocationInstruction` is
+        // special: we need to handle it during dataflow (since it's
+        // effectively a store to an indirection), but it doesn't appear in
+        // source syntax, so dataflow node <-> expression conversion shouldn't
+        // care about it.
+        op = getAUse(instr) and not op = getAnInitializeDynamicAllocationInstructionAddress()
+      |
+        op
+      ), _, false, false) and
     result = getConvertedResultExpressionImpl(instr) and
     n = 0
     or
@@ -1341,6 +1354,9 @@ class ParameterNode extends Node {
    * pointer-indirection parameters are at further negative positions.
    */
   predicate isParameterOf(Function f, ParameterPosition pos) { none() } // overridden by subclasses
+
+  /** Gets the `Parameter` associated with this node, if it exists. */
+  Parameter getParameter() { none() } // overridden by subclasses
 }
 
 /** An explicit positional parameter, including `this`, but not `...`. */
@@ -1363,10 +1379,9 @@ private class ExplicitParameterNode extends ParameterNode, DirectParameterNode {
     f.getParameter(pos.(DirectPosition).getIndex()) = instr.getParameter()
   }
 
-  /** Gets the `Parameter` associated with this node. */
-  Parameter getParameter() { result = instr.getParameter() }
-
   override string toStringImpl() { result = instr.getParameter().toString() }
+
+  override Parameter getParameter() { result = instr.getParameter() }
 }
 
 /** An implicit `this` parameter. */