Support data extensions

asgerf · asgerf · commit 5af1b367c79a · 2022-12-07T11:35:05.000+01:00
diff --git a/config/identical-files.json b/config/identical-files.json
@@ -537,6 +537,11 @@
     "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
     "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
   ],
+  "ApiGraphModelsExtensions": [
+    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsExtensions.qll",
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModelsExtensions.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll"
+  ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
     "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
diff --git a/javascript/ql/lib/qlpack.yml b/javascript/ql/lib/qlpack.yml
@@ -7,3 +7,5 @@ library: true
 upgrades: upgrades
 dependencies:
   codeql/regex: ${workspace}
+dataExtensions:
+  - semmle/javascript/frameworks/**/model.yml
diff --git a/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll b/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll
@@ -72,6 +72,7 @@ private module API = Specific::API;
 private module DataFlow = Specific::DataFlow;
 
 private import Specific::AccessPathSyntax
+private import ApiGraphModelsExtensions as Extensions
 
 /** Module containing hooks for providing input data to be interpreted as a model. */
 module ModelInput {
@@ -236,6 +237,8 @@ predicate sourceModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sourceModel(type, path, kind)
 }
 
 /** Holds if a sink model exists for the given parameters. */
@@ -246,6 +249,8 @@ private predicate sinkModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sinkModel(type, path, kind)
 }
 
 /** Holds if a summary model `row` exists for the given parameters. */
@@ -258,6 +263,8 @@ private predicate summaryModel(string type, string path, string input, string ou
     row.splitAt(";", 3) = output and
     row.splitAt(";", 4) = kind
   )
+  or
+  Extensions::summaryModel(type, path, input, output, kind)
 }
 
 /** Holds if a type model exists for the given parameters. */
@@ -268,6 +275,8 @@ private predicate typeModel(string type1, string type2, string path) {
     row.splitAt(";", 1) = type2 and
     row.splitAt(";", 2) = path
   )
+  or
+  Extensions::typeModel(type1, type2, path)
 }
 
 /** Holds if a type variable model exists for the given parameters. */
@@ -277,6 +286,8 @@ private predicate typeVariableModel(string name, string path) {
     row.splitAt(";", 0) = name and
     row.splitAt(";", 1) = path
   )
+  or
+  Extensions::typeVariableModel(name, path)
 }
 
 /**
diff --git a/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsExtensions.qll b/javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsExtensions.qll
@@ -0,0 +1,36 @@
+/**
+ * Defines extensible predicates for contributing library models from data extensions.
+ */
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a flow
+ * source of the given `kind`.
+ *
+ * The kind `remote` represents a general remote flow source.
+ */
+extensible predicate sourceModel(string type, string path, string kind);
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a sink
+ * of the given `kind`.
+ */
+extensible predicate sinkModel(string type, string path, string kind);
+
+/**
+ * Holds if calls to `(type, path)`, the value referred to by `input`
+ * can flow to the value referred to by `output`.
+ *
+ * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
+ * respectively.
+ */
+extensible predicate summaryModel(string type, string path, string input, string output, string kind);
+
+/**
+ * Holds if `(type2, path)` should be seen as an instance of `type1`.
+ */
+extensible predicate typeModel(string type1, string type2, string path);
+
+/**
+ * Holds if `path` can be substituted for a token `TypeVar[name]`.
+ */
+extensible predicate typeVariableModel(string name, string path);
diff --git a/javascript/ql/lib/semmle/javascript/frameworks/data/internal/model.yml b/javascript/ql/lib/semmle/javascript/frameworks/data/internal/model.yml
@@ -0,0 +1,26 @@
+extensions:
+  # Contribute empty data sets to avoid errors about an undefined extensionals
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: sourceModel
+    data: []
+
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: sinkModel
+    data: []
+
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: summaryModel
+    data: []
+
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: typeModel
+    data: []
+
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: typeVariableModel
+    data: []
diff --git a/python/ql/lib/qlpack.yml b/python/ql/lib/qlpack.yml
@@ -6,4 +6,6 @@ extractor: python
 library: true
 upgrades: upgrades
 dependencies:
-  codeql/regex: ${workspace}
+  codeql/regex: ${workspace}
+dataExtensions:
+  - semmle/python/frameworks/**/model.yml
diff --git a/python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll b/python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll
@@ -72,6 +72,7 @@ private module API = Specific::API;
 private module DataFlow = Specific::DataFlow;
 
 private import Specific::AccessPathSyntax
+private import ApiGraphModelsExtensions as Extensions
 
 /** Module containing hooks for providing input data to be interpreted as a model. */
 module ModelInput {
@@ -236,6 +237,8 @@ predicate sourceModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sourceModel(type, path, kind)
 }
 
 /** Holds if a sink model exists for the given parameters. */
@@ -246,6 +249,8 @@ private predicate sinkModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sinkModel(type, path, kind)
 }
 
 /** Holds if a summary model `row` exists for the given parameters. */
@@ -258,6 +263,8 @@ private predicate summaryModel(string type, string path, string input, string ou
     row.splitAt(";", 3) = output and
     row.splitAt(";", 4) = kind
   )
+  or
+  Extensions::summaryModel(type, path, input, output, kind)
 }
 
 /** Holds if a type model exists for the given parameters. */
@@ -268,6 +275,8 @@ private predicate typeModel(string type1, string type2, string path) {
     row.splitAt(";", 1) = type2 and
     row.splitAt(";", 2) = path
   )
+  or
+  Extensions::typeModel(type1, type2, path)
 }
 
 /** Holds if a type variable model exists for the given parameters. */
@@ -277,6 +286,8 @@ private predicate typeVariableModel(string name, string path) {
     row.splitAt(";", 0) = name and
     row.splitAt(";", 1) = path
   )
+  or
+  Extensions::typeVariableModel(name, path)
 }
 
 /**
diff --git a/python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll b/python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsExtensions.qll
@@ -0,0 +1,36 @@
+/**
+ * Defines extensible predicates for contributing library models from data extensions.
+ */
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a flow
+ * source of the given `kind`.
+ *
+ * The kind `remote` represents a general remote flow source.
+ */
+extensible predicate sourceModel(string type, string path, string kind);
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a sink
+ * of the given `kind`.
+ */
+extensible predicate sinkModel(string type, string path, string kind);
+
+/**
+ * Holds if calls to `(type, path)`, the value referred to by `input`
+ * can flow to the value referred to by `output`.
+ *
+ * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
+ * respectively.
+ */
+extensible predicate summaryModel(string type, string path, string input, string output, string kind);
+
+/**
+ * Holds if `(type2, path)` should be seen as an instance of `type1`.
+ */
+extensible predicate typeModel(string type1, string type2, string path);
+
+/**
+ * Holds if `path` can be substituted for a token `TypeVar[name]`.
+ */
+extensible predicate typeVariableModel(string name, string path);
diff --git a/python/ql/lib/semmle/python/frameworks/data/internal/model.yml b/python/ql/lib/semmle/python/frameworks/data/internal/model.yml
@@ -0,0 +1,26 @@
+extensions:
+  # Contribute empty data sets to avoid errors about an undefined extensionals
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sourceModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sinkModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: summaryModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeVariableModel
+    data: []
diff --git a/ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll b/ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll
@@ -72,6 +72,7 @@ private module API = Specific::API;
 private module DataFlow = Specific::DataFlow;
 
 private import Specific::AccessPathSyntax
+private import ApiGraphModelsExtensions as Extensions
 
 /** Module containing hooks for providing input data to be interpreted as a model. */
 module ModelInput {
@@ -236,6 +237,8 @@ predicate sourceModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sourceModel(type, path, kind)
 }
 
 /** Holds if a sink model exists for the given parameters. */
@@ -246,6 +249,8 @@ private predicate sinkModel(string type, string path, string kind) {
     row.splitAt(";", 1) = path and
     row.splitAt(";", 2) = kind
   )
+  or
+  Extensions::sinkModel(type, path, kind)
 }
 
 /** Holds if a summary model `row` exists for the given parameters. */
@@ -258,6 +263,8 @@ private predicate summaryModel(string type, string path, string input, string ou
     row.splitAt(";", 3) = output and
     row.splitAt(";", 4) = kind
   )
+  or
+  Extensions::summaryModel(type, path, input, output, kind)
 }
 
 /** Holds if a type model exists for the given parameters. */
@@ -268,6 +275,8 @@ private predicate typeModel(string type1, string type2, string path) {
     row.splitAt(";", 1) = type2 and
     row.splitAt(";", 2) = path
   )
+  or
+  Extensions::typeModel(type1, type2, path)
 }
 
 /** Holds if a type variable model exists for the given parameters. */
@@ -277,6 +286,8 @@ private predicate typeVariableModel(string name, string path) {
     row.splitAt(";", 0) = name and
     row.splitAt(";", 1) = path
   )
+  or
+  Extensions::typeVariableModel(name, path)
 }
 
 /**
diff --git a/ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModelsExtensions.qll b/ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModelsExtensions.qll
@@ -0,0 +1,36 @@
+/**
+ * Defines extensible predicates for contributing library models from data extensions.
+ */
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a flow
+ * source of the given `kind`.
+ *
+ * The kind `remote` represents a general remote flow source.
+ */
+extensible predicate sourceModel(string type, string path, string kind);
+
+/**
+ * Holds if the value at `(type, path)` should be seen as a sink
+ * of the given `kind`.
+ */
+extensible predicate sinkModel(string type, string path, string kind);
+
+/**
+ * Holds if calls to `(type, path)`, the value referred to by `input`
+ * can flow to the value referred to by `output`.
+ *
+ * `kind` should be either `value` or `taint`, for value-preserving or taint-preserving steps,
+ * respectively.
+ */
+extensible predicate summaryModel(string type, string path, string input, string output, string kind);
+
+/**
+ * Holds if `(type2, path)` should be seen as an instance of `type1`.
+ */
+extensible predicate typeModel(string type1, string type2, string path);
+
+/**
+ * Holds if `path` can be substituted for a token `TypeVar[name]`.
+ */
+extensible predicate typeVariableModel(string name, string path);
diff --git a/ruby/ql/lib/codeql/ruby/frameworks/data/internal/model.yml b/ruby/ql/lib/codeql/ruby/frameworks/data/internal/model.yml
@@ -0,0 +1,26 @@
+extensions:
+  # Contribute empty data sets to avoid errors about an undefined extensionals
+  - addsTo:
+      pack: codeql/ruby-all
+      extensible: sourceModel
+    data: []
+
+  - addsTo:
+      pack: codeql/ruby-all
+      extensible: sinkModel
+    data: []
+
+  - addsTo:
+      pack: codeql/ruby-all
+      extensible: summaryModel
+    data: []
+
+  - addsTo:
+      pack: codeql/ruby-all
+      extensible: typeModel
+    data: []
+
+  - addsTo:
+      pack: codeql/ruby-all
+      extensible: typeVariableModel
+    data: []
diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml
@@ -8,3 +8,6 @@ library: true
 dependencies:
   codeql/ssa: ${workspace}
   codeql/regex: ${workspace}
+  codeql/ssa: 0.0.1
+dataExtensions:
+  - codeql/ruby/frameworks/**/model.yml

Original file line number	Diff line number	Diff line change
`@@ -72,6 +72,7 @@ private module API = Specific::API;`
`72`	`72`	`private module DataFlow = Specific::DataFlow;`
`73`	`73`
`74`	`74`	`private import Specific::AccessPathSyntax`
	`75`	`+private import ApiGraphModelsExtensions as Extensions`
`75`	`76`
`76`	`77`	`/** Module containing hooks for providing input data to be interpreted as a model. */`
`77`	`78`	`module ModelInput {`
`@@ -236,6 +237,8 @@ predicate sourceModel(string type, string path, string kind) {`
`236`	`237`	`row.splitAt(";", 1) = path and`
`237`	`238`	`row.splitAt(";", 2) = kind`
`238`	`239`	`)`
	`240`	`+ or`
	`241`	`+ Extensions::sourceModel(type, path, kind)`
`239`	`242`	`}`
`240`	`243`
`241`	`244`	`/** Holds if a sink model exists for the given parameters. */`
`@@ -246,6 +249,8 @@ private predicate sinkModel(string type, string path, string kind) {`
`246`	`249`	`row.splitAt(";", 1) = path and`
`247`	`250`	`row.splitAt(";", 2) = kind`
`248`	`251`	`)`
	`252`	`+ or`
	`253`	`+ Extensions::sinkModel(type, path, kind)`
`249`	`254`	`}`
`250`	`255`
`251`	`256`	/** Holds if a summary model `row` exists for the given parameters. */
`@@ -258,6 +263,8 @@ private predicate summaryModel(string type, string path, string input, string ou`
`258`	`263`	`row.splitAt(";", 3) = output and`
`259`	`264`	`row.splitAt(";", 4) = kind`
`260`	`265`	`)`
	`266`	`+ or`
	`267`	`+ Extensions::summaryModel(type, path, input, output, kind)`
`261`	`268`	`}`
`262`	`269`
`263`	`270`	`/** Holds if a type model exists for the given parameters. */`
`@@ -268,6 +275,8 @@ private predicate typeModel(string type1, string type2, string path) {`
`268`	`275`	`row.splitAt(";", 1) = type2 and`
`269`	`276`	`row.splitAt(";", 2) = path`
`270`	`277`	`)`
	`278`	`+ or`
	`279`	`+ Extensions::typeModel(type1, type2, path)`
`271`	`280`	`}`
`272`	`281`
`273`	`282`	`/** Holds if a type variable model exists for the given parameters. */`
`@@ -277,6 +286,8 @@ private predicate typeVariableModel(string name, string path) {`
`277`	`286`	`row.splitAt(";", 0) = name and`
`278`	`287`	`row.splitAt(";", 1) = path`
`279`	`288`	`)`
	`289`	`+ or`
	`290`	`+ Extensions::typeVariableModel(name, path)`
`280`	`291`	`}`
`281`	`292`
`282`	`293`	`/**`