Merge pull request #16490 from yoff/python/rich-type-column-MaD

Python: Rich `type` column in MaD

Author: yoff

python/ql/lib/change-notes/2024-05-17-maD-rich-type-column.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* A Python MaD (Models as Data) row may now contain a dotted path in the `type` column. Like in Ruby, a path to a class will refer to instances of that class. This means that the summary `["foo", "Member[MyClass].Instance.Member[instance_method]", "Argument[0]", "ReturnValue", "value"]` can now be written `["foo.MS_Class", "Member[instance_method]", "Argument[0]", "ReturnValue", "value"]`. To refer to an actual class, one may add a `!` at the end of the path.

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -29,7 +29,11 @@ import semmle.python.dataflow.new.DataFlow::DataFlow as DataFlow
 /**
  * Holds if models describing `type` may be relevant for the analysis of this database.
  */
-predicate isTypeUsed(string type) { API::moduleImportExists(type) }
+bindingset[type]
+predicate isTypeUsed(string type) {
+  // If `type` is a path, then it is the first component that should be imported.
+  API::moduleImportExists(type.splitAt(".", 0))
+}
 
 /**
  * Holds if `type` can be obtained from an instance of `otherType` due to
@@ -41,8 +45,59 @@ predicate hasImplicitTypeModel(string type, string otherType) { none() }
 bindingset[type, path]
 API::Node getExtraNodeFromPath(string type, AccessPath path, int n) { none() }
 
+/**
+ * Holds if `type` = `typePath`+`suffix` and `suffix` is either empty or "!".
+ */
+bindingset[type]
+private predicate parseType(string type, string typePath, string suffix) {
+  exists(string regexp |
+    regexp = "([^!]+)(!|)" and
+    typePath = type.regexpCapture(regexp, 1) and
+    suffix = type.regexpCapture(regexp, 2)
+  )
+}
+
+private predicate parseRelevantType(string type, string typePath, string suffix) {
+  isRelevantType(type) and
+  parseType(type, typePath, suffix)
+}
+
+pragma[nomagic]
+private string getTypePathComponent(string typePath, int n) {
+  parseRelevantType(_, typePath, _) and
+  result = typePath.splitAt(".", n)
+}
+
+private int getNumTypePathComponents(string typePath) {
+  result = strictcount(int n | exists(getTypePathComponent(typePath, n)))
+}
+
+private API::Node getNodeFromTypePath(string typePath, int n) {
+  n = 1 and
+  result = API::moduleImport(getTypePathComponent(typePath, 0))
+  or
+  result = getNodeFromTypePath(typePath, n - 1).getMember(getTypePathComponent(typePath, n - 1))
+}
+
+private API::Node getNodeFromTypePath(string typePath) {
+  result = getNodeFromTypePath(typePath, getNumTypePathComponents(typePath))
+}
+
 /** Gets a Python-specific interpretation of the given `type`. */
-API::Node getExtraNodeFromType(string type) { result = API::moduleImport(type) }
+API::Node getExtraNodeFromType(string type) {
+  result = API::moduleImport(type)
+  or
+  exists(string typePath, string suffix, API::Node node |
+    parseRelevantType(type, typePath, suffix) and
+    node = getNodeFromTypePath(typePath)
+  |
+    suffix = "!" and
+    result = node
+    or
+    suffix = "" and
+    result = node.getAnInstance()
+  )
+}
 
 /**
  * Gets a Python-specific API graph successor of `node` reachable by resolving `token`.

python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml

@@ -18,9 +18,10 @@ extensions:
       - ["foo", "Member[MS_spread]", "Argument[0]", "ReturnValue.TupleElement[0]", "value"]
       - ["foo", "Member[MS_spread]", "Argument[1]", "ReturnValue.TupleElement[1]", "value"]
       - ["foo", "Member[MS_spread_all]", "Argument[0]", "ReturnValue.TupleElement[0,1]", "value"]
-      - ["foo", "Member[MS_Class].Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
-      - ["foo", "Member[MS_Class_transitive].Subclass.Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
-      - ["foo", "Member[MS_Class].Instance.Member[instance_method]", "Argument[self]", "ReturnValue.TupleElement[0]", "value"]
-      - ["foo", "Member[MS_Class].Instance.Member[instance_method]", "Argument[0]", "ReturnValue.TupleElement[1]", "value"]
-      - ["foo", "Member[MS_Class].Instance.Member[explicit_self]", "Argument[self:]", "ReturnValue", "value"]
+      - ["foo.MS_Class!", "Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
+      - ["foo.MS_Class_transitive!", "Subclass.Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
+      - ["foo.MS_Class_transitive", "Member[instance_method]", "Argument[0]", "ReturnValue", "value"]
+      - ["foo.MS_Class", "Member[instance_method]", "Argument[self]", "ReturnValue.TupleElement[0]", "value"]
+      - ["foo.MS_Class", "Member[instance_method]", "Argument[0]", "ReturnValue.TupleElement[1]", "value"]
+      - ["foo.MS_Class", "Member[explicit_self]", "Argument[self:]", "ReturnValue", "value"]
       - ["json", "Member[MS_loads]", "Argument[0]", "ReturnValue", "taint"]

python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml

@@ -18,9 +18,10 @@ extensions:
       - ["foo", "Member[MS_spread]", "Argument[0]", "ReturnValue.TupleElement[0]", "value"]
       - ["foo", "Member[MS_spread]", "Argument[1]", "ReturnValue.TupleElement[1]", "value"]
       - ["foo", "Member[MS_spread_all]", "Argument[0]", "ReturnValue.TupleElement[0,1]", "value"]
-      - ["foo", "Member[MS_Class].Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
-      - ["foo", "Member[MS_Class_transitive].Subclass.Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
-      - ["foo", "Member[MS_Class].Instance.Member[instance_method]", "Argument[self]", "ReturnValue.TupleElement[0]", "value"]
-      - ["foo", "Member[MS_Class].Instance.Member[instance_method]", "Argument[0]", "ReturnValue.TupleElement[1]", "value"]
-      - ["foo", "Member[MS_Class].Instance.Member[explicit_self]", "Argument[self:]", "ReturnValue", "value"]
+      - ["foo.MS_Class!", "Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
+      - ["foo.MS_Class_transitive!", "Subclass.Call", "Argument[0, x:]", "ReturnValue.Attribute[config]", "value"]
+      - ["foo.MS_Class_transitive", "Member[instance_method]", "Argument[0]", "ReturnValue", "value"]
+      - ["foo.MS_Class", "Member[instance_method]", "Argument[self]", "ReturnValue.TupleElement[0]", "value"]
+      - ["foo.MS_Class", "Member[instance_method]", "Argument[0]", "ReturnValue.TupleElement[1]", "value"]
+      - ["foo.MS_Class", "Member[explicit_self]", "Argument[self:]", "ReturnValue", "value"]
       - ["json", "Member[MS_loads]", "Argument[0]", "ReturnValue", "taint"]

python/ql/test/library-tests/dataflow/model-summaries/model_summaries.py

@@ -140,6 +140,8 @@ class C(MS_Class_transitive):
 subclass_via_kw = C(x = SOURCE)
 SINK(subclass_via_kw.config)  # $ flow="SOURCE, l:-1 -> subclass_via_kw.config"
 
+SINK(subclass_via_kw.instance_method(SOURCE))  # $ flow="SOURCE -> subclass_via_kw.instance_method(..)"
+
 class D(MS_Class_transitive):
     def __init__(x, y):
         # special handling of y