Merge pull request #12871 from aibaars/py-yaml

Python: add YAML support

Author: aibaars

python/downgrades/0355ecf0ac589e66467a378e0e9d60f41ee4a757/old.dbscheme

@@ -0,0 +1,10 @@
+description: Add YAML tables
+compatibility: backwards
+
+yaml.rel: delete
+yaml_anchors.rel: delete
+yaml_aliases.rel: delete
+yaml_scalars.rel: delete
+yaml_errors.rel: delete
+yaml_locations.rel: delete
+

python/downgrades/0355ecf0ac589e66467a378e0e9d60f41ee4a757/semmlecode.python.dbscheme

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added support for querying the contents of YAML files.

python/downgrades/0355ecf0ac589e66467a378e0e9d60f41ee4a757/upgrade.properties

@@ -9,5 +9,6 @@ dependencies:
   codeql/regex: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
+  codeql/yaml: ${workspace}
 dataExtensions:
   - semmle/python/frameworks/**/model.yml

python/ql/lib/change-notes/2023-04-20-yaml.md

@@ -8,6 +8,7 @@
 
 import python
 import semmle.python.RegexTreeView
+import semmle.python.Yaml
 
 private newtype TPrintAstConfiguration = MkPrintAstConfiguration()
 
@@ -53,7 +54,9 @@ private newtype TPrintAstNode =
     shouldPrint(list.getAnItem(), _) and
     not list = any(Module mod).getBody() and
     not forall(AstNode child | child = list.getAnItem() | isNotNeeded(child))
-  }
+  } or
+  TYamlNode(YamlNode node) or
+  TYamlMappingNode(YamlMapping mapping, int i) { exists(mapping.getKeyNode(i)) }
 
 /**
  * A node in the output tree.
@@ -633,6 +636,80 @@ private module PrettyPrinting {
   }
 }
 
+/**
+ * Classes for printing YAML AST.
+ */
+module PrintYaml {
+  /**
+   * A print node representing a YAML value in a .yml file.
+   */
+  class YamlNodeNode extends PrintAstNode, TYamlNode {
+    YamlNode node;
+
+    YamlNodeNode() { this = TYamlNode(node) }
+
+    override string toString() {
+      result = "[" + concat(node.getAPrimaryQlClass(), ",") + "] " + node.toString()
+    }
+
+    override Location getLocation() { result = node.getLocation() }
+
+    /**
+     * Gets the `YAMLNode` represented by this node.
+     */
+    final YamlNode getValue() { result = node }
+
+    override PrintAstNode getChild(int childIndex) {
+      exists(YamlNode child | result.(YamlNodeNode).getValue() = child |
+        child = node.getChildNode(childIndex)
+      )
+    }
+  }
+
+  /**
+   * A print node representing a `YAMLMapping`.
+   *
+   * Each child of this node aggregates the key and value of a mapping.
+   */
+  class YamlMappingNode extends YamlNodeNode {
+    override YamlMapping node;
+
+    override PrintAstNode getChild(int childIndex) {
+      exists(YamlMappingMapNode map | map = result | map.maps(node, childIndex))
+    }
+  }
+
+  /**
+   * A print node representing the `i`th mapping in `mapping`.
+   */
+  class YamlMappingMapNode extends PrintAstNode, TYamlMappingNode {
+    YamlMapping mapping;
+    int i;
+
+    YamlMappingMapNode() { this = TYamlMappingNode(mapping, i) }
+
+    override string toString() {
+      result = "(Mapping " + i + ")" and not exists(mapping.getKeyNode(i).(YamlScalar).getValue())
+      or
+      result = "(Mapping " + i + ") " + mapping.getKeyNode(i).(YamlScalar).getValue() + ":"
+    }
+
+    /**
+     * Holds if this print node represents the `index`th mapping of `m`.
+     */
+    predicate maps(YamlMapping m, int index) {
+      m = mapping and
+      index = i
+    }
+
+    override PrintAstNode getChild(int childIndex) {
+      childIndex = 0 and result.(YamlNodeNode).getValue() = mapping.getKeyNode(i)
+      or
+      childIndex = 1 and result.(YamlNodeNode).getValue() = mapping.getValueNode(i)
+    }
+  }
+}
+
 /** Holds if `node` belongs to the output tree, and its property `key` has the given `value`. */
 query predicate nodes(PrintAstNode node, string key, string value) { value = node.getProperty(key) }
 

python/ql/lib/qlpack.yml

@@ -0,0 +1,50 @@
+/**
+ * Provides classes for working with YAML data.
+ *
+ * YAML documents are represented as abstract syntax trees whose nodes
+ * are either YAML values or alias nodes referring to another YAML value.
+ */
+
+private import codeql.yaml.Yaml as LibYaml
+
+private module YamlSig implements LibYaml::InputSig {
+  import semmle.python.Files
+
+  class LocatableBase extends @yaml_locatable {
+    Location getLocation() { yaml_locations(this, result) }
+
+    string toString() { none() }
+  }
+
+  class NodeBase extends LocatableBase, @yaml_node {
+    NodeBase getChildNode(int i) { yaml(result, _, this, i, _, _) }
+
+    string getTag() { yaml(this, _, _, _, result, _) }
+
+    string getAnchor() { yaml_anchors(this, result) }
+
+    override string toString() { yaml(this, _, _, _, _, result) }
+  }
+
+  class ScalarNodeBase extends NodeBase, @yaml_scalar_node {
+    int getStyle() { yaml_scalars(this, result, _) }
+
+    string getValue() { yaml_scalars(this, _, result) }
+  }
+
+  class CollectionNodeBase extends NodeBase, @yaml_collection_node { }
+
+  class MappingNodeBase extends CollectionNodeBase, @yaml_mapping_node { }
+
+  class SequenceNodeBase extends CollectionNodeBase, @yaml_sequence_node { }
+
+  class AliasNodeBase extends NodeBase, @yaml_alias_node {
+    string getTarget() { yaml_aliases(this, result) }
+  }
+
+  class ParseErrorBase extends LocatableBase, @yaml_error {
+    string getMessage() { yaml_errors(this, result) }
+  }
+}
+
+import LibYaml::Make<YamlSig>

python/ql/lib/semmle/python/PrintAst.qll

@@ -1104,3 +1104,44 @@ xmllocations(int xmlElement: @xmllocatable ref,
              int location: @location_default ref);
 
 @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;
+
+/**
+ *  YAML
+ */
+
+#keyset[parent, idx]
+yaml (unique int id: @yaml_node,
+      int kind: int ref,
+      int parent: @yaml_node_parent ref,
+      int idx: int ref,
+      varchar(900) tag: string ref,
+      varchar(900) tostring: string ref);
+
+case @yaml_node.kind of
+  0 = @yaml_scalar_node
+| 1 = @yaml_mapping_node
+| 2 = @yaml_sequence_node
+| 3 = @yaml_alias_node
+;
+
+@yaml_collection_node = @yaml_mapping_node | @yaml_sequence_node;
+
+@yaml_node_parent = @yaml_collection_node | @file;
+
+yaml_anchors (unique int node: @yaml_node ref,
+              varchar(900) anchor: string ref);
+
+yaml_aliases (unique int alias: @yaml_alias_node ref,
+              varchar(900) target: string ref);
+
+yaml_scalars (unique int scalar: @yaml_scalar_node ref,
+              int style: int ref,
+              varchar(900) value: string ref);
+
+yaml_errors (unique int id: @yaml_error,
+             varchar(900) message: string ref);
+
+yaml_locations(unique int locatable: @yaml_locatable ref,
+             int location: @location_default ref);
+
+@yaml_locatable = @yaml_node | @yaml_error;