Swift: generated extractor tests

Author: redsun82

docs/codeql/ql-training/query-examples/java/query-injection-java-1.ql

@@ -4,4 +4,4 @@ from Method m, MethodAccess ma
 where
   m.getName().matches("sparql%Query") and
   ma.getMethod() = m
-select ma, m
+select ma, m

swift/codegen/codegen.py

@@ -31,6 +31,8 @@ def _parse_args() -> argparse.Namespace:
     p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements",
                    help="output directory for QL stub/customization files (default %(default)s). Defines also the "
                         "generated qll file importing every class file")
+    p.add_argument("--ql-test-output", type=_abspath, default=paths.swift_dir / "ql/test/extractor-tests/generated",
+                   help="output directory for QL generated extractor test files (default %(default)s)")
     p.add_argument("--ql-format", action="store_true", default=True,
                    help="use codeql to autoformat QL files (which is the default)")
     p.add_argument("--no-ql-format", action="store_false", dest="ql_format", help="do not format QL files")

swift/codegen/generators/qlgen.py

@@ -3,6 +3,7 @@
 import logging
 import pathlib
 import subprocess
+import typing
 
 import inflection
 
@@ -11,41 +12,46 @@
 log = logging.getLogger(__name__)
 
 
+class FormatError(Exception):
+    pass
+
+
 def get_ql_property(cls: schema.Class, prop: schema.Property):
+    common_args = dict(
+        type=prop.type if not prop.is_predicate else "predicate",
+        skip_qltest="no_qltest" in prop.tags,
+        is_child=prop.is_child,
+        is_optional=prop.is_optional,
+        is_predicate=prop.is_predicate,
+    )
     if prop.is_single:
         return ql.Property(
+            **common_args,
             singular=inflection.camelize(prop.name),
-            type=prop.type,
             tablename=inflection.tableize(cls.name),
             tableparams=["this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
-            is_child=prop.is_child,
         )
     elif prop.is_repeated:
         return ql.Property(
+            **common_args,
             singular=inflection.singularize(inflection.camelize(prop.name)),
             plural=inflection.pluralize(inflection.camelize(prop.name)),
-            type=prop.type,
             tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
             tableparams=["this", "index", "result"],
-            is_optional=prop.is_optional,
-            is_child=prop.is_child,
         )
     elif prop.is_optional:
         return ql.Property(
+            **common_args,
             singular=inflection.camelize(prop.name),
-            type=prop.type,
             tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
             tableparams=["this", "result"],
-            is_optional=True,
-            is_child=prop.is_child,
         )
     elif prop.is_predicate:
         return ql.Property(
+            **common_args,
             singular=inflection.camelize(prop.name, uppercase_first_letter=False),
-            type="predicate",
             tablename=inflection.underscore(f"{cls.name}_{prop.name}"),
             tableparams=["this"],
-            is_predicate=True,
         )
 
 
@@ -77,38 +83,75 @@ def get_classes_used_by(cls: ql.Class):
 
 def is_generated(file):
     with open(file) as contents:
-        return next(contents).startswith("// generated")
+        for line in contents:
+            return line.startswith("// generated")
+        return False
 
 
 def format(codeql, files):
     format_cmd = [codeql, "query", "format", "--in-place", "--"]
-    format_cmd.extend(str(f) for f in files)
-    res = subprocess.run(format_cmd, check=True, stderr=subprocess.PIPE, text=True)
+    format_cmd.extend(str(f) for f in files if f.suffix in (".qll", ".ql"))
+    res = subprocess.run(format_cmd, stderr=subprocess.PIPE, text=True)
+    if res.returncode:
+        for line in res.stderr.splitlines():
+            log.error(line.strip())
+        raise FormatError("QL format failed")
     for line in res.stderr.splitlines():
         log.debug(line.strip())
 
 
+def _get_all_properties(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[ql.Property]:
+    for b in cls.bases:
+        for p in _get_all_properties(lookup[b], lookup):
+            yield p
+    for p in cls.properties:
+        yield p
+
+
+def _get_all_properties_to_be_tested(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
+        ql.PropertyForTest]:
+    # deduplicate using id
+    already_seen = set()
+    for p in _get_all_properties(cls, lookup):
+        if not p.skip_qltest and id(p) not in already_seen:
+            already_seen.add(id(p))
+            yield ql.PropertyForTest(p.getter, p.type, p.is_single, p.is_predicate, p.is_repeated)
+
+
+def _partition(l, pred):
+    """ partitions a list according to boolean predicate """
+    res = ([], [])
+    for x in l:
+        res[not pred(x)].append(x)
+    return res
+
+
 def generate(opts, renderer):
     input = opts.schema
     out = opts.ql_output
     stub_out = opts.ql_stub_output
+    test_out = opts.ql_test_output
+    missing_test_source_filename = "MISSING_SOURCE.txt"
     existing = {q for q in out.rglob("*.qll")}
     existing |= {q for q in stub_out.rglob("*.qll") if is_generated(q)}
+    existing |= {q for q in test_out.rglob("*.ql")}
+    existing |= {q for q in test_out.rglob(missing_test_source_filename)}
 
     data = schema.load(input)
 
     classes = [get_ql_class(cls) for cls in data.classes]
-    classes.sort(key=lambda cls: cls.name)
+    lookup = {cls.name: cls for cls in classes}
+    classes.sort(key=lambda cls: (cls.dir, cls.name))
     imports = {}
 
     for c in classes:
         imports[c.name] = get_import(stub_out / c.path, opts.swift_dir)
 
     for c in classes:
-        qll = (out / c.path).with_suffix(".qll")
+        qll = out / c.path.with_suffix(".qll")
         c.imports = [imports[t] for t in get_classes_used_by(c)]
         renderer.render(c, qll)
-        stub_file = (stub_out / c.path).with_suffix(".qll")
+        stub_file = stub_out / c.path.with_suffix(".qll")
         if not stub_file.is_file() or is_generated(stub_file):
             stub = ql.Stub(name=c.name, base_import=get_import(qll, opts.swift_dir))
             renderer.render(stub, stub_file)
@@ -120,6 +163,21 @@ def generate(opts, renderer):
 
     renderer.render(ql.GetParentImplementation(classes), out / 'GetImmediateParent.qll')
 
+    for c in classes:
+        if not c.final:
+            continue
+        test_dir = test_out / c.path
+        test_dir.mkdir(parents=True, exist_ok=True)
+        if not any(test_dir.glob("*.swift")):
+            log.warning(f"no test source in {c.path}")
+            renderer.render(ql.MissingTestInstructions(), test_dir / missing_test_source_filename)
+            continue
+        total_props, partial_props = _partition(_get_all_properties_to_be_tested(c, lookup),
+                                                lambda p: p.is_single or p.is_predicate)
+        renderer.render(ql.ClassTester(class_name=c.name, properties=total_props), test_dir / f"{c.name}.ql")
+        for p in partial_props:
+            renderer.render(ql.PropertyTester(class_name=c.name, property=p), test_dir / f"{c.name}_{p.getter}.ql")
+
     renderer.cleanup(existing)
     if opts.ql_format:
         format(opts.codeql_binary, renderer.written)

swift/codegen/lib/ql.py

@@ -37,6 +37,7 @@ class Property:
     is_optional: bool = False
     is_predicate: bool = False
     is_child: bool = False
+    skip_qltest: bool = False
 
     def __post_init__(self):
         if self.tableparams:
@@ -63,6 +64,10 @@ def type_is_class(self):
     def is_repeated(self):
         return bool(self.plural)
 
+    @property
+    def is_single(self):
+        return not (self.is_optional or self.is_repeated or self.is_predicate)
+
 
 @dataclass
 class Class:
@@ -113,3 +118,33 @@ class GetParentImplementation:
     template: ClassVar = 'ql_parent'
 
     classes: List[Class] = field(default_factory=list)
+
+
+@dataclass
+class PropertyForTest:
+    getter: str
+    type: str = None
+    is_single: bool = False
+    is_predicate: bool = False
+    is_repeated: bool = False
+
+
+@dataclass
+class ClassTester:
+    template: ClassVar = 'ql_test_class'
+
+    class_name: str
+    properties: List[PropertyForTest] = field(default_factory=list)
+
+
+@dataclass
+class PropertyTester:
+    template: ClassVar = 'ql_test_property'
+
+    class_name: str
+    property: PropertyForTest
+
+
+@dataclass
+class MissingTestInstructions:
+    template: ClassVar = 'ql_test_missing'

swift/codegen/lib/schema.py

@@ -3,10 +3,17 @@
 import pathlib
 import re
 from dataclasses import dataclass, field
-from typing import List, Set, Dict, ClassVar
+from typing import List, Set, Union, Dict, ClassVar
 
 import yaml
 
+
+class Error(Exception):
+
+    def __str__(self):
+        return f"schema.Error{args}"
+
+
 root_class_name = "Element"
 
 
@@ -20,6 +27,7 @@ class Property:
     name: str
     type: str = None
     is_child: bool = False
+    tags: List[str] = field(default_factory=list)
 
 
 @dataclass
@@ -55,6 +63,7 @@ class Class:
     derived: Set[str] = field(default_factory=set)
     properties: List[Property] = field(default_factory=list)
     dir: pathlib.Path = pathlib.Path()
+    tags: List[str] = field(default_factory=list)
 
 
 @dataclass
@@ -63,18 +72,24 @@ class Schema:
     includes: Set[str] = field(default_factory=set)
 
 
-def _parse_property(name: str, type: str, is_child: bool = False):
-    assert not (is_child and type[0].islower()), f"children must have class type, got {type} for {name}"
+def _parse_property(name: str, type: Union[str, Dict[str, str]], is_child: bool = False):
+    if isinstance(type, dict):
+        tags = type.get("_tags", [])
+        type = type["type"]
+    else:
+        tags = []
+    if is_child and type[0].islower():
+        raise Error(f"children must have class type, got {type} for {name}")
     if type.endswith("?*"):
-        return RepeatedOptionalProperty(name, type[:-2], is_child=is_child)
+        return RepeatedOptionalProperty(name, type[:-2], is_child=is_child, tags=tags)
     elif type.endswith("*"):
-        return RepeatedProperty(name, type[:-1], is_child=is_child)
+        return RepeatedProperty(name, type[:-1], is_child=is_child, tags=tags)
     elif type.endswith("?"):
-        return OptionalProperty(name, type[:-1], is_child=is_child)
+        return OptionalProperty(name, type[:-1], is_child=is_child, tags=tags)
     elif type == "predicate":
-        return PredicateProperty(name)
+        return PredicateProperty(name, tags=tags)
     else:
-        return SingleProperty(name, type, is_child=is_child)
+        return SingleProperty(name, type, is_child=is_child, tags=tags)
 
 
 class _DirSelector:
@@ -98,7 +113,8 @@ def load(path):
     for name, info in data.items():
         if name.startswith("_"):
             continue
-        assert name[0].isupper()
+        if not name[0].isupper():
+            raise Error(f"keys in the schema file must be capitalized class names or metadata, got {name}")
         cls = classes[name]
         for k, v in info.items():
             if not k.startswith("_"):
@@ -113,6 +129,10 @@ def load(path):
                 cls.dir = pathlib.Path(v)
             elif k == "_children":
                 cls.properties.extend(_parse_property(kk, vv, is_child=True) for kk, vv in v.items())
+            elif k == "_tags":
+                cls.tags = v
+            else:
+                raise Error(f"unknown metadata {k} for class {name}")
         if not cls.bases and cls.name != root_class_name:
             cls.bases.add(root_class_name)
             classes[root_class_name].derived.add(name)

swift/codegen/schema.yml

@@ -19,9 +19,12 @@ File:
   name: string
 
 Locatable:
-  location: Location
+  location:
+    type: Location
+    _tags: [no_qltest]
 
 Location:
+  _tags: [ no_qltest ]
   file: File
   start_line: int
   start_column: int

swift/codegen/templates/ql_test_class.mustache

@@ -0,0 +1,16 @@
+// generated by {{generator}}
+
+import codeql.swift.elements
+import TestUtils
+
+from {{class_name}} x{{#properties}}, {{#is_single}}{{type}}{{/is_single}}{{#is_predicate}}string{{/is_predicate}} {{getter}}{{/properties}}
+where toBeTested(x)
+{{#properties}}
+{{#is_single}}
+and {{getter}} = x.{{getter}}()
+{{/is_single}}
+{{#is_predicate}}
+and if x.{{getter}}() then {{getter}} = "yes" else {{getter}} = "no"
+{{/is_predicate}}
+{{/properties}}
+select x{{#properties}}, "{{getter}}:", {{getter}}{{/properties}}

swift/codegen/templates/ql_test_missing.mustache

@@ -0,0 +1,4 @@
+// generated by {{generator}}
+
+After a swift source file is added in this directory and {{generator}} is run again, test queries
+will appear and this file will be deleted

swift/codegen/templates/ql_test_property.mustache

@@ -0,0 +1,10 @@
+// generated by {{generator}}
+
+import codeql.swift.elements
+import TestUtils
+
+{{#property}}
+from {{class_name}} x{{#is_repeated}}, int index{{/is_repeated}}
+where toBeTested(x)
+select x, {{#is_repeated}}index, {{/is_repeated}}x.{{getter}}({{#is_repeated}}index{{/is_repeated}})
+{{/property}}

swift/codegen/test/test_ql.py

@@ -45,11 +45,23 @@ def test_property_indefinite_article(name, expected_getter):
     ("", False),
     ("X", True),
 ])
-def test_property_is_plural(plural, expected):
+def test_property_is_repeated(plural, expected):
     prop = ql.Property("foo", "Foo", "props", ["x"], plural=plural)
     assert prop.is_repeated is expected
 
 
+@pytest.mark.parametrize("is_optional,is_predicate,plural,expected", [
+    (False, False, None, True),
+    (False, False, "", True),
+    (False, False, "X", False),
+    (True, False, None, False),
+    (False, True, None, False),
+])
+def test_property_is_repeated(is_optional, is_predicate, plural, expected):
+    prop = ql.Property("foo", "Foo", "props", ["x"], plural=plural, is_predicate=is_predicate, is_optional=is_optional)
+    assert prop.is_single is expected
+
+
 def test_property_no_plural_no_indefinite_getter():
     prop = ql.Property("Prop", "Foo", "props", ["x"])
     assert prop.indefinite_getter is None