Swift: add structured C++ generated classes

This adds `cppgen`, creating structured C++ classes mirroring QL classes
out of `schema.yml`.

An example of generated code at the time of this commit can be found
[in this gist][1].

[1]: https://gist.github.com/redsun82/57304ddb487a8aa40eaa0caa695048fa

Closes github/codeql-c-team#863

Author: redsun82

swift/codegen/BUILD.bazel

@@ -1,5 +1,17 @@
 load("@swift_codegen_deps//:requirements.bzl", "requirement")
 
+filegroup(
+    name = "schema",
+    srcs = ["schema.yml"],
+    visibility = ["//swift:__subpackages__"],
+)
+
+filegroup(
+    name = "schema_includes",
+    srcs = glob(["*.dbscheme"]),
+    visibility = ["//swift:__subpackages__"],
+)
+
 py_binary(
     name = "codegen",
     srcs = glob(
@@ -15,6 +27,17 @@ py_binary(
 py_binary(
     name = "trapgen",
     srcs = ["trapgen.py"],
+    data = ["//swift/codegen/templates:trap"],
+    visibility = ["//swift:__subpackages__"],
+    deps = [
+        "//swift/codegen/lib",
+        requirement("toposort"),
+    ],
+)
+
+py_binary(
+    name = "cppgen",
+    srcs = ["cppgen.py"],
     data = ["//swift/codegen/templates:cpp"],
     visibility = ["//swift:__subpackages__"],
     deps = [

swift/codegen/cppgen.py

@@ -0,0 +1,67 @@
+import functools
+import inflection
+from typing import Dict
+
+from toposort import toposort_flatten
+
+from swift.codegen.lib import cpp, generator, schema
+
+
+def _get_type(t: str) -> str:
+    if t == "string":
+        return "std::string"
+    if t == "boolean":
+        return "bool"
+    if t[0].isupper():
+        return f"TrapLabel<{t}Tag>"
+    return t
+
+
+def _get_field(cls: schema.Class, p: schema.Property) -> cpp.Field:
+    trap_name = None
+    if not p.is_single:
+        trap_name = inflection.pluralize(inflection.camelize(f"{cls.name}_{p.name}")) + "Trap"
+    args = dict(
+        name=p.name + ("_" if p.name in cpp.cpp_keywords else ""),
+        type=_get_type(p.type),
+        is_optional=p.is_optional,
+        is_repeated=p.is_repeated,
+        trap_name=trap_name,
+    )
+    args.update(cpp.get_field_override(p.name))
+    return cpp.Field(**args)
+
+
+class Processor:
+    def __init__(self, data: Dict[str, schema.Class]):
+        self._classmap = data
+
+    @functools.cache
+    def _get_class(self, name: str) -> cpp.Class:
+        cls = self._classmap[name]
+        trap_name = None
+        if not cls.derived or any(p.is_single for p in cls.properties):
+            trap_name = inflection.pluralize(cls.name) + "Trap"
+        return cpp.Class(
+            name=name,
+            bases=[self._get_class(b) for b in cls.bases],
+            fields=[_get_field(cls, p) for p in cls.properties],
+            final=not cls.derived,
+            trap_name=trap_name,
+        )
+
+    def get_classes(self):
+        inheritance_graph = {k: cls.bases for k, cls in self._classmap.items()}
+        return [self._get_class(cls) for cls in toposort_flatten(inheritance_graph)]
+
+
+def generate(opts, renderer):
+    processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes})
+    out = opts.cpp_output
+    renderer.render(cpp.ClassList(processor.get_classes()), out / "TrapClasses.h")
+
+
+tags = ("cpp", "schema")
+
+if __name__ == "__main__":
+    generator.run()

swift/codegen/lib/cpp.py

@@ -1,3 +1,4 @@
+import re
 from dataclasses import dataclass, field
 from typing import List, ClassVar
 
@@ -14,13 +15,35 @@
                 "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", "wchar_t", "while",
                 "xor", "xor_eq"}
 
+_field_overrides = [
+    (re.compile(r"(start|end)_(line|column)|index|num_.*"), {"type": "unsigned"}),
+    (re.compile(r"(.*)_"), lambda m: {"name": m[1]}),
+]
+
+
+def get_field_override(field: str):
+    for r, o in _field_overrides:
+        m = r.fullmatch(field)
+        if m:
+            return o(m) if callable(o) else o
+    return {}
+
 
 @dataclass
 class Field:
     name: str
     type: str
+    is_optional: bool = False
+    is_repeated: bool = False
+    trap_name: str = None
     first: bool = False
 
+    def __post_init__(self):
+        if self.is_optional:
+            self.type = f"std::optional<{self.type}>"
+        elif self.is_repeated:
+            self.type = f"std::vector<{self.type}>"
+
     @property
     def cpp_name(self):
         if self.name in cpp_keywords:
@@ -36,6 +59,12 @@ def get_streamer(self):
         else:
             return lambda x: x
 
+    @property
+    def is_single(self):
+        return not (self.is_optional or self.is_repeated)
+
+
+
 
 @dataclass
 class Trap:
@@ -74,13 +103,48 @@ def has_bases(self):
 
 @dataclass
 class TrapList:
-    template: ClassVar = 'cpp_traps'
+    template: ClassVar = 'trap_traps'
 
     traps: List[Trap] = field(default_factory=list)
 
 
 @dataclass
 class TagList:
-    template: ClassVar = 'cpp_tags'
+    template: ClassVar = 'trap_tags'
 
     tags: List[Tag] = field(default_factory=list)
+
+
+@dataclass
+class ClassBase:
+    ref: 'Class'
+    first: bool = False
+
+
+@dataclass
+class Class:
+    name: str
+    bases: List[ClassBase] = field(default_factory=list)
+    final: bool = False
+    fields: List[Field] = field(default_factory=list)
+    trap_name: str = None
+
+    def __post_init__(self):
+        self.bases = [ClassBase(c) for c in sorted(self.bases, key=lambda cls: cls.name)]
+        if self.bases:
+            self.bases[0].first = True
+
+    @property
+    def has_bases(self):
+        return bool(self.bases)
+
+    @property
+    def single_fields(self):
+        return [f for f in self.fields if f.is_single]
+
+
+@dataclass
+class ClassList:
+    template: ClassVar = "cpp_classes"
+
+    classes: List[Class]

swift/codegen/lib/options.py

@@ -15,7 +15,7 @@ def _init_options():
     Option("--ql-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
     Option("--ql-stub-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
     Option("--codeql-binary", tags=["ql"], default="codeql")
-    Option("--trap-output", tags=["trap"], type=_abspath, required=True)
+    Option("--cpp-output", tags=["cpp"], type=_abspath, required=True)
 
 
 def _abspath(x):

swift/codegen/templates/BUILD.bazel

@@ -1,5 +1,11 @@
+package(default_visibility = ["//swift:__subpackages__"])
+
+filegroup(
+    name = "trap",
+    srcs = glob(["trap_*.mustache"]),
+)
+
 filegroup(
     name = "cpp",
     srcs = glob(["cpp_*.mustache"]),
-    visibility = ["//swift:__subpackages__"],
 )

swift/codegen/templates/cpp_classes.mustache

@@ -0,0 +1,46 @@
+// generated by {{generator}}
+// clang-format off
+#pragma once
+
+#include <iostream>
+#include <optional>
+#include <vector>
+
+#include "swift/extractor/trap/TrapLabel.h"
+#include "swift/extractor/trap/TrapEntries.h"
+
+namespace codeql {
+{{#classes}}
+
+struct {{name}}{{#final}} : Binding<{{name}}Tag>{{#bases}}, {{ref.name}}{{/bases}}{{/final}}{{^final}}{{#has_bases}}: {{#bases}}{{^first}}, {{/first}}{{ref.name}}{{/bases}}{{/has_bases}}{{/final}} {
+  {{#fields}}
+  {{type}} {{name}}{};
+  {{/fields}}
+  {{#final}}
+
+  friend std::ostream& operator<<(std::ostream& out, const {{name}}& x) {
+    x.emit(out);
+    return out;
+  }
+  {{/final}}
+
+ protected:
+  void emit({{^final}}TrapLabel<{{name}}Tag> id, {{/final}}std::ostream& out) const {
+    {{#bases}}
+    {{ref.name}}::emit(id, out);
+    {{/bases}}
+    {{#trap_name}}
+    out << {{.}}{id{{#single_fields}}, {{name}}{{/single_fields}}} << '\n';
+    {{/trap_name}}
+    {{#fields}}
+    {{#is_optional}}
+    if ({{name}}) out << {{trap_name}}{id, *{{name}}} << '\n';
+    {{/is_optional}}
+    {{#is_repeated}}
+    for (auto i = 0u; i < {{name}}.size(); ++i) out << {{trap_name}}{id, i, {{name}}[i]};
+    {{/is_repeated}}
+    {{/fields}}
+  }
+};
+{{/classes}}
+}

swift/codegen/templates/cpp_tags.mustache …ift/codegen/templates/trap_tags.mustacheswift/codegen/templates/cpp_tags.mustache renamed to swift/codegen/templates/trap_tags.mustache swift/codegen/templates/cpp_tags.mustache renamed to swift/codegen/templates/trap_tags.mustache

@@ -27,6 +27,27 @@ def test_field_get_streamer(type, expected):
     assert f.get_streamer()("value") == expected
 
 
+@pytest.mark.parametrize("is_optional,is_repeated,expected", [
+    (False, False, True),
+    (True, False, False),
+    (False, True, False),
+    (True, True, False),
+])
+def test_field_is_single(is_optional, is_repeated, expected):
+    f = cpp.Field("name", "type", is_optional=is_optional, is_repeated=is_repeated)
+    assert f.is_single is expected
+
+
+@pytest.mark.parametrize("is_optional,is_repeated,expected", [
+    (False, False, "bar"),
+    (True, False, "std::optional<bar>"),
+    (False, True, "std::vector<bar>"),
+])
+def test_field_modal_types(is_optional, is_repeated, expected):
+    f = cpp.Field("name", "bar", is_optional=is_optional, is_repeated=is_repeated)
+    assert f.type == expected
+
+
 def test_trap_has_first_field_marked():
     fields = [
         cpp.Field("a", "x"),
@@ -56,5 +77,39 @@ def test_tag_has_bases(bases, expected):
     assert t.has_bases is expected
 
 
+def test_class_has_first_base_marked():
+    bases = [
+        cpp.Class("a"),
+        cpp.Class("b"),
+        cpp.Class("c"),
+    ]
+    expected = [cpp.ClassBase(c) for c in bases]
+    expected[0].first = True
+    c = cpp.Class("foo", bases=bases)
+    assert c.bases == expected
+
+
+@pytest.mark.parametrize("bases,expected", [
+    ([], False),
+    (["a"], True),
+    (["a", "b"], True)
+])
+def test_class_has_bases(bases, expected):
+    t = cpp.Class("name", [cpp.Class(b) for b in bases])
+    assert t.has_bases is expected
+
+
+def test_class_single_fields():
+    fields = [
+        cpp.Field("a", "A"),
+        cpp.Field("b", "B", is_optional=True),
+        cpp.Field("c", "C"),
+        cpp.Field("d", "D", is_repeated=True),
+        cpp.Field("e", "E"),
+    ]
+    c = cpp.Class("foo", fields=fields)
+    assert c.single_fields == fields[::2]
+
+
 if __name__ == '__main__':
-    sys.exit(pytest.main())
+    sys.exit(pytest.main([__file__] + sys.argv[1:]))