Add the AST Viewer

This commit adds the AST Viewer for viewing the QL AST of a file in a
database.

The different components are as follows:

1. There is a new view `codeQLAstViewer`, which displays the AST
2. This view is backed by the `AstViewerDataProvider` and `AstViewer` classes in astView.ts
3. To generate an AST, we use contextual queries, similar to how Find references/declarations are implemented. In particular, in `definitions.ts` there is `TemplatePrintAstProvider` which provides an AST for a given source buffer.
  - Similar to the other queries, we first determine which database the buffer belongs to.
  - Based on that, we generate a synthetic qlpack and run the templatized `printAst.ql` query
  - We plug in the archive-relative path name of the source file.
  - After the query is run, we wrap the results in an `AstBuilder` instance.
  - When requested, the `AstBuilder` will generate the full AST of the file from the BQRS results.
  - The AST roots (all top-level elements, functions, variable declarations, etc, are roots) are passed to the `AstViewer` instance, which handles the display lifecycle and other VS Code-specific functions.

There are a few unrelated pieces here, which can be pulled out to another PR if required:

- The `codeQLQueryHistory` view now has a _welcome_ message to make it more obvious to users how to start.
- `definitions.ts` is moved to the `contextual` subfolder.
- `fileRangeFromURI` is extracted from `definitions.ts` to its own file so it can be reused.

Also, note that this relies on github/codeql#3931 for the C/C++ query to be available in the QL sources. Other languages will need similar queries.

Author: aeisenberg

extensions/ql-vscode/media/dark/clear-all.svg

@@ -25,13 +25,15 @@
     "onLanguage:ql",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
+    "onView:codeQLAstViewer",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
     "onCommand:codeQLDatabases.chooseDatabaseFolder",
     "onCommand:codeQLDatabases.chooseDatabaseArchive",
     "onCommand:codeQLDatabases.chooseDatabaseInternet",
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
+    "onCommand:codeQL.viewAst",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
@@ -218,6 +220,10 @@
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
       },
+      {
+        "command": "codeQL.viewAst",
+        "title": "CodeQL: View AST"
+      },
       {
         "command": "codeQL.upgradeCurrentDatabase",
         "title": "CodeQL: Upgrade Current Database"
@@ -333,6 +339,18 @@
       {
         "command": "codeQLTests.acceptOutput",
         "title": "CodeQL: Accept Test Output"
+      },
+      {
+        "command": "codeQLAstViewer.gotoCode",
+        "title": "Go To Code"
+      },
+      {
+        "command": "codeQLAstViewer.clear",
+        "title": "Clear AST",
+        "icon": {
+          "light": "media/light/clear-all.svg",
+          "dark": "media/dark/clear-all.svg"
+        }
       }
     ],
     "menus": {
@@ -366,6 +384,11 @@
           "command": "codeQLDatabases.chooseDatabaseLgtm",
           "when": "view == codeQLDatabases",
           "group": "navigation"
+        },
+        {
+          "command": "codeQLAstViewer.clear",
+          "when": "view == codeQLAstViewer",
+          "group": "navigation"
         }
       ],
       "view/item/context": [
@@ -446,6 +469,11 @@
           "group": "9_qlCommands",
           "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
         },
+        {
+          "command": "codeQL.viewAst",
+          "group": "9_qlCommands",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQL.runQueries",
           "group": "9_qlCommands"
@@ -468,6 +496,10 @@
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -543,6 +575,14 @@
         {
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
+        },
+        {
+          "command": "codeQLAstViewer.gotoCode",
+          "when": "false"
+        },
+        {
+          "command": "codeQLAstViewer.clear",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -574,9 +614,23 @@
         {
           "id": "codeQLQueryHistory",
           "name": "Query History"
+        },
+        {
+          "id": "codeQLAstViewer",
+          "name": "AST Viewer"
         }
       ]
-    }
+    },
+    "viewsWelcome": [
+      {
+        "view": "codeQLAstViewer",
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+      },
+      {
+        "view": "codeQLQueryHistory",
+        "contents": "Run the 'CodeQL: Run Query' command on QL query.\n[Run Query](command:codeQL.runQuery)"
+      }
+    ]
   },
   "scripts": {
     "build": "gulp",

extensions/ql-vscode/media/light/clear-all.svg

@@ -0,0 +1,102 @@
+import * as vscode from 'vscode';
+
+import { DatabaseItem } from './databases';
+import { UrlValue, BqrsId } from './bqrs-cli-types';
+import fileRangeFromURI from './contextual/fileRangeFromURI';
+import { showLocation } from './interface-utils';
+
+export interface AstItem {
+  id: BqrsId;
+  label?: string;
+  location?: UrlValue;
+  parent: AstItem | RootAstItem;
+  children: AstItem[];
+  order: number;
+}
+
+export type RootAstItem = Omit<AstItem, 'parent'>;
+
+class AstViewerDataProvider implements vscode.TreeDataProvider<AstItem | RootAstItem> {
+
+  public roots: RootAstItem[] = [];
+  public db: DatabaseItem | undefined;
+
+  private _onDidChangeTreeData =
+    new vscode.EventEmitter<AstItem | undefined>();
+  readonly onDidChangeTreeData: vscode.Event<AstItem | undefined> =
+    this._onDidChangeTreeData.event;
+
+  constructor() {
+    vscode.commands.registerCommand('codeQLAstViewer.gotoCode',
+      async (location: UrlValue, db: DatabaseItem) => {
+        if (location) {
+          await showLocation(fileRangeFromURI(location, db));
+        }
+      });
+    }
+
+  refresh(): void {
+    this._onDidChangeTreeData.fire();
+  }
+  getChildren(item?: AstItem): vscode.ProviderResult<(AstItem | RootAstItem)[]> {
+    const children = item ? item.children : this.roots;
+    return children.sort((c1, c2) => (c1.order - c2.order));
+  }
+
+  getParent(item: AstItem): vscode.ProviderResult<AstItem> {
+    return item.parent as AstItem;
+  }
+
+  getTreeItem(item: AstItem): vscode.TreeItem {
+    const line = typeof item.location === 'string'
+      ? item.location
+      : item.location?.startLine;
+
+    const state = item.children.length
+      ? vscode.TreeItemCollapsibleState.Collapsed
+      : vscode.TreeItemCollapsibleState.None;
+    const treeItem = new vscode.TreeItem(item.label || '', state);
+    treeItem.description = line ? `Line ${line}` : '';
+    treeItem.id = String(item.id);
+    treeItem.tooltip = `${treeItem.description} ${treeItem.label}`;
+    treeItem.command = {
+      command: 'codeQLAstViewer.gotoCode',
+      title: 'Go To Code',
+      tooltip: `Go To ${item.location}`,
+      arguments: [item.location, this.db]
+    };
+    return treeItem;
+  }
+}
+
+export class AstViewer {
+  private treeView: vscode.TreeView<AstItem | RootAstItem>;
+  private treeDataProvider: AstViewerDataProvider;
+
+  constructor() {
+    this.treeDataProvider = new AstViewerDataProvider();
+    this.treeView = vscode.window.createTreeView('codeQLAstViewer', {
+      treeDataProvider: this.treeDataProvider,
+      showCollapseAll: true
+    });
+
+    vscode.commands.registerCommand('codeQLAstViewer.clear', () => {
+      this.clear();
+    });
+  }
+
+  updateRoots(roots: RootAstItem[], db: DatabaseItem, fileName: string) {
+    this.treeDataProvider.roots = roots;
+    this.treeDataProvider.db = db;
+    this.treeDataProvider.refresh();
+    this.treeView.message = `AST for ${fileName}`;
+    this.treeView.reveal(roots[0], { focus: true });
+  }
+
+  private clear() {
+    this.treeDataProvider.roots = [];
+    this.treeDataProvider.db = undefined;
+    this.treeDataProvider.refresh();
+    this.treeView.message = undefined;
+  }
+}

extensions/ql-vscode/package.json

@@ -53,9 +53,12 @@ export interface BQRSInfo {
   'result-sets': ResultSetSchema[];
 }
 
+export type BqrsId = number;
+
 export interface EntityValue {
   url?: UrlValue;
   label?: string;
+  id?: BqrsId;
 }
 
 export interface LineColumnLocation {

extensions/ql-vscode/src/astViewer.ts

@@ -499,7 +499,7 @@ export class CodeQLCliServer implements Disposable {
   */
   async bqrsDecode(bqrsPath: string, resultSet: string, pageSize?: number, offset?: number): Promise<DecodedBqrsChunk> {
     const subcommandArgs = [
-      '--entities=url,string',
+      '--entities=id,url,string',
       '--result-set', resultSet,
     ].concat(
       pageSize ? ['--rows', pageSize.toString()] : []

extensions/ql-vscode/src/bqrs-cli-types.ts

@@ -0,0 +1,121 @@
+import { QueryWithResults } from '../run-queries';
+import { CodeQLCliServer } from '../cli';
+import { DecodedBqrsChunk, BqrsId, EntityValue } from '../bqrs-cli-types';
+import { DatabaseItem } from '../databases';
+import { AstItem, RootAstItem } from '../astViewer';
+
+/**
+ * A class that wraps a tree of QL results from a query that
+ * has an @kind of graph
+ */
+// RENAME to ASTParser / ASTCreator
+export default class AstBuilder {
+
+  private roots: RootAstItem[] | undefined;
+  private bqrsPath: string;
+  constructor(
+    queryResults: QueryWithResults,
+    private cli: CodeQLCliServer,
+    public db: DatabaseItem,
+    public fileName: string
+  ) {
+    this.bqrsPath = queryResults.query.resultsPaths.resultsPath;
+  }
+
+  async getRoots(): Promise<RootAstItem[]> {
+    if (!this.roots) {
+      this.roots = await this.parseRoots();
+    }
+    return this.roots;
+  }
+
+  private async parseRoots(): Promise<RootAstItem[]> {
+    const [nodeTuples, edgeTuples, graphProperties] = await Promise.all([
+      await this.cli.bqrsDecode(this.bqrsPath, 'nodes'),
+      await this.cli.bqrsDecode(this.bqrsPath, 'edges'),
+      await this.cli.bqrsDecode(this.bqrsPath, 'graphProperties'),
+    ]);
+
+    if (!this.isValidGraph(graphProperties)) {
+      throw new Error('AST is invalid');
+    }
+
+    const idToItem = new Map<BqrsId, AstItem | RootAstItem>();
+    const parentToChildren = new Map<BqrsId, BqrsId[]>();
+    const childToParent = new Map<BqrsId, BqrsId>();
+    const astOrder = new Map<BqrsId, number>();
+    const roots = [];
+
+    // Build up the parent-child relationships
+    edgeTuples.tuples.forEach(tuple => {
+      const from = tuple[0] as EntityValue;
+      const to = tuple[1] as EntityValue;
+      const toId = to.id!;
+      const fromId = from.id!;
+
+      if (tuple[2] === 'semmle.order') {
+        astOrder.set(toId, Number(tuple[3]));
+      } else if (tuple[2] === 'semmle.label') {
+        childToParent.set(toId, fromId);
+        let children = parentToChildren.get(fromId);
+        if (!children) {
+          parentToChildren.set(fromId, children = []);
+        }
+        children.push(toId);
+      }
+    });
+
+    // populate parents and children
+    nodeTuples.tuples.forEach(tuple => {
+      const entity = tuple[0] as EntityValue;
+      const id = entity.id!;
+
+      if (tuple[1] === 'semmle.order') {
+        astOrder.set(id, Number(tuple[2]));
+
+      } else if (tuple[1] === 'semmle.label') {
+        const item = {
+          id,
+          label: entity.label,
+          location: entity.url,
+          children: [] as AstItem[],
+          order: Number.MAX_SAFE_INTEGER
+        };
+
+        idToItem.set(id, item as RootAstItem);
+        const parent = idToItem.get(childToParent.get(id) || -1);
+
+        if (parent) {
+          const astItem = item as AstItem;
+          astItem.parent = parent;
+          parent.children.push(astItem);
+        }
+        const children = parentToChildren.get(id) || [];
+        children.forEach(childId => {
+          const child = idToItem.get(childId) as AstItem | undefined;
+          if (child) {
+            child.parent = item;
+            item.children.push(child);
+          }
+        });
+      }
+    });
+
+    // find the roots and add the order
+    for(const [, item] of idToItem) {
+      item.order = astOrder.has(item.id)
+        ? astOrder.get(item.id)!
+        : Number.MAX_SAFE_INTEGER;
+
+      if (!('parent' in item)) {
+        roots.push(item);
+      }
+    }
+    return roots;
+  }
+
+  private isValidGraph(graphProperties: DecodedBqrsChunk) {
+    const tuple = graphProperties?.tuples?.find(t => t[0] === 'semmle.graphKind');
+    return tuple?.[1] === 'tree';
+  }
+}