Merge remote-tracking branch 'origin/main' into nickrolfe/oj

Author: nickrolfe

autobuilder/src/main.rs

@@ -26,12 +26,12 @@ fn main() -> std::io::Result<()> {
 
     for line in env::var("LGTM_INDEX_FILTERS")
         .unwrap_or_default()
-        .split("\n")
+        .split('\n')
     {
-        if line.starts_with("include:") {
-            cmd.arg("--include").arg(&line[8..]);
-        } else if line.starts_with("exclude:") {
-            cmd.arg("--exclude").arg(&line[8..]);
+        if let Some(stripped) = line.strip_prefix("include:") {
+            cmd.arg("--include").arg(stripped);
+        } else if let Some(stripped) = line.strip_prefix("exclude:") {
+            cmd.arg("--exclude").arg(stripped);
         }
     }
     let exit = &cmd.spawn()?.wait()?;

extractor/src/extractor.rs

@@ -165,7 +165,7 @@ pub fn extract(
     schema: &NodeTypeMap,
     trap_writer: &mut TrapWriter,
     path: &Path,
-    source: &Vec<u8>,
+    source: &[u8],
     ranges: &[Range],
 ) -> std::io::Result<()> {
     let span = span!(
@@ -180,17 +180,16 @@ pub fn extract(
 
     let mut parser = Parser::new();
     parser.set_language(language).unwrap();
-    parser.set_included_ranges(&ranges).unwrap();
+    parser.set_included_ranges(ranges).unwrap();
     let tree = parser.parse(&source, None).expect("Failed to parse file");
     trap_writer.comment(format!("Auto-generated TRAP file for {}", path.display()));
     let file_label = &trap_writer.populate_file(path);
     let mut visitor = Visitor {
-        source: &source,
-        trap_writer: trap_writer,
+        source,
+        trap_writer,
         // TODO: should we handle path strings that are not valid UTF8 better?
         path: format!("{}", path.display()),
         file_label: *file_label,
-        token_counter: 0,
         toplevel_child_counter: 0,
         stack: Vec::new(),
         language_prefix,
@@ -206,15 +205,7 @@ pub fn extract(
 /// HTML entities.
 fn escape_key<'a, S: Into<Cow<'a, str>>>(key: S) -> Cow<'a, str> {
     fn needs_escaping(c: char) -> bool {
-        match c {
-            '&' => true,
-            '{' => true,
-            '}' => true,
-            '"' => true,
-            '@' => true,
-            '#' => true,
-            _ => false,
-        }
+        matches!(c, '&' | '{' | '}' | '"' | '@' | '#')
     }
 
     let key = key.into();
@@ -297,11 +288,9 @@ struct Visitor<'a> {
     /// source file.
     file_label: Label,
     /// The source code as a UTF-8 byte array
-    source: &'a Vec<u8>,
+    source: &'a [u8],
     /// A TrapWriter to accumulate trap entries
     trap_writer: &'a mut TrapWriter,
-    /// A counter for tokens
-    token_counter: usize,
     /// A counter for top-level child nodes
     toplevel_child_counter: usize,
     /// Language prefix
@@ -345,7 +334,7 @@ impl Visitor<'_> {
         full_error_message: String,
         node: Node,
     ) {
-        let (start_line, start_column, end_line, end_column) = location_for(&self.source, node);
+        let (start_line, start_column, end_line, end_column) = location_for(self.source, node);
         let loc = self.trap_writer.location(
             self.file_label,
             start_line,
@@ -376,15 +365,15 @@ impl Visitor<'_> {
         let id = self.trap_writer.fresh_id();
 
         self.stack.push((id, 0, Vec::new()));
-        return true;
+        true
     }
 
     fn leave_node(&mut self, field_name: Option<&'static str>, node: Node) {
         if node.is_error() || node.is_missing() {
             return;
         }
         let (id, _, child_nodes) = self.stack.pop().expect("Vistor: empty stack");
-        let (start_line, start_column, end_line, end_column) = location_for(&self.source, node);
+        let (start_line, start_column, end_line, end_column) = location_for(self.source, node);
         let loc = self.trap_writer.location(
             self.file_label,
             start_line,
@@ -425,13 +414,10 @@ impl Visitor<'_> {
                     vec![
                         Arg::Label(id),
                         Arg::Int(*kind_id),
-                        Arg::Label(self.file_label),
-                        Arg::Int(self.token_counter),
                         sliced_source_arg(self.source, node),
                         Arg::Label(loc),
                     ],
                 );
-                self.token_counter += 1;
             }
             EntryKind::Table {
                 fields,
@@ -446,11 +432,10 @@ impl Visitor<'_> {
                             Arg::Int(parent_index),
                         ],
                     );
-                    let mut all_args = Vec::new();
-                    all_args.push(Arg::Label(id));
+                    let mut all_args = vec![Arg::Label(id)];
                     all_args.extend(args);
                     all_args.push(Arg::Label(loc));
-                    self.trap_writer.add_tuple(&table_name, all_args);
+                    self.trap_writer.add_tuple(table_name, all_args);
                 }
             }
             _ => {
@@ -485,8 +470,8 @@ impl Visitor<'_> {
     fn complex_node(
         &mut self,
         node: &Node,
-        fields: &Vec<Field>,
-        child_nodes: &Vec<ChildNode>,
+        fields: &[Field],
+        child_nodes: &[ChildNode],
         parent_id: Label,
     ) -> Option<Vec<Arg>> {
         let mut map: Map<&Option<String>, (&Field, Vec<Arg>)> = Map::new();
@@ -523,22 +508,20 @@ impl Visitor<'_> {
                     );
                     self.record_parse_error_for_node(error_message, full_error_message, *node);
                 }
-            } else {
-                if child_node.field_name.is_some() || child_node.type_name.named {
-                    let error_message = format!(
-                        "value for unknown field: {}::{} and type {:?}",
-                        node.kind(),
-                        &child_node.field_name.unwrap_or("child"),
-                        &child_node.type_name
-                    );
-                    let full_error_message = format!(
-                        "{}:{}: {}",
-                        &self.path,
-                        node.start_position().row + 1,
-                        error_message
-                    );
-                    self.record_parse_error_for_node(error_message, full_error_message, *node);
-                }
+            } else if child_node.field_name.is_some() || child_node.type_name.named {
+                let error_message = format!(
+                    "value for unknown field: {}::{} and type {:?}",
+                    node.kind(),
+                    &child_node.field_name.unwrap_or("child"),
+                    &child_node.type_name
+                );
+                let full_error_message = format!(
+                    "{}:{}: {}",
+                    &self.path,
+                    node.start_position().row + 1,
+                    error_message
+                );
+                self.record_parse_error_for_node(error_message, full_error_message, *node);
             }
         }
         let mut args = Vec::new();
@@ -586,13 +569,12 @@ impl Visitor<'_> {
                             );
                             break;
                         }
-                        let mut args = Vec::new();
-                        args.push(Arg::Label(parent_id));
+                        let mut args = vec![Arg::Label(parent_id)];
                         if *has_index {
                             args.push(Arg::Int(index))
                         }
                         args.push(child_value.clone());
-                        self.trap_writer.add_tuple(&table_name, args);
+                        self.trap_writer.add_tuple(table_name, args);
                     }
                 }
             }
@@ -610,13 +592,10 @@ impl Visitor<'_> {
                 if tp == single_type {
                     return true;
                 }
-                match &self.schema.get(single_type).unwrap().kind {
-                    EntryKind::Union { members } => {
-                        if self.type_matches_set(tp, members) {
-                            return true;
-                        }
+                if let EntryKind::Union { members } = &self.schema.get(single_type).unwrap().kind {
+                    if self.type_matches_set(tp, members) {
+                        return true;
                     }
-                    _ => {}
                 }
             }
             node_types::FieldTypeInfo::Multiple { types, .. } => {
@@ -646,15 +625,15 @@ impl Visitor<'_> {
 }
 
 // Emit a slice of a source file as an Arg.
-fn sliced_source_arg(source: &Vec<u8>, n: Node) -> Arg {
+fn sliced_source_arg(source: &[u8], n: Node) -> Arg {
     let range = n.byte_range();
     Arg::String(String::from_utf8_lossy(&source[range.start..range.end]).into_owned())
 }
 
 // Emit a pair of `TrapEntry`s for the provided node, appropriately calibrated.
 // The first is the location and label definition, and the second is the
 // 'Located' entry.
-fn location_for<'a>(source: &Vec<u8>, n: Node) -> (usize, usize, usize, usize) {
+fn location_for(source: &[u8], n: Node) -> (usize, usize, usize, usize) {
     // Tree-sitter row, column values are 0-based while CodeQL starts
     // counting at 1. In addition Tree-sitter's row and column for the
     // end position are exclusive while CodeQL's end positions are inclusive.
@@ -812,18 +791,18 @@ impl fmt::Display for Arg {
 /// the string is sliced at the provided limit. If there is a multi-byte character
 /// at the limit then the returned slice will be slightly shorter than the limit to
 /// avoid splitting that multi-byte character.
-fn limit_string(string: &String, max_size: usize) -> &str {
+fn limit_string(string: &str, max_size: usize) -> &str {
     if string.len() <= max_size {
         return string;
     }
-    let p = string.as_ptr();
+    let p = string.as_bytes();
     let mut index = max_size;
     // We want to clip the string at [max_size]; however, the character at that position
     // may span several bytes. We need to find the first byte of the character. In UTF-8
     // encoded data any byte that matches the bit pattern 10XXXXXX is not a start byte.
     // Therefore we decrement the index as long as there are bytes matching this pattern.
     // This ensures we cut the string at the border between one character and another.
-    while index > 0 && unsafe { (*p.offset(index as isize) & 0b11000000) == 0b10000000 } {
+    while index > 0 && (p[index] & 0b11000000) == 0b10000000 {
         index -= 1;
     }
     &string[0..index]

extractor/src/main.rs

@@ -2,7 +2,6 @@ mod extractor;
 
 extern crate num_cpus;
 
-use clap;
 use flate2::write::GzEncoder;
 use rayon::prelude::*;
 use std::fs;
@@ -57,7 +56,7 @@ impl TrapCompression {
  * (minimum of 1). If unspecified, should be considered as set to -1."
  */
 fn num_codeql_threads() -> usize {
-    let threads_str = std::env::var("CODEQL_THREADS").unwrap_or("-1".to_owned());
+    let threads_str = std::env::var("CODEQL_THREADS").unwrap_or_else(|_| "-1".to_owned());
     match threads_str.parse::<i32>() {
         Ok(num) if num <= 0 => {
             let reduction = -num as usize;
@@ -124,6 +123,10 @@ fn main() -> std::io::Result<()> {
 
     let language = tree_sitter_ruby::language();
     let erb = tree_sitter_embedded_template::language();
+    // Look up tree-sitter kind ids now, to avoid string comparisons when scanning ERB files.
+    let erb_directive_id = erb.id_for_node_kind("directive", true);
+    let erb_output_directive_id = erb.id_for_node_kind("output_directive", true);
+    let erb_code_id = erb.id_for_node_kind("code", true);
     let schema = node_types::read_node_types_str("ruby", tree_sitter_ruby::NODE_TYPES)?;
     let erb_schema =
         node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?;
@@ -149,7 +152,13 @@ fn main() -> std::io::Result<()> {
                     &[],
                 )?;
 
-                let (ranges, line_breaks) = scan_erb(erb, &source);
+                let (ranges, line_breaks) = scan_erb(
+                    erb,
+                    &source,
+                    erb_directive_id,
+                    erb_output_directive_id,
+                    erb_code_id,
+                );
                 for i in line_breaks {
                     if i < source.len() {
                         source[i] = b'\n';
@@ -181,12 +190,12 @@ fn main() -> std::io::Result<()> {
 }
 
 fn write_trap(
-    trap_dir: &PathBuf,
+    trap_dir: &Path,
     path: PathBuf,
     trap_writer: extractor::TrapWriter,
     trap_compression: &TrapCompression,
 ) -> std::io::Result<()> {
-    let trap_file = path_for(&trap_dir, &path, trap_compression.extension());
+    let trap_file = path_for(trap_dir, &path, trap_compression.extension());
     std::fs::create_dir_all(&trap_file.parent().unwrap())?;
     let trap_file = std::fs::File::create(&trap_file)?;
     let mut trap_file = BufWriter::new(trap_file);
@@ -199,18 +208,24 @@ fn write_trap(
     }
 }
 
-fn scan_erb(erb: Language, source: &std::vec::Vec<u8>) -> (Vec<Range>, Vec<usize>) {
+fn scan_erb(
+    erb: Language,
+    source: &[u8],
+    directive_id: u16,
+    output_directive_id: u16,
+    code_id: u16,
+) -> (Vec<Range>, Vec<usize>) {
     let mut parser = Parser::new();
     parser.set_language(erb).unwrap();
     let tree = parser.parse(&source, None).expect("Failed to parse file");
     let mut result = Vec::new();
     let mut line_breaks = vec![];
 
     for n in tree.root_node().children(&mut tree.walk()) {
-        let kind = n.kind();
-        if kind == "directive" || kind == "output_directive" {
+        let kind_id = n.kind_id();
+        if kind_id == directive_id || kind_id == output_directive_id {
             for c in n.children(&mut tree.walk()) {
-                if c.kind() == "code" {
+                if c.kind_id() == code_id {
                     let mut range = c.range();
                     if range.end_byte < source.len() {
                         line_breaks.push(range.end_byte);
@@ -222,7 +237,7 @@ fn scan_erb(erb: Language, source: &std::vec::Vec<u8>) -> (Vec<Range>, Vec<usize
             }
         }
     }
-    if result.len() == 0 {
+    if result.is_empty() {
         let root = tree.root_node();
         // Add an empty range at the end of the file
         result.push(Range {

generator/src/dbscheme.rs

@@ -68,10 +68,10 @@ impl<'a> fmt::Display for Table<'a> {
                 }
                 write!(f, "{}", key)?;
             }
-            write!(f, "]\n")?;
+            writeln!(f, "]")?;
         }
 
-        write!(f, "{}(\n", self.name)?;
+        writeln!(f, "{}(", self.name)?;
         for (column_index, column) in self.columns.iter().enumerate() {
             write!(f, "  ")?;
             if column.unique {
@@ -92,7 +92,7 @@ impl<'a> fmt::Display for Table<'a> {
             if column_index + 1 != self.columns.len() {
                 write!(f, ",")?;
             }
-            write!(f, "\n")?;
+            writeln!(f)?;
         }
         write!(f, ");")?;