JavaScript: Pull ExtractorState out of FileExtractor.

Author: Max Schaefer

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -173,7 +173,6 @@ public class AutoBuild {
 	private final Path LGTM_SRC, SEMMLE_DIST;
 	private final TypeScriptMode typeScriptMode;
 	private final String defaultEncoding;
-	private ExtractorState extractorState;
 
 	public AutoBuild() {
 		this.LGTM_SRC = toRealPath(getPathFromEnvVar("LGTM_SRC"));
@@ -182,7 +181,6 @@ public AutoBuild() {
 		this.trapCache = mkTrapCache();
 		this.typeScriptMode = getEnumFromEnvVar("LGTM_INDEX_TYPESCRIPT", TypeScriptMode.class, TypeScriptMode.BASIC);
 		this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING");
-		this.extractorState = new ExtractorState();
 		setupMatchers();
 	}
 
@@ -413,12 +411,12 @@ public File lookup(String source, ExtractorConfig config, FileType type) {
 			}
 		}
 
-		FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache, extractorState);
+		FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache);
 		FileVisitor<? super Path> visitor = new SimpleFileVisitor<Path>() {
 			@Override
 			public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 				if (".js".equals(FileUtil.extension(file.toString())))
-					extract(extractor, file);
+					extract(extractor, file, null);
 				return super.visitFile(file, attrs);
 			}
 		};
@@ -435,7 +433,7 @@ private void extractSource() throws IOException {
 		config = config.withTypeScriptMode(typeScriptMode);
 		if (defaultEncoding != null)
 			config = config.withDefaultEncoding(defaultEncoding);
-		FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache, extractorState);
+		FileExtractor extractor = new FileExtractor(config, outputConfig, trapCache);
 		Path[] currentRoot = new Path[1];
 		final Set<Path> filesToExtract = new LinkedHashSet<>();
 		final List<Path> tsconfigFiles = new ArrayList<>();
@@ -482,6 +480,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 		}
 
 		// If there are any .ts files, verify that TypeScript is installed.
+		ExtractorState extractorState = new ExtractorState();
 		TypeScriptParser tsParser = extractorState.getTypeScriptParser();
 		boolean hasTypeScriptFiles = false;
 		for (Path file : filesToExtract) {
@@ -494,7 +493,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 			}
 		}
 		if (hasTypeScriptFiles || !tsconfigFiles.isEmpty()) {
-			verifyTypeScriptInstallation();
+			verifyTypeScriptInstallation(extractorState);
 		}
 
 		// Extract TypeScript projects
@@ -515,7 +514,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 					typeScriptFiles.add(sourcePath.toFile());
 				}
 			}
-			extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor);
+			extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor, extractorState);
 			tsParser.closeProject(projectFile);
 		}
 
@@ -533,7 +532,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 			}
 		}
 		if (!remainingTypeScriptFiles.isEmpty()) {
-			extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor);
+			extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor, extractorState);
 		}
 
 		// The TypeScript compiler instance is no longer needed.
@@ -542,7 +541,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 		// Extract non-TypeScript files
 		for (Path f : filesToExtract) {
 			if (extractedFiles.add(f)) {
-				extract(extractor, f);
+				extract(extractor, f, null);
 			}
 		}
 	}
@@ -551,16 +550,17 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 	 * Verifies that Node.js and the TypeScript compiler are installed and can be
 	 * found.
 	 */
-	public void verifyTypeScriptInstallation() {
+	public void verifyTypeScriptInstallation(ExtractorState extractorState) {
 		extractorState.getTypeScriptParser().verifyInstallation(true);
 	}
 
-	public void extractTypeScriptFiles(List<File> files, Set<Path> extractedFiles, FileExtractor extractor) throws IOException {
+	public void extractTypeScriptFiles(List<File> files, Set<Path> extractedFiles,
+			FileExtractor extractor, ExtractorState extractorState) throws IOException {
 		extractorState.getTypeScriptParser().prepareFiles(files);
 		for (File f : files) {
 			Path path = f.toPath();
 			extractedFiles.add(path);
-			extract(extractor, f.toPath());
+			extract(extractor, f.toPath(), extractorState);
 		}
 	}
 
@@ -597,15 +597,15 @@ private SourceType getSourceType() {
 	/**
 	 * Extract a single file.
 	 */
-	protected void extract(FileExtractor extractor, Path file) throws IOException {
+	protected void extract(FileExtractor extractor, Path file, ExtractorState state) throws IOException {
 		File f = file.toFile();
 		if (!f.exists()) {
 			warn("Skipping " + file + ", which does not exist.");
 			return;
 		}
 
 		long start = logBeginProcess("Extracting " + file);
-		extractor.extract(f);
+		extractor.extract(f, state);
 		logEndProcess(start);
 	}
 

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -395,13 +395,11 @@ public boolean isTrapCachingAllowed() {
 	private final ExtractorConfig config;
 	private final ExtractorOutputConfig outputConfig;
 	private final ITrapCache trapCache;
-	private final ExtractorState state;
 
-	public FileExtractor(ExtractorConfig config, ExtractorOutputConfig outputConfig, ITrapCache trapCache, ExtractorState state) {
+	public FileExtractor(ExtractorConfig config, ExtractorOutputConfig outputConfig, ITrapCache trapCache) {
 		this.config = config;
 		this.outputConfig = outputConfig;
 		this.trapCache = trapCache;
-		this.state = state;
 	}
 
 	public ExtractorConfig getConfig() {
@@ -412,7 +410,7 @@ public boolean supports(File f) {
 		return config.hasFileType() || FileType.forFile(f, config) != null;
 	}
 
-	public void extract(File f) throws IOException {
+	public void extract(File f, ExtractorState state) throws IOException {
 		// populate source archive
 		String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
 		outputConfig.getSourceArchive().add(f, source);
@@ -424,7 +422,7 @@ public void extract(File f) throws IOException {
 		locationManager.emitFileLocation(fileLabel, 0, 0, 0, 0);
 
 		// now extract the contents
-		extractContents(f, fileLabel, source, locationManager);
+		extractContents(f, fileLabel, source, locationManager, state);
 	}
 
 
@@ -440,7 +438,7 @@ public void extract(File f) throws IOException {
 	 *
 	 * We only cache the content-dependent part, which makes up the bulk of the TRAP
 	 * file anyway. The location-dependent part is emitted from scratch every time
-	 * by the {@link #extract(File)} method above.
+	 * by the {@link #extract(File, ExtractorState)} method above.
 	 *
 	 * In order to keep labels in the main part independent of the file's location,
 	 * we bump the TRAP label counter to a known value (currently 20000) after the
@@ -451,7 +449,8 @@ public void extract(File f) throws IOException {
 	 * Also note that we support extraction with TRAP writer factories that are not file-backed;
 	 * obviously, no caching is done in that scenario.
 	 */
-	private void extractContents(File f, Label fileLabel, String source, LocationManager locationManager) throws IOException {
+	private void extractContents(File f, Label fileLabel, String source, LocationManager locationManager,
+			ExtractorState state) throws IOException {
 		TrapWriter trapwriter = locationManager.getTrapWriter();
 		FileType fileType = config.hasFileType() ? FileType.valueOf(config.getFileType())
 				: FileType.forFile(f, config);

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -125,7 +125,7 @@ public void run(String[] args) {
 				ap.error(P_TRAP_CACHE_BOUND + " should only be specified together with " + P_TRAP_CACHE + ".");
 			trapCache = new DummyTrapCache();
 		}
-		fileExtractor = new FileExtractor(extractorConfig, extractorOutputConfig, trapCache, extractorState);
+		fileExtractor = new FileExtractor(extractorConfig, extractorOutputConfig, trapCache);
 
 		setupMatchers(ap);
 
@@ -211,7 +211,7 @@ private void ensureFileIsExtracted(File f, ArgsParser ap) {
 		}
 		long start = verboseLogStartTimer(ap, "Extracting " + f);
 		try {
-			fileExtractor.extract(f.getAbsoluteFile());
+			fileExtractor.extract(f.getAbsoluteFile(), extractorState);
 			verboseLogEndTimer(ap, start);
 		} catch (IOException e) {
 			throw new ResourceError("Extraction of " + f + " failed.", e);

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -21,6 +21,7 @@
 import org.junit.Test;
 
 import com.semmle.js.extractor.AutoBuild;
+import com.semmle.js.extractor.ExtractorState;
 import com.semmle.js.extractor.FileExtractor;
 import com.semmle.util.data.StringUtil;
 import com.semmle.util.exception.UserError;
@@ -94,12 +95,12 @@ private void runTest() throws IOException {
 			Set<String> actual = new LinkedHashSet<>();
 			new AutoBuild() {
 				@Override
-				protected void extract(FileExtractor extractor, Path file) {
+				protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
 					actual.add(file.toString());
 				}
 
 				@Override
-				public void verifyTypeScriptInstallation() {
+				public void verifyTypeScriptInstallation(ExtractorState state) {
 				}
 
 				@Override