Swift: store TRAP files in a temporary folder until the extraction is complete

Currently, we have a number of assertions in the codebase and certain
assumptions about the AST. These don't always hold, sometimes leading to
a crash in the extractor.
The crashes leave incomplete TRAP files that cannot be imported into the
database.

With this change, we still get those incomplete TRAP files, but we also
get a database in the end (even thoough it is also incomplete as we
cannot import everything).

Author: AlexDenisov

swift/extractor/SwiftExtractor.cpp

@@ -62,13 +62,13 @@ static void extractDeclarations(const SwiftExtractorConfiguration& config,
   // TODO: find a more robust approach to avoid collisions?
   llvm::StringRef filename = primaryFile ? primaryFile->getFilename() : module.getModuleFilename();
   std::string tempTrapName = filename.str() + '.' + std::to_string(getpid()) + ".trap";
-  llvm::SmallString<PATH_MAX> tempTrapPath(config.trapDir);
+  llvm::SmallString<PATH_MAX> tempTrapPath(config.tempTrapDir);
   llvm::sys::path::append(tempTrapPath, tempTrapName);
 
-  llvm::StringRef trapParent = llvm::sys::path::parent_path(tempTrapPath);
-  if (std::error_code ec = llvm::sys::fs::create_directories(trapParent)) {
-    std::cerr << "Cannot create trap directory '" << trapParent.str() << "': " << ec.message()
-              << "\n";
+  llvm::StringRef tempTrapParent = llvm::sys::path::parent_path(tempTrapPath);
+  if (std::error_code ec = llvm::sys::fs::create_directories(tempTrapParent)) {
+    std::cerr << "Cannot create temp trap directory '" << tempTrapParent.str()
+              << "': " << ec.message() << "\n";
     return;
   }
 
@@ -117,6 +117,13 @@ static void extractDeclarations(const SwiftExtractorConfiguration& config,
   llvm::SmallString<PATH_MAX> trapPath(config.trapDir);
   llvm::sys::path::append(trapPath, trapName);
 
+  llvm::StringRef trapParent = llvm::sys::path::parent_path(trapPath);
+  if (std::error_code ec = llvm::sys::fs::create_directories(trapParent)) {
+    std::cerr << "Cannot create trap directory '" << trapParent.str() << "': " << ec.message()
+              << "\n";
+    return;
+  }
+
   // TODO: The last process wins. Should we do better than that?
   if (std::error_code ec = llvm::sys::fs::rename(tempTrapPath, trapPath)) {
     std::cerr << "Cannot rename temp trap file '" << tempTrapPath.str().str() << "' -> '"

swift/extractor/SwiftExtractorConfiguration.h

@@ -9,7 +9,14 @@ struct SwiftExtractorConfiguration {
   std::string trapDir;
   // The location for storing extracted source files.
   std::string sourceArchiveDir;
-  // The arguments passed to the extractor. Used for debugging.
+  // A temporary directory that exists during database creation, but is deleted once the DB is
+  // finalized.
+  std::string scratchDir;
+  // A temporary directory that contains TRAP files before they moved into the final destination.
+  // Subdirectory of the scratchDir.
+  std::string tempTrapDir;
+
+  // The original arguments passed to the extractor. Used for debugging.
   std::vector<std::string> frontendOptions;
 };
 }  // namespace codeql

swift/extractor/main.cpp

@@ -51,6 +51,10 @@ int main(int argc, char** argv) {
   codeql::SwiftExtractorConfiguration configuration{};
   configuration.trapDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR", ".");
   configuration.sourceArchiveDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_SOURCE_ARCHIVE_DIR", ".");
+  configuration.scratchDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_SCRATCH_DIR", ".");
+
+  configuration.tempTrapDir = configuration.scratchDir + "/swift-trap-temp";
+
   std::vector<const char*> args;
   for (int i = 1; i < argc; i++) {
     args.push_back(argv[i]);