Further add resilliancy in execution

Author: JLLeitschuh

vulnerability_fix_engine.py

@@ -41,6 +41,10 @@ class ForkAlreadyExistsException(Exception):
     pass
 
 
+class AmbiguousObjectNameHeadException(Exception):
+    pass
+
+
 async def subprocess_run(args: List[str], cwd: str) -> Optional[str]:
     proc = await asyncio.create_subprocess_exec(
         args[0],
@@ -70,6 +74,8 @@ async def subprocess_run(args: List[str], cwd: str) -> Optional[str]:
                 raise PullRequestAlreadyExistsException(error_msg)
             if 'Error creating fork' in msg and 'already exists on github.com' in msg:
                 raise ForkAlreadyExistsException(error_msg)
+            if ' Ambiguous object name: \'HEAD\'' in msg:
+                raise AmbiguousObjectNameHeadException(error_msg)
             raise RuntimeError(error_msg)
     else:
         if stderr:
@@ -159,6 +165,7 @@ async def do_fix_vulnerable_file(self, project_name: str, file: str, expected_fi
 class VulnerabilityFixReport:
     files_fixed: int
     vulnerabilities_fixed: int
+    file_name_fixed: List[str]
 
 
 @dataclass
@@ -276,6 +283,7 @@ async def do_fix_vulnerabilities(self) -> VulnerabilityFixReport:
         project_vulnerabilities_fixed = 0
         project_files_fixed = 0
         submodules = self.submodule_files()
+        files_fixed: List[str] = []
         for file in self.project_files.files:
             # Skip submodule files
             skip = next((True for submodule in submodules if file.startswith(submodule)), False)
@@ -284,16 +292,21 @@ async def do_fix_vulnerabilities(self) -> VulnerabilityFixReport:
                 if file_vulnerabilities_fixed > 0:
                     project_vulnerabilities_fixed += file_vulnerabilities_fixed
                     project_files_fixed += 1
-        return VulnerabilityFixReport(project_files_fixed, project_vulnerabilities_fixed)
+                    files_fixed.append(file)
+        return VulnerabilityFixReport(
+            project_files_fixed,
+            project_vulnerabilities_fixed,
+            files_fixed
+        )
 
     async def do_create_branch(self):
         await self.do_run_in(['git', 'checkout', '-b', self.fix_module.branch_name])
 
-    async def do_stage_changes(self):
+    async def do_stage_changes(self, project_report: VulnerabilityFixReport):
         command = ['git', 'add']
         # Only run add on the files we've modified
         # This hopefully limits CRLF changes
-        files_trimmed = [file_name.lstrip('/') for file_name in self.project_files.files.keys()]
+        files_trimmed = [file_name.lstrip('/') for file_name in project_report.file_name_fixed]
         command.extend(files_trimmed)
         await self.do_run_in(command)
 
@@ -375,7 +388,7 @@ async def execute_vulnerability_fixer_engine(engine: VulnerabilityFixerEngine, l
     # If the LGTM data is out-of-date, there can be cases where no vulnerabilities are fixed
     if project_report.vulnerabilities_fixed != 0:
         await engine.do_create_branch()
-        await engine.do_stage_changes()
+        await engine.do_stage_changes(project_report)
         await engine.do_commit_changes()
 
         if not engine.project_files.project_name.lower().startswith('jlleitschuh'):
@@ -387,9 +400,15 @@ async def execute_vulnerability_fixer_engine(engine: VulnerabilityFixerEngine, l
     return project_report
 
 
-async def execute_vulnerability_fixer_engine_checked(engine: VulnerabilityFixerEngine, lock) -> VulnerabilityFixReport:
+async def execute_vulnerability_fixer_engine_checked(
+        engine: VulnerabilityFixerEngine,
+        lock
+) -> Optional[VulnerabilityFixReport]:
     try:
         return await execute_vulnerability_fixer_engine(engine, lock)
+    except AmbiguousObjectNameHeadException:
+        # They named their main branch 'HEAD'... Why?! No fix for them.
+        return None
     except BaseException as e:
         if 'CancelledError' in e.__class__.__name__:
             raise e
@@ -425,6 +444,8 @@ async def _do_execute_engines(engines: List[VulnerabilityFixerEngine]):
         print(f'Processing {len(waiting_reports)} Projects:')
         all_reports = await asyncio.gather(*waiting_reports)
         for report in all_reports:
+            if report is None:
+                continue
             if report.vulnerabilities_fixed > 0:
                 projects_fixed += 1
                 files_fixed += report.files_fixed
@@ -447,12 +468,14 @@ async def _do_execute_fix_module(fix_module: VulnerabilityFixModule, starting_le
     for vulnerable_project in vulnerable_projects:
         if not vulnerable_project.project_name.startswith(starting_letter):
             continue
-        if is_archived_git_hub_repository(vulnerable_project):
-            logging.info(f'Skipping project {vulnerable_project.project_name} since it is archived')
-            continue
+        # Check this first, it's going to be faster
         if os.path.exists(fix_module.save_point_file_name(vulnerable_project)):
             logging.info(f'Skipping project {vulnerable_project.project_name} since save point file already exists')
             continue
+        # Check this second, it's going to be slower
+        if is_archived_git_hub_repository(vulnerable_project):
+            logging.info(f'Skipping project {vulnerable_project.project_name} since it is archived')
+            continue
         print(f'Loading Execution for: {vulnerable_project.project_name}')
         engine = VulnerabilityFixerEngine(
             fix_module=fix_module,
@@ -463,10 +486,11 @@ async def _do_execute_fix_module(fix_module: VulnerabilityFixModule, starting_le
     size = 100
     engine_lists = x = [engines[i:i + size] for i in range(0, len(engines), size)]
     for engine_list in engine_lists:
-        try:
-            await _do_execute_engines(engine_list)
-        except EnginesExecutionException as e:
-            logging.exception(f'Failed while processing engine group. {str(e)}')
+        await _do_execute_engines(engine_list)
+        # try:
+        #     await _do_execute_engines(engine_list)
+        # except EnginesExecutionException as e:
+        #     logging.exception(f'Failed while processing engine group. {str(e)}')
 
 
 def do_execute_fix_module(fix_module: VulnerabilityFixModule):