feat: add bypass-removal tag to prevent runner scale-down

Add support for ghr:bypass-removal EC2 tag that allows engineers
to manually tag runners to prevent them from being scaled down during
debugging or investigation. When this tag is set to 'true', the runner
will be skipped during scale-down operations with appropriate logging.

Author: Brend-Smits

lambdas/functions/control-plane/src/aws/runners.d.ts

@@ -11,6 +11,7 @@ export interface RunnerList {
   org?: string;
   orphan?: boolean;
   runnerId?: string;
+  bypassRemoval?: boolean;
 }
 
 export interface RunnerInfo {

lambdas/functions/control-plane/src/aws/runners.ts

@@ -93,6 +93,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
             org: i.Tags?.find((e) => e.Key === 'ghr:Org')?.Value as string,
             orphan: i.Tags?.find((e) => e.Key === 'ghr:orphan')?.Value === 'true',
             runnerId: i.Tags?.find((e) => e.Key === 'ghr:github_runner_id')?.Value as string,
+            bypassRemoval: i.Tags?.find((e) => e.Key === 'ghr:bypass-removal')?.Value === 'true',
           });
         }
       }

lambdas/functions/control-plane/src/scale-runners/scale-down.ts

@@ -130,6 +130,14 @@ function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean {
 async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promise<void> {
   const githubAppClient = await getOrCreateOctokit(ec2runner);
   try {
+    const runnerList = ec2runner as unknown as RunnerList;
+    if (runnerList.bypassRemoval) {
+      logger.info(
+        `Runner '${ec2runner.instanceId}' has bypass-removal tag set, skipping removal. Remove the tag to allow scale-down.`,
+      );
+      return;
+    }
+
     const states = await Promise.all(
       ghRunnerIds.map(async (ghRunnerId) => {
         // Get busy state instead of using the output of listGitHubRunners(...) to minimize to race condition.