feat: add bypass-removal tag to prevent runner scale-down (#4995)

Add support for ghr:bypass-removal EC2 tag that allows engineers to
manually tag runners to prevent them from being scaled down during
debugging or investigation. When this tag is set to 'true', the runner
will be skipped during scale-down operations with appropriate logging.

Tested on our staging environment and things are working:
<img width="967" height="102" alt="image"
src="https://github.com/user-attachments/assets/2de96e22-4ec7-43ba-aace-6152f4a5fdb8"
/>

This small change makes debugging some issue on the runner significantly
easier...

Author: Brend-Smits

docs/additional_notes.md

@@ -32,3 +32,31 @@ If default labels are removed:
 | 'custom5' | Linux | no match |
 | 'custom5' | [ self-hosted, Linux ] | no match |
 | 'custom5' | [ custom5, self-hosted, Linux ] | no match |
+
+# Preventing Runner Scale-Down for Debugging
+
+The module supports a bypass mechanism that allows you to prevent specific runners from being scaled down during debugging or investigation. This is useful when you need to access a runner instance directly to troubleshoot issues.
+
+## Usage
+
+To prevent a runner from being terminated during scale-down operations, add the `ghr:bypass-removal` tag to the EC2 instance with a value of `true`:
+
+```bash
+aws ec2 create-tags --resources <instance-id> --tags Key=ghr:bypass-removal,Value=true
+```
+
+When this tag is set, the scale-down process will skip the runner and log a message indicating that the runner is protected:
+
+```
+Runner 'i-xxxxxxxxxxxx' has bypass-removal tag set, skipping removal. Remove the tag to allow scale-down.
+```
+
+## Removing the Protection
+
+Once you've finished debugging and want to allow the runner to be scaled down normally, remove the tag or set it to any other value:
+
+```bash
+aws ec2 delete-tags --resources <instance-id> --tags Key=ghr:bypass-removal
+```
+
+**Note:** The bypass-removal tag only prevents automatic scale-down. The runner will still continue to process job(s) as normal. Make sure to remove the tag after debugging to ensure proper resource management. It will also still terminate itself if the instance is empheral and the job is complete.

lambdas/functions/control-plane/src/aws/runners.d.ts

@@ -11,6 +11,7 @@ export interface RunnerList {
   org?: string;
   orphan?: boolean;
   runnerId?: string;
+  bypassRemoval?: boolean;
 }
 
 export interface RunnerInfo {

lambdas/functions/control-plane/src/aws/runners.test.ts

@@ -91,6 +91,7 @@ describe('list instances', () => {
       type: 'Org',
       owner: 'CoderToCat',
       orphan: false,
+      bypassRemoval: false,
     });
   });
 
@@ -105,6 +106,7 @@ describe('list instances', () => {
       owner: 'CoderToCat',
       orphan: false,
       runnerId: '9876543210',
+      bypassRemoval: false,
     });
   });
 
@@ -124,6 +126,7 @@ describe('list instances', () => {
       type: 'Org',
       owner: 'CoderToCat',
       orphan: true,
+      bypassRemoval: false,
     });
   });
 

lambdas/functions/control-plane/src/aws/runners.ts

@@ -93,6 +93,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
             org: i.Tags?.find((e) => e.Key === 'ghr:Org')?.Value as string,
             orphan: i.Tags?.find((e) => e.Key === 'ghr:orphan')?.Value === 'true',
             runnerId: i.Tags?.find((e) => e.Key === 'ghr:github_runner_id')?.Value as string,
+            bypassRemoval: i.Tags?.find((e) => e.Key === 'ghr:bypass-removal')?.Value === 'true',
           });
         }
       }

lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts

@@ -286,6 +286,25 @@ describe('Scale down runners', () => {
         checkNonTerminated(runners);
       });
 
+      it(`Should not terminate runner with bypass-removal tag set.`, async () => {
+        // setup
+        const runners = [
+          createRunnerTestData('idle-with-bypass', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 10, true, false, false),
+        ];
+        // Set bypass-removal tag
+        runners[0].bypassRemoval = true;
+
+        mockGitHubRunners(runners);
+        mockAwsRunners(runners);
+
+        // act
+        await scaleDown();
+
+        // assert
+        expect(terminateRunner).not.toHaveBeenCalled();
+        checkNonTerminated(runners);
+      });
+
       it(`Should not terminate a runner that became busy just before deregister runner.`, async () => {
         // setup
         const runners = [
@@ -813,5 +832,6 @@ function createRunnerTestData(
     orphan,
     shouldBeTerminated,
     runnerId: runnerId !== undefined ? String(runnerId) : undefined,
+    bypassRemoval: false,
   };
 }

lambdas/functions/control-plane/src/scale-runners/scale-down.ts

@@ -130,6 +130,14 @@ function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean {
 async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promise<void> {
   const githubAppClient = await getOrCreateOctokit(ec2runner);
   try {
+    const runnerList = ec2runner as unknown as RunnerList;
+    if (runnerList.bypassRemoval) {
+      logger.info(
+        `Runner '${ec2runner.instanceId}' has bypass-removal tag set, skipping removal. Remove the tag to allow scale-down.`,
+      );
+      return;
+    }
+
     const states = await Promise.all(
       ghRunnerIds.map(async (ghRunnerId) => {
         // Get busy state instead of using the output of listGitHubRunners(...) to minimize to race condition.