Merge branch 'main' into main

Author: wadherv

.github/workflows/mkdocs/requirements.txt

@@ -330,9 +330,9 @@ six==1.17.0 \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
     # via python-dateutil
-urllib3==2.6.0 \
-    --hash=sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f \
-    --hash=sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1
+urllib3==2.6.3 \
+    --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \
+    --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4
     # via requests
 watchdog==6.0.0 \
     --hash=sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a \

MAINTAINERS.md

@@ -10,6 +10,8 @@ Roles and responsibilities of the maintainers of the project.
 | Koen de Laat      | [@koendelaat]       | Philips          |
 | Guilherme Caulada | [@guicaulada]       | Grafana Labs     |
 | Ederson Brilhante | [@edersonbrilhante] | Cisco            |
+| Brend Smits       | [@Brend-Smits]      | Philips          |
+| Stuart Pearson    | [@stuartp44]        | Philips          |
 
 ## Responsibilities
 

docs/additional_notes.md

@@ -32,3 +32,31 @@ If default labels are removed:
 | 'custom5' | Linux | no match |
 | 'custom5' | [ self-hosted, Linux ] | no match |
 | 'custom5' | [ custom5, self-hosted, Linux ] | no match |
+
+# Preventing Runner Scale-Down for Debugging
+
+The module supports a bypass mechanism that allows you to prevent specific runners from being scaled down during debugging or investigation. This is useful when you need to access a runner instance directly to troubleshoot issues.
+
+## Usage
+
+To prevent a runner from being terminated during scale-down operations, add the `ghr:bypass-removal` tag to the EC2 instance with a value of `true`:
+
+```bash
+aws ec2 create-tags --resources <instance-id> --tags Key=ghr:bypass-removal,Value=true
+```
+
+When this tag is set, the scale-down process will skip the runner and log a message indicating that the runner is protected:
+
+```
+Runner 'i-xxxxxxxxxxxx' has bypass-removal tag set, skipping removal. Remove the tag to allow scale-down.
+```
+
+## Removing the Protection
+
+Once you've finished debugging and want to allow the runner to be scaled down normally, remove the tag or set it to any other value:
+
+```bash
+aws ec2 delete-tags --resources <instance-id> --tags Key=ghr:bypass-removal
+```
+
+**Note:** The bypass-removal tag only prevents automatic scale-down. The runner will still continue to process job(s) as normal. Make sure to remove the tag after debugging to ensure proper resource management. It will also still terminate itself if the instance is empheral and the job is complete.

examples/termination-watcher/README.md

@@ -4,7 +4,7 @@ This module shows how to use the termination watcher stand-alone.
 
 ## Usages
 
-Ensure your have the lambda for the termination locally build. By default the one in the lambdas folder will be used.
+Ensure you have the lambda for the termination built locally. By default the one in the lambdas folder will be used.
 
 Build lambda's (requires node and yarn).
 

lambdas/functions/ami-housekeeper/package.json

@@ -17,17 +17,17 @@
     "all": "yarn build && yarn format && yarn lint && yarn test"
   },
   "devDependencies": {
-    "@aws-sdk/types": "^3.953.0",
-    "@types/aws-lambda": "^8.10.155",
+    "@aws-sdk/types": "^3.957.0",
+    "@types/aws-lambda": "^8.10.159",
     "@vercel/ncc": "^0.38.4",
     "aws-sdk-client-mock": "^4.1.0",
     "aws-sdk-client-mock-jest": "^4.1.0"
   },
   "dependencies": {
     "@aws-github-runner/aws-powertools-util": "*",
     "@aws-github-runner/aws-ssm-util": "*",
-    "@aws-sdk/client-ec2": "^3.953.0",
-    "@aws-sdk/client-ssm": "^3.953.0",
+    "@aws-sdk/client-ec2": "^3.958.0",
+    "@aws-sdk/client-ssm": "^3.958.0",
     "cron-parser": "^5.4.0"
   },
   "nx": {

lambdas/functions/control-plane/package.json

@@ -17,10 +17,10 @@
     "all": "yarn build && yarn format && yarn lint && yarn test"
   },
   "devDependencies": {
-    "@aws-sdk/types": "^3.953.0",
+    "@aws-sdk/types": "^3.957.0",
     "@octokit/types": "^16.0.0",
-    "@types/aws-lambda": "^8.10.155",
-    "@types/node": "^22.19.0",
+    "@types/aws-lambda": "^8.10.159",
+    "@types/node": "^22.19.3",
     "@vercel/ncc": "^0.38.4",
     "aws-sdk-client-mock": "^4.1.0",
     "aws-sdk-client-mock-jest": "^4.1.0",
@@ -33,8 +33,8 @@
     "@aws-github-runner/aws-powertools-util": "*",
     "@aws-github-runner/aws-ssm-util": "*",
     "@aws-lambda-powertools/parameters": "^2.30.0",
-    "@aws-sdk/client-ec2": "^3.953.0",
-    "@aws-sdk/client-sqs": "^3.953.0",
+    "@aws-sdk/client-ec2": "^3.958.0",
+    "@aws-sdk/client-sqs": "^3.960.0",
     "@middy/core": "^6.4.5",
     "@octokit/auth-app": "8.1.2",
     "@octokit/core": "7.0.6",

lambdas/functions/control-plane/src/aws/runners.d.ts

@@ -11,6 +11,7 @@ export interface RunnerList {
   org?: string;
   orphan?: boolean;
   runnerId?: string;
+  bypassRemoval?: boolean;
 }
 
 export interface RunnerInfo {

lambdas/functions/control-plane/src/aws/runners.test.ts

@@ -91,6 +91,7 @@ describe('list instances', () => {
       type: 'Org',
       owner: 'CoderToCat',
       orphan: false,
+      bypassRemoval: false,
     });
   });
 
@@ -105,6 +106,7 @@ describe('list instances', () => {
       owner: 'CoderToCat',
       orphan: false,
       runnerId: '9876543210',
+      bypassRemoval: false,
     });
   });
 
@@ -124,6 +126,7 @@ describe('list instances', () => {
       type: 'Org',
       owner: 'CoderToCat',
       orphan: true,
+      bypassRemoval: false,
     });
   });
 

lambdas/functions/control-plane/src/aws/runners.ts

@@ -93,6 +93,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
             org: i.Tags?.find((e) => e.Key === 'ghr:Org')?.Value as string,
             orphan: i.Tags?.find((e) => e.Key === 'ghr:orphan')?.Value === 'true',
             runnerId: i.Tags?.find((e) => e.Key === 'ghr:github_runner_id')?.Value as string,
+            bypassRemoval: i.Tags?.find((e) => e.Key === 'ghr:bypass-removal')?.Value === 'true',
           });
         }
       }

lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts

@@ -286,6 +286,25 @@ describe('Scale down runners', () => {
         checkNonTerminated(runners);
       });
 
+      it(`Should not terminate runner with bypass-removal tag set.`, async () => {
+        // setup
+        const runners = [
+          createRunnerTestData('idle-with-bypass', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 10, true, false, false),
+        ];
+        // Set bypass-removal tag
+        runners[0].bypassRemoval = true;
+
+        mockGitHubRunners(runners);
+        mockAwsRunners(runners);
+
+        // act
+        await scaleDown();
+
+        // assert
+        expect(terminateRunner).not.toHaveBeenCalled();
+        checkNonTerminated(runners);
+      });
+
       it(`Should not terminate a runner that became busy just before deregister runner.`, async () => {
         // setup
         const runners = [
@@ -813,5 +832,6 @@ function createRunnerTestData(
     orphan,
     shouldBeTerminated,
     runnerId: runnerId !== undefined ? String(runnerId) : undefined,
+    bypassRemoval: false,
   };
 }