C-241 Add EventBridge rule for all EC2 terminations

The existing spot-specific rules (BidEvictedEvent, Spot Interruption Warning)
only fire on AWS spot reclamations. Scale-down terminations and manual
terminations — the most common causes of stale runners — were not covered.

Add an EC2 Instance State-change Notification rule (state: shutting-down) that
catches ALL termination types. Reuses the same notification Lambda since both
event types have detail['instance-id']. Gated behind enable_runner_deregistration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jensenbox

lambdas/functions/termination-watcher/src/deregister.ts

@@ -25,10 +25,7 @@ export function createThrottleOptions() {
 
 async function getAppCredentials(): Promise<{ appId: number; privateKey: string }> {
   const appId = parseInt(await getParameter(process.env.PARAMETER_GITHUB_APP_ID_NAME!));
-  const privateKey = Buffer.from(
-    await getParameter(process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME!),
-    'base64',
-  )
+  const privateKey = Buffer.from(await getParameter(process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME!), 'base64')
     .toString()
     .replace('/[\\n]/g', String.fromCharCode(10));
   return { appId, privateKey };
@@ -142,12 +139,7 @@ async function findRunnerByInstanceId(
   return undefined;
 }
 
-async function deleteRunner(
-  octokit: Octokit,
-  owner: string,
-  runnerId: number,
-  runnerType: string,
-): Promise<void> {
+async function deleteRunner(octokit: Octokit, owner: string, runnerId: number, runnerType: string): Promise<void> {
   if (runnerType === 'Repo') {
     const [repoOwner, repo] = owner.split('/');
     await octokit.actions.deleteSelfHostedRunnerFromRepo({

lambdas/vitest.base.config.ts

@@ -9,11 +9,11 @@ const defaultConfig = defineConfig({
       include: ['**/src/**/*.ts'],
       exclude: ['**/*local*.ts', '**/*.d.ts', '**/*.test.ts', '**/node_modules/**'],
       all: true,
-      reportsDirectory: './coverage'
+      reportsDirectory: './coverage',
     },
     globals: true,
-    watch: false
-  }
+    watch: false,
+  },
 });
 
 export default defaultConfig;

modules/termination-watcher/notification/main.tf

@@ -42,6 +42,46 @@ resource "aws_lambda_permission" "main" {
   source_arn    = aws_cloudwatch_event_rule.spot_instance_termination_warning.arn
 }
 
+# EC2 Instance State-change Notification — catches ALL termination types
+# (scale-down, manual, spot reclamation, ASG) not just spot-specific events.
+# Uses "shutting-down" state to deregister runners while instance metadata is still available.
+# Reuses the same Lambda as the spot interruption warning handler since both event
+# types have detail['instance-id'] — the handler extracts it identically.
+resource "aws_cloudwatch_event_rule" "ec2_instance_state_change" {
+  count = var.config._enable_runner_deregistration ? 1 : 0
+
+  name        = "${var.config.prefix != null ? format("%s-", var.config.prefix) : ""}instance-termination"
+  description = "EC2 Instance Termination (all causes) — deregisters runners from GitHub"
+  tags        = local.config.tags
+
+  event_pattern = <<EOF
+{
+  "source": ["aws.ec2"],
+  "detail-type": ["EC2 Instance State-change Notification"],
+  "detail": {
+    "state": ["shutting-down"]
+  }
+}
+EOF
+}
+
+resource "aws_cloudwatch_event_target" "state_change" {
+  count = var.config._enable_runner_deregistration ? 1 : 0
+
+  rule = aws_cloudwatch_event_rule.ec2_instance_state_change[0].name
+  arn  = module.termination_warning_watcher.lambda.function.arn
+}
+
+resource "aws_lambda_permission" "state_change" {
+  count = var.config._enable_runner_deregistration ? 1 : 0
+
+  statement_id  = "AllowExecutionFromCloudWatchStateChange"
+  action        = "lambda:InvokeFunction"
+  function_name = module.termination_warning_watcher.lambda.function.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.ec2_instance_state_change[0].arn
+}
+
 resource "aws_iam_role_policy" "lambda_policy" {
   name = "lambda-policy"
   role = module.termination_warning_watcher.lambda.role.name