C-241 Add EventBridge rule for all EC2 terminations

The existing spot-specific rules (BidEvictedEvent, Spot Interruption Warning)
only fire on AWS spot reclamations. Scale-down terminations and manual
terminations — the most common causes of stale runners — were not covered.

Add an EC2 Instance State-change Notification rule (state: shutting-down) that
catches ALL termination types. Reuses the same notification Lambda since both
event types have detail['instance-id']. Gated behind enable_runner_deregistration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jensenbox

lambdas/functions/termination-watcher/src/deregister.ts

@@ -25,10 +25,7 @@ export function createThrottleOptions() {
 
 async function getAppCredentials(): Promise<{ appId: number; privateKey: string }> {
   const appId = parseInt(await getParameter(process.env.PARAMETER_GITHUB_APP_ID_NAME!));
-  const privateKey = Buffer.from(
-    await getParameter(process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME!),
-    'base64',
-  )
+  const privateKey = Buffer.from(await getParameter(process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME!), 'base64')
     .toString()
     .replace('/[\\n]/g', String.fromCharCode(10));
   return { appId, privateKey };
@@ -142,12 +139,7 @@ async function findRunnerByInstanceId(
   return undefined;
 }
 
-async function deleteRunner(
-  octokit: Octokit,
-  owner: string,
-  runnerId: number,
-  runnerType: string,
-): Promise<void> {
+async function deleteRunner(octokit: Octokit, owner: string, runnerId: number, runnerType: string): Promise<void> {
   if (runnerType === 'Repo') {
     const [repoOwner, repo] = owner.split('/');
     await octokit.actions.deleteSelfHostedRunnerFromRepo({

lambdas/vitest.base.config.ts

@@ -9,11 +9,11 @@ const defaultConfig = defineConfig({
       include: ['**/src/**/*.ts'],
       exclude: ['**/*local*.ts', '**/*.d.ts', '**/*.test.ts', '**/node_modules/**'],
       all: true,
-      reportsDirectory: './coverage'
+      reportsDirectory: './coverage',
     },
     globals: true,
-    watch: false
-  }
+    watch: false,
+  },
 });
 
 export default defaultConfig;

main.tf

@@ -363,26 +363,26 @@ module "ami_housekeeper" {
 
 locals {
   lambda_instance_termination_watcher = {
-    prefix                    = var.prefix
-    tags                      = local.tags
-    aws_partition             = var.aws_partition
-    architecture              = var.lambda_architecture
-    principals                = var.lambda_principals
-    runtime                   = var.lambda_runtime
-    security_group_ids        = var.lambda_security_group_ids
-    subnet_ids                = var.lambda_subnet_ids
-    lambda_tags               = var.lambda_tags
-    log_level                 = var.log_level
-    log_class                 = var.log_class
-    logging_kms_key_id        = var.logging_kms_key_id
-    logging_retention_in_days = var.logging_retention_in_days
-    role_path                 = var.role_path
-    role_permissions_boundary = var.role_permissions_boundary
-    s3_bucket                 = var.lambda_s3_bucket
-    tracing_config            = var.tracing_config
-    metrics                   = var.metrics
+    prefix                       = var.prefix
+    tags                         = local.tags
+    aws_partition                = var.aws_partition
+    architecture                 = var.lambda_architecture
+    principals                   = var.lambda_principals
+    runtime                      = var.lambda_runtime
+    security_group_ids           = var.lambda_security_group_ids
+    subnet_ids                   = var.lambda_subnet_ids
+    lambda_tags                  = var.lambda_tags
+    log_level                    = var.log_level
+    log_class                    = var.log_class
+    logging_kms_key_id           = var.logging_kms_key_id
+    logging_retention_in_days    = var.logging_retention_in_days
+    role_path                    = var.role_path
+    role_permissions_boundary    = var.role_permissions_boundary
+    s3_bucket                    = var.lambda_s3_bucket
+    tracing_config               = var.tracing_config
+    metrics                      = var.metrics
     enable_runner_deregistration = var.instance_termination_watcher.enable_runner_deregistration
-    github_app_parameters        = var.instance_termination_watcher.enable_runner_deregistration ? {
+    github_app_parameters = var.instance_termination_watcher.enable_runner_deregistration ? {
       id         = local.github_app_parameters.id
       key_base64 = local.github_app_parameters.key_base64
     } : null

modules/multi-runner/termination-watcher.tf

@@ -1,25 +1,25 @@
 locals {
   lambda_instance_termination_watcher = {
-    prefix                    = var.prefix
-    tags                      = local.tags
-    aws_partition             = var.aws_partition
-    architecture              = var.lambda_architecture
-    principals                = var.lambda_principals
-    runtime                   = var.lambda_runtime
-    security_group_ids        = var.lambda_security_group_ids
-    subnet_ids                = var.lambda_subnet_ids
-    log_level                 = var.log_level
-    log_class                 = var.log_class
-    logging_kms_key_id        = var.logging_kms_key_id
-    logging_retention_in_days = var.logging_retention_in_days
-    role_path                 = var.role_path
-    role_permissions_boundary = var.role_permissions_boundary
-    s3_bucket                 = var.lambda_s3_bucket
-    tracing_config            = var.tracing_config
-    lambda_tags               = var.lambda_tags
-    metrics                   = var.metrics
+    prefix                       = var.prefix
+    tags                         = local.tags
+    aws_partition                = var.aws_partition
+    architecture                 = var.lambda_architecture
+    principals                   = var.lambda_principals
+    runtime                      = var.lambda_runtime
+    security_group_ids           = var.lambda_security_group_ids
+    subnet_ids                   = var.lambda_subnet_ids
+    log_level                    = var.log_level
+    log_class                    = var.log_class
+    logging_kms_key_id           = var.logging_kms_key_id
+    logging_retention_in_days    = var.logging_retention_in_days
+    role_path                    = var.role_path
+    role_permissions_boundary    = var.role_permissions_boundary
+    s3_bucket                    = var.lambda_s3_bucket
+    tracing_config               = var.tracing_config
+    lambda_tags                  = var.lambda_tags
+    metrics                      = var.metrics
     enable_runner_deregistration = var.instance_termination_watcher.enable_runner_deregistration
-    github_app_parameters        = var.instance_termination_watcher.enable_runner_deregistration ? {
+    github_app_parameters = var.instance_termination_watcher.enable_runner_deregistration ? {
       id         = local.github_app_parameters.id
       key_base64 = local.github_app_parameters.key_base64
     } : null

modules/multi-runner/variables.tf

@@ -699,11 +699,11 @@ variable "instance_termination_watcher" {
       enable_spot_termination_notification_watcher = optional(bool, true)
     }), {})
     enable_runner_deregistration = optional(bool, true)
-    memory_size                 = optional(number, null)
-    s3_key                      = optional(string, null)
-    s3_object_version           = optional(string, null)
-    timeout                     = optional(number, null)
-    zip                         = optional(string, null)
+    memory_size                  = optional(number, null)
+    s3_key                       = optional(string, null)
+    s3_object_version            = optional(string, null)
+    timeout                      = optional(number, null)
+    zip                          = optional(string, null)
   })
   default = {}
 }

modules/termination-watcher/main.tf

@@ -22,13 +22,13 @@ locals {
   }
 
   config = merge(var.config, {
-    name                         = local.name,
-    handler                      = "index.interruptionWarning",
-    zip                          = local.lambda_zip,
-    environment_variables        = local.environment_variables
-    metrics_namespace            = var.config.metrics.namespace
-    _deregistration_env_vars     = local.deregistration_env_vars
-    _ssm_parameter_arns          = local.ssm_parameter_arns
+    name                          = local.name,
+    handler                       = "index.interruptionWarning",
+    zip                           = local.lambda_zip,
+    environment_variables         = local.environment_variables
+    metrics_namespace             = var.config.metrics.namespace
+    _deregistration_env_vars      = local.deregistration_env_vars
+    _ssm_parameter_arns           = local.ssm_parameter_arns
     _enable_runner_deregistration = local.enable_runner_deregistration
   })
 }

modules/termination-watcher/notification/main.tf

@@ -42,6 +42,46 @@ resource "aws_lambda_permission" "main" {
   source_arn    = aws_cloudwatch_event_rule.spot_instance_termination_warning.arn
 }
 
+# EC2 Instance State-change Notification — catches ALL termination types
+# (scale-down, manual, spot reclamation, ASG) not just spot-specific events.
+# Uses "shutting-down" state to deregister runners while instance metadata is still available.
+# Reuses the same Lambda as the spot interruption warning handler since both event
+# types have detail['instance-id'] — the handler extracts it identically.
+resource "aws_cloudwatch_event_rule" "ec2_instance_state_change" {
+  count = var.config._enable_runner_deregistration ? 1 : 0
+
+  name        = "${var.config.prefix != null ? format("%s-", var.config.prefix) : ""}instance-termination"
+  description = "EC2 Instance Termination (all causes) — deregisters runners from GitHub"
+  tags        = local.config.tags
+
+  event_pattern = <<EOF
+{
+  "source": ["aws.ec2"],
+  "detail-type": ["EC2 Instance State-change Notification"],
+  "detail": {
+    "state": ["shutting-down"]
+  }
+}
+EOF
+}
+
+resource "aws_cloudwatch_event_target" "state_change" {
+  count = var.config._enable_runner_deregistration ? 1 : 0
+
+  rule = aws_cloudwatch_event_rule.ec2_instance_state_change[0].name
+  arn  = module.termination_warning_watcher.lambda.function.arn
+}
+
+resource "aws_lambda_permission" "state_change" {
+  count = var.config._enable_runner_deregistration ? 1 : 0
+
+  statement_id  = "AllowExecutionFromCloudWatchStateChange"
+  action        = "lambda:InvokeFunction"
+  function_name = module.termination_warning_watcher.lambda.function.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.ec2_instance_state_change[0].arn
+}
+
 resource "aws_iam_role_policy" "lambda_policy" {
   name = "lambda-policy"
   role = module.termination_warning_watcher.lambda.role.name

modules/termination-watcher/variables.tf

@@ -71,7 +71,7 @@ variable "config" {
       capture_http_requests = optional(bool, false)
       capture_error         = optional(bool, false)
     }), {})
-    zip = optional(string, null)
+    zip                          = optional(string, null)
     enable_runner_deregistration = optional(bool, false)
     github_app_parameters = optional(object({
       id         = map(string)

variables.tf

@@ -972,11 +972,11 @@ variable "instance_termination_watcher" {
       enable_spot_termination_notification_watcher = optional(bool, true)
     }), {})
     enable_runner_deregistration = optional(bool, true)
-    memory_size                 = optional(number, null)
-    s3_key                      = optional(string, null)
-    s3_object_version           = optional(string, null)
-    timeout                     = optional(number, null)
-    zip                         = optional(string, null)
+    memory_size                  = optional(number, null)
+    s3_key                       = optional(string, null)
+    s3_object_version            = optional(string, null)
+    timeout                      = optional(number, null)
+    zip                          = optional(string, null)
   })
   default = {}