Add SQS retry infrastructure for runner deregistration (C-1841)

Add Terraform resources to support the SQS-based deregistration retry
that was added in ed30bf8. When GitHub returns 422 (runner busy), the
termination-watcher Lambda now has infrastructure to queue a delayed
retry:

- SQS queue with 5-minute delivery delay for retry messages
- Dead-letter queue (14-day retention, 3 max receives) for failures
- Dedicated Lambda function (index.deregisterRetry handler)
- SQS event source mapping to trigger the retry Lambda
- IAM policies: SQS send/receive, SSM read, EC2 describe
- IAM policies on notification/termination Lambdas for SQS:SendMessage
- Pass DEREGISTER_RETRY_QUEUE_URL env var to all termination Lambdas
- Rebuild termination-watcher.zip with latest code

Co-Authored-By: Paperclip <noreply@paperclip.ing>

Author: SRE Engineer

lambdas/functions/termination-watcher/termination-watcher.zip

@@ -0,0 +1,155 @@
+# SQS-based deregistration retry for runners that return 422 (busy executing a job).
+# When a runner can't be deregistered immediately, the termination-watcher Lambda
+# sends a message to this queue with a 5-minute delay. By the time the message
+# becomes visible, the EC2 instance has terminated and the runner appears offline,
+# allowing clean GitHub API deletion.
+
+# Dead-letter queue — messages that fail after 3 attempts land here for investigation
+resource "aws_sqs_queue" "deregister_retry_dlq" {
+  count = local.enable_runner_deregistration ? 1 : 0
+
+  name                      = "${var.config.prefix}-deregister-retry-dlq"
+  message_retention_seconds = 1209600 # 14 days
+  tags                      = var.config.tags
+}
+
+# Main retry queue — 5-minute delivery delay gives EC2 time to terminate
+resource "aws_sqs_queue" "deregister_retry" {
+  count = local.enable_runner_deregistration ? 1 : 0
+
+  name                       = "${var.config.prefix}-deregister-retry"
+  delay_seconds              = 300 # 5 minutes
+  message_retention_seconds  = 86400 # 24 hours
+  visibility_timeout_seconds = 60 # Lambda timeout + buffer
+  tags                       = var.config.tags
+
+  redrive_policy = jsonencode({
+    deadLetterTargetArn = aws_sqs_queue.deregister_retry_dlq[0].arn
+    maxReceiveCount     = 3
+  })
+}
+
+# Dedicated Lambda function for processing SQS retry messages.
+# Uses the same code package as the termination-watcher but with
+# handler index.deregisterRetry (SQS event handler).
+module "deregister_retry_lambda" {
+  count  = local.enable_runner_deregistration ? 1 : 0
+  source = "../lambda"
+
+  lambda = merge(local.config, {
+    name    = "deregister-retry"
+    handler = "index.deregisterRetry"
+    environment_variables = merge(
+      local.deregistration_env_vars,
+      var.config.environment_variables,
+      {
+        DEREGISTER_RETRY_QUEUE_URL = aws_sqs_queue.deregister_retry[0].url
+        TAG_FILTERS                = jsonencode(var.config.tag_filters)
+      }
+    )
+  })
+}
+
+# SQS event source mapping — triggers the retry Lambda when messages arrive
+resource "aws_lambda_event_source_mapping" "deregister_retry" {
+  count = local.enable_runner_deregistration ? 1 : 0
+
+  event_source_arn = aws_sqs_queue.deregister_retry[0].arn
+  function_name    = module.deregister_retry_lambda[0].lambda.function.arn
+  batch_size       = 1 # Process one retry at a time to avoid GitHub rate limits
+  enabled          = true
+}
+
+# IAM: Allow the retry Lambda to receive/delete from the retry queue
+resource "aws_iam_role_policy" "deregister_retry_sqs" {
+  count = local.enable_runner_deregistration ? 1 : 0
+
+  name = "sqs-deregister-retry"
+  role = module.deregister_retry_lambda[0].lambda.role.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes",
+          "sqs:SendMessage"
+        ]
+        Resource = [
+          aws_sqs_queue.deregister_retry[0].arn,
+          aws_sqs_queue.deregister_retry_dlq[0].arn
+        ]
+      }
+    ]
+  })
+}
+
+# IAM: Allow the retry Lambda to read SSM parameters (GitHub App credentials)
+resource "aws_iam_role_policy" "deregister_retry_ssm" {
+  count = local.enable_runner_deregistration ? 1 : 0
+
+  name = "ssm-deregister-retry"
+  role = module.deregister_retry_lambda[0].lambda.role.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["ssm:GetParameter"]
+        Resource = local.ssm_parameter_arns
+      }
+    ]
+  })
+}
+
+# IAM: Allow the retry Lambda to describe EC2 instances (for tag lookups)
+resource "aws_iam_role_policy" "deregister_retry_ec2" {
+  count = local.enable_runner_deregistration ? 1 : 0
+
+  name = "ec2-deregister-retry"
+  role = module.deregister_retry_lambda[0].lambda.role.name
+
+  policy = templatefile("${path.module}/policies/lambda.json", {})
+}
+
+# IAM: Allow the notification Lambda to send messages to the retry queue
+resource "aws_iam_role_policy" "notification_sqs_send" {
+  count = local.enable_runner_deregistration && var.config.features.enable_spot_termination_notification_watcher ? 1 : 0
+
+  name = "sqs-deregister-retry-send"
+  role = module.termination_notification[0].lambda.role.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["sqs:SendMessage"]
+        Resource = aws_sqs_queue.deregister_retry[0].arn
+      }
+    ]
+  })
+}
+
+# IAM: Allow the termination handler Lambda to send messages to the retry queue
+resource "aws_iam_role_policy" "termination_sqs_send" {
+  count = local.enable_runner_deregistration && var.config.features.enable_spot_termination_handler ? 1 : 0
+
+  name = "sqs-deregister-retry-send"
+  role = module.termination_handler[0].lambda.role.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["sqs:SendMessage"]
+        Resource = aws_sqs_queue.deregister_retry[0].arn
+      }
+    ]
+  })
+}

modules/termination-watcher/deregister-retry.tf

@@ -4,12 +4,14 @@ locals {
 
   enable_runner_deregistration = var.config.enable_runner_deregistration && var.config.github_app_parameters != null
 
-  deregistration_env_vars = local.enable_runner_deregistration ? {
+  deregistration_env_vars = local.enable_runner_deregistration ? merge({
     ENABLE_RUNNER_DEREGISTRATION         = "true"
     PARAMETER_GITHUB_APP_ID_NAME         = var.config.github_app_parameters.id.name
     PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name
     GHES_URL                             = var.config.ghes_url != null ? var.config.ghes_url : ""
-  } : {}
+  }, length(aws_sqs_queue.deregister_retry) > 0 ? {
+    DEREGISTER_RETRY_QUEUE_URL = aws_sqs_queue.deregister_retry[0].url
+  } : {}) : {}
 
   ssm_parameter_arns = local.enable_runner_deregistration ? [
     var.config.github_app_parameters.id.arn,

modules/termination-watcher/main.tf

@@ -13,3 +13,15 @@ output "spot_termination_handler" {
     lambda_role      = module.termination_handler[0].lambda.role
   } : null
 }
+
+output "deregister_retry" {
+  value = local.enable_runner_deregistration ? {
+    queue_url        = aws_sqs_queue.deregister_retry[0].url
+    queue_arn        = aws_sqs_queue.deregister_retry[0].arn
+    dlq_url          = aws_sqs_queue.deregister_retry_dlq[0].url
+    dlq_arn          = aws_sqs_queue.deregister_retry_dlq[0].arn
+    lambda           = module.deregister_retry_lambda[0].lambda.function
+    lambda_log_group = module.deregister_retry_lambda[0].lambda.log_group
+    lambda_role      = module.deregister_retry_lambda[0].lambda.role
+  } : null
+}