feat(multi-runner)!: support running the scale-down lambda once for every runner group

Iterating the list of active runners in the GitHub API can be slow and
expensive in terms of rate limit consumption. It's a paginated API,
returning up to 100 runners per page. With several thousand runners
across many runner groups, running `scale-down` once per runner group
can quickly eat up large portions of the rate limit.

Here we break the Terraform `scale-down` module into its own sub-module,
so that `multi-runner` can create one instance of the Lambda function
instead of the `runner` module managing it. A flag is added to the
`runner` module to disable the `scale-down` function creation in the
`multi-runner` case.

Then the Lambda's code is modified to accept a list of configurations,
and process them all.

With this, we only need to fetch the list of runners once for all runner
groups.

BREAKING CHANGE: When using the `multi-runner` module, the per-group
`scale_down_schedule_expression` is no longer supported.

Only needed if you are using the `multi-runner` module.

One instance of `scale-down` will now handle all runner groups.

1. Remove any `scale_down_schedule_expression` settings from your
   `multi_runner_config` runner configs.
2. To customise the frequency of the consolidated `scale-down` function,
   set the `scale_down_schedule_expression` variable on the
   `multi-runner` module itself.

Author: iainlane

examples/multi-runner/main.tf

@@ -98,7 +98,6 @@ module "runners" {
   #        runner_extra_labels             = ["amazon"]
   #        runners_maximum_count           = 1
   #        enable_ephemeral_runners        = true
-  #        scale_down_schedule_expression  = "cron(* * * * ? *)"
   #      }
   #    }
   #  }
@@ -107,6 +106,7 @@ module "runners" {
   subnet_ids                        = module.base.vpc.private_subnets
   runners_scale_up_lambda_timeout   = 60
   runners_scale_down_lambda_timeout = 60
+  scale_down_schedule_expression    = "cron(* * * * ? *)"
   prefix                            = local.environment
   tags = {
     Project = "ProjectX"

examples/multi-runner/templates/runner-configs/linux-arm64.yaml

@@ -19,7 +19,6 @@ runner_config:
     id_ssm_parameter_arn: ${ami_id_ssm_parameter_arn}
   runners_maximum_count: 1
   delay_webhook_event: 0
-  scale_down_schedule_expression: cron(* * * * ? *)
   runner_hook_job_started: |
     echo "Running pre job hook as $(whoami)"
   runner_hook_job_completed: |

examples/multi-runner/templates/runner-configs/linux-x64-ubuntu-2204.yaml

@@ -19,7 +19,6 @@ runner_config:
     - m5a.large
   runners_maximum_count: 1
   delay_webhook_event: 0
-  scale_down_schedule_expression: cron(* * * * ? *)
   userdata_template: ./templates/user-data.sh
   ami:
     owners:

examples/multi-runner/templates/runner-configs/linux-x64-ubuntu.yaml

@@ -20,7 +20,6 @@ runner_config:
     - m5a.large
   runners_maximum_count: 1
   delay_webhook_event: 0
-  scale_down_schedule_expression: cron(* * * * ? *)
   userdata_template: ./templates/user-data.sh
   ami:
     owners:

examples/multi-runner/templates/runner-configs/linux-x64.yaml

@@ -21,7 +21,6 @@ runner_config:
   enable_on_demand_failover_for_errors: ['InsufficientInstanceCapacity']
   create_service_linked_role_spot: true
   delay_webhook_event: 0
-  scale_down_schedule_expression: cron(* * * * ? *)
   runner_metadata_options:
     instance_metadata_tags: disabled
     http_endpoint: enabled

examples/multi-runner/templates/runner-configs/windows-x64.yaml

@@ -13,7 +13,6 @@ runner_config:
     - c5.large
   runners_maximum_count: 1
   delay_webhook_event: 5
-  scale_down_schedule_expression: cron(* * * * ? *)
   runner_boot_time_in_minutes: 20
   ami_filter:
     name:

lambdas/functions/control-plane/src/aws/runners.ts

@@ -298,8 +298,7 @@ async function createInstances(
 }
 
 // If launchTime is undefined, this will return false
-export function bootTimeExceeded(ec2Runner: { launchTime?: Date }): boolean {
-  const runnerBootTimeInMinutes = process.env.RUNNER_BOOT_TIME_IN_MINUTES;
+export function bootTimeExceeded(ec2Runner: { launchTime?: Date }, runnerBootTimeInMinutes: number): boolean {
   const launchTimePlusBootTime = moment(ec2Runner.launchTime).utc().add(runnerBootTimeInMinutes, 'minutes');
   return launchTimePlusBootTime < moment(new Date()).utc();
 }

lambdas/functions/control-plane/src/modules.d.ts

@@ -4,6 +4,7 @@ declare namespace NodeJS {
     ENABLE_METRIC_GITHUB_APP_RATE_LIMIT: string;
     ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS: string;
     ENVIRONMENT: string;
+    ENVIRONMENT_CONFIGS: string;
     GHES_URL: string;
     JOB_RETRY_CONFIG: string;
     LAUNCH_TEMPLATE_NAME: string;
@@ -14,8 +15,8 @@ declare namespace NodeJS {
     PARAMETER_GITHUB_APP_CLIENT_SECRET_NAME: string;
     PARAMETER_GITHUB_APP_ID_NAME: string;
     PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string;
+    RUNNER_BOOT_TIME_IN_MINUTES: string;
     RUNNER_OWNER: string;
-    SCALE_DOWN_CONFIG: string;
     SSM_TOKEN_PATH: string;
     SSM_CLEANUP_CONFIG: string;
     SUBNET_IDS: string;

lambdas/functions/control-plane/src/pool/pool.ts

@@ -37,6 +37,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
   const instanceAllocationStrategy = process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price'; // same as AWS default
   const runnerOwner = process.env.RUNNER_OWNER;
   const amiIdSsmParameterName = process.env.AMI_ID_SSM_PARAMETER_NAME;
+  const runnerBootTimeInMinutes = parseInt(process.env.RUNNER_BOOT_TIME_IN_MINUTES || '5');
   const tracingEnabled = yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false });
   const onDemandFailoverOnError = process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS
     ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string])
@@ -63,7 +64,7 @@ export async function adjust(event: PoolEvent): Promise<void> {
     statuses: ['running'],
   });
 
-  const numberOfRunnersInPool = calculatePooSize(ec2runners, runnerStatusses);
+  const numberOfRunnersInPool = calculatePooSize(ec2runners, runnerStatusses, runnerBootTimeInMinutes);
   const topUp = event.poolSize - numberOfRunnersInPool;
 
   if (topUp > 0) {
@@ -115,7 +116,7 @@ async function getInstallationId(ghesApiUrl: string, org: string): Promise<numbe
   ).data.id;
 }
 
-function calculatePooSize(ec2runners: RunnerList[], runnerStatus: Map<string, RunnerStatus>): number {
+function calculatePooSize(ec2runners: RunnerList[], runnerStatus: Map<string, RunnerStatus>, runnerBootTimeInMinutes: number): number {
   // Runner should be considered idle if it is still booting, or is idle in GitHub
   let numberOfRunnersInPool = 0;
   for (const ec2Instance of ec2runners) {
@@ -127,7 +128,7 @@ function calculatePooSize(ec2runners: RunnerList[], runnerStatus: Map<string, Ru
       logger.debug(`Runner ${ec2Instance.instanceId} is idle in GitHub and counted as part of the pool`);
     } else if (runnerStatus.get(ec2Instance.instanceId) != null) {
       logger.debug(`Runner ${ec2Instance.instanceId} is not idle in GitHub and NOT counted as part of the pool`);
-    } else if (!bootTimeExceeded(ec2Instance)) {
+    } else if (!bootTimeExceeded(ec2Instance, runnerBootTimeInMinutes)) {
       numberOfRunnersInPool++;
       logger.info(`Runner ${ec2Instance.instanceId} is still booting and counted as part of the pool`);
     } else {

lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts

@@ -11,6 +11,13 @@ export interface ScalingDownConfig {
   evictionStrategy?: EvictionStrategy;
 }
 
+export interface EnvironmentScaleDownConfig {
+  environment: string;
+  idle_config: ScalingDownConfig[];
+  minimum_running_time_in_minutes: number;
+  runner_boot_time_in_minutes: number;
+}
+
 const logger = createChildLogger('scale-down-config.ts');
 
 function inPeriod(period: ScalingDownConfig): boolean {