feat: implement standby pool management with max age termination for idle runners

Author: Brend-Smits

lambdas/functions/control-plane/src/aws/runners.d.ts

@@ -1,4 +1,4 @@
-import { DefaultTargetCapacityType, SpotAllocationStrategy } from '@aws-sdk/client-ec2';
+import { DefaultTargetCapacityType, SpotAllocationStrategy, Tag } from '@aws-sdk/client-ec2';
 
 export type RunnerType = 'Org' | 'Repo';
 
@@ -11,6 +11,7 @@ export interface RunnerList {
   org?: string;
   orphan?: boolean;
   runnerId?: string;
+  tags?: Tag[];
 }
 
 export interface RunnerInfo {

lambdas/functions/control-plane/src/aws/runners.test.ts

@@ -87,27 +87,27 @@ describe('list instances', () => {
     mockEC2Client.on(DescribeInstancesCommand).resolves(mockRunningInstances);
     const resp = await listEC2Runners();
     expect(resp.length).toBe(1);
-    expect(resp).toContainEqual({
+    expect(resp).toContainEqual(expect.objectContaining({
       instanceId: 'i-1234',
       launchTime: new Date('2020-10-10T14:48:00.000+09:00'),
       type: 'Org',
       owner: 'CoderToCat',
       orphan: false,
-    });
+    }));
   });
 
   it('returns a list of instances (JIT)', async () => {
     mockEC2Client.on(DescribeInstancesCommand).resolves(mockRunningInstancesJit);
     const resp = await listEC2Runners();
     expect(resp.length).toBe(1);
-    expect(resp).toContainEqual({
+    expect(resp).toContainEqual(expect.objectContaining({
       instanceId: 'i-1234',
       launchTime: new Date('2020-10-10T14:48:00.000+09:00'),
       type: 'Org',
       owner: 'CoderToCat',
       orphan: false,
       runnerId: '9876543210',
-    });
+    }));
   });
 
   it('check orphan tag.', async () => {
@@ -120,13 +120,13 @@ describe('list instances', () => {
 
     const resp = await listEC2Runners();
     expect(resp.length).toBe(1);
-    expect(resp).toContainEqual({
+    expect(resp).toContainEqual(expect.objectContaining({
       instanceId: instances.Reservations![0].Instances![0].InstanceId!,
       launchTime: instances.Reservations![0].Instances![0].LaunchTime!,
       type: 'Org',
       owner: 'CoderToCat',
       orphan: true,
-    });
+    }));
   });
 
   it('calls EC2 describe instances', async () => {
@@ -302,7 +302,7 @@ describe('stop runner', () => {
       Resources: [runner.instanceId],
       Tags: [
         { Key: 'ghr:state', Value: 'standby' },
-        { Key: 'ghr:stopped_at', Value: expect.any(String) },
+        { Key: 'ghr:standby_time', Value: expect.any(String) },
       ],
     });
   });
@@ -327,7 +327,7 @@ describe('start runner', () => {
     });
     expect(mockEC2Client).toHaveReceivedCommandWith(DeleteTagsCommand, {
       Resources: [runner.instanceId],
-      Tags: [{ Key: 'ghr:state' }, { Key: 'ghr:stopped_at' }],
+      Tags: [{ Key: 'ghr:state' }, { Key: 'ghr:standby_time' }],
     });
   });
 });

lambdas/functions/control-plane/src/aws/runners.ts

@@ -98,6 +98,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
             org: i.Tags?.find((e) => e.Key === 'ghr:Org')?.Value as string,
             orphan: i.Tags?.find((e) => e.Key === 'ghr:orphan')?.Value === 'true',
             runnerId: i.Tags?.find((e) => e.Key === 'ghr:github_runner_id')?.Value as string,
+            tags: i.Tags,
           });
         }
       }
@@ -119,7 +120,7 @@ export async function stopRunner(instanceId: string): Promise<void> {
   await ec2.send(new StopInstancesCommand({ InstanceIds: [instanceId] }));
   await tag(instanceId, [
     { Key: 'ghr:state', Value: 'standby' },
-    { Key: 'ghr:stopped_at', Value: new Date().toISOString() },
+    { Key: 'ghr:standby_time', Value: new Date().toISOString() },
   ]);
   logger.debug(`Runner ${instanceId} has been stopped.`);
 }
@@ -128,7 +129,7 @@ export async function startRunner(instanceId: string): Promise<void> {
   logger.debug(`Runner '${instanceId}' will be started.`);
   const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION }));
   await ec2.send(new StartInstancesCommand({ InstanceIds: [instanceId] }));
-  await untag(instanceId, [{ Key: 'ghr:state' }, { Key: 'ghr:stopped_at' }]);
+  await untag(instanceId, [{ Key: 'ghr:state' }, { Key: 'ghr:standby_time' }]);
   logger.debug(`Runner ${instanceId} has been started.`);
 }
 

lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts

@@ -290,15 +290,15 @@ describe('Scale down runners', () => {
       it(`Should stop idle runner to standby when standby pool enabled`, async () => {
         process.env.STANDBY_POOL_SIZE = '2';
         process.env.STANDBY_IDLE_TIME_MINUTES = '0';
-        
+
         const runners = [
           createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false),
           createRunnerTestData('idle-2', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false),
         ];
 
         mockGitHubRunners(runners);
         mockAwsRunners(runners);
-        
+
         const originalListRunners = mockListRunners.getMockImplementation();
         mockListRunners.mockImplementation(async (filters) => {
           if (filters?.statuses?.includes('stopped') && filters?.standby) {
@@ -311,11 +311,104 @@ describe('Scale down runners', () => {
 
         expect(mockStopRunners).toHaveBeenCalledTimes(2);
         expect(mockTerminateRunners).not.toHaveBeenCalled();
-        
+
         delete process.env.STANDBY_POOL_SIZE;
         delete process.env.STANDBY_IDLE_TIME_MINUTES;
       });
 
+      it(`Should terminate standby runners older than max age`, async () => {
+        process.env.STANDBY_POOL_SIZE = '2';
+        process.env.STANDBY_MAX_AGE_HOURS = '24';
+
+        const oldStandbyTime = moment().subtract(25, 'hours').toISOString();
+        const recentStandbyTime = moment().subtract(1, 'hours').toISOString();
+
+        const oldStandbyRunner: RunnerList = {
+          instanceId: 'i-old-standby',
+          launchTime: moment().subtract(26, 'hours').toDate(),
+          type,
+          owner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`,
+          tags: [
+            { Key: 'ghr:state', Value: 'standby' },
+            { Key: 'ghr:standby_time', Value: oldStandbyTime },
+          ],
+        };
+
+        const recentStandbyRunner: RunnerList = {
+          instanceId: 'i-recent-standby',
+          launchTime: moment().subtract(2, 'hours').toDate(),
+          type,
+          owner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`,
+          tags: [
+            { Key: 'ghr:state', Value: 'standby' },
+            { Key: 'ghr:standby_time', Value: recentStandbyTime },
+          ],
+        };
+
+        const runners = [
+          createRunnerTestData('active-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false),
+        ];
+
+        mockGitHubRunners(runners);
+        mockAwsRunners(runners);
+
+        const originalListRunners = mockListRunners.getMockImplementation();
+        mockListRunners.mockImplementation(async (filters) => {
+          if (filters?.statuses?.includes('stopped') && filters?.standby) {
+            return [oldStandbyRunner, recentStandbyRunner];
+          }
+          return originalListRunners ? originalListRunners(filters) : runners;
+        });
+
+        await scaleDown();
+
+        expect(mockTerminateRunners).toHaveBeenCalledWith('i-old-standby');
+        expect(mockTerminateRunners).not.toHaveBeenCalledWith('i-recent-standby');
+
+        delete process.env.STANDBY_POOL_SIZE;
+        delete process.env.STANDBY_MAX_AGE_HOURS;
+      });
+
+      it(`Should skip age termination when standby pool disabled`, async () => {
+        process.env.STANDBY_POOL_SIZE = '0';
+        process.env.STANDBY_MAX_AGE_HOURS = '24';
+
+        const oldStandbyTime = moment().subtract(25, 'hours').toISOString();
+
+        const oldStandbyRunner: RunnerList = {
+          instanceId: 'i-old-standby',
+          launchTime: moment().subtract(26, 'hours').toDate(),
+          type,
+          owner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`,
+          tags: [
+            { Key: 'ghr:state', Value: 'standby' },
+            { Key: 'ghr:standby_time', Value: oldStandbyTime },
+          ],
+        };
+
+        const runners = [
+          createRunnerTestData('active-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false),
+        ];
+
+        mockGitHubRunners(runners);
+        mockAwsRunners(runners);
+
+        const originalListRunners = mockListRunners.getMockImplementation();
+        mockListRunners.mockImplementation(async (filters) => {
+          if (filters?.statuses?.includes('stopped') && filters?.standby) {
+            return [oldStandbyRunner];
+          }
+          return originalListRunners ? originalListRunners(filters) : runners;
+        });
+
+        await scaleDown();
+
+        expect(mockTerminateRunners).not.toHaveBeenCalledWith('i-old-standby');
+
+        delete process.env.STANDBY_POOL_SIZE;
+        delete process.env.STANDBY_MAX_AGE_HOURS;
+      });
+
       it(`Should not terminate a runner that became busy just before deregister runner.`, async () => {
         // setup
         const runners = [

lambdas/functions/control-plane/src/scale-runners/scale-down.ts

@@ -130,7 +130,7 @@ function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean {
 function runnerIdleTimeExceeded(runner: RunnerInfo): boolean {
   const idleTimeMinutes = parseInt(process.env.STANDBY_IDLE_TIME_MINUTES || '0');
   if (idleTimeMinutes === 0) return true;
-  
+
   const launchTimePlusIdle = moment(runner.launchTime).utc().add(idleTimeMinutes, 'minutes');
   const now = moment(new Date()).utc();
   return launchTimePlusIdle < now;
@@ -169,7 +169,9 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[], should
 
         if (statuses.every((status) => status == 204)) {
           await terminateRunner(ec2runner.instanceId);
-          logger.info(`AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`);
+          logger.info(
+            `AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`,
+          );
         } else {
           logger.error(`Failed to de-register GitHub runner: ${statuses}`);
         }
@@ -191,7 +193,7 @@ async function evaluateAndRemoveRunners(
   let idleCounter = getIdleRunnerCount(scaleDownConfigs);
   const evictionStrategy = getEvictionStrategy(scaleDownConfigs);
   const ownerTags = new Set(ec2Runners.map((runner) => runner.owner));
-  
+
   const standbyPoolSize = parseInt(process.env.STANDBY_POOL_SIZE || '0');
   const environment = process.env.ENVIRONMENT;
 
@@ -201,15 +203,22 @@ async function evaluateAndRemoveRunners(
       .sort(evictionStrategy === 'oldest_first' ? oldestFirstStrategy : newestFirstStrategy);
     logger.debug(`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`);
     logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ec2RunnersFiltered)}`);
-    
-    const standbyRunners = standbyPoolSize > 0 
-      ? await listEC2Runners({ environment, runnerType: 'Org', runnerOwner: ownerTag, statuses: ['stopped'], standby: true })
-      : [];
+
+    const standbyRunners =
+      standbyPoolSize > 0
+        ? await listEC2Runners({
+            environment,
+            runnerType: 'Org',
+            runnerOwner: ownerTag,
+            statuses: ['stopped'],
+            standby: true,
+          })
+        : [];
     const currentStandbyCount = standbyRunners.length;
     let standbyCounter = Math.max(0, standbyPoolSize - currentStandbyCount);
-    
+
     logger.debug(`Standby pool: target=${standbyPoolSize}, current=${currentStandbyCount}, needed=${standbyCounter}`);
-    
+
     for (const ec2Runner of ec2RunnersFiltered) {
       const ghRunners = await listGitHubRunners(ec2Runner);
       const ghRunnersFiltered = ghRunners.filter((runner: { name: string }) =>
@@ -336,6 +345,42 @@ async function listRunners(environment: string) {
   });
 }
 
+async function terminateOldStandbyRunners(environment: string): Promise<void> {
+  const maxAgeHours = parseInt(process.env.STANDBY_MAX_AGE_HOURS || '168');
+  const standbyPoolSize = parseInt(process.env.STANDBY_POOL_SIZE || '0');
+  
+  if (standbyPoolSize === 0) {
+    return;
+  }
+
+  try {
+    const standbyRunners = await listEC2Runners({
+      environment,
+      statuses: ['stopped'],
+      standby: true,
+    });
+
+    for (const runner of standbyRunners) {
+      const standbyTimeTag = runner.tags?.find((t) => t.Key === 'ghr:standby_time')?.Value;
+      if (standbyTimeTag) {
+        const standbyTime = moment(standbyTimeTag);
+        const ageHours = moment().diff(standbyTime, 'hours');
+        
+        if (ageHours > maxAgeHours) {
+          logger.info(
+            `Terminating standby runner '${runner.instanceId}' - age ${ageHours}h exceeds limit ${maxAgeHours}h`,
+          );
+          await terminateRunner(runner.instanceId).catch((e) => {
+            logger.error(`Failed to terminate old standby runner '${runner.instanceId}'`, { error: e });
+          });
+        }
+      }
+    }
+  } catch (e) {
+    logger.warn(`Failure during old standby runner termination.`, { error: e });
+  }
+}
+
 function filterRunners(ec2runners: RunnerList[]): RunnerInfo[] {
   return ec2runners.filter((ec2Runner) => ec2Runner.type && !ec2Runner.orphan) as RunnerInfo[];
 }
@@ -345,10 +390,10 @@ export async function scaleDown(): Promise<void> {
   const environment = process.env.ENVIRONMENT;
   const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as [ScalingDownConfig];
 
-  // first runners marked to be orphan.
   await terminateOrphan(environment);
 
-  // next scale down idle runners with respect to config and mark potential orphans
+  await terminateOldStandbyRunners(environment);
+
   const ec2Runners = await listRunners(environment);
   const activeEc2RunnersCount = ec2Runners.length;
   logger.info(`Found: '${activeEc2RunnersCount}' active GitHub EC2 runner instances before clean-up.`);

modules/runners/scale-down.tf

@@ -36,6 +36,7 @@ resource "aws_lambda_function" "scale_down" {
       SCALE_DOWN_CONFIG                        = jsonencode(var.idle_config)
       STANDBY_POOL_SIZE                        = var.standby_pool_config.size
       STANDBY_IDLE_TIME_MINUTES                = var.standby_pool_config.idle_time_minutes
+      STANDBY_MAX_AGE_HOURS                    = var.standby_pool_config.max_age_hours
       POWERTOOLS_SERVICE_NAME                  = "runners-scale-down"
       POWERTOOLS_METRICS_NAMESPACE             = var.metrics.namespace
       POWERTOOLS_TRACE_ENABLED                 = var.tracing_config.mode != null ? true : false

modules/runners/variables.tf

@@ -561,16 +561,18 @@ variable "pool_config" {
 }
 
 variable "standby_pool_config" {
-  description = "Configuration for the standby pool. Instances in the standby pool are stopped instead of terminated, allowing faster scale-up. The `size` specifies how many stopped instances to maintain. The `idle_time_minutes` specifies how long an idle runner should wait before being moved to standby (0 means immediate). Stopped instances only incur EBS storage costs."
+  description = "Configuration for the standby pool. Instances in the standby pool are stopped instead of terminated, allowing faster scale-up. The `size` specifies how many stopped instances to maintain. The `idle_time_minutes` specifies how long an idle runner should wait before being moved to standby (0 means immediate). The `max_age_hours` specifies the maximum age of stopped instances before they are terminated (default 168 hours = 7 days). Stopped instances only incur EBS storage costs."
   type = object({
     enabled            = optional(bool, false)
     size               = optional(number, 0)
     idle_time_minutes  = optional(number, 0)
+    max_age_hours      = optional(number, 168)
   })
   default = {
     enabled            = false
     size               = 0
     idle_time_minutes  = 0
+    max_age_hours      = 168
   }
 }