feat(runners): support OnDemandOptions.AllocationStrategy in EC2 Fleet

Previously, createInstances always set SpotOptions.AllocationStrategy
even for on-demand fleets, making instance_allocation_strategy and
instance_types ordering meaningless for on-demand users wanting
`prioritized`. Now the fleet request conditionally sets SpotOptions
or OnDemandOptions based on targetCapacityType, and the spot-to-
on-demand failover path maps spot-only strategies to `lowest-price`.

Author: piotrj

lambdas/functions/control-plane/src/aws/runners.d.ts

@@ -1,4 +1,4 @@
-import { DefaultTargetCapacityType, SpotAllocationStrategy } from '@aws-sdk/client-ec2';
+import { DefaultTargetCapacityType, FleetOnDemandAllocationStrategy, SpotAllocationStrategy } from '@aws-sdk/client-ec2';
 
 export type RunnerType = 'Org' | 'Repo';
 
@@ -39,7 +39,7 @@ export interface RunnerInputParameters {
     instanceTypes: string[];
     targetCapacityType: DefaultTargetCapacityType;
     maxSpotPrice?: string;
-    instanceAllocationStrategy: SpotAllocationStrategy;
+    instanceAllocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy;
   };
   numberOfRunners: number;
   amiIdSsmParameterName?: string;

lambdas/functions/control-plane/src/aws/runners.test.ts

@@ -10,6 +10,7 @@ import {
   DescribeInstancesCommand,
   type DescribeInstancesResult,
   EC2Client,
+  FleetOnDemandAllocationStrategy,
   SpotAllocationStrategy,
   TerminateInstancesCommand,
 } from '@aws-sdk/client-ec2';
@@ -366,11 +367,31 @@ describe('create runner', () => {
   });
 
   it('calls create fleet of 1 instance with the on-demand capacity', async () => {
-    await createRunner(createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand' }));
+    await createRunner(
+      createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand', allocationStrategy: 'lowest-price' }),
+    );
     expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, {
       ...expectedCreateFleetRequest({
         ...defaultExpectedFleetRequestValues,
         capacityType: 'on-demand',
+        allocationStrategy: 'lowest-price',
+      }),
+    });
+  });
+
+  it('calls create fleet with on-demand capacity and prioritized allocation strategy', async () => {
+    await createRunner(
+      createRunnerConfig({
+        ...defaultRunnerConfig,
+        capacityType: 'on-demand',
+        allocationStrategy: FleetOnDemandAllocationStrategy.PRIORITIZED,
+      }),
+    );
+    expect(mockEC2Client).toHaveReceivedCommandWith(CreateFleetCommand, {
+      ...expectedCreateFleetRequest({
+        ...defaultExpectedFleetRequestValues,
+        capacityType: 'on-demand',
+        allocationStrategy: FleetOnDemandAllocationStrategy.PRIORITIZED,
       }),
     });
   });
@@ -581,12 +602,13 @@ describe('create runner with errors fail over to OnDemand', () => {
       }),
     });
 
-    // second call with with OnDemand fallback
+    // second call with with OnDemand fallback, allocation strategy defaults to lowest-price
     expect(mockEC2Client).toHaveReceivedNthCommandWith(2, CreateFleetCommand, {
       ...expectedCreateFleetRequest({
         ...defaultExpectedFleetRequestValues,
         totalTargetCapacity: 1,
         capacityType: 'on-demand',
+        allocationStrategy: 'lowest-price',
       }),
     });
   });
@@ -623,12 +645,13 @@ describe('create runner with errors fail over to OnDemand', () => {
       }),
     });
 
-    // second call with with OnDemand failback, capacity is reduced by 1
+    // second call with with OnDemand failback, capacity is reduced by 1, allocation strategy defaults to lowest-price
     expect(mockEC2Client).toHaveReceivedNthCommandWith(2, CreateFleetCommand, {
       ...expectedCreateFleetRequest({
         ...defaultExpectedFleetRequestValues,
         totalTargetCapacity: 1,
         capacityType: 'on-demand',
+        allocationStrategy: 'lowest-price',
       }),
     });
   });
@@ -698,7 +721,7 @@ function createFleetMockWithWithOnDemandFallback(errors: string[], instances?: s
 interface RunnerConfig {
   type: RunnerType;
   capacityType: DefaultTargetCapacityType;
-  allocationStrategy: SpotAllocationStrategy;
+  allocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy;
   maxSpotPrice?: string;
   amiIdSsmParameterName?: string;
   tracingEnabled?: boolean;
@@ -730,7 +753,7 @@ function createRunnerConfig(runnerConfig: RunnerConfig): RunnerInputParameters {
 interface ExpectedFleetRequestValues {
   type: 'Repo' | 'Org';
   capacityType: DefaultTargetCapacityType;
-  allocationStrategy: SpotAllocationStrategy;
+  allocationStrategy: SpotAllocationStrategy | FleetOnDemandAllocationStrategy;
   maxSpotPrice?: string;
   totalTargetCapacity: number;
   imageId?: string;
@@ -778,10 +801,18 @@ function expectedCreateFleetRequest(expectedValues: ExpectedFleetRequestValues):
         ],
       },
     ],
-    SpotOptions: {
-      AllocationStrategy: expectedValues.allocationStrategy,
-      MaxTotalPrice: expectedValues.maxSpotPrice,
-    },
+    ...(expectedValues.capacityType === 'spot'
+      ? {
+          SpotOptions: {
+            AllocationStrategy: expectedValues.allocationStrategy,
+            MaxTotalPrice: expectedValues.maxSpotPrice,
+          },
+        }
+      : {
+          OnDemandOptions: {
+            AllocationStrategy: expectedValues.allocationStrategy,
+          },
+        }),
     TagSpecifications: [
       {
         ResourceType: 'instance',

lambdas/functions/control-plane/src/aws/runners.ts

@@ -7,6 +7,8 @@ import {
   DescribeInstancesResult,
   EC2Client,
   FleetLaunchTemplateOverridesRequest,
+  FleetOnDemandAllocationStrategy,
+  SpotAllocationStrategy,
   Tag,
   TerminateInstancesCommand,
   _InstanceType,
@@ -187,11 +189,21 @@ async function processFleetResult(
     logger.warn(`Create fleet failed, initatiing fall back to on demand instances.`);
     logger.debug('Create fleet failed.', { data: fleet.Errors });
     const numberOfInstances = runnerParameters.numberOfRunners - instances.length;
+    const onDemandValidStrategies = ['lowest-price', 'prioritized'];
+    const failoverAllocationStrategy = onDemandValidStrategies.includes(
+      runnerParameters.ec2instanceCriteria.instanceAllocationStrategy,
+    )
+      ? runnerParameters.ec2instanceCriteria.instanceAllocationStrategy
+      : 'lowest-price';
     const instancesOnDemand = await createRunner({
       ...runnerParameters,
       numberOfRunners: numberOfInstances,
       onDemandFailoverOnError: ['InsufficientInstanceCapacity'],
-      ec2instanceCriteria: { ...runnerParameters.ec2instanceCriteria, targetCapacityType: 'on-demand' },
+      ec2instanceCriteria: {
+        ...runnerParameters.ec2instanceCriteria,
+        targetCapacityType: 'on-demand',
+        instanceAllocationStrategy: failoverAllocationStrategy,
+      },
     });
     instances.push(...instancesOnDemand);
     return instances;
@@ -268,10 +280,20 @@ async function createInstances(
           ),
         },
       ],
-      SpotOptions: {
-        MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice,
-        AllocationStrategy: runnerParameters.ec2instanceCriteria.instanceAllocationStrategy,
-      },
+      ...(runnerParameters.ec2instanceCriteria.targetCapacityType === 'spot'
+        ? {
+            SpotOptions: {
+              MaxTotalPrice: runnerParameters.ec2instanceCriteria.maxSpotPrice,
+              AllocationStrategy:
+                runnerParameters.ec2instanceCriteria.instanceAllocationStrategy as SpotAllocationStrategy,
+            },
+          }
+        : {
+            OnDemandOptions: {
+              AllocationStrategy:
+                runnerParameters.ec2instanceCriteria.instanceAllocationStrategy as FleetOnDemandAllocationStrategy,
+            },
+          }),
       TargetCapacitySpecification: {
         TotalTargetCapacity: runnerParameters.numberOfRunners,
         DefaultTargetCapacityType: runnerParameters.ec2instanceCriteria.targetCapacityType,

modules/multi-runner/variables.tf

@@ -214,7 +214,7 @@ variable "multi_runner_config" {
         enable_runner_binaries_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
         enable_ssm_on_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
         enable_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
-        instance_allocation_strategy: "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`."
+        instance_allocation_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
         instance_max_spot_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
         instance_target_capacity_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
         instance_types: "List of instance types for the action runner. Defaults are based on runner_os (al2023 for linux and Windows Server Core for win)."

modules/runners/variables.tf

@@ -102,12 +102,12 @@ variable "instance_target_capacity_type" {
 }
 
 variable "instance_allocation_strategy" {
-  description = "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`."
+  description = "The allocation strategy for creating instances. For spot, AWS recommends `capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
   type        = string
   default     = "lowest-price"
 
   validation {
-    condition     = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized"], var.instance_allocation_strategy)
+    condition     = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized", "prioritized"], var.instance_allocation_strategy)
     error_message = "The instance allocation strategy does not match the allowed values."
   }
 }

variables.tf

@@ -554,11 +554,11 @@ variable "instance_target_capacity_type" {
 }
 
 variable "instance_allocation_strategy" {
-  description = "The allocation strategy for spot instances. AWS recommends using `price-capacity-optimized` however the AWS default is `lowest-price`."
+  description = "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
   type        = string
   default     = "lowest-price"
   validation {
-    condition     = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized"], var.instance_allocation_strategy)
+    condition     = contains(["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized", "prioritized"], var.instance_allocation_strategy)
     error_message = "The instance allocation strategy does not match the allowed values."
   }
 }