feat: Add feature to enable dynamic instance types via workflow labels

Author: npalm

lambdas/functions/control-plane/src/scale-runners/scale-up.ts

@@ -29,6 +29,7 @@ export interface ActionRequestMessage {
   installationId: number;
   repoOwnerType: string;
   retryCounter?: number;
+  labels?: string[];
 }
 
 export interface ActionRequestMessageSQS extends ActionRequestMessage {
@@ -216,6 +217,7 @@ export async function createRunners(
   ghClient: Octokit,
 ): Promise<string[]> {
   const instances = await createRunner({
+    environment: ec2RunnerConfig.environment,
     runnerType: githubRunnerConfig.runnerType,
     runnerOwner: githubRunnerConfig.runnerOwner,
     numberOfRunners,
@@ -233,9 +235,30 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise<stri
     n_requests: payloads.length,
   });
 
+  const dynamicEc2TypesEnabled = yn(process.env.ENABLE_DYNAMIC_EC2_CONFIGURATION, { default: false });
+  const requestedInstanceType = payload.labels?.find(label => label.startsWith('ghr-ec2-'))?.replace('ghr-ec2-', '');
+
+  if (dynamicEc2TypesEnabled && requestedInstanceType) {
+    logger.info(`Dynamic EC2 instance type requested: ${requestedInstanceType}`);
+  }
+
+  // Store the requested instance type for use in createRunners
+  const ec2Config = {
+    ...payload,
+    requestedInstanceType: dynamicEc2TypesEnabled ? requestedInstanceType : undefined,
+  };
+
   const enableOrgLevel = yn(process.env.ENABLE_ORGANIZATION_RUNNERS, { default: true });
   const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '3');
-  const runnerLabels = process.env.RUNNER_LABELS || '';
+
+  // Combine configured runner labels with dynamic EC2 instance type label if present
+  let runnerLabels = process.env.RUNNER_LABELS || '';
+  if (dynamicEc2TypesEnabled && requestedInstanceType) {
+    const ec2Label = `ghr-ec2-${requestedInstanceType}`;
+    runnerLabels = runnerLabels ? `${runnerLabels},${ec2Label}` : ec2Label;
+    logger.debug(`Added dynamic EC2 instance type label: ${ec2Label} to runner config.`);
+  }
+
   const runnerGroup = process.env.RUNNER_GROUP_NAME || 'Default';
   const environment = process.env.ENVIRONMENT;
   const ssmTokenPath = process.env.SSM_TOKEN_PATH;
@@ -275,6 +298,7 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise<stri
     githubInstallationClient: Octokit;
   };
 
+<<<<<<< HEAD
   const validMessages = new Map<string, MessagesWithClient>();
   const invalidMessages: string[] = [];
   for (const payload of payloads) {
@@ -283,6 +307,41 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise<stri
       logger.warn(
         'Event is not supported in combination with ephemeral runners. Please ensure you have enabled workflow_job events.',
         { eventType, messageId },
+=======
+    if (scaleUp) {
+      logger.info(`Attempting to launch a new runner`);
+
+      await createRunners(
+        {
+          ephemeral,
+          enableJitConfig,
+          ghesBaseUrl,
+          runnerLabels,
+          runnerGroup,
+          runnerNamePrefix,
+          runnerOwner,
+          runnerType,
+          disableAutoUpdate,
+          ssmTokenPath,
+          ssmConfigPath,
+        },
+        {
+          ec2instanceCriteria: {
+            instanceTypes,
+            targetCapacityType: instanceTargetCapacityType,
+            maxSpotPrice: instanceMaxSpotPrice,
+            instanceAllocationStrategy: instanceAllocationStrategy,
+          },
+          environment,
+          launchTemplateName,
+          subnets,
+          amiIdSsmParameterName,
+          tracingEnabled,
+          onDemandFailoverOnError,
+        },
+        githubInstallationClient,
+        ec2Config.requestedInstanceType,
+>>>>>>> 44737379 (feat: Add feature to enable dynamic instance types via workflow labels)
       );
 
       invalidMessages.push(messageId);

lambdas/functions/webhook/src/runners/dispatch.ts

@@ -81,13 +81,16 @@ export function canRunJob(
   runnerLabelsMatchers: string[][],
   workflowLabelCheckAll: boolean,
 ): boolean {
+  // Filter out ghr-ec2- labels as they are handled by the dynamic EC2 instance type feature
+  const filteredLabels = workflowJobLabels.filter(label => !label.startsWith('ghr-ec2-'));
+
   runnerLabelsMatchers = runnerLabelsMatchers.map((runnerLabel) => {
     return runnerLabel.map((label) => label.toLowerCase());
   });
   const matchLabels = workflowLabelCheckAll
-    ? runnerLabelsMatchers.some((rl) => workflowJobLabels.every((wl) => rl.includes(wl.toLowerCase())))
-    : runnerLabelsMatchers.some((rl) => workflowJobLabels.some((wl) => rl.includes(wl.toLowerCase())));
-  const match = workflowJobLabels.length === 0 ? !matchLabels : matchLabels;
+    ? runnerLabelsMatchers.some((rl) => filteredLabels.every((wl) => rl.includes(wl.toLowerCase())))
+    : runnerLabelsMatchers.some((rl) => filteredLabels.some((wl) => rl.includes(wl.toLowerCase())));
+  const match = filteredLabels.length === 0 ? !matchLabels : matchLabels;
 
   logger.debug(
     `Received workflow job event with labels: '${JSON.stringify(workflowJobLabels)}'. The event does ${

lambdas/functions/webhook/src/sqs/index.ts

@@ -12,6 +12,7 @@ export interface ActionRequestMessage {
   installationId: number;
   queueId: string;
   repoOwnerType: string;
+  labels?: string[];
 }
 
 export interface MatcherConfig {

main.tf

@@ -184,8 +184,9 @@ module "runners" {
   github_app_parameters                = local.github_app_parameters
   enable_organization_runners          = var.enable_organization_runners
   enable_ephemeral_runners             = var.enable_ephemeral_runners
-  enable_jit_config                    = var.enable_jit_config
+  enable_dynamic_ec2_configuration             = var.enable_dynamic_ec2_configuration
   enable_job_queued_check              = var.enable_job_queued_check
+  enable_jit_config                    = var.enable_jit_config
   enable_on_demand_failover_for_errors = var.enable_runner_on_demand_failover_for_errors
   scale_errors                         = var.scale_errors
   disable_runner_autoupdate            = var.disable_runner_autoupdate

modules/multi-runner/variables.tf

@@ -77,6 +77,7 @@ variable "multi_runner_config" {
       disable_runner_autoupdate            = optional(bool, false)
       ebs_optimized                        = optional(bool, false)
       enable_ephemeral_runners             = optional(bool, false)
+      enable_dynamic_ec2_configuration                = optional(bool, false)
       enable_job_queued_check              = optional(bool, null)
       enable_on_demand_failover_for_errors = optional(list(string), [])
       scale_errors = optional(list(string), [
@@ -206,7 +207,8 @@ variable "multi_runner_config" {
         disable_runner_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
         ebs_optimized: "The EC2 EBS optimized configuration."
         enable_ephemeral_runners: "Enable ephemeral runners, runners will only be used once."
-        enable_job_queued_check: "Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
+        enable_dynamic_ec2_configuration: "Enable dynamic EC2 configs based on workflow job labels. When enabled, jobs can request specific configs via the 'gh-ec2-<config type key>:<config type value>' label (e.g., 'gh-ec2-instance-type:t3.large')."
+        enable_job_queued_check: "(Optional) Only scale if the job event received by the scale up lambda is is in the state queued. By default enabled for non ephemeral runners and disabled for ephemeral. Set this variable to overwrite the default behavior."
         enable_on_demand_failover_for_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
         scale_errors: "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts"
         enable_organization_runners: "Register runners to organization, instead of repo level"

modules/runners/scale-up.tf

@@ -28,6 +28,7 @@ resource "aws_lambda_function" "scale_up" {
       AMI_ID_SSM_PARAMETER_NAME                = local.ami_id_ssm_parameter_name
       DISABLE_RUNNER_AUTOUPDATE                = var.disable_runner_autoupdate
       ENABLE_EPHEMERAL_RUNNERS                 = var.enable_ephemeral_runners
+      ENABLE_DYNAMIC_EC2_CONFIGURATION                 = var.enable_dynamic_ec2_configuration
       ENABLE_JIT_CONFIG                        = var.enable_jit_config
       ENABLE_JOB_QUEUED_CHECK                  = local.enable_job_queued_check
       ENABLE_METRIC_GITHUB_APP_RATE_LIMIT      = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit

modules/runners/variables.tf

@@ -520,6 +520,12 @@ variable "enable_ephemeral_runners" {
   default     = false
 }
 
+variable "enable_dynamic_ec2_configuration" {
+  description = "Enable dynamic EC2 instance types based on workflow job labels. When enabled, jobs can request specific instance types via the 'gh:ec2:instance-type' label."
+  type        = bool
+  default     = false
+}
+
 variable "enable_job_queued_check" {
   description = "Only scale if the job event received by the scale up lambda is is in the state queued. By default enabled for non ephemeral runners and disabled for ephemeral. Set this variable to overwrite the default behavior."
   type        = bool

variables.tf

@@ -661,6 +661,12 @@ variable "enable_ephemeral_runners" {
   default     = false
 }
 
+variable "enable_dynamic_ec2_configuration" {
+  description = "Enable dynamic EC2 configs based on workflow job labels. When enabled, jobs can request specific configs via the 'gh-ec2-<config type key>:<config type value>' label (e.g., 'gh-ec2-instance-type:t3.large')."
+  type        = bool
+  default     = false
+}
+
 variable "enable_job_queued_check" {
   description = "Only scale if the job event received by the scale up lambda is in the queued state. By default enabled for non ephemeral runners and disabled for ephemeral. Set this variable to overwrite the default behavior."
   type        = bool