fix: add retry logic for GitHub API calls in scale-down Lambda

Add exponential backoff retry for transient GitHub API failures (5xx, 429)
when de-registering runners during scale-down operations.

Key changes:
- Add withRetry() helper with configurable max retries (3) and delays
- Wrap deleteSelfHostedRunnerFromOrg/Repo calls with retry logic
- Only terminate EC2 instance if GitHub de-registration succeeds
- If de-registration fails after retries, leave instance running for
  next scale-down cycle to retry

This prevents stale runner entries in GitHub org settings caused by
transient API failures (e.g., 502 Server Error) during scale-down.

Tests:
- Add unit tests for successful retry after transient 502 error
- Add unit tests verifying EC2 is NOT terminated when retries exhausted

Author: shivdesh

lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts

@@ -631,6 +631,77 @@ describe('Scale down runners', () => {
         await expect(scaleDown()).resolves.not.toThrow();
       });
 
+      it(`Should retry on 502 error and succeed on second attempt.`, async () => {
+        // setup
+        const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, true)];
+
+        let callCount = 0;
+        const error502 = new RequestError('Server Error', 502, {
+          request: {
+            method: 'DELETE',
+            url: 'https://api.github.com/test',
+            headers: {},
+          },
+        });
+
+        if (type === 'Org') {
+          mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => {
+            callCount++;
+            if (callCount === 1) {
+              throw error502;
+            }
+            return { status: 204 };
+          });
+        } else {
+          mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => {
+            callCount++;
+            if (callCount === 1) {
+              throw error502;
+            }
+            return { status: 204 };
+          });
+        }
+
+        mockGitHubRunners(runners);
+        mockAwsRunners(runners);
+
+        // act
+        await scaleDown();
+
+        // assert - should have retried and succeeded
+        expect(callCount).toBe(2);
+        checkTerminated(runners);
+      });
+
+      it(`Should not terminate instance after all retries exhausted on 502 error.`, async () => {
+        // setup
+        const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)];
+
+        const error502 = new RequestError('Server Error', 502, {
+          request: {
+            method: 'DELETE',
+            url: 'https://api.github.com/test',
+            headers: {},
+          },
+        });
+
+        mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => {
+          throw error502;
+        });
+        mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => {
+          throw error502;
+        });
+
+        mockGitHubRunners(runners);
+        mockAwsRunners(runners);
+
+        // act
+        await expect(scaleDown()).resolves.not.toThrow();
+
+        // assert - should NOT terminate since de-registration failed
+        expect(terminateRunner).not.toHaveBeenCalled();
+      });
+
       const evictionStrategies = ['oldest_first', 'newest_first'];
       describe.each(evictionStrategies)('When idle config defined', (evictionStrategy) => {
         const defaultConfig = {

lambdas/functions/control-plane/src/scale-runners/scale-down.ts

@@ -14,6 +14,54 @@ import { getGitHubEnterpriseApiUrl } from './scale-up';
 
 const logger = createChildLogger('scale-down');
 
+const RETRY_CONFIG = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  maxDelayMs: 10000,
+};
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isRetryableError(error: unknown): boolean {
+  if (error instanceof RequestError) {
+    const status = (error as RequestError).status;
+    // Retry on server errors (5xx) and rate limiting (429)
+    return status >= 500 || status === 429;
+  }
+  return false;
+}
+
+async function withRetry<T>(
+  operation: () => Promise<T>,
+  operationName: string,
+  context: string,
+): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= RETRY_CONFIG.maxRetries; attempt++) {
+    try {
+      return await operation();
+    } catch (error) {
+      lastError = error;
+      if (isRetryableError(error) && attempt < RETRY_CONFIG.maxRetries) {
+        const delay = Math.min(
+          RETRY_CONFIG.initialDelayMs * Math.pow(2, attempt - 1),
+          RETRY_CONFIG.maxDelayMs,
+        );
+        logger.warn(
+          `${operationName} failed for ${context} (attempt ${attempt}/${RETRY_CONFIG.maxRetries}), ` +
+          `retrying in ${delay}ms. Error: ${error}`,
+        );
+        await sleep(delay);
+      } else {
+        throw error;
+      }
+    }
+  }
+  throw lastError;
+}
+
 type OrgRunnerList = Endpoints['GET /orgs/{org}/actions/runners']['response']['data']['runners'];
 type RepoRunnerList = Endpoints['GET /repos/{owner}/{repo}/actions/runners']['response']['data']['runners'];
 type RunnerState = OrgRunnerList[number] | RepoRunnerList[number];
@@ -127,6 +175,33 @@ function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean {
   return launchTimePlusMinimum < now;
 }
 
+async function deleteGitHubRunner(
+  githubAppClient: Octokit,
+  ec2runner: RunnerInfo,
+  ghRunnerId: number,
+): Promise<number> {
+  const deleteOperation = async () => {
+    const response =
+      ec2runner.type === 'Org'
+        ? await githubAppClient.actions.deleteSelfHostedRunnerFromOrg({
+            runner_id: ghRunnerId,
+            org: ec2runner.owner,
+          })
+        : await githubAppClient.actions.deleteSelfHostedRunnerFromRepo({
+            runner_id: ghRunnerId,
+            owner: ec2runner.owner.split('/')[0],
+            repo: ec2runner.owner.split('/')[1],
+          });
+    return response.status;
+  };
+
+  return await withRetry(
+    deleteOperation,
+    'Delete GitHub runner',
+    `runner ${ec2runner.instanceId} (GitHub ID: ${ghRunnerId})`,
+  );
+}
+
 async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promise<void> {
   const githubAppClient = await getOrCreateOctokit(ec2runner);
   try {
@@ -146,28 +221,35 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi
     );
 
     if (states.every((busy) => busy === false)) {
-      const statuses = await Promise.all(
+      const results = await Promise.all(
         ghRunnerIds.map(async (ghRunnerId) => {
-          return (
-            ec2runner.type === 'Org'
-              ? await githubAppClient.actions.deleteSelfHostedRunnerFromOrg({
-                  runner_id: ghRunnerId,
-                  org: ec2runner.owner,
-                })
-              : await githubAppClient.actions.deleteSelfHostedRunnerFromRepo({
-                  runner_id: ghRunnerId,
-                  owner: ec2runner.owner.split('/')[0],
-                  repo: ec2runner.owner.split('/')[1],
-                })
-          ).status;
+          try {
+            const status = await deleteGitHubRunner(githubAppClient, ec2runner, ghRunnerId);
+            return { ghRunnerId, status, success: status === 204 };
+          } catch (error) {
+            logger.error(
+              `Failed to de-register GitHub runner ${ghRunnerId} for instance '${ec2runner.instanceId}' after retries. Error: ${error}`,
+              { error: error as Error },
+            );
+            return { ghRunnerId, status: 0, success: false };
+          }
         }),
       );
 
-      if (statuses.every((status) => status == 204)) {
+      const allSucceeded = results.every((r) => r.success);
+      const failedRunners = results.filter((r) => !r.success);
+
+      if (allSucceeded) {
         await terminateRunner(ec2runner.instanceId);
         logger.info(`AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`);
       } else {
-        logger.error(`Failed to de-register GitHub runner: ${statuses}`);
+        // Only terminate EC2 if we successfully de-registered from GitHub
+        // Otherwise, leave the instance running so the next scale-down cycle can retry
+        logger.error(
+          `Failed to de-register ${failedRunners.length} GitHub runner(s) for instance '${ec2runner.instanceId}'. ` +
+          `Instance will NOT be terminated to allow retry on next scale-down cycle. ` +
+          `Failed runner IDs: ${failedRunners.map((r) => r.ghRunnerId).join(', ')}`,
+        );
       }
     } else {
       logger.info(`Runner '${ec2runner.instanceId}' cannot be de-registered, because it is still busy.`);