fix(x2a): Handeling deletions of jobs (#2348)

* Deletion fixes: k8s job delete -> error on UI, delete k8s jobs before delete on DB

* revert change on Pending state

Author: elai-shalev

workspaces/x2a/plugins/x2a-backend/src/router/common.ts

@@ -124,10 +124,20 @@ async function reconcileJobStatus(
     return job;
   }
 
+  deps.logger.info(
+    `Reconciling job ${job.id} (k8s: ${job.k8sJobName}), DB status: '${job.status}'`,
+  );
   const k8sStatus = await deps.kubeService.getJobStatus(job.k8sJobName);
 
   if (k8sStatus.status === 'success' || k8sStatus.status === 'error') {
-    const log = (await deps.kubeService.getJobLogs(job.k8sJobName)) as string;
+    let log: string | null = null;
+    try {
+      log = (await deps.kubeService.getJobLogs(job.k8sJobName)) as string;
+    } catch {
+      deps.logger.warn(
+        `Could not fetch logs for job ${job.id} (k8s job: ${job.k8sJobName})`,
+      );
+    }
     const updated = await deps.x2aDatabase.updateJob({
       id: job.id,
       status: k8sStatus.status,

workspaces/x2a/plugins/x2a-backend/src/router/modules.ts

@@ -19,8 +19,29 @@ import express from 'express';
 import { randomUUID } from 'node:crypto';
 import { InputError, NotFoundError } from '@backstage/errors';
 
+import type { Module } from '@red-hat-developer-hub/backstage-plugin-x2a-common';
+
 import type { RouterDeps } from './types';
 import { getUserRef, reconcileJobStatus } from './common';
+import { calculateModuleStatus } from '../services/X2ADatabaseService/status';
+
+/**
+ * Reconcile any pending/running phase jobs on a module against K8s state.
+ * Mutates the module in-place and returns it.
+ */
+async function reconcileModuleJobs(
+  module: Module,
+  deps: Pick<RouterDeps, 'kubeService' | 'x2aDatabase' | 'logger'>,
+): Promise<Module> {
+  const phases = ['analyze', 'migrate', 'publish'] as const;
+  for (const phase of phases) {
+    const job = module[phase];
+    if (job && ['pending', 'running'].includes(job.status)) {
+      module[phase] = await reconcileJobStatus(job, deps);
+    }
+  }
+  return module;
+}
 
 export function registerModuleRoutes(
   router: express.Router,
@@ -49,6 +70,24 @@ export function registerModuleRoutes(
     // List modules
     const modules = await x2aDatabase.listModules({ projectId });
 
+    // Reconcile any pending/running jobs against K8s
+    await Promise.all(
+      modules.map(m =>
+        reconcileModuleJobs(m, { kubeService, x2aDatabase, logger }),
+      ),
+    );
+
+    // Recalculate status for each module after reconciliation
+    for (const m of modules) {
+      const { status, errorDetails } = calculateModuleStatus({
+        analyze: m.analyze,
+        migrate: m.migrate,
+        publish: m.publish,
+      });
+      m.status = status;
+      m.errorDetails = errorDetails;
+    }
+
     res.json(modules);
   });
 
@@ -85,6 +124,18 @@ export function registerModuleRoutes(
       );
     }
 
+    // Reconcile any pending/running jobs against K8s
+    await reconcileModuleJobs(module, { kubeService, x2aDatabase, logger });
+
+    // Recalculate status after reconciliation may have updated phase jobs
+    const { status, errorDetails } = calculateModuleStatus({
+      analyze: module.analyze,
+      migrate: module.migrate,
+      publish: module.publish,
+    });
+    module.status = status;
+    module.errorDetails = errorDetails;
+
     res.json(module);
   });
 
@@ -293,7 +344,10 @@ export function registerModuleRoutes(
       });
 
       // Update job with k8s job name
-      await x2aDatabase.updateJob({ id: job.id, k8sJobName });
+      await x2aDatabase.updateJob({
+        id: job.id,
+        k8sJobName,
+      });
 
       logger.info(
         `${phase} job created: jobId=${job.id}, moduleId=${moduleId}, k8sJobName=${k8sJobName}`,

workspaces/x2a/plugins/x2a-backend/src/router/projects.ts

@@ -167,6 +167,25 @@ export function registerProjectRoutes(
     const endpoint = 'DELETE /projects/:projectId';
     const projectId = req.params.projectId;
     logger.info(`${endpoint} request received: projectId=${projectId}`);
+
+    // Cancel any active k8s jobs before deleting DB records
+    const jobs = await x2aDatabase.listJobsForProject({ projectId });
+    const activeJobs = jobs.filter(
+      job => ['pending', 'running'].includes(job.status) && job.k8sJobName,
+    );
+    await Promise.all(
+      activeJobs.map(job => {
+        logger.info(
+          `Cancelling k8s job ${job.k8sJobName} for project ${projectId}`,
+        );
+        return kubeService.deleteJob(job.k8sJobName!).catch(err => {
+          logger.warn(
+            `Failed to cancel k8s job ${job.k8sJobName}: ${err.message}`,
+          );
+        });
+      }),
+    );
+
     const deletedCount = await x2aDatabase.deleteProject(
       { projectId },
       {
@@ -317,7 +336,10 @@ export function registerProjectRoutes(
       });
 
       // Update job with k8s job name
-      await x2aDatabase.updateJob({ id: job.id, k8sJobName });
+      await x2aDatabase.updateJob({
+        id: job.id,
+        k8sJobName,
+      });
 
       logger.info(
         `Init job created: jobId=${job.id}, k8sJobName=${k8sJobName}`,

workspaces/x2a/plugins/x2a-backend/src/services/KubeService.ts

@@ -305,6 +305,12 @@ export class KubeService {
         namespace: this.#namespace,
       });
 
+      // If the job is being deleted, treat it as an error regardless of status
+      if (job.metadata?.deletionTimestamp) {
+        this.#logger.warn(`Job ${k8sJobName} is being deleted`);
+        return { status: 'error', message: 'Job was deleted' };
+      }
+
       const jobStatus = job.status;
 
       // Check if job succeeded
@@ -325,7 +331,9 @@ export class KubeService {
       // Job exists but hasn't started yet
       return { status: 'pending', message: 'Job is pending' };
     } catch (error: any) {
-      if (error.statusCode === 404 || error.code === 404) {
+      const statusCode =
+        error.statusCode ?? error.response?.statusCode ?? error.code;
+      if (statusCode === 404) {
         this.#logger.warn(`Job ${k8sJobName} not found`);
         return { status: 'error', message: 'Job not found' };
       }

workspaces/x2a/plugins/x2a-backend/templates/x2a-job-script.sh

@@ -8,6 +8,7 @@ export GIT_TERMINAL_PROMPT=0
 ERROR_MESSAGE=""
 ARTIFACTS=()
 PUSH_FAILED=""
+TERMINATED=false
 
 # Report job result back to the backend.
 # TODO: Incorporate CALLBACK_TOKEN for request signing (HMAC-SHA256).
@@ -92,7 +93,9 @@ Co-Authored-By: ${GIT_AUTHOR_NAME} <${GIT_AUTHOR_EMAIL}>
     fi
   fi
 
-  if [ ${exit_code} -ne 0 ]; then
+  if [ "$TERMINATED" = true ]; then
+    report_result "error" "Job was terminated"
+  elif [ ${exit_code} -ne 0 ]; then
     report_result "error" "${ERROR_MESSAGE:-Script failed with exit code ${exit_code}}"
   elif [ -n "${PUSH_FAILED}" ]; then
     report_result "error" "${PUSH_FAILED}"
@@ -136,6 +139,7 @@ git_clone_repos() {
 }
 
 trap cleanup EXIT
+trap 'TERMINATED=true' SIGTERM SIGINT
 
 #
 # X2A Job Script