Feat: move automatic labs to another field (#1717)

* feat: move automatic labs to another field

Closes #1647

* refactor: remove lab column from builds table

Removes lab from build table since all current build labs are k8s-all

Author: MarceloRobert

backend/kernelCI_app/constants/ingester.py

@@ -2,6 +2,7 @@
 
 import os
 import logging
+import re
 from utils.validation import is_boolean_or_string_true
 
 logger = logging.getLogger("ingester")
@@ -65,3 +66,8 @@
 except (ValueError, TypeError):
     logger.warning("Invalid INGEST_QUEUE_MAXSIZE, using default 5000")
     INGEST_QUEUE_MAXSIZE = 5000
+
+AUTOMATIC_LABS = re.compile(r"^(shell|k8s.*)$")
+"""Regex pattern to find labs that were named automatically and should not be in the real lab/runtime field"""
+AUTOMATIC_LAB_FIELD = "automatic_lab"
+"""Field name where automatic lab names will be moved to"""

backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py

@@ -8,6 +8,8 @@
 from queue import Queue
 from typing_extensions import Literal
 from kernelCI_app.constants.ingester import (
+    AUTOMATIC_LAB_FIELD,
+    AUTOMATIC_LABS,
     CONVERT_LOG_EXCERPT,
     INGEST_BATCH_SIZE,
     INGEST_FILES_BATCH_SIZE,
@@ -89,6 +91,26 @@ def standardize_tree_names(
                 checkout["tree_name"] = correct_tree
 
 
+def standardize_labs(input_data: dict[str, Any]) -> None:
+    """
+    Standardize labs in data, moving automatic lab names to AUTOMATIC_LAB_FIELD
+    """
+
+    builds: list[dict[str, Any]] = input_data.get("builds", [])
+    for build in builds:
+        lab = build.get("misc", {}).get("lab")
+        if lab and AUTOMATIC_LABS.match(lab):
+            build["misc"][AUTOMATIC_LAB_FIELD] = lab
+            build["misc"].pop("lab", None)
+
+    tests: list[dict[str, Any]] = input_data.get("tests", [])
+    for test in tests:
+        lab = test.get("misc", {}).get("runtime")
+        if lab and AUTOMATIC_LABS.match(lab):
+            test["misc"][AUTOMATIC_LAB_FIELD] = lab
+            test["misc"].pop("runtime", None)
+
+
 def prepare_file_data(
     file: SubmissionFileMetadata, tree_names: dict[str, str]
 ) -> tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]:
@@ -121,6 +143,7 @@ def prepare_file_data(
         standardize_tree_names(data, tree_names)
         kcidb_io.schema.V5_3.validate(data)
         kcidb_io.schema.V5_3.upgrade(data)
+        standardize_labs(data)
 
         processing_time = time.time() - start_time
         return data, {

backend/kernelCI_app/tests/unitTests/commands/monitorSubmissions/kcidbng_ingester_test.py

@@ -13,11 +13,13 @@
     SUBMISSION_FILE_DATA_MOCK,
     SUBMISSION_FILENAME_MOCK,
 )
+from kernelCI_app.constants.ingester import AUTOMATIC_LAB_FIELD
 import pytest
 
 from kernelCI_app.management.commands.helpers.kcidbng_ingester import (
     SubmissionFileMetadata,
     standardize_tree_names,
+    standardize_labs,
     prepare_file_data,
     consume_buffer,
     flush_buffers,
@@ -109,6 +111,111 @@ def test_standardize_tree_names_missing_tree_name(self):
         assert "tree_name" not in input_data["checkouts"][1]
 
 
+class TestStandardizeLabs:
+    """Test cases for standardize_labs function."""
+
+    # Test cases:
+    # - builds/tests with missing misc field
+    # - builds/tests with misc but not lab field
+    # - empty builds/tests data
+    # - builds/tests with real/automatic labs/runtimes
+
+    def test_standardize_labs_missing_misc(self):
+        """Test with builds/tests missing misc field."""
+        input_data = {
+            "builds": [
+                {"id": "build1"},
+            ],
+            "tests": [
+                {"id": "test1"},
+            ],
+        }
+
+        standardize_labs(input_data)
+
+        assert "misc" not in input_data["builds"][0]
+        assert "misc" not in input_data["tests"][0]
+
+    def test_standardize_labs_missing_lab_runtime(self):
+        """Test with builds/tests having misc but missing lab/runtime field."""
+        input_data = {
+            "builds": [
+                {
+                    "misc": {
+                        "other_field": "value",
+                    }
+                },
+            ],
+            "tests": [
+                {
+                    "misc": {
+                        "platform": "x86",
+                    }
+                },
+            ],
+        }
+
+        standardize_labs(input_data)
+
+        assert "lab" not in input_data["builds"][0]["misc"]
+        assert AUTOMATIC_LAB_FIELD not in input_data["builds"][0]["misc"]
+        assert "runtime" not in input_data["tests"][0]["misc"]
+        assert AUTOMATIC_LAB_FIELD not in input_data["tests"][0]["misc"]
+
+    def test_standardize_labs_empty_data(self):
+        """Test with empty builds and tests."""
+        input_data = {
+            "builds": [],
+            "tests": [],
+        }
+
+        standardize_labs(input_data)
+
+        assert input_data["builds"] == []
+        assert input_data["tests"] == []
+
+    def test_standardize_labs_mixed_builds_and_tests(self):
+        """Test with a mix of automatic and real labs/runtimes."""
+        input_data = {
+            "builds": [
+                {"misc": {"lab": "shell"}},
+                {"misc": {"lab": "collabora"}},
+                {"misc": {"lab": "k8s-cluster"}},
+            ],
+            "tests": [
+                {"misc": {"runtime": "shell"}},
+                {"misc": {"runtime": "lava-lab"}},
+                {"misc": {"runtime": "k8s"}},
+            ],
+        }
+
+        standardize_labs(input_data)
+
+        # Build 0: shell -> automatic_lab
+        assert "lab" not in input_data["builds"][0]["misc"]
+        assert input_data["builds"][0]["misc"][AUTOMATIC_LAB_FIELD] == "shell"
+
+        # Build 1: real lab stays
+        assert input_data["builds"][1]["misc"]["lab"] == "collabora"
+        assert AUTOMATIC_LAB_FIELD not in input_data["builds"][1]["misc"]
+
+        # Build 2: k8s* -> automatic_lab
+        assert "lab" not in input_data["builds"][2]["misc"]
+        assert input_data["builds"][2]["misc"][AUTOMATIC_LAB_FIELD] == "k8s-cluster"
+
+        # Test 0: shell -> automatic_lab
+        assert "runtime" not in input_data["tests"][0]["misc"]
+        assert input_data["tests"][0]["misc"][AUTOMATIC_LAB_FIELD] == "shell"
+
+        # Test 1: real runtime stays
+        assert input_data["tests"][1]["misc"]["runtime"] == "lava-lab"
+        assert AUTOMATIC_LAB_FIELD not in input_data["tests"][1]["misc"]
+
+        # Test 2: k8s -> automatic_lab
+        assert "runtime" not in input_data["tests"][2]["misc"]
+        assert input_data["tests"][2]["misc"][AUTOMATIC_LAB_FIELD] == "k8s"
+
+
 class TestPrepareFileData:
     """Test cases for prepare_file_data function."""
 
@@ -145,6 +252,7 @@ def test_prepare_file_data_empty_file(self, mock_remove, mock_logger):
     @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.VERBOSE", False)
     @patch("kcidb_io.schema.V5_3.validate")
     @patch("kcidb_io.schema.V5_3.upgrade")
+    @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.standardize_labs")
     @patch(
         "kernelCI_app.management.commands.helpers.kcidbng_ingester.standardize_tree_names"
     )
@@ -158,7 +266,8 @@ def test_prepare_file_data_success(
         mock_time,
         mock_file_open,
         mock_extract_log,
-        mock_standardize,
+        mock_standardize_tree,
+        mock_standardize_labs,
         mock_upgrade,
         mock_validate,
     ):
@@ -179,9 +288,10 @@ def test_prepare_file_data_success(
         assert result_metadata["processing_time"] == 1
         assert mock_time.call_count == 2
         mock_extract_log.assert_called_once_with(expected_data)
-        mock_standardize.assert_called_once_with(expected_data, tree_names)
+        mock_standardize_tree.assert_called_once_with(expected_data, tree_names)
         mock_validate.assert_called_once()
         mock_upgrade.assert_called_once()
+        mock_standardize_labs.assert_called_once_with(expected_data)
         mock_file_open.assert_called_once_with(SUBMISSION_PATH_MOCK, "r")
 
     @patch("kernelCI_app.management.commands.helpers.kcidbng_ingester.logger")

dashboard/src/components/BuildsTable/DefaultBuildsColumns.tsx

@@ -15,7 +15,6 @@ import {
   MoreDetailsTableHeader,
 } from '@/components/Table/DetailsColumn';
 import { getBuildStatusGroup } from '@/utils/status';
-import { UNKNOWN_STRING } from '@/utils/constants/backend';
 
 export const defaultBuildColumns: ColumnDef<AccordionItemBuilds>[] = [
   {
@@ -40,16 +39,6 @@ export const defaultBuildColumns: ColumnDef<AccordionItemBuilds>[] = [
       <TableHeader column={column} intlKey="global.compiler" />
     ),
   },
-  {
-    id: 'lab',
-    accessorKey: 'lab',
-    header: ({ column }): JSX.Element => (
-      <TableHeader column={column} intlKey="global.lab" />
-    ),
-    cell: ({ row }): string => {
-      return row.getValue('lab') || UNKNOWN_STRING;
-    },
-  },
   {
     accessorKey: 'date',
     header: ({ column }): JSX.Element => (