feat: add param to return full environment_misc (#1833)

Author: gustavobtflores

backend/kernelCI_app/constants/localization.py

@@ -113,6 +113,10 @@ class DocStrings:
     TREE_QUERY_ORIGIN_DESCRIPTION = "Origin of the tree"
     TREE_QUERY_GIT_URL_DESCRIPTION = "Git repository URL of the tree"
 
+    FULL_ENVIRONMENT_MISC_DESCRIPTION = (
+        "When true, returns all fields from environment_misc instead of only platform"
+    )
+
     STATUS_HISTORY_PATH_DESCRIPTION = "Test path filter"
     STATUS_HISTORY_ORIGIN_DESCRIPTION = "Origin filter to retrieve tests"
     STATUS_HISTORY_GIT_URL_DESCRIPTION = "Git repository URL to retrieve tests"

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -20,7 +20,11 @@
     should_increment_test_issue,
 )
 from kernelCI_app.helpers.logger import log_message
-from kernelCI_app.helpers.misc import misc_value_or_default, handle_misc
+from kernelCI_app.helpers.misc import (
+    get_environment_misc_value,
+    handle_misc,
+    misc_value_or_default,
+)
 from kernelCI_app.typeModels.databases import (
     FAIL_STATUS,
     NULL_STATUS,
@@ -272,8 +276,10 @@ def handle_tree_status_summary(
 
 def create_record_test_platform(*, record: Dict) -> str:
     environment_misc = handle_misc(record["environment_misc"])
-    test_platform = misc_value_or_default(environment_misc).get("platform")
+    resolved_misc = misc_value_or_default(environment_misc)
+    test_platform = resolved_misc.get("platform")
     record["test_platform"] = test_platform
+    record["parsed_environment_misc"] = resolved_misc
 
     return test_platform
 
@@ -282,10 +288,17 @@ def handle_test_history(
     *,
     record: Dict,
     task: List[HardwareTestHistoryItem],
+    full_environment_misc: bool = False,
 ) -> None:
     create_record_test_platform(record=record)
     record_misc = sanitize_dict(record.get("misc"))
 
+    environment_misc_dict = get_environment_misc_value(
+        full_environment_misc=full_environment_misc,
+        parsed_environment_misc=record.get("parsed_environment_misc"),
+    )
+    environment_misc = EnvironmentMisc(**environment_misc_dict)
+
     test_history_item = HardwareTestHistoryItem(
         id=record["id"],
         origin=record["test_origin"],
@@ -298,7 +311,7 @@ def handle_test_history(
         log_url=record["log_url"],
         architecture=record["build__architecture"],
         compiler=record["build__compiler"],
-        environment_misc=EnvironmentMisc(platform=record["test_platform"]),
+        environment_misc=environment_misc,
         tree_name=record["build__checkout__tree_name"],
         git_repository_branch=record["build__checkout__git_repository_branch"],
         lab=(

backend/kernelCI_app/helpers/misc.py

@@ -3,25 +3,23 @@
 from kernelCI_app.utils import sanitize_dict
 
 
-class Misc(TypedDict):
+class Misc(TypedDict, total=False):
     platform: str
     lab: Optional[str]
 
 
-def handle_misc(misc: Union[str, dict, None]) -> Optional[Misc]:
+def handle_misc(misc: Union[str, dict, None]) -> Optional[dict]:
     """
     Handle misc data (environment or build) by parsing JSON string or dict.
-    Also sets some default values and returns only the necessary fields.
+    Preserves all fields and sets default values for known fields.
     """
 
     misc = sanitize_dict(misc)
     if misc is None:
         return None
 
-    parsed_misc: Misc = {}
-    parsed_misc["platform"] = misc.get("platform", UNKNOWN_STRING)
-    if "lab" in misc:
-        parsed_misc["lab"] = misc.get("lab")
+    parsed_misc = dict(misc)
+    parsed_misc.setdefault("platform", UNKNOWN_STRING)
 
     return parsed_misc
 
@@ -35,3 +33,15 @@ def misc_value_or_default(misc: Optional[Misc]) -> Misc:
     if misc is not None:
         return misc
     return default_misc
+
+
+def get_environment_misc_value(
+    *,
+    full_environment_misc: bool,
+    parsed_environment_misc: Optional[dict],
+) -> dict:
+    """Return the appropriate environment_misc value based on the full_environment_misc flag."""
+    platform = parsed_environment_misc.get("platform")
+    if full_environment_misc:
+        return parsed_environment_misc or {"platform": platform}
+    return {"platform": platform}

backend/kernelCI_app/helpers/treeDetails.py

@@ -18,6 +18,7 @@
 from kernelCI_app.typeModels.issues import Issue, IssueDict
 from kernelCI_app.utils import create_issue_typed, extract_error_message, sanitize_dict
 from kernelCI_app.helpers.misc import (
+    get_environment_misc_value,
     handle_misc,
     misc_value_or_default,
 )
@@ -64,7 +65,9 @@ def create_checkouts_where_clauses(
     }
 
 
-def get_current_row_data(current_row: dict) -> dict:
+def get_current_row_data(
+    current_row: dict, full_environment_misc: bool = False
+) -> dict:
     tmp_test_env_comp_key = 12
     current_row_data = {
         "test_id": current_row[0],
@@ -107,10 +110,11 @@ def get_current_row_data(current_row: dict) -> dict:
         "issue_report_url": current_row[37],
     }
 
-    environment_misc = handle_misc(
+    parsed_environment_misc = handle_misc(
         misc_value_or_default(current_row_data["test_environment_misc"])
     )
-    current_row_data["test_platform"] = environment_misc.get("platform")
+    current_row_data["test_platform"] = parsed_environment_misc.get("platform")
+    current_row_data["parsed_environment_misc"] = parsed_environment_misc
     test_misc = sanitize_dict(current_row_data["test_misc"])
     test_runtime_lab = UNKNOWN_STRING
     if test_misc is not None:
@@ -145,6 +149,11 @@ def get_current_row_data(current_row: dict) -> dict:
     if current_row_data["test_path"] is None:
         current_row_data["test_path"] = UNKNOWN_STRING
 
+    environment_misc_value = get_environment_misc_value(
+        full_environment_misc=full_environment_misc,
+        parsed_environment_misc=parsed_environment_misc,
+    )
+
     current_row_data["history_item"] = {
         "id": current_row_data["test_id"],
         "origin": current_row_data["test_origin"],
@@ -157,7 +166,7 @@ def get_current_row_data(current_row: dict) -> dict:
         "log_url": current_row_data["test_log_url"],
         "architecture": current_row_data["build_architecture"],
         "compiler": current_row_data["build_compiler"],
-        "environment_misc": {"platform": current_row_data["test_platform"]},
+        "environment_misc": environment_misc_value,
         "lab": test_runtime_lab,
     }
 

backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py

@@ -477,6 +477,9 @@ def test_create_record_test_platform(
 
         assert result == "hp-x360-14a-cb0001xx-zork"
         assert record["test_platform"] == "hp-x360-14a-cb0001xx-zork"
+        assert record["parsed_environment_misc"] == {
+            "platform": "hp-x360-14a-cb0001xx-zork"
+        }
         mock_handle_env_misc.assert_called_once_with("{}")
         mock_env_misc_value.assert_called_once_with(
             {"platform": "hp-x360-14a-cb0001xx-zork"}
@@ -486,10 +489,11 @@ def test_create_record_test_platform(
 class TestHandleTestHistory:
     @patch("kernelCI_app.helpers.hardwareDetails.create_record_test_platform")
     def test_handle_test_history(self, mock_create_platform):
-        """Test handle_test_history function."""
+        """Test handle_test_history function with default behavior."""
 
         def mock_create_platform_side_effect(record):
             record["test_platform"] = "x86_64"
+            record["parsed_environment_misc"] = {"platform": "x86_64"}
             return "x86_64"
 
         mock_create_platform.side_effect = mock_create_platform_side_effect
@@ -521,6 +525,96 @@ def mock_create_platform_side_effect(record):
         assert task[0].environment_misc.platform == "x86_64"
         mock_create_platform.assert_called_once_with(record=record)
 
+    @patch("kernelCI_app.helpers.hardwareDetails.create_record_test_platform")
+    def test_handle_test_history_full_environment_misc(self, mock_create_platform):
+        """Test handle_test_history with full_environment_misc=True includes all fields."""
+
+        def mock_create_platform_side_effect(record):
+            record["test_platform"] = "x86_64"
+            record["parsed_environment_misc"] = {
+                "platform": "x86_64",
+                "laa_uid": "uid-1",
+                "dut_uid": "dut-1",
+                "device": "dev-a",
+            }
+            return "x86_64"
+
+        mock_create_platform.side_effect = mock_create_platform_side_effect
+
+        record = {
+            "id": "test123",
+            "test_origin": "test",
+            "status": "PASS",
+            "duration": 100,
+            "path": "test.specific",
+            "start_time": "2024-01-15T10:00:00Z",
+            "environment_compatible": "x86_64",
+            "build__config_name": "defconfig",
+            "log_url": "http://example.com/log",
+            "build__architecture": "x86_64",
+            "build__compiler": "gcc",
+            "environment_misc": "{}",
+            "build__checkout__tree_name": "mainline",
+            "build__checkout__git_repository_branch": "master",
+        }
+
+        task = []
+
+        handle_test_history(record=record, task=task, full_environment_misc=True)
+
+        assert len(task) == 1
+        assert task[0].id == "test123"
+        assert task[0].environment_misc.platform == "x86_64"
+        assert task[0].environment_misc.laa_uid == "uid-1"
+        assert task[0].environment_misc.dut_uid == "dut-1"
+        assert task[0].environment_misc.device == "dev-a"
+        mock_create_platform.assert_called_once_with(record=record)
+
+    @patch("kernelCI_app.helpers.hardwareDetails.create_record_test_platform")
+    def test_handle_test_history_default_no_extra_fields(self, mock_create_platform):
+        """Test handle_test_history with full_environment_misc=False only includes platform."""
+
+        def mock_create_platform_side_effect(record):
+            record["test_platform"] = "x86_64"
+            record["parsed_environment_misc"] = {
+                "platform": "x86_64",
+                "laa_uid": "uid-1",
+                "dut_uid": "dut-1",
+                "device": "dev-a",
+            }
+            return "x86_64"
+
+        mock_create_platform.side_effect = mock_create_platform_side_effect
+
+        record = {
+            "id": "test123",
+            "test_origin": "test",
+            "status": "PASS",
+            "duration": 100,
+            "path": "test.specific",
+            "start_time": "2024-01-15T10:00:00Z",
+            "environment_compatible": "x86_64",
+            "build__config_name": "defconfig",
+            "log_url": "http://example.com/log",
+            "build__architecture": "x86_64",
+            "build__compiler": "gcc",
+            "environment_misc": "{}",
+            "build__checkout__tree_name": "mainline",
+            "build__checkout__git_repository_branch": "master",
+        }
+
+        task = []
+
+        # Default behavior (full_environment_misc=False or omitted)
+        handle_test_history(record=record, task=task, full_environment_misc=False)
+
+        assert len(task) == 1
+        assert task[0].id == "test123"
+        assert task[0].environment_misc.platform == "x86_64"
+        # Verify extra fields are NOT present when flag is False
+        assert not hasattr(task[0].environment_misc, "laa_uid")
+        mock_create_platform.assert_called_once_with(record=record)
+
 
 class TestHandleTestSummary:
     @patch("kernelCI_app.helpers.hardwareDetails.process_issue")

backend/kernelCI_app/tests/unitTests/helpers/misc_test.py

@@ -28,12 +28,12 @@ def test_handle_misc_with_dict(self):
         assert result == {"platform": "arm64"}
 
     def test_handle_misc_with_dict_missing_platform(self):
-        """Test handle_misc with dict missing platform key."""
+        """Test handle_misc with dict missing platform key preserves other fields."""
         misc_dict = {"other": "value"}
 
         result = handle_misc(misc_dict)
 
-        assert result == {"platform": UNKNOWN_STRING}
+        assert result == {"other": "value", "platform": UNKNOWN_STRING}
 
     def test_handle_misc_with_none(self):
         """Test handle_misc with None input."""
@@ -55,6 +55,24 @@ def test_handle_misc_with_lab(self):
 
         assert result == {"platform": "x86_64", "lab": "lab1"}
 
+    def test_handle_misc_preserves_extra_fields(self):
+        """Test handle_misc preserves all extra fields like laa_uid, dut_uid, device."""
+        misc_dict = {
+            "platform": "x86_64",
+            "laa_uid": "uid-1",
+            "dut_uid": "dut-1",
+            "device": "dev-a",
+        }
+
+        result = handle_misc(misc_dict)
+
+        assert result == {
+            "platform": "x86_64",
+            "laa_uid": "uid-1",
+            "dut_uid": "dut-1",
+            "device": "dev-a",
+        }
+
 
 class TestMiscValueOrDefault:
     def test_misc_value_or_default_with_valid_misc(self):