feat(notifications): adjust hardware summary template with 3-level layout (#1726)

ximiali · web-flow · commit a1359f2c63e8 · 2026-02-04T12:05:24.000Z
- Adjust the hardware summary template with a 3-level layout.

- Add configuration to filter test results by labs.

Signed-off-by: Yushan Li &lt;yushan.li@oss.qualcomm.com&gt;
diff --git a/backend/data/notifications/example-hardware-subscription.yaml b/backend/data/notifications/example-hardware-subscription.yaml
@@ -4,6 +4,8 @@
 
 qcs9100-ride: # The hardware platform name
   origin: maestro
+  labs:
+    lava-kci-qualcomm: https://lava-oss.qualcomm.com
   default_recipients:
     - Recipient One <Recipient1@example.com>
     - Recipient Two <Recipient2@example.com>
diff --git a/backend/data/notifications/subscriptions/qcs615_hardware.yaml b/backend/data/notifications/subscriptions/qcs615_hardware.yaml
@@ -1,5 +1,7 @@
 qcs615-ride:
   origin: maestro
+  labs:
+    lava-kci-qualcomm: https://lava-oss.qualcomm.com
   default_recipients:
     - Trilok Soni <tsoni@quicinc.com>
     - Shiraz Hashim <shashim@qti.qualcomm.com>
diff --git a/backend/data/notifications/subscriptions/qcs6490_hardware.yaml b/backend/data/notifications/subscriptions/qcs6490_hardware.yaml
@@ -1,5 +1,7 @@
 qcs6490-rb3gen2:
   origin: maestro
+  labs:
+    lava-kci-qualcomm: https://lava-oss.qualcomm.com
   default_recipients:
     - Trilok Soni <tsoni@quicinc.com>
     - Shiraz Hashim <shashim@qti.qualcomm.com>
diff --git a/backend/data/notifications/subscriptions/qcs8300_hardware.yaml b/backend/data/notifications/subscriptions/qcs8300_hardware.yaml
@@ -1,5 +1,7 @@
 qcs8300-ride:
   origin: maestro
+  labs:
+    lava-kci-qualcomm: https://lava-oss.qualcomm.com
   default_recipients:
     - Trilok Soni <tsoni@quicinc.com>
     - Shiraz Hashim <shashim@qti.qualcomm.com>
diff --git a/backend/data/notifications/subscriptions/qcs9100_hardware.yaml b/backend/data/notifications/subscriptions/qcs9100_hardware.yaml
@@ -1,5 +1,7 @@
 qcs9100-ride:
   origin: maestro
+  labs:
+    lava-kci-qualcomm: https://lava-oss.qualcomm.com
   default_recipients:
     - Trilok Soni <tsoni@quicinc.com>
     - Shiraz Hashim <shashim@qti.qualcomm.com>
diff --git a/backend/data/notifications/subscriptions/x1e80100_hardware.yaml b/backend/data/notifications/subscriptions/x1e80100_hardware.yaml
@@ -1,5 +1,7 @@
 x1e80100:
   origin: maestro
+  labs:
+    lava-kci-qualcomm: https://lava-oss.qualcomm.com
   default_recipients:
     - Trilok Soni <tsoni@quicinc.com>
     - Shiraz Hashim <shashim@qti.qualcomm.com>
diff --git a/backend/kernelCI_app/management/commands/helpers/summary.py b/backend/kernelCI_app/management/commands/helpers/summary.py
@@ -176,7 +176,7 @@ def process_hardware_submissions_files(
 ) -> tuple[set[HardwareKey], dict[HardwareKey, dict[str, Any]]]:
     """
     Processes all hardware submission files and returns the set of HardwareKey
-    and the dict linking each hardware to its default recipients
+    and the dict linking each hardware to its default recipients and labs
     """
     (base_dir, signup_folder) = _assign_default_folders(
         base_dir=base_dir, signup_folder=signup_folder
@@ -189,7 +189,10 @@ def process_hardware_submissions_files(
         "default_recipients": [
             "Recipient One <Recipient1@example.com>",
             "Recipient Two <Recipient2@example.com>"
-        ]
+        ],
+        "labs": {
+            "lava-collabora": "https://lava.collabora.dev",
+        },
     }
     """
 
@@ -200,6 +203,8 @@ def process_hardware_submissions_files(
 
         imx6q-sabrelite:
         origin: maestro
+        labs:
+            lava-collabora: https://lava.collabora.dev
         default_recipients:
             - Recipient One <Recipient1@example.com>
             - Recipient Two <Recipient2@example.com>
@@ -212,11 +217,13 @@ def process_hardware_submissions_files(
                 if hardware_origins is not None and origin not in hardware_origins:
                     continue
                 default_recipients = hardware_values.get("default_recipients", [])
+                labs = hardware_values.get("labs")
 
                 hardware_key = (hardware_name, origin)
                 hardware_key_set.add(hardware_key)
                 hardware_prop_map[hardware_key] = {
-                    "default_recipients": default_recipients
+                    "default_recipients": default_recipients,
+                    "labs": labs,
                 }
         else:
             log_message(
diff --git a/backend/kernelCI_app/management/commands/notifications.py b/backend/kernelCI_app/management/commands/notifications.py
@@ -637,20 +637,31 @@ def generate_hardware_summary_report(
     start_date = now - timedelta(days=7)
     end_date = now
 
-    # process the hardware submission files
+    # Process the hardware submission files
     hardware_key_set, hardware_prop_map = process_hardware_submissions_files(
         signup_folder=signup_folder,
         hardware_origins=hardware_origins,
     )
-
-    # get detailed data for all hardware
+    # Map used to filter tests by lab
+    labs_by_key = {}
+    for key, props in hardware_prop_map.items():
+        labs = props.get("labs")
+        lab_names = set(labs.keys())
+        labs_by_key[key] = lab_names
+
+    # Get detailed data for all hardware
     hardwares_data_raw = get_hardware_summary_data(
         keys=list(hardware_key_set),
         start_date=start_date,
         end_date=end_date,
+        labs_by_key=labs_by_key,
     )
 
-    hardwares_data_dict = defaultdict(list)
+    hardwares_data_dict = defaultdict(
+        lambda: defaultdict(
+            lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+        )
+    )
     for raw in hardwares_data_raw:
         try:
             environment_misc = json.loads(raw.get("environment_misc", "{}"))
@@ -663,50 +674,65 @@ def generate_hardware_summary_report(
         raw["runtime"] = misc.get("runtime")
         origin = raw.get("test_origin")
         key = (hardware_id, origin)
-        hardwares_data_dict[key].append(raw)
+        tree = raw.get("tree_name")
+        branch = raw.get("git_repository_branch")
+        commit = raw.get("git_commit_hash")
+        path = raw.get("path")
+        dot = path.find(".")
+        suite = path[:dot] if dot != -1 else path
 
-    # get the total build/boot/test counts for each hardware
+        hardwares_data_dict[key][suite][tree][branch][commit].append(raw)
+
+    # Get the total build/boot/test counts for each hardware
     hardwares_list_raw = get_hardware_listing_data_bulk(
         keys=list(hardware_key_set),
         start_date=start_date,
         end_date=end_date,
+        labs_by_key=labs_by_key,
     )
 
     # Iterate through each hardware record to render report, extract recipient, send email
     for hardware_id, origin in hardware_key_set:
-        hardware_data = hardwares_data_dict.get((hardware_id, origin), [])
+        hardware_data = hardwares_data_dict.get((hardware_id, origin), {})
         hardware_raw = next(
-            (row for row in hardwares_list_raw if row.get("platform") == hardware_id),
+            (
+                row
+                for row in hardwares_list_raw
+                if row.get("platform") == hardware_id
+                and row.get("test_origin") == origin
+            ),
             None,
         )
         if hardware_raw is None:
-            print(f"Hardware {hardware_id} not found in listing data")
+            print(f"Hardware {hardware_id} from {origin} not found in listing data")
             continue
 
         hardware_item = sanitize_hardware(hardware_raw)
-        build_status_group = group_status(hardware_item.build_status_summary)
-        boot_status_group = group_status(hardware_item.boot_status_summary)
-        test_status_group = group_status(hardware_item.test_status_summary)
+        build_status_group_all = group_status(hardware_item.build_status_summary)
+        boot_status_group_all = group_status(hardware_item.boot_status_summary)
+        test_status_group_all = group_status(hardware_item.test_status_summary)
 
-        # render the template
+        # Render the template
         template = setup_jinja_template("hardware_report.txt.j2")
         report = {}
+        labs_for_key = hardware_prop_map[(hardware_id, origin)]["labs"]
         report["content"] = template.render(
             hardware_id=hardware_id,
             hardware_data=hardware_data,
-            build_status_group=build_status_group,
-            boot_status_group=boot_status_group,
-            test_status_group=test_status_group,
+            labs_for_key=labs_for_key,
+            build_status_group_all=build_status_group_all,
+            boot_status_group_all=boot_status_group_all,
+            test_status_group_all=test_status_group_all,
         )
         report["title"] = (
             f"hardware {hardware_id} summary - {now.strftime("%Y-%m-%d %H:%M %Z")}"
         )
 
-        # extract recipient
+        # Extract recipient
         hardware_report = hardware_prop_map.get((hardware_id, origin), {})
         recipients = hardware_report.get("default_recipients", [])
 
-        # send email
+        # Send email
         send_email_report(
             service=service,
             report=report,
diff --git a/backend/kernelCI_app/management/commands/templates/hardware_report.txt.j2 b/backend/kernelCI_app/management/commands/templates/hardware_report.txt.j2
@@ -1,32 +1,42 @@
 {% extends "base.txt" %}
+{% set IND = ' ' %}
 {% block header %}{% endblock %}
 {% block content %}
 Hello,
 
 Status summary for {{ hardware_id }}
+Builds:{{ "\t" }}{{ "{:>5}".format(build_status_group_all["success"]) }} ✅
+{{- "{:>5}".format(build_status_group_all["failed"]) }} ❌
+{{- "{:>5}".format(build_status_group_all["inconclusive"]) }} ⚠️
+Boots: {{ "\t" }}{{ "{:>5}".format(boot_status_group_all["success"]) }} ✅
+{{- "{:>5}".format(boot_status_group_all["failed"]) }} ❌
+{{- "{:>5}".format(boot_status_group_all["inconclusive"]) }} ⚠️
+Tests: {{ "\t" }}{{ "{:>5}".format(test_status_group_all["success"]) }} ✅
+{{- "{:>5}".format(test_status_group_all["failed"]) }} ❌
+{{- "{:>5}".format(test_status_group_all["inconclusive"]) }} ⚠️
+================================================================
+{% for suite_name, trees in hardware_data.items() %}
+>{{ IND }}TEST SUITE: {{ suite_name }}
+------------------------------------------------------------------------------------------------------------
+{%- for tree_name, branches in trees.items() %}
+{%- for branch_name, commits in branches.items() %}
+{{ IND * 4 }}>>{{ IND }}TREE: {{ tree_name }}{{ IND }}|{{ IND }}BRANCH: {{ branch_name }}
 
-Builds:{{ "\t" }}{{ "{:>5}".format(build_status_group["success"]) }} ✅
-{{- "{:>5}".format(build_status_group["failed"]) }} ❌
-{{- "{:>5}".format(build_status_group["inconclusive"]) }} ⚠️
-Boots: {{ "\t" }}{{ "{:>5}".format(boot_status_group["success"]) }} ✅
-{{- "{:>5}".format(boot_status_group["failed"]) }} ❌
-{{- "{:>5}".format(boot_status_group["inconclusive"]) }} ⚠️
-Tests: {{ "\t" }}{{ "{:>5}".format(test_status_group["success"]) }} ✅
-{{- "{:>5}".format(test_status_group["failed"]) }} ❌
-{{- "{:>5}".format(test_status_group["inconclusive"]) }} ⚠️
--------------
-{% for data in hardware_data %}
-#kernelci test {{ data["id"] }}
-- Path: {{ data["path"] }}
-- Status: {{ data["status"] }}
-- Comment: {{ data["comment"] }}
-- Starttime: {{ data["start_time"] }}
-- Tree: {{ data["tree_name"] }}/{{ data["git_repository_branch"] }}
-- Origin: {{ data["test_origin"] }}
-- Test_lab: {{ data["runtime"] }}
-- Lava_job_id: {{ data["job_id"] }}
-- Commit: {{ data["git_commit_hash"] }}
-- Dashboard: https://dashboard.kernelci.org/test/{{ data["id"] }}
--------------
+{%- for commit_hash, commit_data in commits.items() %}
+{{ IND * 8 }}>>>{{ IND }}COMMIT: {{ commit_hash }}
+
+{%- for data in commit_data | sort(attribute='path') %}
+{{ IND * 12 }}- Path: {{ data["path"] }}
+{{ IND * 12 }}- Status: {{ data["status"] }}
+{{ IND * 12 }}- Origin: {{ data["test_origin"] }}
+{{ IND * 12 }}- Starttime: {{ data["start_time"] }}
+{{ IND * 12 }}- Test_lab: {{ data["runtime"] }}
+{{ IND * 12 }}- Lava_job: {{ labs_for_key[data["runtime"]]}}/scheduler/job/{{ data["job_id"] }}
+{{ IND * 12 }}- Dashboard: https://dashboard.kernelci.org/test/{{ data["id"] }}
+{{ IND * 12 }}------------------------------------------------------------
+{% endfor %}
+{%- endfor -%}
+{%- endfor -%}
+{%- endfor -%}
 {% endfor %}
 {%- endblock -%}
diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py
@@ -175,14 +175,27 @@ def get_hardware_listing_data(
 
 
 def get_hardware_listing_data_bulk(
-    keys: list[tuple[str, str]], start_date: datetime, end_date: datetime
+    keys: list[tuple[str, str]],
+    start_date: datetime,
+    end_date: datetime,
+    labs_by_key: dict[tuple[str, str], set[str]],
 ) -> list[dict]:
     if not keys:
         return []
 
     count_clauses = _get_hardware_listing_count_clauses()
+
+    triples: list[tuple[str, str, str]] = []
+    for hardware_id, origin in keys:
+        lab_names = labs_by_key[(hardware_id, origin)]
+        for lab in lab_names:
+            triples.append((hardware_id, origin, lab))
+
     values_clause = ", ".join(
-        [f"(%(hardware_id_{i})s, %(origin_{i})s)" for i in range(len(keys))]
+        [
+            f"(%(hardware_id_{i})s, %(origin_{i})s, %(lab_name_{i})s)"
+            for i in range(len(triples))
+        ]
     )
 
     query = f"""
@@ -206,12 +219,13 @@ def get_hardware_listing_data_bulk(
                     AND "tests"."start_time" <= %(end_date)s
                     AND EXISTS (
                     SELECT 1
-                    FROM (VALUES {values_clause}) AS key_list(hardware_id, origin)
+                    FROM (VALUES {values_clause}) AS key_list(hardware_id, origin, lab_name)
                     WHERE(
                         tests.environment_compatible @> ARRAY[key_list.hardware_id]::TEXT[]
                         OR tests.environment_misc ->> 'platform' = key_list.hardware_id
                     )
                     AND tests.origin = key_list.origin
+                    AND tests.misc ->> 'runtime' = key_list.lab_name
                     )
             )
         SELECT
@@ -231,9 +245,10 @@ def get_hardware_listing_data_bulk(
         "start_date": start_date,
         "end_date": end_date,
     }
-    for i, (hardware_id, origin) in enumerate(keys):
+    for i, (hardware_id, origin, lab_name) in enumerate(triples):
         params[f"hardware_id_{i}"] = hardware_id
         params[f"origin_{i}"] = origin
+        params[f"lab_name_{i}"] = lab_name
 
     with connection.cursor() as cursor:
         cursor.execute(query, params)
@@ -412,7 +427,11 @@ def query_records(
 
 
 def get_hardware_summary_data(
-    *, keys: list[tuple[str, str]], start_date: datetime, end_date: datetime
+    *,
+    keys: list[tuple[str, str]],
+    start_date: datetime,
+    end_date: datetime,
+    labs_by_key: dict[tuple[str, str], set[str]],
 ) -> list[dict] | None:
     """
     Get hardware summary data for given keys within a date range.
@@ -421,8 +440,14 @@ def get_hardware_summary_data(
     if not keys:
         return []
 
+    triples: list[tuple[str, str, str]] = []
+    for hardware_id, origin in keys:
+        lab_names = labs_by_key[(hardware_id, origin)]
+        for lab in lab_names:
+            triples.append((hardware_id, origin, lab))
+
     with connection.cursor() as cursor:
-        values_clause = ", ".join(["(%s, %s)"] * len(keys))
+        values_clause = ", ".join(["(%s, %s, %s)"] * len(triples))
 
         query = f"""
             SELECT
@@ -466,20 +491,21 @@ def get_hardware_summary_data(
                 AND tests.status = 'FAIL'
                 AND EXISTS (
                 SELECT 1
-                FROM (VALUES {values_clause}) AS key_list(hardware_id, origin)
+                FROM (VALUES {values_clause}) AS key_list(hardware_id, origin, lab_name)
                 WHERE(
                     tests.environment_compatible @> ARRAY[key_list.hardware_id]::TEXT[]
                     OR tests.environment_misc ->> 'platform' = key_list.hardware_id
                 )
                 AND tests.origin = key_list.origin
+                AND tests.misc ->> 'runtime' = key_list.lab_name
                 )
             ORDER BY
                 tests.start_time DESC
             """
 
         params = [start_date, end_date]
-        for hardware_id, origin in keys:
-            params.extend([hardware_id, origin])
+        for hardware_id, origin, lab_name in triples:
+            params.extend([hardware_id, origin, lab_name])
 
         cursor.execute(query, params)
 
