feat(tree): add lazy loading for commit history endpoint (#1729)

Author: gustavobtflores

backend/kernelCI_app/queries/tree.py

@@ -518,7 +518,13 @@ def get_tree_commit_history(
     git_url: Optional[str],
     git_branch: Optional[str],
     tree_name: Optional[str],
+    include_types: Optional[list[str]] = None,
 ) -> Optional[list[tuple]]:
+    if not include_types:
+        include_types = ["builds", "boots", "tests"]
+
+    include_types = [t.lower() for t in include_types]
+
     field_values = {
         "commit_hash": commit_hash,
         "origin_param": origin,
@@ -541,6 +547,62 @@ def get_tree_commit_history(
         git_url=git_url, git_branch=git_branch, tree_name=tree_name
     )
 
+    include_builds = "builds" in include_types
+    include_boots = "boots" in include_types
+    include_tests = "tests" in include_types
+    include_test_data = include_tests or include_boots
+
+    build_prefix = "b." if include_builds else "NULL AS "
+    test_prefix = "t." if include_test_data else "NULL AS "
+    build_id = "b.id" if include_builds else "NULL"
+    build_misc = "b.misc" if include_builds else "NULL"
+    test_misc_runtime = "t.misc->>'runtime'" if include_test_data else "NULL"
+    test_id = "t.id" if include_test_data else "NULL"
+
+    select_clause = f"""c.git_commit_hash,
+        c.git_commit_name,
+        c.git_commit_tags,
+        c.start_time,
+        {build_prefix}duration,
+        {build_prefix}architecture,
+        {build_prefix}compiler,
+        {build_prefix}config_name,
+        {build_prefix}status,
+        {build_prefix}origin,
+        {build_id} AS build_id,
+        {build_misc} AS build_misc,
+        {test_prefix}path,
+        {test_prefix}status,
+        {test_prefix}duration,
+        {test_prefix}environment_compatible,
+        {test_prefix}environment_misc,
+        {test_prefix}origin,
+        {test_misc_runtime} AS test_lab,
+        {test_id} AS test_id,
+        ic.id AS incidents_id,
+        ic.test_id AS incidents_test_id,
+        i.id AS issues_id,
+        i.version AS issues_version"""
+
+    if include_boots and not include_tests:
+        test_filter = "AND (t.path IS NULL OR t.path LIKE 'boot%%')"
+    elif include_tests and not include_boots:
+        test_filter = "AND (t.path IS NULL OR t.path NOT LIKE 'boot%%')"
+    else:
+        test_filter = ""
+
+    if include_test_data:
+        test_join = f"LEFT JOIN tests AS t ON t.build_id = b.id {test_filter}"
+        incidents_condition = "t.id = ic.test_id OR b.id = ic.build_id"
+    else:
+        test_join = ""
+        incidents_condition = "b.id = ic.build_id"
+
+    join_clause = f"""LEFT JOIN builds AS b ON c.id = b.checkout_id
+        {test_join}
+        LEFT JOIN incidents AS ic ON {incidents_condition}
+        LEFT JOIN issues AS i ON ic.issue_id = i.id"""
+
     query = f"""
     WITH HEAD_START_TIME AS (
         SELECT
@@ -630,37 +692,10 @@ def get_tree_commit_history(
             c.start_time DESC
     )
     SELECT
-        c.git_commit_hash,
-        c.git_commit_name,
-        c.git_commit_tags,
-        c.start_time,
-        b.duration,
-        b.architecture,
-        b.compiler,
-        b.config_name,
-        b.status,
-        b.origin,
-        t.path,
-        t.status,
-        t.duration,
-        t.environment_compatible,
-        t.environment_misc,
-        t.origin,
-        t.misc->>'runtime' AS test_lab,
-        b.id AS build_id,
-        b.misc AS build_misc,
-        t.id AS test_id,
-        ic.id AS incidents_id,
-        ic.test_id AS incidents_test_id,
-        i.id AS issues_id,
-        i.version AS issues_version
+        {select_clause}
     FROM
         SELECTED_CHECKOUTS AS c
-        LEFT JOIN builds AS b ON c.id = b.checkout_id
-        LEFT JOIN tests AS t ON t.build_id = b.id
-        LEFT JOIN incidents AS ic ON t.id = ic.test_id
-        OR b.id = ic.build_id
-        LEFT JOIN issues AS i ON ic.issue_id = i.id
+        {join_clause}
     """
 
     with connection.cursor() as cursor:

backend/kernelCI_app/typeModels/treeCommits.py

@@ -1,6 +1,7 @@
 from datetime import datetime
+from enum import Enum
 from typing import List, Optional
-from pydantic import BaseModel, RootModel
+from pydantic import BaseModel, RootModel, field_validator
 
 from kernelCI_app.constants.general import DEFAULT_ORIGIN
 from kernelCI_app.constants.localization import DocStrings
@@ -14,6 +15,12 @@
 )
 
 
+class TreeEntityTypes(str, Enum):
+    BUILDS = "builds"
+    BOOTS = "boots"
+    TESTS = "tests"
+
+
 class DirectTreeCommitsQueryParameters(BaseModel):
     origin: str = Field(
         DEFAULT_ORIGIN, description=DocStrings.TREE_COMMIT_ORIGIN_DESCRIPTION
@@ -24,8 +31,21 @@ class DirectTreeCommitsQueryParameters(BaseModel):
     end_time_stamp_in_seconds: Optional[str] = Field(
         None, description=DocStrings.TREE_COMMIT_END_TS_DESCRIPTION
     )
+    types: Optional[list[TreeEntityTypes]] = Field(
+        None,
+        description="List of types to include (builds, boots, tests)",
+    )
     # TODO: Add filters field in this model
 
+    @field_validator("types", mode="before")
+    @classmethod
+    def validate_types(cls, value):
+        if not value:
+            return []
+        if isinstance(value, str):
+            value = [t.strip() for t in value.split(",") if t.strip()]
+        return value
+
 
 class TreeCommitsQueryParameters(DirectTreeCommitsQueryParameters):
     git_branch: Optional[str] = Field(

backend/kernelCI_app/views/treeCommitsHistory.py

@@ -68,7 +68,7 @@ def setup_filters(self):
     def sanitize_rows(self, rows: dict) -> list:
         result = []
         for row in rows:
-            build_misc = row[18]
+            build_misc = row[11]
             sanitized_build_misc = sanitize_dict(build_misc)
             build_lab = (
                 sanitized_build_misc.get("lab", UNKNOWN_STRING)
@@ -88,15 +88,15 @@ def sanitize_rows(self, rows: dict) -> list:
                     "config_name": row[7],
                     "build_status": NULL_STATUS if row[8] is None else row[8],
                     "build_origin": row[9],
-                    "test_path": row[10],
-                    "test_status": row[11],
-                    "test_duration": row[12],
-                    "hardware_compatibles": row[13],
-                    "test_environment_misc": row[14],
-                    "test_origin": row[15],
-                    "test_lab": row[16],
-                    "build_id": row[17],
+                    "build_id": row[10],
                     "build_misc": build_misc,
+                    "test_path": row[12],
+                    "test_status": row[13],
+                    "test_duration": row[14],
+                    "hardware_compatibles": row[15],
+                    "test_environment_misc": row[16],
+                    "test_origin": row[17],
+                    "test_lab": row[18],
                     "test_id": row[19],
                     "incidents_id": row[20],
                     "incidents_test_id": row[21],
@@ -403,6 +403,7 @@ def get(
                     "start_timestamp_in_seconds"
                 ),
                 end_timestamp_in_seconds=request.GET.get("end_timestamp_in_seconds"),
+                types=request.GET.get("types"),
             )
         except ValidationError as e:
             return create_api_error_response(
@@ -430,6 +431,7 @@ def get(
             git_url=params.git_url,
             git_branch=params.git_branch or git_branch,
             tree_name=tree_name,
+            include_types=params.types,
         )
 
         if not rows:

backend/schema.yml

@@ -1207,6 +1207,17 @@ paths:
           default: null
           title: Start Time Stamp In Seconds
         description: Start time filter in seconds for tree commits
+      - in: query
+        name: types
+        schema:
+          anyOf:
+          - items:
+              $ref: '#/components/schemas/TreeEntityTypes'
+            type: array
+          - type: 'null'
+          default: null
+          title: Types
+        description: List of types to include (builds, boots, tests)
       tags:
       - tree
       security:
@@ -1528,6 +1539,17 @@ paths:
           type: string
         description: Name of the tree
         required: true
+      - in: query
+        name: types
+        schema:
+          anyOf:
+          - items:
+              $ref: '#/components/schemas/TreeEntityTypes'
+            type: array
+          - type: 'null'
+          default: null
+          title: Types
+        description: List of types to include (builds, boots, tests)
       tags:
       - tree
       security:
@@ -3976,6 +3998,13 @@ components:
       - builds
       title: TreeDetailsFullResponse
       type: object
+    TreeEntityTypes:
+      enum:
+      - builds
+      - boots
+      - tests
+      title: TreeEntityTypes
+      type: string
     TreeLatestResponse:
       properties:
         git_repository_url:

dashboard/src/api/commitHistory.ts

@@ -11,7 +11,7 @@ import type {
 import { mapFiltersKeysToBackendCompatible } from '@/utils/utils';
 
 import { getTargetFilter } from '@/types/general';
-import type { TFilter } from '@/types/general';
+import type { TFilter, TreeEntityTypes } from '@/types/general';
 
 import { RequestData } from './commonRequest';
 
@@ -25,6 +25,7 @@ const fetchCommitHistory = async (
   endTimestampInSeconds: number | undefined,
   treeName?: string,
   treeUrlFrom?: TreeDetailsRouteFrom,
+  types?: TreeEntityTypes[],
 ): Promise<TTreeCommitHistoryResponse> => {
   const filtersFormatted = mapFiltersKeysToBackendCompatible(filters);
 
@@ -34,6 +35,7 @@ const fetchCommitHistory = async (
     git_branch: gitBranch,
     start_time_stamp_in_seconds: startTimestampInSeconds,
     end_time_stamp_in_seconds: endTimestampInSeconds,
+    types: types?.join(','),
     ...filtersFormatted,
   };
 
@@ -62,6 +64,7 @@ export const useCommitHistory = ({
   startTimestampInSeconds,
   treeName,
   treeUrlFrom,
+  types,
 }: {
   commitHash: string;
   origin: string;
@@ -72,6 +75,7 @@ export const useCommitHistory = ({
   endTimestampInSeconds?: number;
   treeName?: string;
   treeUrlFrom?: TreeDetailsRouteFrom;
+  types?: TreeEntityTypes[];
 }): UseQueryResult<TTreeCommitHistoryResponse> => {
   const testFilter = getTargetFilter(filter, 'test');
   const treeDetailsFilter = getTargetFilter(filter, 'treeDetails');
@@ -93,6 +97,7 @@ export const useCommitHistory = ({
       endTimestampInSeconds,
       treeName,
       treeUrlFrom,
+      types,
     ],
     queryFn: () =>
       fetchCommitHistory(
@@ -105,6 +110,7 @@ export const useCommitHistory = ({
         endTimestampInSeconds,
         treeName,
         treeUrlFrom,
+        types,
       ),
   });
 };

dashboard/src/components/CommitNavigationGraph/CommitNavigationGraph.tsx

@@ -13,7 +13,7 @@ import type { MessagesKey } from '@/locales/messages';
 import { formatDate } from '@/utils/utils';
 import { mapFilterToReq } from '@/components/Tabs/Filters';
 import { useCommitHistory } from '@/api/commitHistory';
-import type { TFilter } from '@/types/general';
+import type { TFilter, TreeEntityTypes } from '@/types/general';
 
 import { MemoizedSectionError } from '@/components/DetailsPages/SectionError';
 
@@ -74,6 +74,19 @@ const CommitNavigationGraph = ({
 
   const reqFilter = mapFilterToReq(diffFilter);
 
+  const types: TreeEntityTypes[] = useMemo(() => {
+    switch (currentPageTab) {
+      case 'global.builds':
+        return ['builds'];
+      case 'global.boots':
+        return ['boots'];
+      case 'global.tests':
+        return ['tests'];
+      default:
+        return ['builds'];
+    }
+  }, [currentPageTab]);
+
   const { data, status, error, isLoading } = useCommitHistory({
     gitBranch: gitBranch ?? '',
     gitUrl: gitUrl ?? '',
@@ -84,6 +97,7 @@ const CommitNavigationGraph = ({
     startTimestampInSeconds,
     treeName,
     treeUrlFrom,
+    types,
   });
 
   const displayableData = data ? data : null;

dashboard/src/types/general.ts

@@ -398,3 +398,5 @@ export type PossibleMonitorPath =
   | '/hardware/v1'
   | '/tree/v1'
   | '/tree/v2';
+
+export type TreeEntityTypes = 'builds' | 'boots' | 'tests';