fix: Improve performance on TestStatusHistory endpoint (#1863)

* Moving to a raw SQL query, instead of ORM.
  * Force the query to use a hash based comparison, instead of a slower
    nested loop

    Closes #1131

Author: alanpeixinho

backend/kernelCI_app/models.py

@@ -54,6 +54,7 @@ class Meta:
 
 
 class Checkouts(models.Model):
+    # time this entry was ingested
     field_timestamp = models.DateTimeField(
         db_column="_timestamp", blank=True, null=True
     )
@@ -68,6 +69,7 @@ class Checkouts(models.Model):
     patchset_hash = models.TextField(blank=True, null=True)
     message_id = models.TextField(blank=True, null=True)
     comment = models.TextField(blank=True, null=True)
+    # time reported by the submission
     start_time = models.DateTimeField(blank=True, null=True)
     log_url = models.TextField(blank=True, null=True)
     log_excerpt = models.CharField(max_length=16384, blank=True, null=True)

backend/kernelCI_app/queries/test.py

@@ -1,9 +1,9 @@
 from typing import Optional
 
-from django.db import connection
+from django.db import connection, transaction
 
+from kernelCI_app.cache import get_query_cache, set_query_cache
 from kernelCI_app.helpers.database import dict_fetchall
-from kernelCI_app.models import Tests
 from kernelCI_app.typeModels.databases import (
     Origin,
     Test__StartTime,
@@ -64,33 +64,66 @@ def get_test_status_history(
     field_timestamp: Timestamp,
     group_size: int,
 ):
-    query = Tests.objects.filter(
-        path=path,
-        build__checkout__origin=origin,
-        build__checkout__git_repository_url=git_repository_url,
-        build__checkout__git_repository_branch=git_repository_branch,
-        build__config_name=config_name,
-    ).values(
-        "start_time",
-        "id",
-        "status",
-        "build__checkout__git_commit_hash",
-    )
+    cache_key = "TestStatusHistory"
+    params = {
+        "path": path,
+        "origin": origin,
+        "git_repository_url": git_repository_url,
+        "git_repository_branch": git_repository_branch,
+        "config_name": config_name,
+        "group_size": group_size,
+        "field_timestamp": field_timestamp,
+        "platform": platform,
+        "test_start_time": test_start_time,
+    }
+
+    if rows := get_query_cache(key=cache_key, params=params):
+        return rows
 
     if platform is None:
-        query = query.filter(environment_misc__platform__isnull=True)
+        platform_clause = "AND T.ENVIRONMENT_MISC ->> 'platform' IS NULL"
     else:
-        query = query.filter(environment_misc__platform=platform)
+        platform_clause = "AND T.ENVIRONMENT_MISC ->> 'platform' = %(platform)s"
 
     if test_start_time is None:
         if field_timestamp is None:
-            query = query.filter(
-                start_time__isnull=True,
-                field_timestamp__isnull=True,
-            )
+            time_clause = "AND T.START_TIME IS NULL AND T._TIMESTAMP IS NULL"
+            order_clause = "ORDER BY T._TIMESTAMP DESC"
         else:
-            query = query.filter(field_timestamp__lte=field_timestamp)
-        return query.order_by("-field_timestamp")[:group_size]
+            time_clause = "AND T._TIMESTAMP <= %(field_timestamp)s"
+            order_clause = "ORDER BY T._TIMESTAMP DESC"
     else:
-        query = query.filter(start_time__lte=test_start_time)
-        return query.order_by("-start_time")[:group_size]
+        time_clause = "AND T.START_TIME <= %(test_start_time)s"
+        order_clause = "ORDER BY T.START_TIME DESC"
+
+    query = f"""
+        SELECT
+            T.START_TIME,
+            T.ID,
+            COALESCE(T.STATUS, 'NULL') AS status,
+            C.GIT_COMMIT_HASH AS build__checkout__git_commit_hash
+        FROM
+            TESTS T
+        INNER JOIN BUILDS B ON T.BUILD_ID = B.ID
+        INNER JOIN CHECKOUTS C ON B.CHECKOUT_ID = C.ID
+        WHERE
+            T.PATH = %(path)s
+            AND C.ORIGIN = %(origin)s
+            AND C.GIT_REPOSITORY_URL = %(git_repository_url)s
+            AND C.GIT_REPOSITORY_BRANCH = %(git_repository_branch)s
+            AND B.CONFIG_NAME = %(config_name)s
+            {platform_clause}
+            {time_clause}
+        {order_clause}
+        LIMIT %(group_size)s;
+    """
+
+    with transaction.atomic():
+        with connection.cursor() as cursor:
+            # do not let postgres planner use a slow nested loop in the query
+            # a hashed approach should always be faster here
+            cursor.execute("SET LOCAL enable_nestloop = off")
+            cursor.execute(query, params)
+            rows = dict_fetchall(cursor)
+            set_query_cache(key=cache_key, params=params, rows=rows)
+            return rows

backend/kernelCI_app/tests/integrationTests/testStatusHistory_test.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta, timezone
 from http import HTTPStatus
 
 import pytest
@@ -47,6 +48,22 @@
             HTTPStatus.BAD_REQUEST,
             True,
         ),
+        (
+            TestStatusHistoryRequest(
+                path="fluster.debian.v4l2.gstreamer_av1.validate-fluster-results",
+                origin="maestro",
+                git_repository_url="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git",
+                git_repository_branch="master",
+                platform="mt8195-cherry-tomato-r2",
+                field_timestamp=datetime.now(timezone.utc)
+                + timedelta(days=365),  # 1 year ahead
+                current_test_start_time=None,
+                config_name="defconfig+lab-setup+arm64-chromebook"
+                "+CONFIG_MODULE_COMPRESS=n+CONFIG_MODULE_COMPRESS_NONE=y",
+            ),
+            HTTPStatus.OK,
+            False,
+        ),
     ],
 )
 def test_get(

backend/kernelCI_app/tests/unitTests/queries/conftest.py

@@ -19,18 +19,6 @@ def setup_mock_queryset(mock_model, return_value):
     return mock_queryset
 
 
-def setup_mock_test_queryset(mock_tests_model):
-    mock_queryset = Mock()
-    mock_queryset.values.return_value = mock_queryset
-    mock_queryset.filter.return_value = mock_queryset
-    mock_queryset.order_by.return_value = mock_queryset
-    mock_sliced = Mock()
-    mock_sliced.__iter__ = Mock(return_value=iter([]))
-    mock_queryset.__getitem__ = Mock(return_value=mock_sliced)
-    mock_tests_model.objects.filter.return_value = mock_queryset
-    return mock_queryset
-
-
 TEST_TREE = Tree(
     index="0",
     tree_name="mainline",

backend/kernelCI_app/tests/unitTests/queries/test_queries_test.py

@@ -1,9 +1,8 @@
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 from kernelCI_app.queries.test import get_test_details_data, get_test_status_history
 from kernelCI_app.tests.unitTests.queries.conftest import (
     setup_mock_cursor,
-    setup_mock_test_queryset,
 )
 
 
@@ -34,9 +33,25 @@ def test_get_test_details_data_empty_result(
 
 
 class TestGetTestStatusHistory:
-    @patch("kernelCI_app.queries.test.Tests")
-    def test_get_test_status_history_with_platform(self, mock_tests_model):
-        mock_queryset = setup_mock_test_queryset(mock_tests_model)
+    @patch("kernelCI_app.queries.test.transaction.atomic")
+    @patch("kernelCI_app.queries.test.set_query_cache")
+    @patch("kernelCI_app.queries.test.get_query_cache")
+    @patch("kernelCI_app.queries.test.dict_fetchall")
+    @patch("kernelCI_app.queries.test.connection")
+    def test_get_test_status_history_with_platform(
+        self,
+        mock_connection,
+        mock_dict_fetchall,
+        mock_get_cache,
+        mock_set_cache,
+        mock_transaction,
+    ):
+        mock_transaction.return_value.__enter__ = Mock(return_value=None)
+        mock_transaction.return_value.__exit__ = Mock(return_value=None)
+        expected_result = [{"id": "test", "status": "PASS"}]
+        mock_dict_fetchall.return_value = expected_result
+        mock_get_cache.return_value = None
+        mock_cursor = setup_mock_cursor(mock_connection)
 
         result = get_test_status_history(
             path="boot",
@@ -50,12 +65,34 @@ def test_get_test_status_history_with_platform(self, mock_tests_model):
             group_size=10,
         )
 
-        assert list(result) == []
-        mock_queryset.filter.assert_called()
-
-    @patch("kernelCI_app.queries.test.Tests")
-    def test_get_test_status_history_without_platform(self, mock_tests_model):
-        setup_mock_test_queryset(mock_tests_model)
+        assert result == expected_result
+        assert mock_cursor.execute.call_count > 1
+        query_call = mock_cursor.execute.call_args_list[1]
+        assert "platform" in query_call[0][1]
+        assert query_call[0][1]["platform"] == "x86_64"
+        mock_get_cache.assert_called_once()
+        mock_set_cache.assert_called_once()
+        mock_transaction.assert_called_once()
+
+    @patch("kernelCI_app.queries.test.transaction.atomic")
+    @patch("kernelCI_app.queries.test.set_query_cache")
+    @patch("kernelCI_app.queries.test.get_query_cache")
+    @patch("kernelCI_app.queries.test.dict_fetchall")
+    @patch("kernelCI_app.queries.test.connection")
+    def test_get_test_status_history_without_platform(
+        self,
+        mock_connection,
+        mock_dict_fetchall,
+        mock_get_cache,
+        mock_set_cache,
+        mock_transaction,
+    ):
+        mock_transaction.return_value.__enter__ = Mock(return_value=None)
+        mock_transaction.return_value.__exit__ = Mock(return_value=None)
+        expected_result = []
+        mock_dict_fetchall.return_value = expected_result
+        mock_get_cache.return_value = None
+        mock_cursor = setup_mock_cursor(mock_connection)
 
         result = get_test_status_history(
             path="boot",
@@ -69,13 +106,32 @@ def test_get_test_status_history_without_platform(self, mock_tests_model):
             group_size=10,
         )
 
-        assert list(result) == []
-
-    @patch("kernelCI_app.queries.test.Tests")
+        assert result == expected_result
+        query_call = mock_cursor.execute.call_args_list[1]
+        assert "field_timestamp" in query_call[0][1]
+        assert query_call[0][1]["field_timestamp"] == "2025-11-11T10:00:00Z"
+        mock_get_cache.assert_called_once()
+        mock_set_cache.assert_called_once()
+        mock_transaction.assert_called_once()
+
+    @patch("kernelCI_app.queries.test.transaction.atomic")
+    @patch("kernelCI_app.queries.test.set_query_cache")
+    @patch("kernelCI_app.queries.test.get_query_cache")
+    @patch("kernelCI_app.queries.test.dict_fetchall")
+    @patch("kernelCI_app.queries.test.connection")
     def test_get_test_status_history_uses_start_time_when_provided(
-        self, mock_tests_model
+        self,
+        mock_connection,
+        mock_dict_fetchall,
+        mock_get_cache,
+        mock_set_cache,
+        mock_transaction,
     ):
-        mock_queryset = setup_mock_test_queryset(mock_tests_model)
+        mock_transaction.return_value.__enter__ = Mock(return_value=None)
+        mock_transaction.return_value.__exit__ = Mock(return_value=None)
+        mock_dict_fetchall.return_value = []
+        mock_get_cache.return_value = None
+        mock_cursor = setup_mock_cursor(mock_connection)
 
         get_test_status_history(
             path="boot",
@@ -89,11 +145,32 @@ def test_get_test_status_history_uses_start_time_when_provided(
             group_size=10,
         )
 
-        mock_queryset.order_by.assert_called_with("-start_time")
+        query_call = mock_cursor.execute.call_args_list[1]
+        assert "test_start_time" in query_call[0][1]
+        assert query_call[0][1]["test_start_time"] == "2025-11-10T10:00:00Z"
+        mock_get_cache.assert_called_once()
+        mock_set_cache.assert_called_once()
+        mock_transaction.assert_called_once()
 
-    @patch("kernelCI_app.queries.test.Tests")
-    def test_get_test_status_history_with_null_timestamps(self, mock_tests_model):
-        mock_queryset = setup_mock_test_queryset(mock_tests_model)
+    @patch("kernelCI_app.queries.test.transaction.atomic")
+    @patch("kernelCI_app.queries.test.set_query_cache")
+    @patch("kernelCI_app.queries.test.get_query_cache")
+    @patch("kernelCI_app.queries.test.dict_fetchall")
+    @patch("kernelCI_app.queries.test.connection")
+    def test_get_test_status_history_with_null_timestamps(
+        self,
+        mock_connection,
+        mock_dict_fetchall,
+        mock_get_cache,
+        mock_set_cache,
+        mock_transaction,
+    ):
+        mock_transaction.return_value.__enter__ = Mock(return_value=None)
+        mock_transaction.return_value.__exit__ = Mock(return_value=None)
+        expected_result = []
+        mock_dict_fetchall.return_value = expected_result
+        mock_get_cache.return_value = None
+        mock_cursor = setup_mock_cursor(mock_connection)
 
         result = get_test_status_history(
             path="boot",
@@ -107,7 +184,40 @@ def test_get_test_status_history_with_null_timestamps(self, mock_tests_model):
             group_size=10,
         )
 
-        assert list(result) == []
-        filter_calls = [str(call) for call in mock_queryset.filter.call_args_list]
-        assert any("start_time__isnull" in call for call in filter_calls)
-        assert any("field_timestamp__isnull" in call for call in filter_calls)
+        assert result == expected_result
+        query_call = mock_cursor.execute.call_args_list[1]
+        sql = query_call[0][0]
+        assert "T.START_TIME IS NULL" in sql
+        assert "T._TIMESTAMP IS NULL" in sql
+        mock_get_cache.assert_called_once()
+        mock_set_cache.assert_called_once()
+        mock_transaction.assert_called_once()
+
+    @patch("kernelCI_app.queries.test.set_query_cache")
+    @patch("kernelCI_app.queries.test.get_query_cache")
+    @patch("kernelCI_app.queries.test.dict_fetchall")
+    def test_get_test_status_history_returns_cached_result(
+        self,
+        mock_dict_fetchall,
+        mock_get_cache,
+        mock_set_cache,
+    ):
+        cached_result = [{"id": "cached", "status": "PASS"}]
+        mock_get_cache.return_value = cached_result
+
+        result = get_test_status_history(
+            path="boot",
+            origin="maestro",
+            git_repository_url="https://my_url.com",
+            git_repository_branch="master",
+            platform="x86_64",
+            test_start_time="2025-11-11T10:00:00Z",
+            config_name="defconfig",
+            field_timestamp=None,
+            group_size=10,
+        )
+
+        assert result == cached_result
+        mock_dict_fetchall.assert_not_called()
+        mock_set_cache.assert_not_called()
+        mock_get_cache.assert_called_once()

backend/kernelCI_app/typeModels/testDetails.py

@@ -29,7 +29,6 @@
     Test__OutputFiles,
     Test__Path,
     Test__StartTime,
-    Test__Status,
     Timestamp,
 )
 from kernelCI_app.utils import validate_str_to_dict
@@ -72,7 +71,7 @@ class TestDetailsResponse(BaseModel):
 class TestStatusHistoryItem(BaseModel):
     start_time: Test__StartTime
     id: Test__Id
-    status: Test__Status
+    status: StatusValues
     git_commit_hash: Checkout__GitCommitHash = Field(
         validation_alias="build__checkout__git_commit_hash"
     )