kselftest: report WARNING status on nonfatal failures

When a nonfatal target has command failures, report WARNING
instead of PASS. This makes partial build failures visible
to CI without failing the build.

Suggested-by: Ben Copeland <ben.copeland@linaro.org>
Signed-off-by: Anders Roxell <anders.roxell@linaro.org>

Author: roxell

test/test_build.py

@@ -1103,6 +1103,37 @@ def test_nonfatal_continues_on_failure(self, linux, mocker):
         mocker.patch.object(b, "check_artifacts", return_value=True)
         result = b.build(target)
         assert result.passed
+        assert result.warning
+        assert not result.failed
+
+    def test_warning_does_not_block_dependent_targets(self, linux, mocker):
+        b = Build(tree=linux, targets=["config"])
+        b.status["kselftest"] = BuildInfo("WARNING")
+        target = mocker.MagicMock()
+        target.name = "next"
+        target.nonfatal = False
+        target.dependencies = ["kselftest"]
+        target.preconditions = []
+        target.commands = [Command(["true"])]
+        mocker.patch.object(b, "run_cmd", return_value=True)
+        mocker.patch.object(b, "check_artifacts", return_value=True)
+        result = b.build(target)
+        assert result.passed
+        assert not result.skipped
+
+    def test_warning_target_build_not_failed(self, linux):
+        b = Build(tree=linux, targets=["config"])
+        b.status["config"] = BuildInfo("PASS")
+        b.status["kselftest"] = BuildInfo("WARNING")
+        assert not b.failed
+        assert b.passed
+
+    def test_fail_overrides_warning(self, linux):
+        b = Build(tree=linux, targets=["config"])
+        b.status["config"] = BuildInfo("FAIL")
+        b.status["kselftest"] = BuildInfo("WARNING")
+        assert b.failed
+        assert not b.passed
 
 
 class TestHeaders:

tuxmake/build.py

@@ -79,7 +79,7 @@ def passed(self):
         """
         `True` if this target passed.
         """
-        return self.status == "PASS"
+        return self.status in ("PASS", "WARNING")
 
     @property
     def skipped(self):
@@ -88,6 +88,13 @@ def skipped(self):
         """
         return self.status == "SKIP"
 
+    @property
+    def warning(self):
+        """
+        `True` if this target passed with warnings.
+        """
+        return self.status == "WARNING"
+
 
 class Build:
     """
@@ -591,6 +598,7 @@ def build(self, target):
         target.prepare()
 
         fail = False
+        had_nonfatal_failure = False
         exclude_keys = getattr(target, "exclude_build_makevars", set())
         for cmd in target.commands:
             if not self.run_cmd(
@@ -601,6 +609,7 @@ def build(self, target):
             ):
                 if target.nonfatal:
                     self.log("W: command failed, continuing (nonfatal target)")
+                    had_nonfatal_failure = True
                 else:
                     fail = True
                     break
@@ -611,6 +620,9 @@ def build(self, target):
         if fail:
             return BuildInfo("FAIL")
 
+        if had_nonfatal_failure:
+            return BuildInfo("WARNING")
+
         return BuildInfo("PASS")
 
     def check_artifacts(self, target):
@@ -667,7 +679,15 @@ def collect_metadata(self):
         }
         errors, warnings = self.parse_log()
         self.metadata["results"] = {
-            "status": "PASS" if self.passed else "FAIL",
+            "status": (
+                "FAIL"
+                if self.failed
+                else (
+                    "WARNING"
+                    if any(info.warning for info in self.status.values())
+                    else "PASS"
+                )
+            ),
             "targets": {
                 name: {"status": s.status, "duration": s.duration}
                 for name, s in self.status.items()