unittest: Add more tests, including test output.

Make it possible to run the unittest from within itself. This allows us
to write tests that assert:
- That the TestSuite had the correct outcomes (failed, errored, skipped,
  etc)
- That the test output the correct information to the user (test names,
  test outcomes, etc)

With this change, we no longer need to rely on raising assertions that
are not subclasses of `AssertionError` to show unexpected behaviour in
tests.

This commit makes a (small) change to how tests for the unittest
framework are written. It is now possible to run the entire test
framework within itself, allowing us to have assertions against the
TestResults and the generated text output.

To write assertions against the output TestSuite, we need a way to
redirect the output of `print`. This commit adds the `_stdout` variable
to the unittest module to allow us to redirect the output. While this is
hacky, this variable will go away in a later commit in the series.

NOTE: Some of the added tests either had to be skipped or have the
incorrect result encoded in them as the existing framework does not
behave as expected. These will be fixed in latter commits in the series.

Signed-off-by: Greg Darke <micropython@me.tsukasa.au>

Author: tsukasa-au

python-stdlib/unittest/tests/helpers.py

@@ -0,0 +1,90 @@
+import io
+import unittest
+import collections
+
+
+def _run_tests(suite: unittest.TestSuite) -> tuple[unittest.TestResult, str]:
+    """Runs a TestSuite, capturing its output.
+
+    Args:
+        suite: The TestSuite to run
+
+    Returns:
+        A tuple of (test_result, text_output)
+    """
+    stdout = io.StringIO()
+    tmp_stdout = unittest._stdout
+    tmp_current_test = unittest.__current_test__
+    tmp_test_result = unittest.__test_result__
+    try:
+        unittest._stdout = stdout
+        result = unittest.TestResult()
+        suite.run(result)
+        return result, stdout.getvalue()
+    finally:
+        unittest._stdout = tmp_stdout
+        unittest.__current_test__ = tmp_current_test
+        unittest.__test_result__ = tmp_test_result
+
+
+def run_tests_in_module(parent_test: unittest.TestCase, module) -> tuple[unittest.TestResult, str]:
+    test_name, parent_suite_name = unittest.__current_test__
+    parent_suite_name = f"{parent_suite_name[1:-1]}.{test_name}"
+    suite = unittest.TestSuite(name=parent_suite_name)
+    suite._load_module(module)
+    return _run_tests(suite)
+
+
+def run_tests_in_testcase(
+    parent_test: unittest.TestCase, *testcase_classes: type[unittest.TestCase]
+) -> tuple[unittest.TestResult, str]:
+    """Runs tests in the given TestCase classes."""
+
+    class _FakeModule: ...
+
+    for tc in testcase_classes:
+        setattr(_FakeModule, tc.__name__, tc)
+    return run_tests_in_module(parent_test, _FakeModule)
+
+
+class _TestResultSummary(
+    collections.namedtuple(
+        "TestResultSummary", ("testsRun", "numFailures", "numErrors", "numSkipped")
+    )
+):
+    @classmethod
+    def convert(cls, test_result: unittest.TestResult):
+        return cls(
+            test_result.testsRun,
+            len(test_result.failures),
+            len(test_result.errors),
+            len(test_result.skipped),
+        )
+
+
+class BaseTestCase(unittest.TestCase):
+    def full_test_name(self):
+        my_name, cls_name = unittest.__current_test__
+        return f"{cls_name[1:-1]}.{my_name}"
+
+    def assertTestResult(
+        self,
+        result: unittest.TestResult,
+        *,
+        testsRun: int,
+        numFailures: int,
+        numErrors: int,
+        numSkipped: int,
+    ):
+        expected = _TestResultSummary(testsRun, numFailures, numErrors, numSkipped)
+        actual = _TestResultSummary.convert(result)
+        if expected == actual:
+            return
+        err_parts = [f"{actual} != (expected) {expected}"]
+        if actual.numFailures != expected.numFailures:
+            err_parts.append(f"failures={result.failures!r}")
+        if actual.numErrors != expected.numErrors:
+            err_parts.append(f"errors={result.errors!r}")
+        if actual.numSkipped != expected.numSkipped:
+            err_parts.append(f"skipped={result.skipped!r}")
+        raise AssertionError("\n".join(err_parts))

python-stdlib/unittest/tests/test_basics.py

@@ -1,28 +1,215 @@
 import unittest
+import helpers
 
 
-class TestWithRunTest(unittest.TestCase):
-    run = False
+class Basics(helpers.BaseTestCase):
+    def test_bare_function__passes(self):
+        class FakeModule:
+            @staticmethod
+            def test_func1(): ...
+            @staticmethod
+            def test_func2(): ...
 
-    def runTest(self):
-        TestWithRunTest.run = True
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=2, numFailures=0, numErrors=0, numSkipped=0)
+        self.assertTrue(result.wasSuccessful())
+        self.assertEqual(
+            output,
+            f"test_func1 ({self.full_test_name()}) ... ok\ntest_func2 ({self.full_test_name()}) ... ok\n",
+        )
 
-    def testRunTest(self):
-        self.fail()
+    def test_bare_function__fail(self):
+        class FakeModule:
+            @staticmethod
+            def test_func_fail():
+                assert False
 
-    @staticmethod
-    def tearDownClass():
-        if not TestWithRunTest.run:
-            raise ValueError()
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=1, numErrors=0, numSkipped=0)
+        self.assertFalse(result.wasSuccessful())
+        self.assertEqual(output, f"test_func_fail ({self.full_test_name()}) ... FAIL\n")
 
+    def test_bare_function__error(self):
+        class FakeModule:
+            @staticmethod
+            def test_func_error():
+                raise ValueError
 
-def test_func():
-    pass
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=1, numSkipped=0)
+        self.assertFalse(result.wasSuccessful())
+        self.assertEqual(output, f"test_func_error ({self.full_test_name()}) ... ERROR\n")
 
+    def test_bare_function__expect_failure__fail(self):
+        class FakeModule:
+            @unittest.expectedFailure
+            def test_func_fail():
+                assert False
 
-@unittest.expectedFailure
-def test_foo():
-    raise ValueError()
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=0, numSkipped=0)
+        self.assertTrue(result.wasSuccessful())
+        # FIXME: This should be "test_func_fail", but the existing
+        # implementation pulls the wrong name
+        self.assertEqual(output, f"test_exp_fail ({self.full_test_name()}) ... ok\n")
+
+    @unittest.skip("expectedFailure incorrectly consumes all failure types")
+    def test_bare_function__expect_failure__error(self):
+        class FakeModule:
+            @unittest.expectedFailure
+            def test_func_error():
+                raise ValueError
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=1, numSkipped=0)
+        self.assertFalse(result.wasSuccessful())
+        self.assertEqual(output, f"test_func_error ({self.full_test_name()}) ... ERROR\n")
+
+    @unittest.skip("expectedFailure incorrectly consumes the SkipTest exception")
+    def test_bare_function__expect_failure__skip(self):
+        class FakeModule:
+            @unittest.expectedFailure
+            @unittest.skip("reason1")
+            def test_func_error():
+                raise ValueError
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=0, numSkipped=1)
+        self.assertTrue(result.wasSuccessful())
+        self.assertEqual(
+            output, f"test_func_error ({self.full_test_name()}) ... skipped: reason1\n"
+        )
+
+    @unittest.skip("expectedFailure incorrectly consumes the SkipTest exception")
+    def test_testcase__expect_failure__skip(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                @unittest.expectedFailure
+                def test_func_skip_in_test(self):
+                    self.skipTest("reason1")
+
+                @unittest.expectedFailure
+                @unittest.skip("reason2")
+                def test_func_skip_wrap_test(self):
+                    pass
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=2, numFailures=0, numErrors=0, numSkipped=2)
+        self.assertTrue(result.wasSuccessful())
+        self.assertEqual(
+            output,
+            f"test_func_skip_in_test ({self.full_test_name()}.Test) ... skipped: reason1\n"
+            f"test_func_skip_wrap_test ({self.full_test_name()}.Test) ... skipped: reason2\n",
+        )
+
+
+class TestTestCase(helpers.BaseTestCase):
+    def test_method_called__passes(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def test1(self): ...
+                def test2(self): ...
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=2, numFailures=0, numErrors=0, numSkipped=0)
+        self.assertEqual(
+            output,
+            f"test1 ({self.full_test_name()}.Test) ... ok\n"
+            f"test2 ({self.full_test_name()}.Test) ... ok\n",
+        )
+
+    def test_method_called__fail(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def test1(self):
+                    self.fail("reason1")
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=1, numErrors=0, numSkipped=0)
+        self.assertEqual(output, f"test1 ({self.full_test_name()}.Test) ... FAIL\n")
+
+    def test_method_called__error(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def test1(self):
+                    raise ValueError("reason1")
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=1, numSkipped=0)
+        self.assertEqual(output, f"test1 ({self.full_test_name()}.Test) ... ERROR\n")
+
+    @unittest.skip("unittest framework incorrectly calls `Test.test3`")
+    def test_only_calls_methods(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def test1(self): ...
+
+                test_copy = test1
+
+                test2 = None
+
+                @property
+                def test3(self):
+                    raise ValueError
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=2, numFailures=0, numErrors=0, numSkipped=0)
+        self.assertEqual(
+            output,
+            f"test1 ({self.full_test_name()}.Test) ... ok\n"
+            f"test_copy ({self.full_test_name()}.Test) ... ok\n",
+        )
+
+    def test_prefers_runTest(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def test1(self):
+                    self.fail("wrong method called")
+
+                def runTest(self):
+                    pass
+
+                def __repr__(self):
+                    # FIXME: Remove this method, it should not be needed
+                    return "runTest"
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=0, numSkipped=0)
+        self.assertEqual(output, f"runTest ({self.full_test_name()}.Test) ... ok\n")
+
+    def test_keyboard_interrupt_not_captured(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def test(self):
+                    raise KeyboardInterrupt
+
+        with self.assertRaises(KeyboardInterrupt):
+            helpers.run_tests_in_module(self, FakeModule)
+
+    @unittest.skip("unittest framework does not call `TestCase.run` method")
+    def test_run_method_overridable(self):
+        class FakeModule:
+            class Test(unittest.TestCase):
+                def run(self, result: unittest.TestResult | None = None):
+                    if result is None:
+                        result = self.defaultTestResult()
+
+                    tmp_result = unittest.TestResult(stream=result._stream)
+                    super().run(tmp_result)
+                    if tmp_result.failures:
+                        test, err_msg = tmp_result.failures[-1]
+                        err_msg += "\nSome extra debugging info"
+                        tmp_result.failures[-1] = test, err_msg
+                    result += tmp_result
+                    return result
+
+                def test1(self):
+                    self.fail("reason1")
+
+        result, output = helpers.run_tests_in_module(self, FakeModule)
+        self.assertTestResult(result, testsRun=1, numFailures=1, numErrors=0, numSkipped=0)
+        self.assertEqual(output, f"test1 ({self.full_test_name()}.Test) ... FAIL\n")
+        self.assertIn("\nSome extra debugging info", result.failures[-1][1])
 
 
 if __name__ == "__main__":