unittest: Make implementation closer to CPython.

NOTE: This code is based on the CPython unittest implementation, and
probably falls under the Python License.

The main things I wanted to accomplish with this were:
- Ensure that each test runs in its own `TestCase` instance
  + This makes it possible to move some per-test setup code from `setUp`
    to `__init__`.
- Add `TestCase.run(test_result)`
  + In the CPython implementation, this method gives people a single
    place to add custom logic that can run _after_ the tests have
    completed, with access to the `unittest.TestResult` instance. This
    is useful, as it allows people to only output debug information on
    test failures.

While here I ended up making the following changes:
- Moved the test executing code into `TestCase` (and `_Outcome`), from
  `_run_suite`.
  + As part of this, I also migrated the exception handling logic to use
    `ucontextlib.contextmanager`
- Generate nicer error messages for failures in class setup/teardown
  methods.
- Removed `__test_result__` and `__current_test__` hidden variables
- Migrate `TestCase.subTest` to use `ucontextlib.contextmanager`
  + This significantly simplifies the logic
- Make subtest failures output show inline similar to CPython.

Bugs fixed:
- Allow `skip` decorator to work on bare functions
- Stop `expectedFailure` decorator from masking non-assertion failures
- Show correct test name when wrapped with `expectedFailure` decorator
- Show correct test name when TestCase.runTest method exists
- Exceptions in setUp/tearDown should show as test failures
  + Previously they went up the stack, likely causing the process to
    exit
- Exceptions in class setUp/tearDown should show as test failures
  + Previously they went up the stack, likely causing the process to
    exit
- Don't invoke properties with names that start with `test`
- Non-AssertionError exceptions in subtests show up as "FAIL" instead of
  "ERROR".
- Tests now execute in an explicit order
  + This makes it easier to write output tests for the unittest
    framework. We _may_ want to explicitly shuffle this order so that
    user's don't have cross-test dependencies.

Signed-off-by: Greg Darke <micropython@me.tsukasa.au>

Author: tsukasa-au

python-stdlib/unittest/manifest.py

@@ -1,3 +1,4 @@
 metadata(version="0.10.4")
 
+require("ucontextlib")
 package("unittest")

python-stdlib/unittest/tests/helpers.py

@@ -13,24 +13,19 @@ def _run_tests(suite: unittest.TestSuite) -> tuple[unittest.TestResult, str]:
         A tuple of (test_result, text_output)
     """
     stdout = io.StringIO()
-    tmp_stdout = unittest._stdout
-    tmp_current_test = unittest.__current_test__
-    tmp_test_result = unittest.__test_result__
-    try:
-        unittest._stdout = stdout
-        result = unittest.TestResult()
-        suite.run(result)
-        return result, stdout.getvalue()
-    finally:
-        unittest._stdout = tmp_stdout
-        unittest.__current_test__ = tmp_current_test
-        unittest.__test_result__ = tmp_test_result
+    result = unittest.TestResult(stream=stdout)
+    suite.run(result)
+    return result, stdout.getvalue()
 
 
 def run_tests_in_module(parent_test: unittest.TestCase, module) -> tuple[unittest.TestResult, str]:
-    test_name, parent_suite_name = unittest.__current_test__
-    parent_suite_name = f"{parent_suite_name[1:-1]}.{test_name}"
-    suite = unittest.TestSuite(name=parent_suite_name)
+    """Runs all tests in the given module-like object.
+
+    Args:
+        module: An object that can have its attributes listed with the `dir` function.
+    """
+    _, parent_suite_name = parent_test._test_name
+    suite = unittest.TestSuite(name=f"{parent_suite_name[1:-1]}.{parent_test._test_method_name}")
     suite._load_module(module)
     return _run_tests(suite)
 
@@ -64,8 +59,7 @@ def convert(cls, test_result: unittest.TestResult):
 
 class BaseTestCase(unittest.TestCase):
     def full_test_name(self):
-        my_name, cls_name = unittest.__current_test__
-        return f"{cls_name[1:-1]}.{my_name}"
+        return f"{self.__module__}.{self.__class__.__name__}.{self._test_method_name}"
 
     def assertTestResult(
         self,

python-stdlib/unittest/tests/test_basics.py

@@ -49,11 +49,8 @@ def test_func_fail():
         result, output = helpers.run_tests_in_module(self, FakeModule)
         self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=0, numSkipped=0)
         self.assertTrue(result.wasSuccessful())
-        # FIXME: This should be "test_func_fail", but the existing
-        # implementation pulls the wrong name
-        self.assertEqual(output, f"test_exp_fail ({self.full_test_name()}) ... ok\n")
+        self.assertEqual(output, f"test_func_fail ({self.full_test_name()}) ... ok\n")
 
-    @unittest.skip("expectedFailure incorrectly consumes all failure types")
     def test_bare_function__expect_failure__error(self):
         class FakeModule:
             @unittest.expectedFailure
@@ -65,7 +62,6 @@ def test_func_error():
         self.assertFalse(result.wasSuccessful())
         self.assertEqual(output, f"test_func_error ({self.full_test_name()}) ... ERROR\n")
 
-    @unittest.skip("expectedFailure incorrectly consumes the SkipTest exception")
     def test_bare_function__expect_failure__skip(self):
         class FakeModule:
             @unittest.expectedFailure
@@ -80,7 +76,6 @@ def test_func_error():
             output, f"test_func_error ({self.full_test_name()}) ... skipped: reason1\n"
         )
 
-    @unittest.skip("expectedFailure incorrectly consumes the SkipTest exception")
     def test_testcase__expect_failure__skip(self):
         class FakeModule:
             class Test(unittest.TestCase):
@@ -138,7 +133,6 @@ def test1(self):
         self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=1, numSkipped=0)
         self.assertEqual(output, f"test1 ({self.full_test_name()}.Test) ... ERROR\n")
 
-    @unittest.skip("unittest framework incorrectly calls `Test.test3`")
     def test_only_calls_methods(self):
         class FakeModule:
             class Test(unittest.TestCase):
@@ -169,10 +163,6 @@ def test1(self):
                 def runTest(self):
                     pass
 
-                def __repr__(self):
-                    # FIXME: Remove this method, it should not be needed
-                    return "runTest"
-
         result, output = helpers.run_tests_in_module(self, FakeModule)
         self.assertTestResult(result, testsRun=1, numFailures=0, numErrors=0, numSkipped=0)
         self.assertEqual(output, f"runTest ({self.full_test_name()}.Test) ... ok\n")
@@ -186,7 +176,6 @@ def test(self):
         with self.assertRaises(KeyboardInterrupt):
             helpers.run_tests_in_module(self, FakeModule)
 
-    @unittest.skip("unittest framework does not call `TestCase.run` method")
     def test_run_method_overridable(self):
         class FakeModule:
             class Test(unittest.TestCase):

python-stdlib/unittest/tests/test_setup.py

@@ -112,15 +112,14 @@ def test(self):
         result, output = helpers.run_tests_in_testcase(self, Test)
         self.assertTestResult(result, testsRun=2, numFailures=0, numErrors=0, numSkipped=0)
         self.assertTestMethodCountsEqual(
-            Test, init=1, setUpClass=1, setUp=2, test=2, tearDown=2, tearDownClass=1
+            Test, init=2, setUpClass=1, setUp=2, test=2, tearDown=2, tearDownClass=1
         )
         self.assertEqual(
             output,
             f"test ({self.full_test_name()}.Test) ... ok\n"
             f"test2 ({self.full_test_name()}.Test) ... ok\n",
         )
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_setup_failure_skips_test(self):
         class Test(self.__class__.TracingTestCase):
             def setUp(self):
@@ -134,7 +133,6 @@ def setUp(self):
         )
         self.assertEqual(output, f"test ({self.full_test_name()}.Test) ... FAIL\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_setup_errors_skips_test(self):
         class Test(self.__class__.TracingTestCase):
             def setUp(self):
@@ -148,7 +146,6 @@ def setUp(self):
         )
         self.assertEqual(output, f"test ({self.full_test_name()}.Test) ... ERROR\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_setup_skiptest_skips_test(self):
         class Test(self.__class__.TracingTestCase):
             def setUp(self):
@@ -162,7 +159,6 @@ def setUp(self):
         )
         self.assertEqual(output, f"test ({self.full_test_name()}.Test) ... skipped: reason1\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_class_setup_failure_skips_test(self):
         class Test(self.__class__.TracingTestCase):
             @classmethod
@@ -171,13 +167,14 @@ def setUpClass(cls):
                 raise AssertionError
 
         result, output = helpers.run_tests_in_testcase(self, Test)
-        self.assertTestResult(result, testsRun=0, numFailures=0, numErrors=1, numSkipped=0)
+        # NOTE: CPython counts this as an error, but this is neither documented,
+        # nor tested.
+        self.assertTestResult(result, testsRun=0, numFailures=1, numErrors=0, numSkipped=0)
         self.assertTestMethodCountsEqual(
             Test, init=1, setUpClass=1, setUp=0, test=0, tearDown=0, tearDownClass=0
         )
         self.assertEqual(output, f"setUpClass ({self.full_test_name()}.Test) ... FAIL\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_class_setup_errors_skips_test(self):
         class Test(self.__class__.TracingTestCase):
             @classmethod
@@ -192,7 +189,6 @@ def setUpClass(cls):
         )
         self.assertEqual(output, f"setUpClass ({self.full_test_name()}.Test) ... ERROR\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_class_setup_skiptest_skips_test(self):
         class Test(self.__class__.TracingTestCase):
             @classmethod
@@ -209,7 +205,6 @@ def setUpClass(cls):
             output, f"setUpClass ({self.full_test_name()}.Test) ... skipped: reason1\n"
         )
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_teardown_failure_is_failure(self):
         class Test(self.__class__.TracingTestCase):
             def tearDown(self):
@@ -223,7 +218,6 @@ def tearDown(self):
         )
         self.assertEqual(output, f"test ({self.full_test_name()}.Test) ... FAIL\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_teardown_errors_are_errors(self):
         class Test(self.__class__.TracingTestCase):
             def tearDown(self):
@@ -237,7 +231,6 @@ def tearDown(self):
         )
         self.assertEqual(output, f"test ({self.full_test_name()}.Test) ... ERROR\n")
 
-    @unittest.skip("Exceptions/Assertions in setUp are not caught")
     def test_teardown_skiptest_is_skip(self):
         class Test(self.__class__.TracingTestCase):
             def tearDown(self):
@@ -264,7 +257,7 @@ def test(self):
             Test.method_call_order,
             (
                 "__init__",
-                # NOTE: CPython calls __init__ here a second time
+                "__init__",
                 "setUpClass",
                 "setUp",
                 "test",

python-stdlib/unittest/tests/test_subtest.py

@@ -21,7 +21,10 @@ def test(self):
         result, output = helpers.run_tests_in_testcase(self, _Test)
         self.assertTestResult(result, testsRun=1, numFailures=2, numErrors=0, numSkipped=0)
         self.assertEqual(
-            output, "test (test_subtest.Test.test_subtest_catches_failures._Test) ... FAIL\n"
+            output,
+            "test (test_subtest.Test.test_subtest_catches_failures._Test) ... \n"
+            "  test (test_subtest.Test.test_subtest_catches_failures._Test) (inner=1) ... FAIL\n"
+            "  test (test_subtest.Test.test_subtest_catches_failures._Test) (inner=2) ... FAIL\n",
         )
 
     def test_subtest_catches_exceptions(self):
@@ -37,10 +40,11 @@ def test(self):
         self.assertEqual(result.testsRun, 1)
         self.assertEqual(len(result.failures), 0)
         self.assertEqual(len(result.errors), 2)
-        # FIXME: unittest framework incorrectly prints the test `FAIL`ed, rather
-        # than `ERROR`ed.
         self.assertEqual(
-            output, "test (test_subtest.Test.test_subtest_catches_exceptions._Test) ... FAIL\n"
+            output,
+            "test (test_subtest.Test.test_subtest_catches_exceptions._Test) ... \n"
+            "  test (test_subtest.Test.test_subtest_catches_exceptions._Test) (inner=1) ... ERROR\n"
+            "  test (test_subtest.Test.test_subtest_catches_exceptions._Test) (inner=2) ... ERROR\n",
         )