Merge pull request #113 from gnikonorov/issue_101

Add --only-rerun flag to add option to rerun specific errors

Author: gnikonorov

CHANGES.rst

@@ -5,7 +5,7 @@ Changelog
 ----------------
 
 - Drop dependency on ``mock``.
-
+- Add in new flag ``--only-rerun`` to allow for users to rerun only certain errors.
 
 9.0 (2020-03-18)
 ----------------

README.rst

@@ -55,6 +55,24 @@ test re-run is launched:
 
    $ pytest --reruns 5 --reruns-delay 1
 
+Re-run all failures matching certain expressions
+------------------------------------------------
+
+To re-run only those failures that match a certain list of expressions, use the
+``--only-rerun`` flag and pass it a regular expression. For example, the following would
+only rerun those errors that match ``AssertionError``:
+
+.. code-block:: bash
+
+   $ pytest --reruns 5 --only-rerun AssertionError
+   
+Passing the flag multiple times accumulates the arguments, so the following would only rerun
+those errors that match ``AssertionError`` or ``ValueError``:
+
+.. code-block:: bash
+
+   $ pytest --reruns 5 --only-rerun AssertionError --only-rerun ValueError
+
 Re-run individual failures
 --------------------------
 

pytest_rerunfailures.py

@@ -1,4 +1,5 @@
 import pkg_resources
+import re
 import time
 import warnings
 
@@ -30,6 +31,14 @@ def pytest_addoption(parser):
     group = parser.getgroup(
         "rerunfailures",
         "re-run failing tests to eliminate flaky failures")
+    group._addoption(
+        '--only-rerun',
+        action='append',
+        dest='only_rerun',
+        type=str,
+        default=None,
+        help='If passed, only rerun errors matching the regex provided. Pass this flag multiple times to accumulate a list of regexes to match'
+    )
     group._addoption(
         '--reruns',
         action="store",
@@ -173,6 +182,21 @@ def _remove_failed_setup_state_from_session(item):
     setup_state.stack = list()
 
 
+def _should_hard_fail_on_error(session_config, report):
+    if report.outcome != 'failed':
+        return False
+
+    rerun_errors = session_config.option.only_rerun
+    if not rerun_errors:
+        return False
+
+    for rerun_regex in rerun_errors:
+        if re.search(rerun_regex, report.longrepr.reprcrash.message):
+            return False
+
+    return True
+
+
 def pytest_runtest_protocol(item, nextitem):
     """
     Note: when teardown fails, two reports are generated for the case, one for
@@ -200,9 +224,10 @@ def pytest_runtest_protocol(item, nextitem):
         reports = runtestprotocol(item, nextitem=nextitem, log=False)
 
         for report in reports:  # 3 reports: setup, call, teardown
+            is_terminal_error = _should_hard_fail_on_error(item.session.config, report)
             report.rerun = item.execution_count - 1
             xfail = hasattr(report, 'wasxfail')
-            if item.execution_count > reruns or not report.failed or xfail:
+            if item.execution_count > reruns or not report.failed or xfail or is_terminal_error:
                 # last run or no failure detected, log normally
                 item.ihook.pytest_runtest_logreport(report=report)
             else:

test_pytest_rerunfailures.py

@@ -407,3 +407,33 @@ def pytest_runtest_logfinish(nodeid, location):
     """.format(hook_message))
     result = testdir.runpytest('--reruns', '1', '-s')
     result.stdout.fnmatch_lines(hook_message)
+
+@pytest.mark.parametrize(
+        "file_text, only_rerun_texts, should_rerun",
+        [
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['AssertionError'], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['Assertion*'], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['Assertion'], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['ValueError'], False),
+            ('def test_only_rerun(): raise AssertionError("ERR")', [''], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['AssertionError: '], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['AssertionError: ERR'], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['ERR'], True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['AssertionError,ValueError'], False),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['AssertionError ValueError'], False),
+            ('def test_only_rerun(): raise AssertionError("ERR")', ['AssertionError', 'ValueError'], True),
+        ]
+)
+def test_only_rerun_flag(testdir, file_text, only_rerun_texts, should_rerun):
+    testdir.makepyfile(file_text)
+
+    num_failed = 1
+    num_passed = 0
+    num_reruns = 1
+    num_reruns_actual = num_reruns if should_rerun else 0
+
+    pytest_args = ['--reruns', str(num_reruns)]
+    for only_rerun_text in only_rerun_texts:
+        pytest_args.extend(['--only-rerun', only_rerun_text])
+    result = testdir.runpytest(*pytest_args)
+    assert_outcomes(result, passed=num_passed, failed=num_failed, rerun=num_reruns_actual)