add new flag --only-rerun to rerun only certain failures

Author: gnikonorov

pytest_rerunfailures.py

@@ -1,4 +1,5 @@
 import pkg_resources
+import re
 import time
 import warnings
 
@@ -30,6 +31,14 @@ def pytest_addoption(parser):
     group = parser.getgroup(
         "rerunfailures",
         "re-run failing tests to eliminate flaky failures")
+    group._addoption(
+        '--only-rerun',
+        action='store',
+        dest='only_rerun',
+        type=str,
+        default=None,
+        help='Optional comma seperated list. If passed, only rerun errors matching the regexes provided.'
+    )
     group._addoption(
         '--reruns',
         action="store",
@@ -173,6 +182,21 @@ def _remove_failed_setup_state_from_session(item):
     setup_state.stack = list()
 
 
+def _should_fail_on_error(session_config, report):
+    if report.outcome != 'failed':
+        return False
+
+    rerun_errors = session_config.option.only_rerun
+    if not rerun_errors:
+        return False
+
+    for rerun_regex in rerun_errors.split(','):
+        if re.search(rerun_regex, report.longrepr.reprcrash.message):
+            return False
+
+    return True
+
+
 def pytest_runtest_protocol(item, nextitem):
     """
     Note: when teardown fails, two reports are generated for the case, one for
@@ -200,9 +224,10 @@ def pytest_runtest_protocol(item, nextitem):
         reports = runtestprotocol(item, nextitem=nextitem, log=False)
 
         for report in reports:  # 3 reports: setup, call, teardown
+            is_terminal_error = _should_fail_on_error(item.session.config, report)
             report.rerun = item.execution_count - 1
             xfail = hasattr(report, 'wasxfail')
-            if item.execution_count > reruns or not report.failed or xfail:
+            if item.execution_count > reruns or not report.failed or xfail or is_terminal_error:
                 # last run or no failure detected, log normally
                 item.ihook.pytest_runtest_logreport(report=report)
             else:

test_pytest_rerunfailures.py

@@ -407,3 +407,29 @@ def pytest_runtest_logfinish(nodeid, location):
     """.format(hook_message))
     result = testdir.runpytest('--reruns', '1', '-s')
     result.stdout.fnmatch_lines(hook_message)
+
+@pytest.mark.parametrize(
+        "file_text, only_rerun_text, should_rerun",
+        [
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'AssertionError', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'Assertion*', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'Assertion', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'ValueError', False),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'AssertionError,ValueError', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'AssertionError ValueError', False),
+            ('def test_only_rerun(): raise AssertionError("ERR")', '', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'AssertionError: ', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'AssertionError: ERR', True),
+            ('def test_only_rerun(): raise AssertionError("ERR")', 'ERR', True),
+        ]
+)
+def test_pytest_only_rerun(testdir, file_text, only_rerun_text, should_rerun):
+    testdir.makepyfile(file_text)
+
+    num_failed = 1
+    num_passed = 0
+    num_reruns = 1
+    num_reruns_actual = num_reruns if should_rerun else 0
+
+    result = testdir.runpytest('--reruns', str(num_reruns), '--only-rerun', only_rerun_text)
+    assert_outcomes(result, passed=num_passed, failed=num_failed, rerun=num_reruns_actual)