Add only_rerun and rerun_except kwargs to marker (#221)

This allows the command line arguments with the same names to be
overridden on a per-test basis.  The functionality is the same -
each argument takes a regex string or list of regex strings.

Author: cfulljames

CHANGES.rst

@@ -4,7 +4,10 @@ Changelog
 11.2 (unreleased)
 -----------------
 
-- Nothing changed yet.
+Features
+++++++++
+
+- Add ``only_rerun`` and ``rerun_except`` arguments to ``@pytest.mark.flaky`` marker.
 
 
 11.1.2 (2023-03-09)

README.rst

@@ -142,6 +142,27 @@ You can also specify an optional ``condition`` in the re-run marker:
       import random
       assert random.choice([True, False])
 
+Exception filtering can be accomplished by specifying regular expressions for
+``only_rerun`` and ``rerun_except``. They override the ``--only-rerun`` and
+``--rerun-except`` command line arguments, respectively.
+
+Arguments can be a single string:
+
+.. code-block:: python
+
+   @pytest.mark.flaky(rerun_except="AssertionError")
+   def test_example():
+       raise AssertionError()
+
+Or a list of strings:
+
+.. code-block:: python
+
+   @pytest.mark.flaky(only_rerun=["AssertionError", "ValueError"])
+   def test_example():
+       raise AssertionError()
+
+
 You can use ``@pytest.mark.flaky(condition)`` similarly as ``@pytest.mark.skipif(condition)``, see `pytest-mark-skipif <https://docs.pytest.org/en/6.2.x/reference.html#pytest-mark-skipif>`_
 
 .. code-block:: python

pytest_rerunfailures.py

@@ -302,12 +302,25 @@ def _remove_failed_setup_state_from_session(item):
         setup_state.stack = []
 
 
-def _should_hard_fail_on_error(session_config, report):
+def _get_rerun_filter_regex(item, regex_name):
+    rerun_marker = _get_marker(item)
+
+    if rerun_marker is not None and regex_name in rerun_marker.kwargs:
+        regex = rerun_marker.kwargs[regex_name]
+        if isinstance(regex, str):
+            regex = [regex]
+    else:
+        regex = getattr(item.session.config.option, regex_name)
+
+    return regex
+
+
+def _should_hard_fail_on_error(item, report):
     if report.outcome != "failed":
         return False
 
-    rerun_errors = session_config.option.only_rerun
-    rerun_except_errors = session_config.option.rerun_except
+    rerun_errors = _get_rerun_filter_regex(item, "only_rerun")
+    rerun_except_errors = _get_rerun_filter_regex(item, "rerun_except")
 
     if not rerun_errors and not rerun_except_errors:
 
@@ -333,7 +346,7 @@ def _should_hard_fail_on_error(session_config, report):
 
 def _should_not_rerun(item, report, reruns):
     xfail = hasattr(report, "wasxfail")
-    is_terminal_error = _should_hard_fail_on_error(item.session.config, report)
+    is_terminal_error = _should_hard_fail_on_error(item, report)
     condition = get_reruns_condition(item)
     return (
         item.execution_count > reruns

test_pytest_rerunfailures.py

@@ -659,6 +659,76 @@ def test_fail_two():
     assert_outcomes(result, passed=0, failed=1, rerun=2)
 
 
+@pytest.mark.parametrize(
+    "marker_only_rerun,cli_only_rerun,should_rerun",
+    [
+        ("AssertionError", None, True),
+        ("AssertionError: ERR", None, True),
+        (["AssertionError"], None, True),
+        (["AssertionError: ABC"], None, False),
+        ("ValueError", None, False),
+        (["ValueError"], None, False),
+        (["AssertionError", "ValueError"], None, True),
+        # CLI override behavior
+        ("AssertionError", "ValueError", True),
+        ("ValueError", "AssertionError", False),
+    ],
+)
+def test_only_rerun_flag_in_flaky_marker(
+    testdir, marker_only_rerun, cli_only_rerun, should_rerun
+):
+    testdir.makepyfile(
+        f"""
+        import pytest
+
+        @pytest.mark.flaky(reruns=1, only_rerun={marker_only_rerun!r})
+        def test_fail():
+            raise AssertionError("ERR")
+        """
+    )
+    args = []
+    if cli_only_rerun:
+        args.extend(["--only-rerun", cli_only_rerun])
+    result = testdir.runpytest()
+    num_reruns = 1 if should_rerun else 0
+    assert_outcomes(result, passed=0, failed=1, rerun=num_reruns)
+
+
+@pytest.mark.parametrize(
+    "marker_rerun_except,cli_rerun_except,should_rerun",
+    [
+        ("AssertionError", None, False),
+        ("AssertionError: ERR", None, False),
+        (["AssertionError"], None, False),
+        (["AssertionError: ABC"], None, True),
+        ("ValueError", None, True),
+        (["ValueError"], None, True),
+        (["OSError", "ValueError"], None, True),
+        # CLI override behavior
+        ("AssertionError", "ValueError", False),
+        ("ValueError", "AssertionError", True),
+    ],
+)
+def test_rerun_except_flag_in_flaky_marker(
+    testdir, marker_rerun_except, cli_rerun_except, should_rerun
+):
+    testdir.makepyfile(
+        f"""
+        import pytest
+
+        @pytest.mark.flaky(reruns=1, rerun_except={marker_rerun_except!r})
+        def test_fail():
+            raise AssertionError("ERR")
+        """
+    )
+    args = []
+    if cli_rerun_except:
+        args.extend(["--rerun-except", cli_rerun_except])
+    result = testdir.runpytest(*args)
+    num_reruns = 1 if should_rerun else 0
+    assert_outcomes(result, passed=0, failed=1, rerun=num_reruns)
+
+
 def test_ini_file_parameters(testdir):
     testdir.makepyfile(
         """