Make Flux scenarios private in DIT (#364)

* Relax Flux tests in DIT

* Keep Flux scenarios private

* Add Flux tests on pre-release

Author: gdalle

.github/workflows/Test.yml

@@ -84,8 +84,6 @@ jobs:
             group: Back/SecondOrder
           - version: 'pre'
             group: Down/Detector
-          - version: 'pre'
-            group: Down/Flux
 
     steps:
       - uses: actions/checkout@v4

DifferentiationInterface/test/Down/Flux/test.jl

@@ -13,7 +13,7 @@ test_differentiation(
         AutoZygote(),
         # AutoEnzyme()  # TODO: fix
     ],
-    flux_scenarios();
+    DIT.flux_scenarios();
     isequal=DIT.flux_isequal,
     isapprox=DIT.flux_isapprox,
     rtol=1e-2,

DifferentiationInterfaceTest/docs/src/api.md

@@ -19,13 +19,14 @@ DifferentiationBenchmarkDataRow
 
 ## Pre-made scenario lists
 
+The precise contents of the scenario lists are not part of the API, only their existence.
+
 ```@docs
 default_scenarios
 sparse_scenarios
 component_scenarios
 gpu_scenarios
 static_scenarios
-flux_scenarios
 ```
 
 ## Scenario types

DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl

@@ -109,6 +109,5 @@ export DifferentiationBenchmarkDataRow
 export static_scenarios
 export component_scenarios
 export gpu_scenarios
-export flux_scenarios
 
 end

DifferentiationInterfaceTest/src/scenarios/extensions.jl

@@ -35,6 +35,10 @@ Create a vector of [`Scenario`](@ref)s with neural networks from [Flux.jl](https
 
 !!! warning
     This function requires Flux.jl and FiniteDifferences.jl to be loaded (it is implemented in a package extension).
+
+!!! danger
+    These scenarios are still experimental and not part of the public API.
+    Their ground truth values are computed with finite differences, and thus subject to imprecision.
 """
 function flux_scenarios end
 

DifferentiationInterfaceTest/test/weird.jl

@@ -25,10 +25,10 @@ test_differentiation(
 
 test_differentiation(
     AutoZygote(),
-    flux_scenarios();
+    DIT.flux_scenarios();
     isequal=DIT.flux_isequal,
     isapprox=DIT.flux_isapprox,
-    rtol=1e-2,
-    atol=1e-3,
+    rtol=5e-2,
+    atol=1e-2,
     logging=LOGGING,
 )