chore: revamp testing CI with dedicated environments

.github/workflows/Test.yml

@@ -16,8 +16,8 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-  test-DI:
-    name: ${{ matrix.version }} - DI (${{ matrix.group }})
+  test-DI-Core:
+    name: ${{ matrix.version }} - DI Core (${{ matrix.group }})
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skipci') }}
     timeout-minutes: 120
@@ -32,25 +32,9 @@ jobs:
           - '1.11'
           - '1.12'
         group:
-          - Core/Internals
-          - Back/DifferentiateWith
-          - Core/SimpleFiniteDiff
-          - Back/SparsityDetector
-          - Core/ZeroBackends
-          - Back/ChainRules
-          # - Back/Diffractor
-          - Back/Enzyme
-          - Back/FastDifferentiation
-          - Back/FiniteDiff
-          - Back/FiniteDifferences
-          - Back/ForwardDiff
-          - Back/GTPSA
-          - Back/Mooncake
-          - Back/PolyesterForwardDiff
-          - Back/ReverseDiff
-          - Back/Symbolics
-          - Back/Tracker
-          - Back/Zygote
+          - Internals
+          - SimpleFiniteDiff
+          - ZeroBackends
         skip_lts:
           - ${{ github.event.pull_request.draft }}
         skip_pre:
@@ -60,13 +44,8 @@ jobs:
             version: '1.10'
           - skip_pre: true
             version: '1.12'
-          - version: '1.11'
-            group: Back/ChainRules
-          - version: '1.12'
-            group: Back/Enzyme
-          - version: '1.12'
-            group: Back/DifferentiateWith
     env:
+      JULIA_DI_TEST_TYPE: 'Core'
       JULIA_DI_TEST_GROUP: ${{ matrix.group }}
       JULIA_DI_PR_DRAFT: ${{ github.event.pull_request.draft }}
     steps:
@@ -79,7 +58,6 @@ jobs:
       - name: Install dependencies & run tests
         run: julia --color=yes -e '
           using Pkg;
-          Pkg.Registry.update();
           Pkg.activate("./DifferentiationInterface/test");
           if VERSION < v"1.11";
           Pkg.rm("DifferentiationInterfaceTest");
@@ -104,6 +82,81 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
 
+  test-DI-Backend:
+    name: ${{ matrix.version }} - DI Back (${{ matrix.group }})
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skipci') }}
+    timeout-minutes: 120
+    permissions: # needed to allow julia-actions/cache to proactively delete old caches that it has created
+      actions: write
+      contents: read
+    strategy:
+      fail-fast: false # TODO: toggle
+      matrix:
+        version:
+          - '1.10'
+          - '1.11'
+          - '1.12'
+        group:
+          - ChainRules
+          - DifferentiateWith
+          # - Diffractor
+          - Enzyme
+          - FastDifferentiation
+          - FiniteDiff
+          - FiniteDifferences
+          - ForwardDiff
+          - GTPSA
+          - Mooncake
+          - PolyesterForwardDiff
+          - ReverseDiff
+          - SparsityDetector
+          - Symbolics
+          - Tracker
+          - Zygote
+        skip_lts:
+          - ${{ github.event.pull_request.draft }}
+        skip_pre:
+          - ${{ github.event.pull_request.draft }}
+        exclude:
+          - skip_lts: true
+            version: '1.10'
+          - skip_pre: true
+            version: '1.12'
+          - version: '1.11'
+            group: ChainRules
+          - version: '1.12'
+            group: Enzyme
+          - version: '1.12'
+            group: DifferentiateWith
+    env:
+      JULIA_DI_TEST_TYPE: 'Back'
+      JULIA_DI_TEST_GROUP: ${{ matrix.group }}
+      JULIA_DI_PR_DRAFT: ${{ github.event.pull_request.draft }}
+    steps:
+      - uses: actions/checkout@v5
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: x64
+      - uses: julia-actions/cache@v2
+      - name: Install dependencies & run tests
+        run: julia --code-coverage=user --color=yes -e '
+          using Pkg;
+          group = ENV["JULIA_DI_TEST_GROUP"];
+          Pkg.activate("./DifferentiationInterface/test/Back/$group");
+          Pkg.develop([PackageSpec(path="./DifferentiationInterface"), PackageSpec(path="./DifferentiationInterfaceTest")]);
+          include("./DifferentiationInterface/test/Back/run_backend.jl");'
+      - uses: julia-actions/julia-processcoverage@v1
+        with:
+          directories: ./DifferentiationInterface/src,./DifferentiationInterface/ext,./DifferentiationInterface/test
+      - uses: codecov/codecov-action@v5
+        with:
+          files: lcov.info
+          flags: DI
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false
+
   test-DIT:
     name: ${{ matrix.version }} - DIT (${{ matrix.group }})
     runs-on: ubuntu-latest

DifferentiationInterface/test/Back/ChainRules/Project.toml

@@ -0,0 +1,10 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"

DifferentiationInterface/test/Back/ChainRules/zygote.jl → DifferentiationInterface/test/Back/ChainRules/test.jl

@@ -1,16 +1,12 @@
-using Pkg
-Pkg.add(["ChainRulesCore", "Zygote"])
+include("../../testutils.jl")
 
-using ChainRulesCore
 using DifferentiationInterface, DifferentiationInterfaceTest
 using Test
 using Zygote: ZygoteRuleConfig
 
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 for backend in [AutoChainRules(ZygoteRuleConfig())]
     @test check_available(backend)
     @test !check_inplace(backend)

DifferentiationInterface/test/Back/DifferentiateWith/Project.toml

@@ -0,0 +1,14 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"

DifferentiationInterface/test/Back/DifferentiateWith/test.jl

@@ -1,5 +1,4 @@
-using Pkg
-Pkg.add(["ChainRulesTestUtils", "FiniteDiff", "ForwardDiff", "Zygote", "Mooncake"])
+include("../../testutils.jl")
 
 using ChainRulesTestUtils: ChainRulesTestUtils
 using DifferentiationInterface, DifferentiationInterfaceTest
@@ -11,8 +10,6 @@ using Mooncake: Mooncake
 using StableRNGs
 using Test
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 struct ADBreaker{F}
     f::F
 end

DifferentiationInterface/test/Back/Enzyme/Project.toml

@@ -0,0 +1,11 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

DifferentiationInterface/test/Back/Enzyme/test.jl

@@ -1,7 +1,4 @@
-# see https://github.com/JuliaDiff/DifferentiationInterface.jl/issues/855
-
-using Pkg
-Pkg.add("Enzyme")
+include("../../testutils.jl")
 
 using ADTypes: ADTypes
 using DifferentiationInterface, DifferentiationInterfaceTest
@@ -14,8 +11,6 @@ using Test
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 backends = [
     AutoEnzyme(; mode = nothing),
     AutoEnzyme(; mode = Enzyme.Forward),

DifferentiationInterface/test/Back/FastDifferentiation/Project.toml

@@ -0,0 +1,10 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+FastDifferentiation = "eb9bf01b-bf85-4b60-bf87-ee5de06c00be"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

DifferentiationInterface/test/Back/FastDifferentiation/fastdifferentiation.jl → DifferentiationInterface/test/Back/FastDifferentiation/test.jl

@@ -1,5 +1,4 @@
-using Pkg
-Pkg.add("FastDifferentiation")
+include("../../testutils.jl")
 
 using DifferentiationInterface, DifferentiationInterfaceTest
 using SparseMatrixColorings
@@ -10,8 +9,6 @@ using Test
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 for backend in [AutoFastDifferentiation(), AutoSparse(AutoFastDifferentiation())]
     @test check_available(backend)
     @test check_inplace(backend)

DifferentiationInterface/test/Back/FiniteDiff/Project.toml

@@ -0,0 +1,9 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl

@@ -1,5 +1,4 @@
 using Pkg
-Pkg.add("FiniteDiff")
 
 using ADTypes: ADTypes
 using DifferentiationInterface, DifferentiationInterfaceTest
@@ -8,8 +7,6 @@ import DifferentiationInterfaceTest as DIT
 using FiniteDiff: FiniteDiff
 using Test
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 @testset "Benchmarking sparse" begin
     filtered_sparse_scenarios = filter(sparse_scenarios(; band_sizes = [])) do scen
         DIT.function_place(scen) == :in &&

DifferentiationInterface/test/Back/FiniteDiff/test.jl

@@ -1,5 +1,4 @@
-using Pkg
-Pkg.add("FiniteDiff")
+include("../../testutils.jl")
 
 using DifferentiationInterface, DifferentiationInterfaceTest
 using DifferentiationInterface: DenseSparsityDetector
@@ -10,8 +9,6 @@ using Test
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 for backend in [AutoFiniteDiff()]
     @test check_available(backend)
     @test check_inplace(backend)
@@ -114,3 +111,5 @@ end;
     @test prep.relstep_g == 0.1
     @test prep.relstep_h == 0.1
 end
+
+include("benchmark.jl")

DifferentiationInterface/test/Back/FiniteDifferences/Project.toml

@@ -0,0 +1,9 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

DifferentiationInterface/test/Back/FiniteDifferences/test.jl

@@ -1,5 +1,4 @@
-using Pkg
-Pkg.add("FiniteDifferences")
+include("../../testutils.jl")
 
 using DifferentiationInterface, DifferentiationInterfaceTest
 using FiniteDifferences: FiniteDifferences
@@ -8,8 +7,6 @@ using Test
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 for backend in [AutoFiniteDifferences(; fdm = FiniteDifferences.central_fdm(3, 1))]
     @test check_available(backend)
     @test !check_inplace(backend)

DifferentiationInterface/test/Back/ForwardDiff/Project.toml

@@ -0,0 +1,12 @@
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
+SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

DifferentiationInterface/test/Back/ForwardDiff/benchmark.jl

@@ -1,5 +1,4 @@
 using Pkg
-Pkg.add("ForwardDiff")
 
 using ADTypes: ADTypes
 using DifferentiationInterface, DifferentiationInterfaceTest
@@ -9,8 +8,6 @@ using ForwardDiff: ForwardDiff
 using StaticArrays: StaticArrays, @SVector
 using Test
 
-LOGGING = get(ENV, "CI", "false") == "false"
-
 @testset verbose = true "Benchmarking static" begin
     filtered_static_scenarios = filter(static_scenarios(; include_batchified = false)) do scen
         DIT.function_place(scen) == :out && DIT.operator_place(scen) == :out