fix!: turn Chairmarks and JET into weakdeps

DifferentiationInterface/test/Back/FiniteDiff/Project.toml

@@ -1,5 +1,7 @@
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"

DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl

@@ -1,11 +1,13 @@
 using Pkg
 
 using ADTypes: ADTypes
+using DataFrames: DataFrame
 using DifferentiationInterface, DifferentiationInterfaceTest
 import DifferentiationInterface as DI
 import DifferentiationInterfaceTest as DIT
 using FiniteDiff: FiniteDiff
 using Test
+import Chairmarks
 
 @testset "Benchmarking sparse" begin
     filtered_sparse_scenarios = filter(sparse_scenarios(; band_sizes = [])) do scen
@@ -21,7 +23,7 @@ using Test
         benchmark = :prepared,
         excluded = SECOND_ORDER,
         logging = LOGGING,
-    )
+    ) |> DataFrame
     @testset "Analyzing benchmark results" begin
         @testset "$(row[:scenario])" for row in eachrow(data)
             @test row[:allocs] == 0

DifferentiationInterface/test/Back/ForwardDiff/Project.toml

@@ -1,10 +1,13 @@
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb"
 SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"

DifferentiationInterface/test/Back/ForwardDiff/benchmark.jl

@@ -1,6 +1,7 @@
 using Pkg
 
 using ADTypes: ADTypes
+using DataFrames: DataFrame
 using DifferentiationInterface, DifferentiationInterfaceTest
 import DifferentiationInterface as DI
 import DifferentiationInterfaceTest as DIT
@@ -18,7 +19,7 @@ using Test
         benchmark = :prepared,
         excluded = [:hessian, :pullback],  # TODO: figure this out
         logging = LOGGING,
-    )
+    ) |> DataFrame
     @testset "Analyzing benchmark results" begin
         @testset "$(row[:scenario])" for row in eachrow(data)
             @test row[:allocs] == 0
@@ -40,7 +41,7 @@ end
         benchmark = :prepared,
         excluded = SECOND_ORDER,
         logging = LOGGING,
-    )
+    ) |> DataFrame
     @testset "Analyzing benchmark results" begin
         @testset "$(row[:scenario])" for row in eachrow(data)
             @test row[:allocs] == 0

DifferentiationInterface/test/Back/ForwardDiff/test.jl

@@ -9,6 +9,7 @@ using ForwardDiff: ForwardDiff
 using StaticArrays: StaticArrays, @SVector
 using JLArrays: JLArrays
 using Test
+import JET, Chairmarks
 
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)

DifferentiationInterface/test/Back/GTPSA/Project.toml

@@ -4,6 +4,7 @@ DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
 GTPSA = "b27dd330-f138-47c5-815b-40db9dd9b6e8"
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

DifferentiationInterface/test/Back/GTPSA/test.jl

@@ -3,6 +3,7 @@ include("../../testutils.jl")
 using DifferentiationInterface, DifferentiationInterfaceTest
 using GTPSA: GTPSA
 using Test
+import JET
 
 using ExplicitImports
 check_no_implicit_imports(DifferentiationInterface)

DifferentiationInterface/test/Core/ZeroBackends/test.jl

@@ -7,7 +7,7 @@ using JLArrays: JLArrays
 using SparseMatrixColorings
 using StaticArrays: StaticArrays
 using Test
-
+using JET
 
 zero_backends = [AutoZeroForward(), AutoZeroReverse()]
 

DifferentiationInterface/test/Project.toml

@@ -1,6 +1,7 @@
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -23,11 +24,12 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 [compat]
 ADTypes = "1.18"
 Aqua = "0.8.12"
+Chairmarks = "1.2.1"
 ComponentArrays = "0.15.27"
 DataFrames = "1.7.0"
 Dates = "1"
 DifferentiationInterface = "0.7.10"
-DifferentiationInterfaceTest = "0.10.3"
+DifferentiationInterfaceTest = "0.11.0"
 ExplicitImports = "1.10.1"
 InteractiveUtils = "1"
 JET = "0.9,0.10,0.11"
@@ -42,4 +44,4 @@ Test = "1"
 julia = "1.10.10"
 
 [sources]
-DifferentiationInterface = { path = ".." }
+DifferentiationInterface = { path = ".." }

DifferentiationInterfaceTest/CHANGELOG.md

@@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.4...main)
+## [Unreleased](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.11.0...main)
+
+## [0.11.0](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.4...DifferentiationInterfaceTest-v0.11.0)
+
+### Changed
+
+- Benchmarking and type stability analysis are now implemented in package extensions, which depend on Chairmarks and JET respectively
+- Benchmarking no longer returns a `DataFrame` but a Tables.jl-compatible object.
 
 ## [0.10.4](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.3...DifferentiationInterfaceTest-v0.10.4)
 

DifferentiationInterfaceTest/Project.toml

@@ -1,33 +1,35 @@
 name = "DifferentiationInterfaceTest"
 uuid = "a82114a7-5aa3-49a8-9643-716bb13727a3"
+version = "0.11.0"
 authors = ["Guillaume Dalle", "Adrian Hill"]
-version = "0.10.4"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
-Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
-DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
 DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
-Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [weakdeps]
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [extensions]
+DifferentiationInterfaceTestChairmarksExt = "Chairmarks"
 DifferentiationInterfaceTestComponentArraysExt = "ComponentArrays"
+DifferentiationInterfaceTestJETExt = "JET"
 DifferentiationInterfaceTestJLArraysExt = "JLArrays"
 DifferentiationInterfaceTestStaticArraysExt = "StaticArrays"
 
@@ -36,7 +38,7 @@ ADTypes = "1.7.0"
 AllocCheck = "0.2"
 Chairmarks = "1.2.1"
 ComponentArrays = "0.15"
-DataFrames = "1.6.1"
+DataAPI = "1.16.0"
 DifferentiationInterface = "0.7.7"
 DocStringExtensions = "0.8,0.9"
 ForwardDiff = "0.10.36,1"
@@ -48,7 +50,7 @@ ProgressMeter = "1"
 Random = "1"
 SparseArrays = "1"
 StaticArrays = "1.9"
-Statistics = "1"
+Tables = "1.12.1"
 Test = "1"
 Zygote = "0.6, 0.7"
 julia = "1.10"

DifferentiationInterfaceTest/docs/Project.toml

@@ -1,6 +1,7 @@
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
@@ -15,9 +16,10 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 [compat]
 ADTypes = "1.18"
 BenchmarkTools = "1.6.3"
+Chairmarks = "1.2.1"
 DataFrames = "1.8.1"
 DifferentiationInterface = "0.7.10"
-DifferentiationInterfaceTest = "0.10.2"
+DifferentiationInterfaceTest = "0.11.0"
 Documenter = "1"
 DocumenterInterLinks = "1.1"
 ForwardDiff = "1.2.2"

DifferentiationInterfaceTest/docs/src/tutorial.md

@@ -3,6 +3,8 @@
 We present a typical workflow with DifferentiationInterfaceTest.jl, building on the tutorial of the [DifferentiationInterface.jl documentation](https://juliadiff.org/DifferentiationInterface.jl/DifferentiationInterface) (which we encourage you to read first).
 
 ```@repl tuto
+import Chairmarks
+using DataFrames
 using DifferentiationInterface, DifferentiationInterfaceTest
 using ForwardDiff: ForwardDiff
 using Zygote: Zygote
@@ -30,12 +32,12 @@ Of course we know the true gradient mapping:
 
 DifferentiationInterfaceTest.jl relies with so-called [`Scenario`](@ref)s, in which you encapsulate the information needed for your test:
 
-  - the operator category (here `:gradient`)
-  - the behavior of the operator (either `:in` or `:out` of place)
-  - the function `f`
-  - the input `x` of the function `f` (and possible tangents or contexts)
-  - the reference first-order result `res1` (and possible second-order result `res2`) of the operator
-  - the arguments `prep_args` passed during preparation
+- the operator category (here `:gradient`)
+- the behavior of the operator (either `:in` or `:out` of place)
+- the function `f`
+- the input `x` of the function `f` (and possible tangents or contexts)
+- the reference first-order result `res1` (and possible second-order result `res2`) of the operator
+- the arguments `prep_args` passed during preparation
 
 ```@example tuto
 xv = rand(Float32, 3)
@@ -68,7 +70,12 @@ Once you are confident that your backends give the correct answers, you probably
 This is made easy by the [`benchmark_differentiation`](@ref) function, whose syntax should feel familiar:
 
 ```@example tuto
-df = benchmark_differentiation(backends, scenarios);
+table = benchmark_differentiation(backends, scenarios);
 ```
 
-The resulting object is a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl), whose columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref).
+The resulting object is a table, which can easily be converted into a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl).
+Its columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref).
+
+```@example tuto
+df = DataFrame(table)
+```

DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl

@@ -0,0 +1,127 @@
+module DifferentiationInterfaceTestChairmarksExt
+
+using ADTypes: AbstractADType
+using Chairmarks: @be, Benchmark, Sample
+import DifferentiationInterface as DI
+using DifferentiationInterface:
+    prepare_pushforward,
+    prepare_pushforward_same_point,
+    prepare!_pushforward,
+    pushforward,
+    pushforward!,
+    value_and_pushforward,
+    value_and_pushforward!,
+    prepare_pullback,
+    prepare_pullback_same_point,
+    prepare!_pullback,
+    pullback,
+    pullback!,
+    value_and_pullback,
+    value_and_pullback!,
+    prepare_derivative,
+    prepare!_derivative,
+    derivative,
+    derivative!,
+    value_and_derivative,
+    value_and_derivative!,
+    prepare_gradient,
+    prepare!_gradient,
+    gradient,
+    gradient!,
+    value_and_gradient,
+    value_and_gradient!,
+    prepare_jacobian,
+    prepare!_jacobian,
+    jacobian,
+    jacobian!,
+    value_and_jacobian,
+    value_and_jacobian!,
+    prepare_second_derivative,
+    prepare!_second_derivative,
+    second_derivative,
+    second_derivative!,
+    value_derivative_and_second_derivative,
+    value_derivative_and_second_derivative!,
+    prepare_hvp,
+    prepare_hvp_same_point,
+    prepare!_hvp,
+    hvp,
+    hvp!,
+    gradient_and_hvp,
+    gradient_and_hvp!,
+    prepare_hessian,
+    prepare!_hessian,
+    hessian,
+    hessian!,
+    value_gradient_and_hessian,
+    value_gradient_and_hessian!
+import DifferentiationInterfaceTest as DIT
+using DifferentiationInterfaceTest:
+    ALL_OPS,
+    CallCounter, CallsResult, DifferentiationBenchmarkDataRow, DifferentiationBenchmark, Scenario,
+    mysimilar, reset_count!
+using Test: Test, @test
+
+function failed_bench()
+    evals = 0.0
+    time = NaN
+    allocs = NaN
+    bytes = NaN
+    gc_fraction = NaN
+    compile_fraction = NaN
+    recompile_fraction = NaN
+    warmup = NaN
+    checksum = NaN
+    sample = Sample(
+        evals,
+        time,
+        allocs,
+        bytes,
+        gc_fraction,
+        compile_fraction,
+        recompile_fraction,
+        warmup,
+        checksum,
+    )
+    return Benchmark([sample])
+end
+
+@kwdef struct BenchmarkResult
+    prepared_valop::Benchmark = failed_bench()
+    prepared_op::Benchmark = failed_bench()
+    preparation::Benchmark = failed_bench()
+    unprepared_valop::Benchmark = failed_bench()
+    unprepared_op::Benchmark = failed_bench()
+end
+
+
+function record!(
+        data::DifferentiationBenchmark;
+        backend::AbstractADType,
+        scenario::Scenario,
+        operator::String,
+        prepared::Union{Nothing, Bool},
+        bench::Benchmark,
+        calls::Integer,
+        aggregation,
+    )
+    row = DifferentiationBenchmarkDataRow(;
+        backend = backend,
+        scenario = scenario,
+        operator = Symbol(operator),
+        prepared = prepared,
+        calls = calls,
+        samples = length(bench.samples),
+        evals = Int(bench.samples[1].evals),
+        time = aggregation(getfield.(bench.samples, :time)),
+        allocs = aggregation(getfield.(bench.samples, :allocs)),
+        bytes = aggregation(getfield.(bench.samples, :bytes)),
+        gc_fraction = aggregation(getfield.(bench.samples, :gc_fraction)),
+        compile_fraction = aggregation(getfield.(bench.samples, :compile_fraction)),
+    )
+    return push!(data.rows, row)
+end
+
+include("benchmark_eval.jl")
+
+end