From 39a32d677bf456c315f92f00cf7ffa435d8825a4 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Mon, 26 Jan 2026 21:33:18 +0100 Subject: [PATCH 01/12] fix!: turn Chairmarks and JET into weakdeps --- DifferentiationInterface/test/Project.toml | 4 +- DifferentiationInterface/test/testutils.jl | 2 + DifferentiationInterfaceTest/CHANGELOG.md | 9 +- DifferentiationInterfaceTest/Project.toml | 12 +- .../docs/Project.toml | 7 +- .../docs/src/tutorial.md | 23 ++-- ...fferentiationInterfaceTestChairmarksExt.jl | 127 ++++++++++++++++++ .../benchmark_eval.jl | 20 +-- .../DifferentiationInterfaceTestJETExt.jl | 10 ++ .../type_stability_eval.jl | 24 ++-- .../src/DifferentiationInterfaceTest.jl | 7 +- .../src/test_differentiation.jl | 8 +- .../src/tests/benchmark.jl | 87 ++++++------ .../src/tests/type_stability.jl | 10 ++ .../test/Project.toml | 6 +- DifferentiationInterfaceTest/test/runtests.jl | 2 + 16 files changed, 250 insertions(+), 108 deletions(-) create mode 100644 DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl rename DifferentiationInterfaceTest/{src/tests => ext/DifferentiationInterfaceTestChairmarksExt}/benchmark_eval.jl (98%) create mode 100644 DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl rename DifferentiationInterfaceTest/{src/tests => ext/DifferentiationInterfaceTestJETExt}/type_stability_eval.jl (97%) create mode 100644 DifferentiationInterfaceTest/src/tests/type_stability.jl diff --git a/DifferentiationInterface/test/Project.toml b/DifferentiationInterface/test/Project.toml index 350fcbb58..de83dd6db 100644 --- a/DifferentiationInterface/test/Project.toml +++ b/DifferentiationInterface/test/Project.toml @@ -1,6 +1,7 @@ [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -23,6 +24,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] ADTypes = "1.18" Aqua = "0.8.12" +Chairmarks = "1.2.1" ComponentArrays = "0.15.27" DataFrames = "1.7.0" Dates = "1" @@ -42,4 +44,4 @@ Test = "1" julia = "1.10.10" [sources] -DifferentiationInterface = { path = ".." } \ No newline at end of file +DifferentiationInterface = { path = ".." } diff --git a/DifferentiationInterface/test/testutils.jl b/DifferentiationInterface/test/testutils.jl index 7329f4abf..f782551a1 100644 --- a/DifferentiationInterface/test/testutils.jl +++ b/DifferentiationInterface/test/testutils.jl @@ -14,6 +14,8 @@ using DifferentiationInterfaceTest using SparseConnectivityTracer using SparseMatrixColorings using Test +import JET +import Chairmarks using DifferentiationInterfaceTest: default_scenarios, diff --git a/DifferentiationInterfaceTest/CHANGELOG.md b/DifferentiationInterfaceTest/CHANGELOG.md index b314043ee..46857eaa5 100644 --- a/DifferentiationInterfaceTest/CHANGELOG.md +++ b/DifferentiationInterfaceTest/CHANGELOG.md @@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.4...main) +## [Unreleased](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.11.0...main) + +## [0.11.0](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.4...DifferentiationInterfaceTest-v0.11.0) + +### Changed + +- Benchmarking and type stability analysis are now implemented in package extensions, which depend on Chairmarks and JET respectively +- Benchmarking no longer returns a `DataFrame` but a Tables.jl-compatible object. ## [0.10.4](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.3...DifferentiationInterfaceTest-v0.10.4) diff --git a/DifferentiationInterfaceTest/Project.toml b/DifferentiationInterfaceTest/Project.toml index a417d9eac..3994b4d29 100644 --- a/DifferentiationInterfaceTest/Project.toml +++ b/DifferentiationInterfaceTest/Project.toml @@ -1,33 +1,35 @@ name = "DifferentiationInterfaceTest" uuid = "a82114a7-5aa3-49a8-9643-716bb13727a3" +version = "0.11.0" authors = ["Guillaume Dalle", "Adrian Hill"] -version = "0.10.4" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" -Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [extensions] +DifferentiationInterfaceTestChairmarksExt = "Chairmarks" DifferentiationInterfaceTestComponentArraysExt = "ComponentArrays" +DifferentiationInterfaceTestJETExt = "JET" DifferentiationInterfaceTestJLArraysExt = "JLArrays" DifferentiationInterfaceTestStaticArraysExt = "StaticArrays" @@ -36,7 +38,6 @@ ADTypes = "1.7.0" AllocCheck = "0.2" Chairmarks = "1.2.1" ComponentArrays = "0.15" -DataFrames = "1.6.1" DifferentiationInterface = "0.7.7" DocStringExtensions = "0.8,0.9" ForwardDiff = "0.10.36,1" @@ -49,6 +50,7 @@ Random = "1" SparseArrays = "1" StaticArrays = "1.9" Statistics = "1" +Tables = "1.12.1" Test = "1" Zygote = "0.6, 0.7" julia = "1.10" diff --git a/DifferentiationInterfaceTest/docs/Project.toml b/DifferentiationInterfaceTest/docs/Project.toml index 70209d812..5e0465b64 100644 --- a/DifferentiationInterfaceTest/docs/Project.toml +++ b/DifferentiationInterfaceTest/docs/Project.toml @@ -1,6 +1,7 @@ [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" @@ -15,15 +16,13 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] ADTypes = "1.18" BenchmarkTools = "1.6.3" +Chairmarks = "1.2.1" DataFrames = "1.8.1" DifferentiationInterface = "0.7.10" -DifferentiationInterfaceTest = "0.10.2" +DifferentiationInterfaceTest = "0.11.0" Documenter = "1" DocumenterInterLinks = "1.1" ForwardDiff = "1.2.2" PrettyTables = "3.1" Zygote = "0.7.10" julia = "1.10.10" - -[sources] -DifferentiationInterfaceTest = { path = ".." } diff --git a/DifferentiationInterfaceTest/docs/src/tutorial.md b/DifferentiationInterfaceTest/docs/src/tutorial.md index 57ca9e2d4..79737802a 100644 --- a/DifferentiationInterfaceTest/docs/src/tutorial.md +++ b/DifferentiationInterfaceTest/docs/src/tutorial.md @@ -3,6 +3,8 @@ We present a typical workflow with DifferentiationInterfaceTest.jl, building on the tutorial of the [DifferentiationInterface.jl documentation](https://juliadiff.org/DifferentiationInterface.jl/DifferentiationInterface) (which we encourage you to read first). ```@repl tuto +import Chairmarks +using DataFrames using DifferentiationInterface, DifferentiationInterfaceTest using ForwardDiff: ForwardDiff using Zygote: Zygote @@ -30,12 +32,12 @@ Of course we know the true gradient mapping: DifferentiationInterfaceTest.jl relies with so-called [`Scenario`](@ref)s, in which you encapsulate the information needed for your test: - - the operator category (here `:gradient`) - - the behavior of the operator (either `:in` or `:out` of place) - - the function `f` - - the input `x` of the function `f` (and possible tangents or contexts) - - the reference first-order result `res1` (and possible second-order result `res2`) of the operator - - the arguments `prep_args` passed during preparation +- the operator category (here `:gradient`) +- the behavior of the operator (either `:in` or `:out` of place) +- the function `f` +- the input `x` of the function `f` (and possible tangents or contexts) +- the reference first-order result `res1` (and possible second-order result `res2`) of the operator +- the arguments `prep_args` passed during preparation ```@example tuto xv = rand(Float32, 3) @@ -68,7 +70,12 @@ Once you are confident that your backends give the correct answers, you probably This is made easy by the [`benchmark_differentiation`](@ref) function, whose syntax should feel familiar: ```@example tuto -df = benchmark_differentiation(backends, scenarios); +table = benchmark_differentiation(backends, scenarios); ``` -The resulting object is a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl), whose columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref). +The resulting object is a table, which can easily be converted into a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl). +Its columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref). + +```@example tuto +df = DataFrame(table) +``` diff --git a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl new file mode 100644 index 000000000..64fcc3447 --- /dev/null +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl @@ -0,0 +1,127 @@ +module DifferentiationInterfaceTestChairmarksExt + +using ADTypes: AbstractADType +using Chairmarks: @be, Benchmark, Sample +import DifferentiationInterface as DI +using DifferentiationInterface: + prepare_pushforward, + prepare_pushforward_same_point, + prepare!_pushforward, + pushforward, + pushforward!, + value_and_pushforward, + value_and_pushforward!, + prepare_pullback, + prepare_pullback_same_point, + prepare!_pullback, + pullback, + pullback!, + value_and_pullback, + value_and_pullback!, + prepare_derivative, + prepare!_derivative, + derivative, + derivative!, + value_and_derivative, + value_and_derivative!, + prepare_gradient, + prepare!_gradient, + gradient, + gradient!, + value_and_gradient, + value_and_gradient!, + prepare_jacobian, + prepare!_jacobian, + jacobian, + jacobian!, + value_and_jacobian, + value_and_jacobian!, + prepare_second_derivative, + prepare!_second_derivative, + second_derivative, + second_derivative!, + value_derivative_and_second_derivative, + value_derivative_and_second_derivative!, + prepare_hvp, + prepare_hvp_same_point, + prepare!_hvp, + hvp, + hvp!, + gradient_and_hvp, + gradient_and_hvp!, + prepare_hessian, + prepare!_hessian, + hessian, + hessian!, + value_gradient_and_hessian, + value_gradient_and_hessian! +import DifferentiationInterfaceTest as DIT +using DifferentiationInterfaceTest: + ALL_OPS, + CallCounter, CallsResult, DifferentiationBenchmarkDataRow, DifferentiationBenchmark, Scenario, + mysimilar, reset_count! +using Test + +function failed_bench() + evals = 0.0 + time = NaN + allocs = NaN + bytes = NaN + gc_fraction = NaN + compile_fraction = NaN + recompile_fraction = NaN + warmup = NaN + checksum = NaN + sample = Sample( + evals, + time, + allocs, + bytes, + gc_fraction, + compile_fraction, + recompile_fraction, + warmup, + checksum, + ) + return Benchmark([sample]) +end + +@kwdef struct BenchmarkResult + prepared_valop::Benchmark = failed_bench() + prepared_op::Benchmark = failed_bench() + preparation::Benchmark = failed_bench() + unprepared_valop::Benchmark = failed_bench() + unprepared_op::Benchmark = failed_bench() +end + + +function record!( + data::DifferentiationBenchmark; + backend::AbstractADType, + scenario::Scenario, + operator::String, + prepared::Union{Nothing, Bool}, + bench::Benchmark, + calls::Integer, + aggregation, + ) + row = DifferentiationBenchmarkDataRow(; + backend = backend, + scenario = scenario, + operator = Symbol(operator), + prepared = prepared, + calls = calls, + samples = length(bench.samples), + evals = Int(bench.samples[1].evals), + time = aggregation(getfield.(bench.samples, :time)), + allocs = aggregation(getfield.(bench.samples, :allocs)), + bytes = aggregation(getfield.(bench.samples, :bytes)), + gc_fraction = aggregation(getfield.(bench.samples, :gc_fraction)), + compile_fraction = aggregation(getfield.(bench.samples, :compile_fraction)), + ) + return push!(data.rows, row) +end + +include("benchmark_eval.jl") + +end diff --git a/DifferentiationInterfaceTest/src/tests/benchmark_eval.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/benchmark_eval.jl similarity index 98% rename from DifferentiationInterfaceTest/src/tests/benchmark_eval.jl rename to DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/benchmark_eval.jl index b8a01fc29..d679f58da 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark_eval.jl +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/benchmark_eval.jl @@ -1,19 +1,3 @@ -@kwdef struct BenchmarkResult - prepared_valop::Benchmark = failed_bench() - prepared_op::Benchmark = failed_bench() - preparation::Benchmark = failed_bench() - unprepared_valop::Benchmark = failed_bench() - unprepared_op::Benchmark = failed_bench() -end - -@kwdef struct CallsResult - preparation::Int = -1 - prepared_valop::Int = -1 - prepared_op::Int = -1 - unprepared_valop::Int = -1 - unprepared_op::Int = -1 -end - for op in ALL_OPS op! = Symbol(op, "!") val_prefix = if op == :second_derivative @@ -34,8 +18,8 @@ for op in ALL_OPS S2out = Scenario{op, :out, :in} S2in = Scenario{op, :in, :in} - @eval function run_benchmark!( - data::Vector{DifferentiationBenchmarkDataRow}, + @eval function DIT.run_benchmark!( + data::DifferentiationBenchmark, backend::AbstractADType, scenario::Union{$S1out, $S1in, $S2out, $S2in}; logging::Bool, diff --git a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl new file mode 100644 index 000000000..84e671d71 --- /dev/null +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl @@ -0,0 +1,10 @@ +module DifferentiationInterfaceTestJETExt + +using ADTypes: AbstractADType +using DifferentiationInterfaceTest: ALL_OPS, Scenario +import DifferentiationInterfaceTest as DIT +using JET: @test_opt + +include("type_stability_eval.jl") + +end diff --git a/DifferentiationInterfaceTest/src/tests/type_stability_eval.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/type_stability_eval.jl similarity index 97% rename from DifferentiationInterfaceTest/src/tests/type_stability_eval.jl rename to DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/type_stability_eval.jl index 911f6463e..949ea1855 100644 --- a/DifferentiationInterfaceTest/src/tests/type_stability_eval.jl +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/type_stability_eval.jl @@ -19,7 +19,7 @@ for op in ALL_OPS S2in = Scenario{op, :in, :in} if op in [:derivative, :gradient, :jacobian] - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1out; subset::Symbol, @@ -46,7 +46,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1in; subset::Symbol, @@ -77,7 +77,7 @@ for op in ALL_OPS op == :gradient && continue - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S2out; subset::Symbol, @@ -106,7 +106,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S2in; subset::Symbol, @@ -138,7 +138,7 @@ for op in ALL_OPS end elseif op in [:second_derivative, :hessian] - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1out; subset::Symbol, @@ -165,7 +165,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1in; subset::Symbol, @@ -197,7 +197,7 @@ for op in ALL_OPS end elseif op in [:pushforward, :pullback] - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1out; subset::Symbol, @@ -226,7 +226,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1in; subset::Symbol, @@ -257,7 +257,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S2out; subset::Symbol, @@ -288,7 +288,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S2in; subset::Symbol, @@ -324,7 +324,7 @@ for op in ALL_OPS end elseif op in [:hvp] - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1out; subset::Symbol, @@ -353,7 +353,7 @@ for op in ALL_OPS return nothing end - @eval function test_jet( + @eval function DIT.test_jet( ba::AbstractADType, scen::$S1in; subset::Symbol, diff --git a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl index 51b216915..4fbc6abd2 100644 --- a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl +++ b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl @@ -16,8 +16,6 @@ using ADTypes: SymbolicMode, mode using AllocCheck: check_allocs -using Chairmarks: @be, Benchmark, Sample -using DataFrames: DataFrame import DifferentiationInterface as DI using DifferentiationInterface: prepare_pushforward, @@ -92,13 +90,13 @@ using DifferentiationInterface: using DifferentiationInterface: Rewrap, Context, Constant, Cache, ConstantOrCache, unwrap using DifferentiationInterface: PreparationMismatchError using DocStringExtensions: TYPEDFIELDS, TYPEDSIGNATURES -using JET: @test_opt using LinearAlgebra: Adjoint, Diagonal, Transpose, I, dot, parent using PrecompileTools: @compile_workload using ProgressMeter: ProgressUnknown, next! using Random: AbstractRNG, default_rng, rand! using SparseArrays: SparseArrays, AbstractSparseMatrix, SparseMatrixCSC, nnz, sparse, spdiagm +using Tables: Tables, AbstractRow using Test: @testset, @test, @test_throws """ @@ -139,9 +137,8 @@ include("scenarios/extensions.jl") include("tests/correctness_eval.jl") include("tests/prep_eval.jl") -include("tests/type_stability_eval.jl") +include("tests/type_stability.jl") include("tests/benchmark.jl") -include("tests/benchmark_eval.jl") include("tests/allocs_eval.jl") include("test_differentiation.jl") diff --git a/DifferentiationInterfaceTest/src/test_differentiation.jl b/DifferentiationInterfaceTest/src/test_differentiation.jl index daf59b405..af5ef60e9 100644 --- a/DifferentiationInterfaceTest/src/test_differentiation.jl +++ b/DifferentiationInterfaceTest/src/test_differentiation.jl @@ -8,7 +8,7 @@ Apply a list of `backends` on a list of `scenarios`, running a variety of differ This function always creates and runs a `@testset`, though its contents may vary. - if `benchmark == :none`, it returns `nothing`. - - if `benchmark != :none`, it returns a `DataFrame` of benchmark results, whose columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref). + - if `benchmark != :none`, it returns a table of benchmark results, compatible with the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface. # Positional arguments @@ -52,11 +52,15 @@ Each setting tests/benchmarks a different subset of calls: **Type stability options:** +Type stability checks are implemented in a package extension: please call `import JET` beforehand if you want to use them. + - `ignored_modules=nothing`: list of modules that JET.jl should ignore - `function_filter`: filter for functions that JET.jl should ignore (with a reasonable default) **Benchmark options:** +Benchmarking is implemented in a package extension: please call `import Chairmarks` beforehand if you want to use it. + - `count_calls=true`: whether to also count function calls during benchmarking - `benchmark_test=true`: whether to include tests which succeed iff benchmark doesn't error - `benchmark_seconds=1`: how long to run each benchmark for @@ -120,7 +124,7 @@ function test_differentiation( title = testset_name end - benchmark_data = DifferentiationBenchmarkDataRow[] + benchmark_data = DifferentiationBenchmark() prog = ProgressUnknown(; desc = "$title", spinner = true, enabled = logging) diff --git a/DifferentiationInterfaceTest/src/tests/benchmark.jl b/DifferentiationInterfaceTest/src/tests/benchmark.jl index d71a54383..daf0a4377 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark.jl +++ b/DifferentiationInterfaceTest/src/tests/benchmark.jl @@ -21,28 +21,12 @@ function reset_count!(cc::CallCounter) return count end -function failed_bench() - evals = 0.0 - time = NaN - allocs = NaN - bytes = NaN - gc_fraction = NaN - compile_fraction = NaN - recompile_fraction = NaN - warmup = NaN - checksum = NaN - sample = Sample( - evals, - time, - allocs, - bytes, - gc_fraction, - compile_fraction, - recompile_fraction, - warmup, - checksum, - ) - return Benchmark([sample]) +@kwdef struct CallsResult + preparation::Int = -1 + prepared_valop::Int = -1 + prepared_op::Int = -1 + unprepared_valop::Int = -1 + unprepared_op::Int = -1 end """ @@ -56,7 +40,7 @@ $(TYPEDFIELDS) See the documentation of [Chairmarks.jl](https://github.com/LilithHafner/Chairmarks.jl) for more details on the measurement fields. """ -Base.@kwdef struct DifferentiationBenchmarkDataRow{T} +Base.@kwdef struct DifferentiationBenchmarkDataRow{T} <: AbstractRow "backend used for benchmarking" backend::AbstractADType "scenario used for benchmarking" @@ -83,29 +67,36 @@ Base.@kwdef struct DifferentiationBenchmarkDataRow{T} compile_fraction::T end -function record!( - data::Vector{DifferentiationBenchmarkDataRow}; - backend::AbstractADType, - scenario::Scenario, - operator::String, - prepared::Union{Nothing, Bool}, - bench::Benchmark, - calls::Integer, - aggregation, - ) - row = DifferentiationBenchmarkDataRow(; - backend = backend, - scenario = scenario, - operator = Symbol(operator), - prepared = prepared, - calls = calls, - samples = length(bench.samples), - evals = Int(bench.samples[1].evals), - time = aggregation(getfield.(bench.samples, :time)), - allocs = aggregation(getfield.(bench.samples, :allocs)), - bytes = aggregation(getfield.(bench.samples, :bytes)), - gc_fraction = aggregation(getfield.(bench.samples, :gc_fraction)), - compile_fraction = aggregation(getfield.(bench.samples, :compile_fraction)), - ) - return push!(data, row) +Tables.getcolumn(row::DifferentiationBenchmarkDataRow, i::Int) = getfield(row, i) +Tables.getcolumn(row::DifferentiationBenchmarkDataRow, nm::Symbol) = getfield(row, nm) +Tables.columnnames(row::DifferentiationBenchmarkDataRow) = fieldnames(typeof(row)) + +""" + DifferentiationBenchmark + +# Fields + +$(TYPEDFIELDS) +""" +struct DifferentiationBenchmark{T} + rows::Vector{DifferentiationBenchmarkDataRow{T}} end + +function DifferentiationBenchmark() + return DifferentiationBenchmark(DifferentiationBenchmarkDataRow{Float64}[]) +end + +Tables.istable(::Type{DifferentiationBenchmark}) = true +Tables.rowaccess(::Type{DifferentiationBenchmark}) = true +Tables.rows(data::DifferentiationBenchmark) = data.rows + +""" + run_benchmark!(...) + +Perform the actual measurement of preparation and differentiation efficiency. + +!!! warning + Implemented in a package extension that depends on [Chairmarks.jl](https://github.com/LilithHafner/Chairmarks.jl). + If this function fails with a `MethodError`, try `import Chairmarks` before running it again. +""" +function run_benchmark! end diff --git a/DifferentiationInterfaceTest/src/tests/type_stability.jl b/DifferentiationInterfaceTest/src/tests/type_stability.jl new file mode 100644 index 000000000..2f9f04b59 --- /dev/null +++ b/DifferentiationInterfaceTest/src/tests/type_stability.jl @@ -0,0 +1,10 @@ +""" + test_jet(backend, scenario; subset, ignored_modules, function_filter) + +Perform the actual diagnosis of type stability for preparation and differentiatioj. + +!!! warning + Implemented in a package extension that depends on [JET.jl](https://github.com/aviatesk/JET.jl). + If this function fails with a `MethodError`, try `import JET` before running it again. +""" +function test_jet end diff --git a/DifferentiationInterfaceTest/test/Project.toml b/DifferentiationInterfaceTest/test/Project.toml index 416ae978c..57b5a54a4 100644 --- a/DifferentiationInterfaceTest/test/Project.toml +++ b/DifferentiationInterfaceTest/test/Project.toml @@ -1,6 +1,7 @@ [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" @@ -24,10 +25,10 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] ADTypes = "1.18" Aqua = "0.8.12" +Chairmarks = "1.2.1" ComponentArrays = "0.15.30" DataFrames = "1.8.1" DifferentiationInterface = "0.7.10" -DifferentiationInterfaceTest = "0.10.2" ExplicitImports = "1.10.1" FiniteDiff = "2.27.0" FiniteDifferences = "0.12.33" @@ -40,6 +41,3 @@ SparseMatrixColorings = "0.4.9" StaticArrays = "1.9.15" Zygote = "0.7.10" julia = "1.10.10" - -[sources] -DifferentiationInterfaceTest = { path = ".." } diff --git a/DifferentiationInterfaceTest/test/runtests.jl b/DifferentiationInterfaceTest/test/runtests.jl index 4691fee52..82b4bd80e 100644 --- a/DifferentiationInterfaceTest/test/runtests.jl +++ b/DifferentiationInterfaceTest/test/runtests.jl @@ -4,6 +4,8 @@ using DifferentiationInterfaceTest using Pkg using SparseConnectivityTracer using Test +import JET +import Chairmarks using DifferentiationInterfaceTest: default_scenarios, From 52325dc62855325782fe24ea4d0a27bf92db57ad Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Mon, 26 Jan 2026 23:15:34 +0100 Subject: [PATCH 02/12] Fixes --- DifferentiationInterface/test/Project.toml | 2 +- DifferentiationInterfaceTest/src/test_differentiation.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DifferentiationInterface/test/Project.toml b/DifferentiationInterface/test/Project.toml index de83dd6db..bda4eef35 100644 --- a/DifferentiationInterface/test/Project.toml +++ b/DifferentiationInterface/test/Project.toml @@ -29,7 +29,7 @@ ComponentArrays = "0.15.27" DataFrames = "1.7.0" Dates = "1" DifferentiationInterface = "0.7.10" -DifferentiationInterfaceTest = "0.10.3" +DifferentiationInterfaceTest = "0.11.0" ExplicitImports = "1.10.1" InteractiveUtils = "1" JET = "0.9,0.10,0.11" diff --git a/DifferentiationInterfaceTest/src/test_differentiation.jl b/DifferentiationInterfaceTest/src/test_differentiation.jl index af5ef60e9..fd835f2b0 100644 --- a/DifferentiationInterfaceTest/src/test_differentiation.jl +++ b/DifferentiationInterfaceTest/src/test_differentiation.jl @@ -209,7 +209,7 @@ function test_differentiation( end end if benchmark != :none - return DataFrame(benchmark_data) + return benchmark_data else return nothing end From 775ff9599038f03aece905f172c80ab281517f9b Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Mon, 26 Jan 2026 23:47:49 +0100 Subject: [PATCH 03/12] No column access --- DifferentiationInterfaceTest/src/tests/benchmark.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/DifferentiationInterfaceTest/src/tests/benchmark.jl b/DifferentiationInterfaceTest/src/tests/benchmark.jl index daf0a4377..28ee67117 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark.jl +++ b/DifferentiationInterfaceTest/src/tests/benchmark.jl @@ -88,6 +88,7 @@ end Tables.istable(::Type{DifferentiationBenchmark}) = true Tables.rowaccess(::Type{DifferentiationBenchmark}) = true +Tables.columnaccess(::Type{DifferentiationBenchmark}) = false Tables.rows(data::DifferentiationBenchmark) = data.rows """ From c883dc0902db77a972a84606f1b277f256c797bd Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Mon, 26 Jan 2026 23:55:02 +0100 Subject: [PATCH 04/12] JET --- DifferentiationInterface/test/Back/ForwardDiff/Project.toml | 1 + DifferentiationInterface/test/Back/GTPSA/Project.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/DifferentiationInterface/test/Back/ForwardDiff/Project.toml b/DifferentiationInterface/test/Back/ForwardDiff/Project.toml index c780a32bc..591f2723d 100644 --- a/DifferentiationInterface/test/Back/ForwardDiff/Project.toml +++ b/DifferentiationInterface/test/Back/ForwardDiff/Project.toml @@ -5,6 +5,7 @@ DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35" diff --git a/DifferentiationInterface/test/Back/GTPSA/Project.toml b/DifferentiationInterface/test/Back/GTPSA/Project.toml index 432414730..90caf1b17 100644 --- a/DifferentiationInterface/test/Back/GTPSA/Project.toml +++ b/DifferentiationInterface/test/Back/GTPSA/Project.toml @@ -4,6 +4,7 @@ DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" GTPSA = "b27dd330-f138-47c5-815b-40db9dd9b6e8" +JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From f19b406c2b974da9f5b685395e0b07acd19cdcf2 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Mon, 26 Jan 2026 23:56:28 +0100 Subject: [PATCH 05/12] Fix --- DifferentiationInterface/test/Back/ForwardDiff/test.jl | 1 + DifferentiationInterface/test/Back/GTPSA/test.jl | 1 + DifferentiationInterface/test/Core/ZeroBackends/test.jl | 3 ++- DifferentiationInterface/test/testutils.jl | 2 -- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/DifferentiationInterface/test/Back/ForwardDiff/test.jl b/DifferentiationInterface/test/Back/ForwardDiff/test.jl index 5260c9cea..59321b309 100644 --- a/DifferentiationInterface/test/Back/ForwardDiff/test.jl +++ b/DifferentiationInterface/test/Back/ForwardDiff/test.jl @@ -9,6 +9,7 @@ using ForwardDiff: ForwardDiff using StaticArrays: StaticArrays, @SVector using JLArrays: JLArrays using Test +import JET, Chairmarks using ExplicitImports check_no_implicit_imports(DifferentiationInterface) diff --git a/DifferentiationInterface/test/Back/GTPSA/test.jl b/DifferentiationInterface/test/Back/GTPSA/test.jl index 6bdbcacbb..192ad5e8f 100644 --- a/DifferentiationInterface/test/Back/GTPSA/test.jl +++ b/DifferentiationInterface/test/Back/GTPSA/test.jl @@ -3,6 +3,7 @@ include("../../testutils.jl") using DifferentiationInterface, DifferentiationInterfaceTest using GTPSA: GTPSA using Test +import JET, Chairmarks using ExplicitImports check_no_implicit_imports(DifferentiationInterface) diff --git a/DifferentiationInterface/test/Core/ZeroBackends/test.jl b/DifferentiationInterface/test/Core/ZeroBackends/test.jl index 24aeb0d2c..486ac1dc1 100644 --- a/DifferentiationInterface/test/Core/ZeroBackends/test.jl +++ b/DifferentiationInterface/test/Core/ZeroBackends/test.jl @@ -7,7 +7,8 @@ using JLArrays: JLArrays using SparseMatrixColorings using StaticArrays: StaticArrays using Test - +using JET +import JET, Chairmarks zero_backends = [AutoZeroForward(), AutoZeroReverse()] diff --git a/DifferentiationInterface/test/testutils.jl b/DifferentiationInterface/test/testutils.jl index f782551a1..7329f4abf 100644 --- a/DifferentiationInterface/test/testutils.jl +++ b/DifferentiationInterface/test/testutils.jl @@ -14,8 +14,6 @@ using DifferentiationInterfaceTest using SparseConnectivityTracer using SparseMatrixColorings using Test -import JET -import Chairmarks using DifferentiationInterfaceTest: default_scenarios, From 815b5cbb9b4cdc0a77ccb2e58f77cc1d66df3ed0 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:41:35 +0100 Subject: [PATCH 06/12] Fixes --- .../test/Back/FiniteDiff/Project.toml | 1 + .../test/Back/FiniteDiff/benchmark.jl | 3 ++- .../test/Back/ForwardDiff/Project.toml | 2 ++ .../test/Back/ForwardDiff/benchmark.jl | 5 +++-- .../test/Back/GTPSA/test.jl | 2 +- .../test/Core/ZeroBackends/test.jl | 1 - DifferentiationInterfaceTest/Project.toml | 2 ++ .../src/DifferentiationInterfaceTest.jl | 9 ++++---- .../src/tests/benchmark.jl | 21 +++++++++++++------ .../test/zero_backends.jl | 1 + 10 files changed, 32 insertions(+), 15 deletions(-) diff --git a/DifferentiationInterface/test/Back/FiniteDiff/Project.toml b/DifferentiationInterface/test/Back/FiniteDiff/Project.toml index e8c5eb3bc..1f1046491 100644 --- a/DifferentiationInterface/test/Back/FiniteDiff/Project.toml +++ b/DifferentiationInterface/test/Back/FiniteDiff/Project.toml @@ -1,5 +1,6 @@ [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" diff --git a/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl b/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl index cc24f47dc..cec565f66 100644 --- a/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl +++ b/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl @@ -1,6 +1,7 @@ using Pkg using ADTypes: ADTypes +using DataFrames: DataFrame using DifferentiationInterface, DifferentiationInterfaceTest import DifferentiationInterface as DI import DifferentiationInterfaceTest as DIT @@ -21,7 +22,7 @@ using Test benchmark = :prepared, excluded = SECOND_ORDER, logging = LOGGING, - ) + ) |> DataFrame @testset "Analyzing benchmark results" begin @testset "$(row[:scenario])" for row in eachrow(data) @test row[:allocs] == 0 diff --git a/DifferentiationInterface/test/Back/ForwardDiff/Project.toml b/DifferentiationInterface/test/Back/ForwardDiff/Project.toml index 591f2723d..c09a0ec60 100644 --- a/DifferentiationInterface/test/Back/ForwardDiff/Project.toml +++ b/DifferentiationInterface/test/Back/ForwardDiff/Project.toml @@ -1,6 +1,8 @@ [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" diff --git a/DifferentiationInterface/test/Back/ForwardDiff/benchmark.jl b/DifferentiationInterface/test/Back/ForwardDiff/benchmark.jl index a4a0b25f0..ec247057b 100644 --- a/DifferentiationInterface/test/Back/ForwardDiff/benchmark.jl +++ b/DifferentiationInterface/test/Back/ForwardDiff/benchmark.jl @@ -1,6 +1,7 @@ using Pkg using ADTypes: ADTypes +using DataFrames: DataFrame using DifferentiationInterface, DifferentiationInterfaceTest import DifferentiationInterface as DI import DifferentiationInterfaceTest as DIT @@ -18,7 +19,7 @@ using Test benchmark = :prepared, excluded = [:hessian, :pullback], # TODO: figure this out logging = LOGGING, - ) + ) |> DataFrame @testset "Analyzing benchmark results" begin @testset "$(row[:scenario])" for row in eachrow(data) @test row[:allocs] == 0 @@ -40,7 +41,7 @@ end benchmark = :prepared, excluded = SECOND_ORDER, logging = LOGGING, - ) + ) |> DataFrame @testset "Analyzing benchmark results" begin @testset "$(row[:scenario])" for row in eachrow(data) @test row[:allocs] == 0 diff --git a/DifferentiationInterface/test/Back/GTPSA/test.jl b/DifferentiationInterface/test/Back/GTPSA/test.jl index 192ad5e8f..fb074f9f3 100644 --- a/DifferentiationInterface/test/Back/GTPSA/test.jl +++ b/DifferentiationInterface/test/Back/GTPSA/test.jl @@ -3,7 +3,7 @@ include("../../testutils.jl") using DifferentiationInterface, DifferentiationInterfaceTest using GTPSA: GTPSA using Test -import JET, Chairmarks +import JET using ExplicitImports check_no_implicit_imports(DifferentiationInterface) diff --git a/DifferentiationInterface/test/Core/ZeroBackends/test.jl b/DifferentiationInterface/test/Core/ZeroBackends/test.jl index 486ac1dc1..90c8a8037 100644 --- a/DifferentiationInterface/test/Core/ZeroBackends/test.jl +++ b/DifferentiationInterface/test/Core/ZeroBackends/test.jl @@ -8,7 +8,6 @@ using SparseMatrixColorings using StaticArrays: StaticArrays using Test using JET -import JET, Chairmarks zero_backends = [AutoZeroForward(), AutoZeroReverse()] diff --git a/DifferentiationInterfaceTest/Project.toml b/DifferentiationInterfaceTest/Project.toml index 3994b4d29..7fc7d2355 100644 --- a/DifferentiationInterfaceTest/Project.toml +++ b/DifferentiationInterfaceTest/Project.toml @@ -6,6 +6,7 @@ authors = ["Guillaume Dalle", "Adrian Hill"] [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" +DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -38,6 +39,7 @@ ADTypes = "1.7.0" AllocCheck = "0.2" Chairmarks = "1.2.1" ComponentArrays = "0.15" +DataAPI = "1.16.0" DifferentiationInterface = "0.7.7" DocStringExtensions = "0.8,0.9" ForwardDiff = "0.10.36,1" diff --git a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl index 4fbc6abd2..18136665b 100644 --- a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl +++ b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl @@ -16,6 +16,7 @@ using ADTypes: SymbolicMode, mode using AllocCheck: check_allocs +using DataAPI: DataAPI import DifferentiationInterface as DI using DifferentiationInterface: prepare_pushforward, @@ -96,7 +97,7 @@ using ProgressMeter: ProgressUnknown, next! using Random: AbstractRNG, default_rng, rand! using SparseArrays: SparseArrays, AbstractSparseMatrix, SparseMatrixCSC, nnz, sparse, spdiagm -using Tables: Tables, AbstractRow +using Tables: Tables, AbstractRow, AbstractColumns using Test: @testset, @test, @test_throws """ @@ -148,8 +149,8 @@ export Scenario, compute_results export test_differentiation, benchmark_differentiation export DifferentiationBenchmarkDataRow -@compile_workload begin - default_scenarios(; include_constantified = true, include_cachified = true) -end +# @compile_workload begin +# default_scenarios(; include_constantified = true, include_cachified = true) +# end end diff --git a/DifferentiationInterfaceTest/src/tests/benchmark.jl b/DifferentiationInterfaceTest/src/tests/benchmark.jl index 28ee67117..bed333c24 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark.jl +++ b/DifferentiationInterfaceTest/src/tests/benchmark.jl @@ -40,7 +40,7 @@ $(TYPEDFIELDS) See the documentation of [Chairmarks.jl](https://github.com/LilithHafner/Chairmarks.jl) for more details on the measurement fields. """ -Base.@kwdef struct DifferentiationBenchmarkDataRow{T} <: AbstractRow +Base.@kwdef struct DifferentiationBenchmarkDataRow{T} "backend used for benchmarking" backend::AbstractADType "scenario used for benchmarking" @@ -67,10 +67,6 @@ Base.@kwdef struct DifferentiationBenchmarkDataRow{T} <: AbstractRow compile_fraction::T end -Tables.getcolumn(row::DifferentiationBenchmarkDataRow, i::Int) = getfield(row, i) -Tables.getcolumn(row::DifferentiationBenchmarkDataRow, nm::Symbol) = getfield(row, nm) -Tables.columnnames(row::DifferentiationBenchmarkDataRow) = fieldnames(typeof(row)) - """ DifferentiationBenchmark @@ -87,10 +83,23 @@ function DifferentiationBenchmark() end Tables.istable(::Type{DifferentiationBenchmark}) = true +DataAPI.nrow(data::DifferentiationBenchmark) = length(data.rows) +DataAPI.ncol(data::DifferentiationBenchmark) = 12 + Tables.rowaccess(::Type{DifferentiationBenchmark}) = true -Tables.columnaccess(::Type{DifferentiationBenchmark}) = false Tables.rows(data::DifferentiationBenchmark) = data.rows +Tables.getcolumn(row::DifferentiationBenchmarkDataRow, i::Int) = getfield(row, i) +Tables.getcolumn(row::DifferentiationBenchmarkDataRow, nm::Symbol) = getproperty(row, nm) +Tables.columnnames(row::DifferentiationBenchmarkDataRow) = fieldnames(typeof(row)) + +Tables.columnaccess(::Type{DifferentiationBenchmark}) = true +Tables.columns(data::DifferentiationBenchmark) = data + +Tables.getcolumn(cols::DifferentiationBenchmark, i::Int) = getfield.(cols.rows, i) +Tables.getcolumn(cols::DifferentiationBenchmark, nm::Symbol) = getproperty.(cols.rows, nm) +Tables.columnnames(cols::DifferentiationBenchmark) = fieldnames(eltype(cols.rows)) + """ run_benchmark!(...) diff --git a/DifferentiationInterfaceTest/test/zero_backends.jl b/DifferentiationInterfaceTest/test/zero_backends.jl index b4a0f1bd1..6cb1afb7c 100644 --- a/DifferentiationInterfaceTest/test/zero_backends.jl +++ b/DifferentiationInterfaceTest/test/zero_backends.jl @@ -1,4 +1,5 @@ using ADTypes +import Chairmarks, JET using DifferentiationInterface using DifferentiationInterface: AutoZeroForward, AutoZeroReverse using DifferentiationInterfaceTest From 241f4f284818e9c4261b46b89bb34a94442fbb2f Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:11:44 +0100 Subject: [PATCH 07/12] Fix imports --- .../test/Back/FiniteDiff/Project.toml | 1 + .../DifferentiationInterfaceTestJETExt.jl | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/DifferentiationInterface/test/Back/FiniteDiff/Project.toml b/DifferentiationInterface/test/Back/FiniteDiff/Project.toml index 1f1046491..21730280a 100644 --- a/DifferentiationInterface/test/Back/FiniteDiff/Project.toml +++ b/DifferentiationInterface/test/Back/FiniteDiff/Project.toml @@ -1,6 +1,7 @@ [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" diff --git a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl index 84e671d71..774a3f895 100644 --- a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl @@ -1,6 +1,58 @@ module DifferentiationInterfaceTestJETExt using ADTypes: AbstractADType +using DifferentiationInterface: + prepare_pushforward, + prepare_pushforward_same_point, + prepare!_pushforward, + pushforward, + pushforward!, + value_and_pushforward, + value_and_pushforward!, + prepare_pullback, + prepare_pullback_same_point, + prepare!_pullback, + pullback, + pullback!, + value_and_pullback, + value_and_pullback!, + prepare_derivative, + prepare!_derivative, + derivative, + derivative!, + value_and_derivative, + value_and_derivative!, + prepare_gradient, + prepare!_gradient, + gradient, + gradient!, + value_and_gradient, + value_and_gradient!, + prepare_jacobian, + prepare!_jacobian, + jacobian, + jacobian!, + value_and_jacobian, + value_and_jacobian!, + prepare_second_derivative, + prepare!_second_derivative, + second_derivative, + second_derivative!, + value_derivative_and_second_derivative, + value_derivative_and_second_derivative!, + prepare_hvp, + prepare_hvp_same_point, + prepare!_hvp, + hvp, + hvp!, + gradient_and_hvp, + gradient_and_hvp!, + prepare_hessian, + prepare!_hessian, + hessian, + hessian!, + value_gradient_and_hessian, + value_gradient_and_hessian! using DifferentiationInterfaceTest: ALL_OPS, Scenario import DifferentiationInterfaceTest as DIT using JET: @test_opt From 9caa52e75b28fdaaf570b0c6e7d0723502c1c494 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:31:35 +0100 Subject: [PATCH 08/12] Fixes --- DifferentiationInterfaceTest/Project.toml | 2 -- DifferentiationInterfaceTest/docs/Project.toml | 3 +++ .../DifferentiationInterfaceTestChairmarksExt.jl | 2 +- .../DifferentiationInterfaceTestJETExt.jl | 2 +- DifferentiationInterfaceTest/test/Project.toml | 3 +++ DifferentiationInterfaceTest/test/formalities.jl | 1 + 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/DifferentiationInterfaceTest/Project.toml b/DifferentiationInterfaceTest/Project.toml index 7fc7d2355..20ac8126a 100644 --- a/DifferentiationInterfaceTest/Project.toml +++ b/DifferentiationInterfaceTest/Project.toml @@ -14,7 +14,6 @@ PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" @@ -51,7 +50,6 @@ ProgressMeter = "1" Random = "1" SparseArrays = "1" StaticArrays = "1.9" -Statistics = "1" Tables = "1.12.1" Test = "1" Zygote = "0.6, 0.7" diff --git a/DifferentiationInterfaceTest/docs/Project.toml b/DifferentiationInterfaceTest/docs/Project.toml index 5e0465b64..c0613420f 100644 --- a/DifferentiationInterfaceTest/docs/Project.toml +++ b/DifferentiationInterfaceTest/docs/Project.toml @@ -26,3 +26,6 @@ ForwardDiff = "1.2.2" PrettyTables = "3.1" Zygote = "0.7.10" julia = "1.10.10" + +[sources] +DifferentiationInterfaceTest = { path = ".." } diff --git a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl index 64fcc3447..3be1a5357 100644 --- a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl @@ -60,7 +60,7 @@ using DifferentiationInterfaceTest: ALL_OPS, CallCounter, CallsResult, DifferentiationBenchmarkDataRow, DifferentiationBenchmark, Scenario, mysimilar, reset_count! -using Test +using Test: Test, @test function failed_bench() evals = 0.0 diff --git a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl index 774a3f895..40e06046a 100644 --- a/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl +++ b/DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl @@ -53,7 +53,7 @@ using DifferentiationInterface: hessian!, value_gradient_and_hessian, value_gradient_and_hessian! -using DifferentiationInterfaceTest: ALL_OPS, Scenario +using DifferentiationInterfaceTest: ALL_OPS, Scenario, mysimilar import DifferentiationInterfaceTest as DIT using JET: @test_opt diff --git a/DifferentiationInterfaceTest/test/Project.toml b/DifferentiationInterfaceTest/test/Project.toml index 57b5a54a4..aad1ab84f 100644 --- a/DifferentiationInterfaceTest/test/Project.toml +++ b/DifferentiationInterfaceTest/test/Project.toml @@ -41,3 +41,6 @@ SparseMatrixColorings = "0.4.9" StaticArrays = "1.9.15" Zygote = "0.7.10" julia = "1.10.10" + +[sources] +DifferentiationInterfaceTest = { path = ".." } diff --git a/DifferentiationInterfaceTest/test/formalities.jl b/DifferentiationInterfaceTest/test/formalities.jl index 862c0b66d..d0118d36a 100644 --- a/DifferentiationInterfaceTest/test/formalities.jl +++ b/DifferentiationInterfaceTest/test/formalities.jl @@ -5,6 +5,7 @@ using ExplicitImports using JET: JET using SparseMatrixColorings: SparseMatrixColorings using Test +import Chairmarks @testset "Aqua" begin Aqua.test_all(DifferentiationInterfaceTest; ambiguities = false, undocumented_names = true) From 80ec41d625cafbc849a7318db3ed9b6c30a8cce5 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 27 Jan 2026 13:20:50 +0100 Subject: [PATCH 09/12] Fixes --- .../test/Back/FiniteDiff/benchmark.jl | 1 + DifferentiationInterfaceTest/test/formalities.jl | 9 ++++++++- DifferentiationInterfaceTest/test/zero_backends.jl | 11 ++++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl b/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl index cec565f66..9a322c5fe 100644 --- a/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl +++ b/DifferentiationInterface/test/Back/FiniteDiff/benchmark.jl @@ -7,6 +7,7 @@ import DifferentiationInterface as DI import DifferentiationInterfaceTest as DIT using FiniteDiff: FiniteDiff using Test +import Chairmarks @testset "Benchmarking sparse" begin filtered_sparse_scenarios = filter(sparse_scenarios(; band_sizes = [])) do scen diff --git a/DifferentiationInterfaceTest/test/formalities.jl b/DifferentiationInterfaceTest/test/formalities.jl index d0118d36a..0379ed360 100644 --- a/DifferentiationInterfaceTest/test/formalities.jl +++ b/DifferentiationInterfaceTest/test/formalities.jl @@ -13,7 +13,14 @@ end @testset verbose = true "JET" begin # until https://github.com/JuliaLang/julia/pull/59321 is released if VERSION <= v"1.12-" - JET.test_package(DifferentiationInterfaceTest; target_defined_modules = true) + JET.test_package( + DifferentiationInterfaceTest; + target_modules = ( + DifferentiationInterfaceTest, + Base.get_extension(DifferentiationInterfaceTest, :DifferentiationInterfaceTestChairmarksExt), + Base.get_extension(DifferentiationInterfaceTest, :DifferentiationInterfaceTestJETExt), + ) + ) end end diff --git a/DifferentiationInterfaceTest/test/zero_backends.jl b/DifferentiationInterfaceTest/test/zero_backends.jl index 6cb1afb7c..0bdafa3a2 100644 --- a/DifferentiationInterfaceTest/test/zero_backends.jl +++ b/DifferentiationInterfaceTest/test/zero_backends.jl @@ -1,5 +1,6 @@ using ADTypes import Chairmarks, JET +using DataFrames using DifferentiationInterface using DifferentiationInterface: AutoZeroForward, AutoZeroReverse using DifferentiationInterfaceTest @@ -33,7 +34,7 @@ data0 = benchmark_differentiation( AutoZeroForward(), no_matrices(default_scenarios(; include_batchified = false, include_constantified = true)); logging = LOGGING, -); +) |> DataFrame; data1 = benchmark_differentiation( AutoZeroForward(), @@ -42,7 +43,7 @@ data1 = benchmark_differentiation( logging = LOGGING, benchmark_seconds = 0.05, benchmark_aggregation = maximum, -); +) |> DataFrame; struct FakeBackend <: ADTypes.AbstractADType end ADTypes.mode(::FakeBackend) = ADTypes.ForwardMode() @@ -52,7 +53,7 @@ data2 = benchmark_differentiation( no_matrices(default_scenarios(; include_batchified = false)); logging = false, benchmark_test = false, -); +) |> DataFrame; @testset "Benchmarking DataFrame" begin for col in eachcol(data1) @@ -77,14 +78,14 @@ end excluded = [:pullback, :gradient], benchmark = :prepared, logging = LOGGING, - ), + ) |> DataFrame, benchmark_differentiation( AutoZeroReverse(), allocfree_scenarios(); excluded = [:pushforward, :derivative], benchmark = :prepared, logging = LOGGING, - ), + ) |> DataFrame, ) @testset "$(collect(row[1:4]))" for row in collect(eachrow(data_allocfree)) @test row[:allocs] == 0 From 8ca6f2e42d128aa985734bfce4da888b01b9ad8b Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:29:19 +0100 Subject: [PATCH 10/12] Fix and test Tables API --- .../src/tests/benchmark.jl | 6 +-- .../test/Project.toml | 12 +++--- .../test/benchmark.jl | 40 +++++++++++++++++++ DifferentiationInterfaceTest/test/runtests.jl | 1 + 4 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 DifferentiationInterfaceTest/test/benchmark.jl diff --git a/DifferentiationInterfaceTest/src/tests/benchmark.jl b/DifferentiationInterfaceTest/src/tests/benchmark.jl index bed333c24..3dafadeca 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark.jl +++ b/DifferentiationInterfaceTest/src/tests/benchmark.jl @@ -82,18 +82,18 @@ function DifferentiationBenchmark() return DifferentiationBenchmark(DifferentiationBenchmarkDataRow{Float64}[]) end -Tables.istable(::Type{DifferentiationBenchmark}) = true +Tables.istable(::Type{<:DifferentiationBenchmark}) = true DataAPI.nrow(data::DifferentiationBenchmark) = length(data.rows) DataAPI.ncol(data::DifferentiationBenchmark) = 12 -Tables.rowaccess(::Type{DifferentiationBenchmark}) = true +Tables.rowaccess(::Type{<:DifferentiationBenchmark}) = true Tables.rows(data::DifferentiationBenchmark) = data.rows Tables.getcolumn(row::DifferentiationBenchmarkDataRow, i::Int) = getfield(row, i) Tables.getcolumn(row::DifferentiationBenchmarkDataRow, nm::Symbol) = getproperty(row, nm) Tables.columnnames(row::DifferentiationBenchmarkDataRow) = fieldnames(typeof(row)) -Tables.columnaccess(::Type{DifferentiationBenchmark}) = true +Tables.columnaccess(::Type{<:DifferentiationBenchmark}) = true Tables.columns(data::DifferentiationBenchmark) = data Tables.getcolumn(cols::DifferentiationBenchmark, i::Int) = getfield.(cols.rows, i) diff --git a/DifferentiationInterfaceTest/test/Project.toml b/DifferentiationInterfaceTest/test/Project.toml index aad1ab84f..19033f282 100644 --- a/DifferentiationInterfaceTest/test/Project.toml +++ b/DifferentiationInterfaceTest/test/Project.toml @@ -3,13 +3,12 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" -FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" -Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" @@ -19,9 +18,13 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +[sources] +DifferentiationInterfaceTest = {path = ".."} + [compat] ADTypes = "1.18" Aqua = "0.8.12" @@ -31,8 +34,6 @@ DataFrames = "1.8.1" DifferentiationInterface = "0.7.10" ExplicitImports = "1.10.1" FiniteDiff = "2.27.0" -FiniteDifferences = "0.12.33" -Flux = "0.16.5" ForwardDiff = "1.2.2" JET = "0.9, 0.10, 0.11" JLArrays = "0.3" @@ -41,6 +42,3 @@ SparseMatrixColorings = "0.4.9" StaticArrays = "1.9.15" Zygote = "0.7.10" julia = "1.10.10" - -[sources] -DifferentiationInterfaceTest = { path = ".." } diff --git a/DifferentiationInterfaceTest/test/benchmark.jl b/DifferentiationInterfaceTest/test/benchmark.jl new file mode 100644 index 000000000..ed8fd7775 --- /dev/null +++ b/DifferentiationInterfaceTest/test/benchmark.jl @@ -0,0 +1,40 @@ +using Pkg; Pkg.activate(@__DIR__) + +using ADTypes +using DifferentiationInterfaceTest +import DifferentiationInterfaceTest as DIT +using Tables, DataAPI +using Test + +row1 = DIT.DifferentiationBenchmarkDataRow(; + backend = AutoForwardDiff(), + scenario = Scenario{:gradient, :out}(sum, ones(2)), + operator = :gradient, + prepared = true, + calls = 2, + samples = 1, + evals = 1, + time = 1.0, + allocs = 10.0, + bytes = 100.0, + gc_fraction = 0.5, + compile_fraction = 0.1 +) + +tab = DIT.DifferentiationBenchmark([row1, row1]) + +@testset "Tables API" begin + @test Tables.istable(typeof(tab)) + @test Tables.rowaccess(typeof(tab)) + @test Tables.columnaccess(typeof(tab)) + @test DataAPI.nrow(tab) == 2 + @test DataAPI.ncol(tab) == 12 + @test Tables.rows(tab) == tab.rows + @test Tables.columns(tab) == tab + @test Tables.getcolumn(tab, :samples) == [1, 1] + @test Tables.getcolumn(row1, :samples) == 1 + @test Tables.getcolumn(tab, 5) == [2, 2] + @test Tables.getcolumn(row1, 5) == 2 + @test Tables.columnnames(tab) |> length == 12 + @test Tables.columnnames(row1) |> length == 12 +end diff --git a/DifferentiationInterfaceTest/test/runtests.jl b/DifferentiationInterfaceTest/test/runtests.jl index 82b4bd80e..7aa5fdab5 100644 --- a/DifferentiationInterfaceTest/test/runtests.jl +++ b/DifferentiationInterfaceTest/test/runtests.jl @@ -29,6 +29,7 @@ LOGGING = get(ENV, "CI", "false") == "false" if GROUP == "Formalities" || GROUP == "All" @testset verbose = true "Formalities" begin include("formalities.jl") + include("benchmark.jl") end @testset verbose = true "Scenarios" begin include("scenario.jl") From 368086030043a841586766ab1263d50e287c8663 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:19:49 +0100 Subject: [PATCH 11/12] Fix --- DifferentiationInterfaceTest/test/benchmark.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DifferentiationInterfaceTest/test/benchmark.jl b/DifferentiationInterfaceTest/test/benchmark.jl index ed8fd7775..452d62282 100644 --- a/DifferentiationInterfaceTest/test/benchmark.jl +++ b/DifferentiationInterfaceTest/test/benchmark.jl @@ -1,8 +1,7 @@ -using Pkg; Pkg.activate(@__DIR__) - using ADTypes using DifferentiationInterfaceTest import DifferentiationInterfaceTest as DIT +import ForwardDiff using Tables, DataAPI using Test From 4981006d9debf501592dfb957f1a199eb2431651 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:24:36 +0100 Subject: [PATCH 12/12] Fix stuff --- .../src/DifferentiationInterfaceTest.jl | 6 +++--- DifferentiationInterfaceTest/src/tests/benchmark.jl | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl index 18136665b..40f460320 100644 --- a/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl +++ b/DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl @@ -149,8 +149,8 @@ export Scenario, compute_results export test_differentiation, benchmark_differentiation export DifferentiationBenchmarkDataRow -# @compile_workload begin -# default_scenarios(; include_constantified = true, include_cachified = true) -# end +@compile_workload begin + default_scenarios(; include_constantified = true, include_cachified = true) +end end diff --git a/DifferentiationInterfaceTest/src/tests/benchmark.jl b/DifferentiationInterfaceTest/src/tests/benchmark.jl index 3dafadeca..6709c4445 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark.jl +++ b/DifferentiationInterfaceTest/src/tests/benchmark.jl @@ -106,6 +106,7 @@ Tables.columnnames(cols::DifferentiationBenchmark) = fieldnames(eltype(cols.rows Perform the actual measurement of preparation and differentiation efficiency. !!! warning + Implemented in a package extension that depends on [Chairmarks.jl](https://github.com/LilithHafner/Chairmarks.jl). If this function fails with a `MethodError`, try `import Chairmarks` before running it again. """