Skip to content

Commit 39a32d6

Browse files
gdallegdalle
committed
fix!: turn Chairmarks and JET into weakdeps
1 parent 8d33550 commit 39a32d6

16 files changed

Lines changed: 250 additions & 108 deletions

File tree

‎DifferentiationInterface/test/Project.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
[deps]
22
ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
33
Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
4+
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
45
ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
56
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
67
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -23,6 +24,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2324
[compat]
2425
ADTypes = "1.18"
2526
Aqua = "0.8.12"
27+
Chairmarks = "1.2.1"
2628
ComponentArrays = "0.15.27"
2729
DataFrames = "1.7.0"
2830
Dates = "1"
@@ -42,4 +44,4 @@ Test = "1"
4244
julia = "1.10.10"
4345

4446
[sources]
45-
DifferentiationInterface = { path = ".." }
47+
DifferentiationInterface = { path = ".." }

‎DifferentiationInterface/test/testutils.jl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ using DifferentiationInterfaceTest
1414
using SparseConnectivityTracer
1515
using SparseMatrixColorings
1616
using Test
17+
import JET
18+
import Chairmarks
1719

1820
using DifferentiationInterfaceTest:
1921
default_scenarios,

‎DifferentiationInterfaceTest/CHANGELOG.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8-
## [Unreleased](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.4...main)
8+
## [Unreleased](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.11.0...main)
9+
10+
## [0.11.0](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.4...DifferentiationInterfaceTest-v0.11.0)
11+
12+
### Changed
13+
14+
- Benchmarking and type stability analysis are now implemented in package extensions, which depend on Chairmarks and JET respectively
15+
- Benchmarking no longer returns a `DataFrame` but a Tables.jl-compatible object.
916

1017
## [0.10.4](https://github.com/JuliaDiff/DifferentiationInterface.jl/compare/DifferentiationInterfaceTest-v0.10.3...DifferentiationInterfaceTest-v0.10.4)
1118

‎DifferentiationInterfaceTest/Project.toml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,35 @@
11
name = "DifferentiationInterfaceTest"
22
uuid = "a82114a7-5aa3-49a8-9643-716bb13727a3"
3+
version = "0.11.0"
34
authors = ["Guillaume Dalle", "Adrian Hill"]
4-
version = "0.10.4"
55

66
[deps]
77
ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
88
AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
9-
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
10-
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
119
DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
1210
DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
13-
JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
1411
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1512
PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
1613
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
1714
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
1815
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
1916
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
17+
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
2018
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2119

2220
[weakdeps]
21+
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
2322
ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
2423
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
24+
JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
2525
JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb"
2626
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
2727
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
2828

2929
[extensions]
30+
DifferentiationInterfaceTestChairmarksExt = "Chairmarks"
3031
DifferentiationInterfaceTestComponentArraysExt = "ComponentArrays"
32+
DifferentiationInterfaceTestJETExt = "JET"
3133
DifferentiationInterfaceTestJLArraysExt = "JLArrays"
3234
DifferentiationInterfaceTestStaticArraysExt = "StaticArrays"
3335

@@ -36,7 +38,6 @@ ADTypes = "1.7.0"
3638
AllocCheck = "0.2"
3739
Chairmarks = "1.2.1"
3840
ComponentArrays = "0.15"
39-
DataFrames = "1.6.1"
4041
DifferentiationInterface = "0.7.7"
4142
DocStringExtensions = "0.8,0.9"
4243
ForwardDiff = "0.10.36,1"
@@ -49,6 +50,7 @@ Random = "1"
4950
SparseArrays = "1"
5051
StaticArrays = "1.9"
5152
Statistics = "1"
53+
Tables = "1.12.1"
5254
Test = "1"
5355
Zygote = "0.6, 0.7"
5456
julia = "1.10"

‎DifferentiationInterfaceTest/docs/Project.toml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
[deps]
22
ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
33
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
4+
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
45
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
56
DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
67
DifferentiationInterfaceTest = "a82114a7-5aa3-49a8-9643-716bb13727a3"
@@ -15,15 +16,13 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
1516
[compat]
1617
ADTypes = "1.18"
1718
BenchmarkTools = "1.6.3"
19+
Chairmarks = "1.2.1"
1820
DataFrames = "1.8.1"
1921
DifferentiationInterface = "0.7.10"
20-
DifferentiationInterfaceTest = "0.10.2"
22+
DifferentiationInterfaceTest = "0.11.0"
2123
Documenter = "1"
2224
DocumenterInterLinks = "1.1"
2325
ForwardDiff = "1.2.2"
2426
PrettyTables = "3.1"
2527
Zygote = "0.7.10"
2628
julia = "1.10.10"
27-
28-
[sources]
29-
DifferentiationInterfaceTest = { path = ".." }

‎DifferentiationInterfaceTest/docs/src/tutorial.md‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
We present a typical workflow with DifferentiationInterfaceTest.jl, building on the tutorial of the [DifferentiationInterface.jl documentation](https://juliadiff.org/DifferentiationInterface.jl/DifferentiationInterface) (which we encourage you to read first).
44

55
```@repl tuto
6+
import Chairmarks
7+
using DataFrames
68
using DifferentiationInterface, DifferentiationInterfaceTest
79
using ForwardDiff: ForwardDiff
810
using Zygote: Zygote
@@ -30,12 +32,12 @@ Of course we know the true gradient mapping:
3032

3133
DifferentiationInterfaceTest.jl relies with so-called [`Scenario`](@ref)s, in which you encapsulate the information needed for your test:
3234

33-
- the operator category (here `:gradient`)
34-
- the behavior of the operator (either `:in` or `:out` of place)
35-
- the function `f`
36-
- the input `x` of the function `f` (and possible tangents or contexts)
37-
- the reference first-order result `res1` (and possible second-order result `res2`) of the operator
38-
- the arguments `prep_args` passed during preparation
35+
- the operator category (here `:gradient`)
36+
- the behavior of the operator (either `:in` or `:out` of place)
37+
- the function `f`
38+
- the input `x` of the function `f` (and possible tangents or contexts)
39+
- the reference first-order result `res1` (and possible second-order result `res2`) of the operator
40+
- the arguments `prep_args` passed during preparation
3941

4042
```@example tuto
4143
xv = rand(Float32, 3)
@@ -68,7 +70,12 @@ Once you are confident that your backends give the correct answers, you probably
6870
This is made easy by the [`benchmark_differentiation`](@ref) function, whose syntax should feel familiar:
6971

7072
```@example tuto
71-
df = benchmark_differentiation(backends, scenarios);
73+
table = benchmark_differentiation(backends, scenarios);
7274
```
7375

74-
The resulting object is a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl), whose columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref).
76+
The resulting object is a table, which can easily be converted into a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl).
77+
Its columns correspond to the fields of [`DifferentiationBenchmarkDataRow`](@ref).
78+
79+
```@example tuto
80+
df = DataFrame(table)
81+
```

‎DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/DifferentiationInterfaceTestChairmarksExt.jl‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
module DifferentiationInterfaceTestChairmarksExt
2+
3+
using ADTypes: AbstractADType
4+
using Chairmarks: @be, Benchmark, Sample
5+
import DifferentiationInterface as DI
6+
using DifferentiationInterface:
7+
prepare_pushforward,
8+
prepare_pushforward_same_point,
9+
prepare!_pushforward,
10+
pushforward,
11+
pushforward!,
12+
value_and_pushforward,
13+
value_and_pushforward!,
14+
prepare_pullback,
15+
prepare_pullback_same_point,
16+
prepare!_pullback,
17+
pullback,
18+
pullback!,
19+
value_and_pullback,
20+
value_and_pullback!,
21+
prepare_derivative,
22+
prepare!_derivative,
23+
derivative,
24+
derivative!,
25+
value_and_derivative,
26+
value_and_derivative!,
27+
prepare_gradient,
28+
prepare!_gradient,
29+
gradient,
30+
gradient!,
31+
value_and_gradient,
32+
value_and_gradient!,
33+
prepare_jacobian,
34+
prepare!_jacobian,
35+
jacobian,
36+
jacobian!,
37+
value_and_jacobian,
38+
value_and_jacobian!,
39+
prepare_second_derivative,
40+
prepare!_second_derivative,
41+
second_derivative,
42+
second_derivative!,
43+
value_derivative_and_second_derivative,
44+
value_derivative_and_second_derivative!,
45+
prepare_hvp,
46+
prepare_hvp_same_point,
47+
prepare!_hvp,
48+
hvp,
49+
hvp!,
50+
gradient_and_hvp,
51+
gradient_and_hvp!,
52+
prepare_hessian,
53+
prepare!_hessian,
54+
hessian,
55+
hessian!,
56+
value_gradient_and_hessian,
57+
value_gradient_and_hessian!
58+
import DifferentiationInterfaceTest as DIT
59+
using DifferentiationInterfaceTest:
60+
ALL_OPS,
61+
CallCounter, CallsResult, DifferentiationBenchmarkDataRow, DifferentiationBenchmark, Scenario,
62+
mysimilar, reset_count!
63+
using Test
64+
65+
function failed_bench()
66+
evals = 0.0
67+
time = NaN
68+
allocs = NaN
69+
bytes = NaN
70+
gc_fraction = NaN
71+
compile_fraction = NaN
72+
recompile_fraction = NaN
73+
warmup = NaN
74+
checksum = NaN
75+
sample = Sample(
76+
evals,
77+
time,
78+
allocs,
79+
bytes,
80+
gc_fraction,
81+
compile_fraction,
82+
recompile_fraction,
83+
warmup,
84+
checksum,
85+
)
86+
return Benchmark([sample])
87+
end
88+
89+
@kwdef struct BenchmarkResult
90+
prepared_valop::Benchmark = failed_bench()
91+
prepared_op::Benchmark = failed_bench()
92+
preparation::Benchmark = failed_bench()
93+
unprepared_valop::Benchmark = failed_bench()
94+
unprepared_op::Benchmark = failed_bench()
95+
end
96+
97+
98+
function record!(
99+
data::DifferentiationBenchmark;
100+
backend::AbstractADType,
101+
scenario::Scenario,
102+
operator::String,
103+
prepared::Union{Nothing, Bool},
104+
bench::Benchmark,
105+
calls::Integer,
106+
aggregation,
107+
)
108+
row = DifferentiationBenchmarkDataRow(;
109+
backend = backend,
110+
scenario = scenario,
111+
operator = Symbol(operator),
112+
prepared = prepared,
113+
calls = calls,
114+
samples = length(bench.samples),
115+
evals = Int(bench.samples[1].evals),
116+
time = aggregation(getfield.(bench.samples, :time)),
117+
allocs = aggregation(getfield.(bench.samples, :allocs)),
118+
bytes = aggregation(getfield.(bench.samples, :bytes)),
119+
gc_fraction = aggregation(getfield.(bench.samples, :gc_fraction)),
120+
compile_fraction = aggregation(getfield.(bench.samples, :compile_fraction)),
121+
)
122+
return push!(data.rows, row)
123+
end
124+
125+
include("benchmark_eval.jl")
126+
127+
end

DifferentiationInterfaceTest/src/tests/benchmark_eval.jl renamed to DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestChairmarksExt/benchmark_eval.jl

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,3 @@
1-
@kwdef struct BenchmarkResult
2-
prepared_valop::Benchmark = failed_bench()
3-
prepared_op::Benchmark = failed_bench()
4-
preparation::Benchmark = failed_bench()
5-
unprepared_valop::Benchmark = failed_bench()
6-
unprepared_op::Benchmark = failed_bench()
7-
end
8-
9-
@kwdef struct CallsResult
10-
preparation::Int = -1
11-
prepared_valop::Int = -1
12-
prepared_op::Int = -1
13-
unprepared_valop::Int = -1
14-
unprepared_op::Int = -1
15-
end
16-
171
for op in ALL_OPS
182
op! = Symbol(op, "!")
193
val_prefix = if op == :second_derivative
@@ -34,8 +18,8 @@ for op in ALL_OPS
3418
S2out = Scenario{op, :out, :in}
3519
S2in = Scenario{op, :in, :in}
3620

37-
@eval function run_benchmark!(
38-
data::Vector{DifferentiationBenchmarkDataRow},
21+
@eval function DIT.run_benchmark!(
22+
data::DifferentiationBenchmark,
3923
backend::AbstractADType,
4024
scenario::Union{$S1out, $S1in, $S2out, $S2in};
4125
logging::Bool,

‎DifferentiationInterfaceTest/ext/DifferentiationInterfaceTestJETExt/DifferentiationInterfaceTestJETExt.jl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
module DifferentiationInterfaceTestJETExt
2+
3+
using ADTypes: AbstractADType
4+
using DifferentiationInterfaceTest: ALL_OPS, Scenario
5+
import DifferentiationInterfaceTest as DIT
6+
using JET: @test_opt
7+
8+
include("type_stability_eval.jl")
9+
10+
end

0 commit comments

Comments
 (0)