Skip to content

Commit e14bdfc

Browse files
gdallegdalle
authored
Simplify benchmark table (#257)
* Simplify benchmark table * Fix uses of benchmarking
1 parent 65e449d commit e14bdfc

10 files changed

Lines changed: 154 additions & 155 deletions

File tree

‎DifferentiationInterface/test/Single/ForwardDiff.jl‎

Lines changed: 1 addition & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -24,80 +24,6 @@ test_differentiation(
2424
logging=LOGGING,
2525
);
2626

27-
## Efficiency
28-
2927
if VERSION >= v"1.10"
30-
@testset verbose = false "Dense efficiency" begin
31-
# derivative and gradient for `f(x)`
32-
33-
results1 = benchmark_differentiation(
34-
[AutoForwardDiff()];
35-
outofplace=false,
36-
twoarg=false,
37-
input_type=Union{Number,AbstractVector},
38-
output_type=Number,
39-
second_order=false,
40-
excluded=[PullbackScenario],
41-
logging=get(ENV, "CI", "false") == "false",
42-
)
43-
44-
# derivative and jacobian for f!(x, y)
45-
46-
results2 = benchmark_differentiation(
47-
[AutoForwardDiff()];
48-
outofplace=false,
49-
onearg=false,
50-
input_type=Union{Number,AbstractVector},
51-
output_type=AbstractVector,
52-
second_order=false,
53-
excluded=[PullbackScenario],
54-
logging=get(ENV, "CI", "false") == "false",
55-
)
56-
57-
data = vcat(DataFrame(results1), DataFrame(results2))
58-
59-
useless_rows =
60-
startswith.(string.(data[!, :operator]), Ref("prepare")) .|
61-
startswith.(string.(data[!, :operator]), Ref("value_and"))
62-
63-
useful_data = data[.!useless_rows, :]
64-
65-
@testset "$(row[:operator]) - $(row[:func]) : $(row[:input_type]) -> $(row[:output_type])" for row in
66-
eachrow(
67-
useful_data
68-
)
69-
@test row[:allocs] == 0
70-
end
71-
end
72-
73-
@testset verbose = false "Sparse efficiency" begin
74-
# sparse jacobian for f!(x, y)
75-
76-
b_sparse = MyAutoSparse(AutoForwardDiff(; chunksize=1);)
77-
78-
results1 = benchmark_differentiation(
79-
[b_sparse],
80-
sparse_scenarios();
81-
input_type=AbstractVector,
82-
output_type=AbstractVector,
83-
outofplace=false,
84-
onearg=false,
85-
second_order=false,
86-
logging=get(ENV, "CI", "false") == "false",
87-
)
88-
89-
data = vcat(DataFrame(results1))
90-
91-
useless_rows = startswith.(string.(data[!, :operator]), Ref("prepare"))
92-
93-
useful_data = data[.!useless_rows, :]
94-
95-
@testset "$(row[:operator]) - $(row[:func]) : $(row[:input_type]) -> $(row[:output_type])" for row in
96-
eachrow(
97-
useful_data
98-
)
99-
@test row[:allocs] == 0
100-
@test row[:calls] < prod(row[:input_size])
101-
end
102-
end
28+
include("ForwardDiff/efficiency.jl")
10329
end

‎DifferentiationInterface/test/Single/ForwardDiff/efficiency.jl‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
using DifferentiationInterface, DifferentiationInterfaceTest
2+
using ForwardDiff: ForwardDiff
3+
using DataFrames: DataFrame
4+
5+
@testset verbose = false "Dense efficiency" begin
6+
# derivative and gradient for `f(x)`
7+
8+
results1 = benchmark_differentiation(
9+
[AutoForwardDiff()],
10+
default_scenarios();
11+
outofplace=false,
12+
twoarg=false,
13+
input_type=Union{Number,AbstractVector},
14+
output_type=Number,
15+
second_order=false,
16+
excluded=[PullbackScenario],
17+
logging=get(ENV, "CI", "false") == "false",
18+
)
19+
20+
# derivative and jacobian for f!(x, y)
21+
22+
results2 = benchmark_differentiation(
23+
[AutoForwardDiff()],
24+
default_scenarios();
25+
outofplace=false,
26+
onearg=false,
27+
input_type=Union{Number,AbstractVector},
28+
output_type=AbstractVector,
29+
second_order=false,
30+
excluded=[PullbackScenario],
31+
logging=get(ENV, "CI", "false") == "false",
32+
)
33+
34+
data = vcat(DataFrame(results1), DataFrame(results2))
35+
36+
useless_rows =
37+
startswith.(string.(data[!, :operator]), Ref("prepare")) .|
38+
startswith.(string.(data[!, :operator]), Ref("value_and"))
39+
40+
useful_data = data[.!useless_rows, :]
41+
42+
for row in eachrow(useful_data)
43+
scen = row[:scenario]
44+
@testset "$(row[:operator]) - $(string(scen.f)) : $(typeof(scen.x)) -> $(typeof(scen.y))" begin
45+
@test row[:allocs] == 0
46+
end
47+
end
48+
end
49+
50+
@testset verbose = false "Sparse efficiency" begin
51+
# sparse jacobian for f!(x, y)
52+
53+
results1 = benchmark_differentiation(
54+
[MyAutoSparse(AutoForwardDiff(; chunksize=1);)],
55+
sparse_scenarios();
56+
input_type=AbstractVector,
57+
output_type=AbstractVector,
58+
outofplace=false,
59+
onearg=false,
60+
second_order=false,
61+
logging=get(ENV, "CI", "false") == "false",
62+
)
63+
64+
data = vcat(DataFrame(results1))
65+
66+
useless_rows = startswith.(string.(data[!, :operator]), Ref("prepare"))
67+
68+
useful_data = data[.!useless_rows, :]
69+
70+
for row in eachrow(useful_data)
71+
scen = row[:scenario]
72+
@testset "$(row[:operator]) - $(string(scen.f)) : $(typeof(scen.x)) -> $(typeof(scen.y))" begin
73+
@test row[:allocs] == 0
74+
@test row[:calls] < prod(size(scen.x))
75+
end
76+
end
77+
end

‎DifferentiationInterface/test/runtests.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ LOGGING = get(ENV, "CI", "false") == "false"
6161
@testset verbose = true "$folder" for folder in ["Single", "Double", "Internals"]
6262
folder_path = joinpath(@__DIR__, folder)
6363
@testset verbose = true "$(file[1:end-3])" for file in readdir(folder_path)
64+
endswith(file, ".jl") || continue
6465
if (
6566
VERSION < v"1.10" && any(
6667
part == backend for part in split(file[1:(end - 3)], '-') for

‎DifferentiationInterfaceTest/docs/src/api.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ DifferentiationInterfaceTest
1414
```@docs
1515
test_differentiation
1616
benchmark_differentiation
17+
DifferentiationBenchmarkDataRow
1718
```
1819

1920
## Pre-made scenario lists

‎DifferentiationInterfaceTest/docs/src/tutorial.md‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ This is made easy by the [`benchmark_differentiation`](@ref) function, whose syn
7373
benchmark_result = benchmark_differentiation(backends, scenarios);
7474
```
7575

76-
The resulting object is a `Vector` of structs, which can easily be converted into a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl):
76+
The resulting object is a `Vector` of [`DifferentiationBenchmarkDataRow`](@ref), which can easily be converted into a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl):
7777

7878
```@repl tuto
7979
df = DataFrames.DataFrame(benchmark_result)
@@ -83,22 +83,11 @@ Here's what the resulting `DataFrame` looks like with all its columns.
8383
Note that we only compare (possibly) in-place operators, because they are always more efficient.
8484

8585
```@example tuto
86-
function formatter(v, i, j)
87-
if j in (15, 16) # time, bytes
88-
return Printf.@sprintf("%.1e", v)
89-
elseif j == 17 # allocs
90-
return Printf.@sprintf("%.1f", v)
91-
else
92-
return v
93-
end
94-
end
95-
9686
table = PrettyTables.pretty_table(
9787
String,
9888
df;
9989
backend=Val(:markdown),
10090
header=names(df),
101-
formatters=formatter
10291
)
10392
10493
Markdown.parse(table)

‎DifferentiationInterfaceTest/src/DifferentiationInterfaceTest.jl‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ using ADTypes:
1818
ReverseMode,
1919
SymbolicMode
2020
using Chairmarks: @be, Benchmark, Sample
21+
using Compat
2122
using ComponentArrays: ComponentVector
2223
using DifferentiationInterface
2324
using DifferentiationInterface:
@@ -45,7 +46,6 @@ using JET: JET
4546
using JLArrays: jl
4647
using LinearAlgebra: Diagonal, dot
4748
using ProgressMeter: ProgressUnknown, next!
48-
using Compat
4949
using SparseArrays: SparseArrays, SparseMatrixCSC, nnz, spdiagm
5050
using StaticArrays: MMatrix, MVector, SMatrix, SVector
5151
using Test: @testset, @test
@@ -82,5 +82,6 @@ export PushforwardScenario,
8282
export default_scenarios, sparse_scenarios
8383
export static_scenarios, component_scenarios, gpu_scenarios
8484
export test_differentiation, benchmark_differentiation
85+
export DifferentiationBenchmarkDataRow
8586

8687
end

‎DifferentiationInterfaceTest/src/scenarios/scenario.jl‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,10 @@ function group_by_scen_type(scenarios)
6161
)
6262
end
6363

64-
function Base.string(scen::S) where {args,place,F,X,Y,S<:AbstractScenario{args,place,F,X,Y}}
65-
return "$(S.name.name){$args,$place} $(string(scen.f)) : $X -> $Y"
64+
function Base.print(
65+
io::IO, scen::S
66+
) where {args,place,F,X,Y,S<:AbstractScenario{args,place,F,X,Y}}
67+
return print(io, "$(nameof(S)){$args,$place}($(string(scen.f)) : $X -> $Y)")
6668
end
6769

6870
## Struct definitions

‎DifferentiationInterfaceTest/src/test_differentiation.jl‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,14 +132,13 @@ $(TYPEDSIGNATURES)
132132
133133
Benchmark a list of `backends` for a list of `operators` on a list of `scenarios`.
134134
135-
# Keyword arguments
135+
The object returned is a `Vector` of [`DifferentiationBenchmarkDataRow`](@ref).
136136
137-
- filtering: same as [`test_differentiation`](@ref) for the filtering part.
138-
- `logging=false`: whether to log progress
137+
The keyword arguments available here have the same meaning as those in [`test_differentiation`](@ref).
139138
"""
140139
function benchmark_differentiation(
141140
backends::Vector{<:AbstractADType},
142-
scenarios::Vector{<:AbstractScenario}=default_scenarios();
141+
scenarios::Vector{<:AbstractScenario};
143142
# filtering
144143
input_type::Type=Any,
145144
output_type::Type=Any,
@@ -166,7 +165,7 @@ function benchmark_differentiation(
166165
excluded,
167166
)
168167

169-
benchmark_data = BenchmarkDataRow[]
168+
benchmark_data = DifferentiationBenchmarkDataRow[]
170169
prog = ProgressUnknown(; desc="Benchmarking", spinner=true, enabled=logging)
171170
for (i, backend) in enumerate(backends)
172171
filtered_scenarios = filter(s -> compatible(backend, s), scenarios)

0 commit comments

Comments
 (0)