Better printing of benchmarks (#158)

Author: gdalle

.github/workflows/CI.yml

@@ -48,6 +48,7 @@ jobs:
       - name: Install dependencies (replaces buildpkg)
         run: julia --project=${{ matrix.pkg.dir}}/ -e '
           using Pkg;
+          Pkg.Registry.update();
           if "${{ matrix.pkg.name}}" == "DifferentiationInterfaceTest";
               Pkg.develop(PackageSpec(path="./DifferentiationInterface"));
           end;
@@ -88,6 +89,7 @@ jobs:
       - name: Install dependencies
         run: julia --project=${{ matrix.pkg.dir}}/docs/ -e '
           using Pkg;
+          Pkg.Registry.update();
           if "${{ matrix.pkg.name}}" == "DifferentiationInterface";
               Pkg.develop(PackageSpec(path="./DifferentiationInterface"));
           else;

DifferentiationInterfaceTest/Project.toml

@@ -19,7 +19,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 ADTypes = "0.2.7"
-Chairmarks = "1.2"
+Chairmarks = "1.2.1"
 ComponentArrays = "0.15"
 DocStringExtensions = "0.9"
 JET = "0.8"

DifferentiationInterfaceTest/docs/Project.toml

@@ -8,3 +8,4 @@ Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"

DifferentiationInterfaceTest/docs/src/tutorial.md

@@ -9,7 +9,7 @@ We present a typical workflow with DifferentiationInterfaceTest.jl, building on
 ```@repl tuto
 using DifferentiationInterface, DifferentiationInterfaceTest
 import ForwardDiff, Enzyme
-import DataFrames
+import DataFrames, Markdown, PrettyTables, Printf
 ```
 
 ## Introduction
@@ -84,6 +84,15 @@ Here's what the resulting `DataFrame` looks like with all its columns.
 Note that we only compare (possibly) in-place operators, because they are always more efficient.
 
 ```@example tuto
-import Markdown, PrettyTables  # hide
-Markdown.parse(PrettyTables.pretty_table(String, df; backend=Val(:markdown), header=names(df)))  # hide
+function formatter(v, i, j)
+    if j in (14, 15)  # time, bytes
+        return Printf.@sprintf("%.1e", v)
+    elseif j == 16  # allocs
+        return Printf.@sprintf("%.1f", v)
+    else
+        return v
+    end
+end
+table = PrettyTables.pretty_table(String, df; backend=Val(:markdown), header=names(df), formatters=formatter)
+Markdown.parse(table)  # hide
 ```

DifferentiationInterfaceTest/src/tests/benchmark.jl

@@ -50,12 +50,12 @@ $(TYPEDFIELDS)
     output_size::Tuple
     calls::Int
     samples::Int
+    evals::Int
     time::Float64
     bytes::Float64
     allocs::Float64
     compile_fraction::Float64
     gc_fraction::Float64
-    evals::Float64
 end
 
 function record!(
@@ -80,12 +80,12 @@ function record!(
         output_size=size(scenario.y),
         calls=calls,
         samples=length(bench.samples),
+        evals=Int(bench_min.evals),
         time=bench_min.time,
         bytes=bench_min.bytes,
         allocs=bench_min.allocs,
         compile_fraction=bench_min.compile_fraction,
         gc_fraction=bench_min.gc_fraction,
-        evals=bench_min.evals,
     )
     return push!(data, row)
 end
@@ -98,7 +98,7 @@ function run_benchmark!(
     (; f, x, y, dx) = deepcopy(scen)
     # benchmark
     extras = prepare_pushforward(f, ba, x)
-    bench0 = @be prepare_pushforward(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_pushforward(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(y) value_and_pushforward!!(f, _, ba, x, dx, extras)
     bench2 = @be mysimilar(y) pushforward!!(f, _, ba, x, dx, extras)
     # count
@@ -123,7 +123,7 @@ function run_benchmark!(
     f! = f
     # benchmark
     extras = prepare_pushforward(f!, ba, y, x)
-    bench0 = @be prepare_pushforward(f!, ba, y, x) evals = 1 samples = 2
+    bench0 = @be prepare_pushforward(f!, ba, y, x) evals = 1 samples = 1
     bench1 = @be (mysimilar(y), mysimilar(y)) value_and_pushforward!!(
         f!, _[1], _[2], ba, x, dx, extras
     )
@@ -153,7 +153,7 @@ function run_benchmark!(
     (; f, x, y, dy) = deepcopy(scen)
     # benchmark
     extras = prepare_pullback(f, ba, x)
-    bench0 = @be prepare_pullback(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_pullback(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(x) value_and_pullback!!(f, _, ba, x, dy, extras)
     bench2 = @be mysimilar(x) pullback!!(f, _, ba, x, dy, extras)
     bench3 = @be value_and_pullback!!_split(f, ba, x, extras)
@@ -188,7 +188,7 @@ function run_benchmark!(
     f! = f
     # benchmark
     extras = prepare_pullback(f!, ba, y, x)
-    bench0 = @be prepare_pullback(f!, ba, y, x) evals = 1 samples = 2
+    bench0 = @be prepare_pullback(f!, ba, y, x) evals = 1 samples = 1
     bench1 = @be (mysimilar(y), mysimilar(x)) value_and_pullback!!(
         f!, _[1], _[2], ba, x, dy, extras
     )
@@ -216,7 +216,7 @@ function run_benchmark!(
     (; f, x, y) = deepcopy(scen)
     # benchmark
     extras = prepare_derivative(f, ba, x)
-    bench0 = @be prepare_derivative(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_derivative(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(y) value_and_derivative!!(f, _, ba, x, extras)
     bench2 = @be mysimilar(y) derivative!!(f, _, ba, x, extras)
     # count
@@ -241,7 +241,7 @@ function run_benchmark!(
     f! = f
     # benchmark
     extras = prepare_derivative(f!, ba, y, x)
-    bench0 = @be prepare_derivative(f!, ba, y, x) evals = 1 samples = 2
+    bench0 = @be prepare_derivative(f!, ba, y, x) evals = 1 samples = 1
     bench1 = @be (mysimilar(y), mysimilar(y)) value_and_derivative!!(
         f!, _[1], _[2], ba, x, extras
     )
@@ -265,7 +265,7 @@ function run_benchmark!(
     (; f, x) = deepcopy(scen)
     # benchmark
     extras = prepare_gradient(f, ba, x)
-    bench0 = @be prepare_gradient(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_gradient(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(x) value_and_gradient!!(f, _, ba, x, extras)
     bench2 = @be mysimilar(x) gradient!!(f, _, ba, x, extras)
     # count
@@ -292,7 +292,7 @@ function run_benchmark!(
     jac_template = Matrix{eltype(y)}(undef, length(y), length(x))
     # benchmark
     extras = prepare_jacobian(f, ba, x)
-    bench0 = @be prepare_jacobian(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_jacobian(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(jac_template) value_and_jacobian!!(f, _, ba, x, extras)
     bench2 = @be mysimilar(jac_template) jacobian!!(f, _, ba, x, extras)
     # count
@@ -318,7 +318,7 @@ function run_benchmark!(
     jac_template = Matrix{eltype(y)}(undef, length(y), length(x))
     # benchmark
     extras = prepare_jacobian(f!, ba, y, x)
-    bench0 = @be prepare_jacobian(f!, ba, y, x) evals = 1 samples = 2
+    bench0 = @be prepare_jacobian(f!, ba, y, x) evals = 1 samples = 1
     bench1 = @be (mysimilar(y), mysimilar(jac_template)) value_and_jacobian!!(
         f!, _[1], _[2], ba, x, extras
     )
@@ -344,7 +344,7 @@ function run_benchmark!(
     (; f, x, y) = deepcopy(scen)
     # benchmark
     extras = prepare_second_derivative(f, ba, x)
-    bench0 = @be prepare_second_derivative(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_second_derivative(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(y) second_derivative!!(f, _, ba, x, extras)
     # count
     cc = CallCounter(f)
@@ -366,7 +366,7 @@ function run_benchmark!(
     (; f, x, y, dx) = deepcopy(scen)
     # benchmark
     extras = prepare_hvp(f, ba, x, dx)
-    bench0 = @be prepare_hvp(f, ba, x, dx) evals = 1 samples = 2
+    bench0 = @be prepare_hvp(f, ba, x, dx) evals = 1 samples = 1
     bench1 = @be mysimilar(x) hvp!!(f, _, ba, x, dx, extras)
     # count
     cc = CallCounter(f)
@@ -389,7 +389,7 @@ function run_benchmark!(
     hess_template = Matrix{typeof(y)}(undef, length(x), length(x))
     # benchmark
     extras = prepare_hessian(f, ba, x)
-    bench0 = @be prepare_hessian(f, ba, x) evals = 1 samples = 2
+    bench0 = @be prepare_hessian(f, ba, x) evals = 1 samples = 1
     bench1 = @be mysimilar(hess_template) hessian!!(f, _, ba, x, extras)
     # count
     cc = CallCounter(f)