From 10acafb11d6df61503d1794871bf9345ec7c20a4 Mon Sep 17 00:00:00 2001
From: STEVAN Antoine <antoine.stevan@isae-supaero.fr>
Date: Wed, 24 Apr 2024 10:36:55 +0000
Subject: [PATCH] write manual atomic benchmarks (dragoon/komodo!82)
this idea is to not use `criterion` and measure exactly what we want
## results
---
Cargo.toml | 20 +--
benches/README.md | 67 ++++------
benches/operations/curve_group.rs | 112 ----------------
benches/operations/field.rs | 132 ------------------
examples/benches/operations/curve_group.rs | 138 +++++++++++++++++++
examples/benches/operations/field.rs | 148 +++++++++++++++++++++
6 files changed, 322 insertions(+), 295 deletions(-)
delete mode 100644 benches/operations/curve_group.rs
delete mode 100644 benches/operations/field.rs
create mode 100644 examples/benches/operations/curve_group.rs
create mode 100644 examples/benches/operations/field.rs
diff --git a/Cargo.toml b/Cargo.toml
index 8ac0f15e..09da5682 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,16 +66,6 @@ harness = false
name = "commit"
harness = false
-[[bench]]
-name = "field_operations"
-path = "benches/operations/field.rs"
-harness = false
-
-[[bench]]
-name = "curve_group_operations"
-path = "benches/operations/curve_group.rs"
-harness = false
-
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
@@ -83,3 +73,13 @@ path = "examples/benches/commit.rs"
[[example]]
name = "bench_setup_size"
path = "examples/benches/setup_size.rs"
+
+[[example]]
+name = "bench_field_operations"
+path = "examples/benches/operations/field.rs"
+harness = false
+
+[[example]]
+name = "bench_curve_group_operations"
+path = "examples/benches/operations/curve_group.rs"
+harness = false
diff --git a/benches/README.md b/benches/README.md
index 48ac922f..cf3482aa 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -16,52 +16,30 @@ python scripts/plot/benches.py results.ndjson --bench setup
## atomic operations
```nushell
-cargo criterion --output-format verbose --message-format json --bench field_operations out> field.ndjson
-cargo criterion --output-format verbose --message-format json --bench curve_group_operations out> curve.ndjson
+cargo run --example bench_field_operations -- --nb-measurements 1000
+ | lines
+ | each { from nuon }
+ | to ndjson
+ | save --force field.ndjson
+cargo run --example bench_curve_group_operations -- --nb-measurements 1000
+ | lines
+ | each { from nuon }
+ | to ndjson
+ | save --force curve_group.ndjson
```
```nushell
def read-atomic-ops [
- --clean, --include: list<string> = [], --exclude: list<string> = []
+ --include: list<string> = [], --exclude: list<string> = []
]: list -> record {
let raw = $in
- | where reason == "benchmark-complete"
- | select id mean.estimate
- | sort-by id
- | update id { parse "{op} on {curve}" }
- | flatten --all
- | rename --column { op: "group", curve: "species", mean_estimate: "measurement" }
-
- let clean = if $clean {
- $raw | update measurement {|it|
- let species = $it.species
- # FIXME: bug when no parentheses
- let r = (
- $raw
- | where group == 'random sampling' and species == $species
- | into record
- | get measurement
- )
- let l = (
- $raw
- | where group == 'legendre' and species == $species
- | into record
- | get measurement
- )
- match $it.group {
- "addition" | "multiplication" | "substraction" => ($it.measurement - 2 * $r),
- "random sampling" => $it.measurement,
- "sqrt" => ($it.measurement - $r - $l),
- _ => ($it.measurement - $r),
- }
- }
- } else {
- $raw
- }
+ | insert t {|it| $it.times |math avg}
+ | reject times
+ | rename --column { op: "group", curve: "species", t: "measurement" }
let included = if $include != [] {
- $clean | where group in $include
+ $raw | where group in $include
} else {
- $clean
+ $raw
}
$included
@@ -76,7 +54,7 @@ def read-atomic-ops [
```nushell
python scripts/plot/multi_bar.py --title "simple field operations" -l "time (in ns)" (
open field.ndjson
- | read-atomic-ops --clean --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
+ | read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
| to json
)
python scripts/plot/multi_bar.py --title "complex field operations" -l "time (in ns)" (
@@ -84,8 +62,15 @@ python scripts/plot/multi_bar.py --title "complex field operations" -l "time (in
| read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ]
| to json
)
-python scripts/plot/multi_bar.py --title "curve group operations" -l "time (in ns)" (
- open curve.ndjson | read-atomic-ops | to json
+python scripts/plot/multi_bar.py --title "simple curve group operations" -l "time (in ns)" (
+ open curve_group.ndjson
+ | read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
+ | to json
+)
+python scripts/plot/multi_bar.py --title "complex curve group operations" -l "time (in ns)" (
+ open curve_group.ndjson
+ | read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
+ | to json
)
```
diff --git a/benches/operations/curve_group.rs b/benches/operations/curve_group.rs
deleted file mode 100644
index f5374edb..00000000
--- a/benches/operations/curve_group.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ec::CurveGroup;
-use ark_ff::PrimeField;
-
-use criterion::{criterion_group, criterion_main, Criterion};
-
-fn random_sampling_template<G: CurveGroup>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("random sampling on {}", curve), |b| {
- b.iter(|| {
- let _ = G::rand(&mut rng);
- });
- });
-}
-
-fn group_template<F: PrimeField, G: CurveGroup<ScalarField = F>>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("addition on {}", curve), |b| {
- b.iter(|| {
- let g1 = G::rand(&mut rng);
- let g2 = G::rand(&mut rng);
- g1 + g2
- });
- });
-
- c.bench_function(&format!("substraction on {}", curve), |b| {
- b.iter(|| {
- let g1 = G::rand(&mut rng);
- let g2 = G::rand(&mut rng);
- g1 - g2
- });
- });
-
- c.bench_function(&format!("double on {}", curve), |b| {
- b.iter(|| {
- let g1 = G::rand(&mut rng);
- g1.double()
- });
- });
-
- c.bench_function(&format!("scalar multiplication on {}", curve), |b| {
- b.iter(|| {
- let g1 = G::rand(&mut rng);
- let f1 = F::rand(&mut rng);
- g1.mul(f1)
- });
- });
-}
-
-fn curve_group_template<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- curve: &str,
-) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("into affine on {}", curve), |b| {
- b.iter(|| {
- let g1 = G::rand(&mut rng);
- g1.into_affine()
- });
- });
-
- c.bench_function(&format!("from affine on {}", curve), |b| {
- b.iter(|| {
- let g1_affine = G::rand(&mut rng).into_affine();
- let _: G = g1_affine.into();
- });
- });
-
- c.bench_function(&format!("affine addition on {}", curve), |b| {
- b.iter(|| {
- let g1_affine = G::rand(&mut rng).into_affine();
- let g2_affine = G::rand(&mut rng).into_affine();
- g1_affine + g2_affine
- });
- });
-
- c.bench_function(&format!("affine scalar multiplication on {}", curve), |b| {
- b.iter(|| {
- let g1_affine = G::rand(&mut rng).into_affine();
- let f1 = F::rand(&mut rng);
- g1_affine * f1
- });
- });
-}
-
-macro_rules! bench {
- ($c:ident, $b:ident, G1=$g:ident, name=$n:expr) => {
- random_sampling_template::<$c::$g>($b, $n);
- group_template::<$c::Fr, $c::$g>($b, $n);
- curve_group_template::<$c::Fr, $c::$g>($b, $n);
- };
-}
-
-fn bench(c: &mut Criterion) {
- bench!(ark_bls12_381, c, G1 = G1Projective, name = "BLS12-381");
- bench!(ark_bn254, c, G1 = G1Projective, name = "BN-254");
- bench!(ark_pallas, c, G1 = Projective, name = "PALLAS");
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(3.0))
- .sample_size(100);
- targets = bench
-);
-criterion_main!(benches);
diff --git a/benches/operations/field.rs b/benches/operations/field.rs
deleted file mode 100644
index e0514608..00000000
--- a/benches/operations/field.rs
+++ /dev/null
@@ -1,132 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ff::PrimeField;
-
-use criterion::{criterion_group, criterion_main, Criterion};
-
-fn random_sampling_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("random sampling on {}", curve), |b| {
- b.iter(|| {
- let _ = F::rand(&mut rng);
- });
- });
-}
-
-fn additive_group_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("addition on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- let f2 = F::rand(&mut rng);
- f1 + f2
- });
- });
-
- c.bench_function(&format!("substraction on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- let f2 = F::rand(&mut rng);
- f1 - f2
- });
- });
-
- c.bench_function(&format!("double on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- f1.double()
- });
- });
-
- c.bench_function(&format!("multiplication on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- let f2 = F::rand(&mut rng);
- f1 * f2
- });
- });
-}
-
-fn field_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("square on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- f1.square()
- });
- });
-
- c.bench_function(&format!("inverse on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- f1.inverse()
- });
- });
-
- c.bench_function(&format!("legendre on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- f1.legendre()
- });
- });
-
- c.bench_function(&format!("sqrt on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- if f1.legendre().is_qr() {
- let _ = f1.sqrt();
- }
- });
- });
-}
-
-fn prime_field_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("exponentiation on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- f1.pow(F::MODULUS)
- });
- });
-}
-
-fn conversions_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
- let mut rng = ark_std::test_rng();
-
- c.bench_function(&format!("bigint conversion on {}", curve), |b| {
- b.iter(|| {
- let f1 = F::rand(&mut rng);
- f1.into_bigint()
- });
- });
-}
-
-macro_rules! bench {
- ($c:ident, $b:ident, F=$f:ident, name=$n:expr) => {
- random_sampling_template::<$c::$f>($b, $n);
- additive_group_template::<$c::$f>($b, $n);
- field_template::<$c::$f>($b, $n);
- prime_field_template::<$c::$f>($b, $n);
- conversions_template::<$c::$f>($b, $n);
- };
-}
-
-fn bench(c: &mut Criterion) {
- bench!(ark_bls12_381, c, F = Fr, name = "BLS12-381");
- bench!(ark_bn254, c, F = Fr, name = "BN-254");
- bench!(ark_pallas, c, F = Fr, name = "PALLAS");
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(3.0))
- .sample_size(100);
- targets = bench
-);
-criterion_main!(benches);
diff --git a/examples/benches/operations/curve_group.rs b/examples/benches/operations/curve_group.rs
new file mode 100644
index 00000000..d6d2771a
--- /dev/null
+++ b/examples/benches/operations/curve_group.rs
@@ -0,0 +1,138 @@
+// see `benches/README.md`
+use std::time::{Duration, Instant};
+
+use ark_ec::CurveGroup;
+use ark_ff::PrimeField;
+use clap::{command, Parser};
+use rand::RngCore;
+
+fn bench(b: &Bencher, op: &str, thing: fn(&mut dyn RngCore) -> Duration) {
+ let mut rng = ark_std::test_rng();
+
+ let mut times = vec![];
+ for i in 0..b.nb_measurements {
+ eprint!(
+ "{} on {} [{:>5}/{}]\r",
+ op,
+ b.name,
+ i + 1,
+ b.nb_measurements
+ );
+
+ times.push(thing(&mut rng).as_nanos());
+ }
+ eprintln!();
+
+ println!(
+ r#"{{op: "{}", curve: "{}", times: {:?}}}"#,
+ op, b.name, times
+ );
+}
+
+macro_rules! timeit {
+ ($f:tt) => {{
+ let start_time = Instant::now();
+ #[allow(clippy::redundant_closure_call)]
+ let _ = $f();
+ Instant::now().duration_since(start_time)
+ }};
+}
+
+#[derive(Clone)]
+struct Bencher {
+ nb_measurements: usize,
+ name: String,
+}
+
+impl Bencher {
+ fn new(nb_measurements: usize) -> Self {
+ Self {
+ nb_measurements,
+ name: "".to_string(),
+ }
+ }
+
+ fn with_name(&self, name: impl ToString) -> Self {
+ let mut new = self.clone();
+ new.name = name.to_string();
+ new
+ }
+}
+
+fn bench_template<F: PrimeField, G: CurveGroup<ScalarField = F>>(b: &Bencher) {
+ bench(b, "random sampling", |rng| timeit!((|| G::rand(rng))));
+
+ bench(b, "addition", |rng| {
+ let g1 = G::rand(rng);
+ let g2 = G::rand(rng);
+
+ timeit!((|| g1 + g2))
+ });
+
+ bench(b, "substraction", |rng| {
+ let g1 = G::rand(rng);
+ let g2 = G::rand(rng);
+
+ timeit!((|| g1 - g2))
+ });
+
+ bench(b, "double", |rng| {
+ let g1 = G::rand(rng);
+
+ timeit!((|| g1.double()))
+ });
+
+ bench(b, "scalar multiplication", |rng| {
+ let g1 = G::rand(rng);
+ let f1 = F::rand(rng);
+
+ timeit!((|| g1.mul(f1)))
+ });
+
+ bench(b, "into affine", |rng| {
+ let g1 = G::rand(rng);
+
+ timeit!((|| g1.into_affine()))
+ });
+
+ bench(b, "from affine", |rng| {
+ let g1_affine = G::rand(rng).into_affine();
+
+ timeit!((|| Into::<G>::into(g1_affine)))
+ });
+
+ bench(b, "affine addition", |rng| {
+ let g1_affine = G::rand(rng).into_affine();
+ let g2_affine = G::rand(rng).into_affine();
+
+ timeit!((|| g1_affine + g2_affine))
+ });
+
+ bench(b, "affine scalar multiplication", |rng| {
+ let g1_affine = G::rand(rng).into_affine();
+ let f1 = F::rand(rng);
+
+ timeit!((|| g1_affine * f1))
+ });
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long, default_value_t = 10)]
+ nb_measurements: usize,
+}
+
+fn main() {
+ let cli = Cli::parse();
+
+ let bencher = Bencher::new(cli.nb_measurements);
+
+ bench_template::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(
+ &bencher.with_name("BLS12-381"),
+ );
+ bench_template::<ark_bn254::Fr, ark_bn254::G1Projective>(&bencher.with_name("BN-254"));
+ bench_template::<ark_pallas::Fr, ark_pallas::Projective>(&bencher.with_name("PALLAS"));
+}
diff --git a/examples/benches/operations/field.rs b/examples/benches/operations/field.rs
new file mode 100644
index 00000000..5d4a8b6a
--- /dev/null
+++ b/examples/benches/operations/field.rs
@@ -0,0 +1,148 @@
+// see `benches/README.md`
+use std::time::{Duration, Instant};
+
+use ark_ff::PrimeField;
+use clap::{arg, command, Parser};
+use rand::RngCore;
+
+fn bench(b: &Bencher, op: &str, thing: fn(&mut dyn RngCore) -> Duration) {
+ let mut rng = ark_std::test_rng();
+
+ let mut times = vec![];
+ for i in 0..b.nb_measurements {
+ eprint!(
+ "{} on {} [{:>5}/{}]\r",
+ op,
+ b.name,
+ i + 1,
+ b.nb_measurements
+ );
+
+ times.push(thing(&mut rng).as_nanos());
+ }
+ eprintln!();
+
+ println!(
+ r#"{{op: "{}", curve: "{}", times: {:?}}}"#,
+ op, b.name, times
+ );
+}
+
+macro_rules! timeit {
+ ($f:tt) => {{
+ let start_time = Instant::now();
+ #[allow(clippy::redundant_closure_call)]
+ let _ = $f();
+ Instant::now().duration_since(start_time)
+ }};
+}
+
+#[derive(Clone)]
+struct Bencher {
+ nb_measurements: usize,
+ name: String,
+}
+
+impl Bencher {
+ fn new(nb_measurements: usize) -> Self {
+ Self {
+ nb_measurements,
+ name: "".to_string(),
+ }
+ }
+
+ fn with_name(&self, name: impl ToString) -> Self {
+ let mut new = self.clone();
+ new.name = name.to_string();
+ new
+ }
+}
+
+fn bench_template<F: PrimeField>(b: &Bencher) {
+ bench(b, "random sampling", |rng| timeit!((|| F::rand(rng))));
+
+ bench(b, "addition", |rng| {
+ let f1 = F::rand(rng);
+ let f2 = F::rand(rng);
+
+ timeit!((|| f1 + f2))
+ });
+
+ bench(b, "substraction", |rng| {
+ let f1 = F::rand(rng);
+ let f2 = F::rand(rng);
+
+ timeit!((|| f1 - f2))
+ });
+
+ bench(b, "double", |rng| {
+ let f1 = F::rand(rng);
+
+ timeit!((|| f1.double()))
+ });
+
+ bench(b, "multiplication", |rng| {
+ let f1 = F::rand(rng);
+ let f2 = F::rand(rng);
+
+ timeit!((|| f1 * f2))
+ });
+
+ bench(b, "square", |rng| {
+ let f1 = F::rand(rng);
+
+ timeit!((|| f1.square()))
+ });
+
+ bench(b, "inverse", |rng| {
+ let f1 = F::rand(rng);
+
+ timeit!((|| f1.inverse()))
+ });
+
+ bench(b, "legendre", |rng| {
+ let f1 = F::rand(rng);
+
+ timeit!((|| f1.legendre()))
+ });
+
+ bench(b, "sqrt", |rng| {
+ let f1 = F::rand(rng);
+ if f1.legendre().is_qr() {
+ timeit!((|| f1.sqrt()))
+ } else {
+ Duration::default()
+ }
+ });
+
+ bench(b, "exponentiation", |rng| {
+ let f1 = F::rand(rng);
+
+ timeit!((|| f1.pow(F::MODULUS)))
+ });
+
+ bench(b, "into bigint", |rng| {
+ let f1 = F::rand(rng);
+
+ timeit!((|| f1.into_bigint()))
+ });
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long, default_value_t = 10)]
+ nb_measurements: usize,
+}
+
+fn main() {
+ let cli = Cli::parse();
+
+ let bencher = Bencher::new(cli.nb_measurements);
+
+ bench_template::<ark_bls12_381::Fr>(&bencher.with_name("BLS12-381"));
+ bench_template::<ark_bn254::Fr>(&bencher.with_name("BN-254"));
+ bench_template::<ark_pallas::Fr>(&bencher.with_name("PALLAS"));
+}
--
GitLab