Skip to content
Snippets Groups Projects
Commit 10acafb1 authored by STEVAN Antoine's avatar STEVAN Antoine :crab:
Browse files

write manual atomic benchmarks (dragoon/komodo!82) 

this idea is to not use `criterion` and measure exactly what we want

## results
![simple_field](/uploads/c114cee9ef4b0e56e9ab290d1cbbab2e/simple_field.png)
![complex_field](/uploads/7bc8ea613793718f6460438f1df6fd24/complex_field.png)
![simple_curve_group](/uploads/a9f8642fc1f1fa558c2bee366622646f/simple_curve_group.png)
![complex_curve_group](/uploads/76a3fbe2430959e20150070beeb196a5/complex_curve_group.png)
parent b711ad5b
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Cargo.toml
+ 10
− 10
View file @ 10acafb1
......@@ -66,16 +66,6 @@ harness = false
name = "commit"
harness = false
[[bench]]
name = "field_operations"
path = "benches/operations/field.rs"
harness = false
[[bench]]
name = "curve_group_operations"
path = "benches/operations/curve_group.rs"
harness = false
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
......@@ -83,3 +73,13 @@ path = "examples/benches/commit.rs"
[[example]]
name = "bench_setup_size"
path = "examples/benches/setup_size.rs"
[[example]]
name = "bench_field_operations"
path = "examples/benches/operations/field.rs"
harness = false
[[example]]
name = "bench_curve_group_operations"
path = "examples/benches/operations/curve_group.rs"
harness = false
benches/README.md
+ 26
− 41
View file @ 10acafb1
......@@ -16,52 +16,30 @@ python scripts/plot/benches.py results.ndjson --bench setup
## atomic operations
```nushell
cargo criterion --output-format verbose --message-format json --bench field_operations out> field.ndjson
cargo criterion --output-format verbose --message-format json --bench curve_group_operations out> curve.ndjson
cargo run --example bench_field_operations -- --nb-measurements 1000
| lines
| each { from nuon }
| to ndjson
| save --force field.ndjson
cargo run --example bench_curve_group_operations -- --nb-measurements 1000
| lines
| each { from nuon }
| to ndjson
| save --force curve_group.ndjson
```
```nushell
def read-atomic-ops [
--clean, --include: list<string> = [], --exclude: list<string> = []
--include: list<string> = [], --exclude: list<string> = []
]: list -> record {
let raw = $in
| where reason == "benchmark-complete"
| select id mean.estimate
| sort-by id
| update id { parse "{op} on {curve}" }
| flatten --all
| rename --column { op: "group", curve: "species", mean_estimate: "measurement" }
let clean = if $clean {
$raw | update measurement {|it|
let species = $it.species
# FIXME: bug when no parentheses
let r = (
$raw
| where group == 'random sampling' and species == $species
| into record
| get measurement
)
let l = (
$raw
| where group == 'legendre' and species == $species
| into record
| get measurement
)
match $it.group {
"addition" | "multiplication" | "substraction" => ($it.measurement - 2 * $r),
"random sampling" => $it.measurement,
"sqrt" => ($it.measurement - $r - $l),
_ => ($it.measurement - $r),
}
}
} else {
$raw
}
| insert t {|it| $it.times |math avg}
| reject times
| rename --column { op: "group", curve: "species", t: "measurement" }
let included = if $include != [] {
$clean | where group in $include
$raw | where group in $include
} else {
$clean
$raw
}
$included
......@@ -76,7 +54,7 @@ def read-atomic-ops [
```nushell
python scripts/plot/multi_bar.py --title "simple field operations" -l "time (in ns)" (
open field.ndjson
| read-atomic-ops --clean --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
| read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
| to json
)
python scripts/plot/multi_bar.py --title "complex field operations" -l "time (in ns)" (
......@@ -84,8 +62,15 @@ python scripts/plot/multi_bar.py --title "complex field operations" -l "time (in
| read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ]
| to json
)
python scripts/plot/multi_bar.py --title "curve group operations" -l "time (in ns)" (
open curve.ndjson | read-atomic-ops | to json
python scripts/plot/multi_bar.py --title "simple curve group operations" -l "time (in ns)" (
open curve_group.ndjson
| read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
| to json
)
python scripts/plot/multi_bar.py --title "complex curve group operations" -l "time (in ns)" (
open curve_group.ndjson
| read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
| to json
)
```
......
benches/operations/curve_group.rs deleted 100644 → 0
+ 0
− 112
View file @ b711ad5b
// see `benches/README.md`
use std::time::Duration;
use ark_ec::CurveGroup;
use ark_ff::PrimeField;
use criterion::{criterion_group, criterion_main, Criterion};
fn random_sampling_template<G: CurveGroup>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("random sampling on {}", curve), |b| {
b.iter(|| {
let _ = G::rand(&mut rng);
});
});
}
fn group_template<F: PrimeField, G: CurveGroup<ScalarField = F>>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("addition on {}", curve), |b| {
b.iter(|| {
let g1 = G::rand(&mut rng);
let g2 = G::rand(&mut rng);
g1 + g2
});
});
c.bench_function(&format!("substraction on {}", curve), |b| {
b.iter(|| {
let g1 = G::rand(&mut rng);
let g2 = G::rand(&mut rng);
g1 - g2
});
});
c.bench_function(&format!("double on {}", curve), |b| {
b.iter(|| {
let g1 = G::rand(&mut rng);
g1.double()
});
});
c.bench_function(&format!("scalar multiplication on {}", curve), |b| {
b.iter(|| {
let g1 = G::rand(&mut rng);
let f1 = F::rand(&mut rng);
g1.mul(f1)
});
});
}
fn curve_group_template<F: PrimeField, G: CurveGroup<ScalarField = F>>(
c: &mut Criterion,
curve: &str,
) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("into affine on {}", curve), |b| {
b.iter(|| {
let g1 = G::rand(&mut rng);
g1.into_affine()
});
});
c.bench_function(&format!("from affine on {}", curve), |b| {
b.iter(|| {
let g1_affine = G::rand(&mut rng).into_affine();
let _: G = g1_affine.into();
});
});
c.bench_function(&format!("affine addition on {}", curve), |b| {
b.iter(|| {
let g1_affine = G::rand(&mut rng).into_affine();
let g2_affine = G::rand(&mut rng).into_affine();
g1_affine + g2_affine
});
});
c.bench_function(&format!("affine scalar multiplication on {}", curve), |b| {
b.iter(|| {
let g1_affine = G::rand(&mut rng).into_affine();
let f1 = F::rand(&mut rng);
g1_affine * f1
});
});
}
macro_rules! bench {
($c:ident, $b:ident, G1=$g:ident, name=$n:expr) => {
random_sampling_template::<$c::$g>($b, $n);
group_template::<$c::Fr, $c::$g>($b, $n);
curve_group_template::<$c::Fr, $c::$g>($b, $n);
};
}
fn bench(c: &mut Criterion) {
bench!(ark_bls12_381, c, G1 = G1Projective, name = "BLS12-381");
bench!(ark_bn254, c, G1 = G1Projective, name = "BN-254");
bench!(ark_pallas, c, G1 = Projective, name = "PALLAS");
}
criterion_group!(
name = benches;
config = Criterion::default()
.warm_up_time(Duration::from_secs_f32(3.0))
.sample_size(100);
targets = bench
);
criterion_main!(benches);
benches/operations/field.rs deleted 100644 → 0
+ 0
− 132
View file @ b711ad5b
// see `benches/README.md`
use std::time::Duration;
use ark_ff::PrimeField;
use criterion::{criterion_group, criterion_main, Criterion};
fn random_sampling_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("random sampling on {}", curve), |b| {
b.iter(|| {
let _ = F::rand(&mut rng);
});
});
}
fn additive_group_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("addition on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
let f2 = F::rand(&mut rng);
f1 + f2
});
});
c.bench_function(&format!("substraction on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
let f2 = F::rand(&mut rng);
f1 - f2
});
});
c.bench_function(&format!("double on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
f1.double()
});
});
c.bench_function(&format!("multiplication on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
let f2 = F::rand(&mut rng);
f1 * f2
});
});
}
fn field_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("square on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
f1.square()
});
});
c.bench_function(&format!("inverse on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
f1.inverse()
});
});
c.bench_function(&format!("legendre on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
f1.legendre()
});
});
c.bench_function(&format!("sqrt on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
if f1.legendre().is_qr() {
let _ = f1.sqrt();
}
});
});
}
fn prime_field_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("exponentiation on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
f1.pow(F::MODULUS)
});
});
}
fn conversions_template<F: PrimeField>(c: &mut Criterion, curve: &str) {
let mut rng = ark_std::test_rng();
c.bench_function(&format!("bigint conversion on {}", curve), |b| {
b.iter(|| {
let f1 = F::rand(&mut rng);
f1.into_bigint()
});
});
}
macro_rules! bench {
($c:ident, $b:ident, F=$f:ident, name=$n:expr) => {
random_sampling_template::<$c::$f>($b, $n);
additive_group_template::<$c::$f>($b, $n);
field_template::<$c::$f>($b, $n);
prime_field_template::<$c::$f>($b, $n);
conversions_template::<$c::$f>($b, $n);
};
}
fn bench(c: &mut Criterion) {
bench!(ark_bls12_381, c, F = Fr, name = "BLS12-381");
bench!(ark_bn254, c, F = Fr, name = "BN-254");
bench!(ark_pallas, c, F = Fr, name = "PALLAS");
}
criterion_group!(
name = benches;
config = Criterion::default()
.warm_up_time(Duration::from_secs_f32(3.0))
.sample_size(100);
targets = bench
);
criterion_main!(benches);
examples/benches/operations/curve_group.rs 0 → 100644
+ 138
− 0
View file @ 10acafb1
// see `benches/README.md`
use std::time::{Duration, Instant};
use ark_ec::CurveGroup;
use ark_ff::PrimeField;
use clap::{command, Parser};
use rand::RngCore;
fn bench(b: &Bencher, op: &str, thing: fn(&mut dyn RngCore) -> Duration) {
let mut rng = ark_std::test_rng();
let mut times = vec![];
for i in 0..b.nb_measurements {
eprint!(
"{} on {} [{:>5}/{}]\r",
op,
b.name,
i + 1,
b.nb_measurements
);
times.push(thing(&mut rng).as_nanos());
}
eprintln!();
println!(
r#"{{op: "{}", curve: "{}", times: {:?}}}"#,
op, b.name, times
);
}
macro_rules! timeit {
($f:tt) => {{
let start_time = Instant::now();
#[allow(clippy::redundant_closure_call)]
let _ = $f();
Instant::now().duration_since(start_time)
}};
}
#[derive(Clone)]
struct Bencher {
nb_measurements: usize,
name: String,
}
impl Bencher {
fn new(nb_measurements: usize) -> Self {
Self {
nb_measurements,
name: "".to_string(),
}
}
fn with_name(&self, name: impl ToString) -> Self {
let mut new = self.clone();
new.name = name.to_string();
new
}
}
fn bench_template<F: PrimeField, G: CurveGroup<ScalarField = F>>(b: &Bencher) {
bench(b, "random sampling", |rng| timeit!((|| G::rand(rng))));
bench(b, "addition", |rng| {
let g1 = G::rand(rng);
let g2 = G::rand(rng);
timeit!((|| g1 + g2))
});
bench(b, "substraction", |rng| {
let g1 = G::rand(rng);
let g2 = G::rand(rng);
timeit!((|| g1 - g2))
});
bench(b, "double", |rng| {
let g1 = G::rand(rng);
timeit!((|| g1.double()))
});
bench(b, "scalar multiplication", |rng| {
let g1 = G::rand(rng);
let f1 = F::rand(rng);
timeit!((|| g1.mul(f1)))
});
bench(b, "into affine", |rng| {
let g1 = G::rand(rng);
timeit!((|| g1.into_affine()))
});
bench(b, "from affine", |rng| {
let g1_affine = G::rand(rng).into_affine();
timeit!((|| Into::<G>::into(g1_affine)))
});
bench(b, "affine addition", |rng| {
let g1_affine = G::rand(rng).into_affine();
let g2_affine = G::rand(rng).into_affine();
timeit!((|| g1_affine + g2_affine))
});
bench(b, "affine scalar multiplication", |rng| {
let g1_affine = G::rand(rng).into_affine();
let f1 = F::rand(rng);
timeit!((|| g1_affine * f1))
});
}
#[derive(Parser)]
#[command(version, about, long_about = None)]
struct Cli {
/// the number of measurements to repeat each case, larger values will reduce the variance of
/// the measurements
#[arg(short, long, default_value_t = 10)]
nb_measurements: usize,
}
fn main() {
let cli = Cli::parse();
let bencher = Bencher::new(cli.nb_measurements);
bench_template::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(
&bencher.with_name("BLS12-381"),
);
bench_template::<ark_bn254::Fr, ark_bn254::G1Projective>(&bencher.with_name("BN-254"));
bench_template::<ark_pallas::Fr, ark_pallas::Projective>(&bencher.with_name("PALLAS"));
}
examples/benches/operations/field.rs 0 → 100644
+ 148
− 0
View file @ 10acafb1
// see `benches/README.md`
use std::time::{Duration, Instant};
use ark_ff::PrimeField;
use clap::{arg, command, Parser};
use rand::RngCore;
fn bench(b: &Bencher, op: &str, thing: fn(&mut dyn RngCore) -> Duration) {
let mut rng = ark_std::test_rng();
let mut times = vec![];
for i in 0..b.nb_measurements {
eprint!(
"{} on {} [{:>5}/{}]\r",
op,
b.name,
i + 1,
b.nb_measurements
);
times.push(thing(&mut rng).as_nanos());
}
eprintln!();
println!(
r#"{{op: "{}", curve: "{}", times: {:?}}}"#,
op, b.name, times
);
}
macro_rules! timeit {
($f:tt) => {{
let start_time = Instant::now();
#[allow(clippy::redundant_closure_call)]
let _ = $f();
Instant::now().duration_since(start_time)
}};
}
#[derive(Clone)]
struct Bencher {
nb_measurements: usize,
name: String,
}
impl Bencher {
fn new(nb_measurements: usize) -> Self {
Self {
nb_measurements,
name: "".to_string(),
}
}
fn with_name(&self, name: impl ToString) -> Self {
let mut new = self.clone();
new.name = name.to_string();
new
}
}
fn bench_template<F: PrimeField>(b: &Bencher) {
bench(b, "random sampling", |rng| timeit!((|| F::rand(rng))));
bench(b, "addition", |rng| {
let f1 = F::rand(rng);
let f2 = F::rand(rng);
timeit!((|| f1 + f2))
});
bench(b, "substraction", |rng| {
let f1 = F::rand(rng);
let f2 = F::rand(rng);
timeit!((|| f1 - f2))
});
bench(b, "double", |rng| {
let f1 = F::rand(rng);
timeit!((|| f1.double()))
});
bench(b, "multiplication", |rng| {
let f1 = F::rand(rng);
let f2 = F::rand(rng);
timeit!((|| f1 * f2))
});
bench(b, "square", |rng| {
let f1 = F::rand(rng);
timeit!((|| f1.square()))
});
bench(b, "inverse", |rng| {
let f1 = F::rand(rng);
timeit!((|| f1.inverse()))
});
bench(b, "legendre", |rng| {
let f1 = F::rand(rng);
timeit!((|| f1.legendre()))
});
bench(b, "sqrt", |rng| {
let f1 = F::rand(rng);
if f1.legendre().is_qr() {
timeit!((|| f1.sqrt()))
} else {
Duration::default()
}
});
bench(b, "exponentiation", |rng| {
let f1 = F::rand(rng);
timeit!((|| f1.pow(F::MODULUS)))
});
bench(b, "into bigint", |rng| {
let f1 = F::rand(rng);
timeit!((|| f1.into_bigint()))
});
}
#[derive(Parser)]
#[command(version, about, long_about = None)]
struct Cli {
/// the number of measurements to repeat each case, larger values will reduce the variance of
/// the measurements
#[arg(short, long, default_value_t = 10)]
nb_measurements: usize,
}
fn main() {
let cli = Cli::parse();
let bencher = Bencher::new(cli.nb_measurements);
bench_template::<ark_bls12_381::Fr>(&bencher.with_name("BLS12-381"));
bench_template::<ark_bn254::Fr>(&bencher.with_name("BN-254"));
bench_template::<ark_pallas::Fr>(&bencher.with_name("PALLAS"));
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment