diff --git a/Cargo.toml b/Cargo.toml
index 31e8a7330f2dd2fc325730376d532fc8f9e818b5..df73a393782fb60769b8eb6d8295ef546fb7b6fb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,18 +55,6 @@ criterion = "0.3"
name = "recoding"
harness = false
-[[bench]]
-name = "linalg"
-harness = false
-
-[[bench]]
-name = "setup"
-harness = false
-
-[[bench]]
-name = "commit"
-harness = false
-
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
@@ -78,9 +66,15 @@ path = "examples/benches/setup_size.rs"
[[example]]
name = "bench_field_operations"
path = "examples/benches/operations/field.rs"
-harness = false
[[example]]
name = "bench_curve_group_operations"
path = "examples/benches/operations/curve_group.rs"
-harness = false
+
+[[example]]
+name = "bench_setup"
+path = "examples/benches/setup.rs"
+
+[[example]]
+name = "bench_linalg"
+path = "examples/benches/linalg.rs"
diff --git a/benches/README.md b/benches/README.md
index b9df21b961262ff72631674c32e345c33eb0ab71..93be82238e07a343c819e31d4f690e08017a6c87 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -1,57 +1,19 @@
-## run the benchmarks
-```shell
-nushell> cargo criterion --output-format verbose --message-format json out> results.ndjson
-```
-
-## add the _trusted setup_ sizes
-```shell
-nushell> cargo run --example bench_setup_size out>> results.ndjson
-```
-
-## plot the results
-```shell
-python scripts/plot/benches.py results.ndjson --bench linalg
-python scripts/plot/benches.py results.ndjson --bench setup
-```
-
## atomic operations
```nushell
cargo run --example bench_field_operations -- --nb-measurements 1000
| lines
| each { from json }
- | to ndjson
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
| save --force field.ndjson
cargo run --example bench_curve_group_operations -- --nb-measurements 1000
| lines
| each { from json }
- | to ndjson
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
| save --force curve_group.ndjson
```
```nushell
-def read-atomic-ops [
- --include: list<string> = [], --exclude: list<string> = []
-]: list -> record {
- let raw = $in
- | insert t {|it| $it.times |math avg}
- | reject times
- | rename --column { label: "group", name: "species", t: "measurement" }
+use scripts/parse.nu read-atomic-ops
- let included = if $include != [] {
- $raw | where group in $include
- } else {
- $raw
- }
-
- $included
- | where group not-in $exclude
- | group-by group --to-table
- | reject items.group
- | update items { transpose -r | into record }
- | transpose -r
- | into record
-}
-```
-```nushell
python scripts/plot/multi_bar.py --title "simple field operations" -l "time (in ns)" (
open field.ndjson
| read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
@@ -74,23 +36,105 @@ python scripts/plot/multi_bar.py --title "complex curve group operations" -l "ti
)
```
-## oneshot benchmarks
-these are benchmarks that run a single measurement, implemented as _examples_ in
-`examples/benches/`.
+## linear algebra
+```nushell
+let sizes = seq 0 7 | each { 2 ** $in }
+cargo run --example bench_linalg -- --nb-measurements 10 ...$sizes
+ | lines
+ | each { from json }
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
+ | save --force linalg.ndjson
+```
+```nushell
+let linalg = open linalg.ndjson
+ | update times { each { $in / 1_000_000 } }
+ | insert mean {|it| $it.times | math avg}
+ | insert stddev {|it| $it.times | into float | math stddev}
+ | update label { parse "{op} {n}"}
+ | flatten --all label
+ | into int n
+
+for graph in [
+ [op, title];
+
+ ["inverse", "time to inverse an nxn matrix on certain curves"],
+ ["transpose", "time to transpose an nxn matrix on certain curves"],
+ ["mul", "time to multiply two nxn matrices on certain curves"]
+] {
+ python scripts/plot/plot.py ...[
+ --title $graph.title
+ --x-label "size"
+ --y-label "time (in ms)"
+ (
+ $linalg
+ | where op == $graph.op
+ | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+ ]
+}
+```
+
+## trusted setup
+```nushell
+let degrees = seq 0 13 | each { 2 ** $in }
+cargo run --example bench_setup -- --nb-measurements 10 ...$degrees
+ | lines
+ | each { from json }
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
+ | save --force setup.ndjson
+```
+```nushell
+python scripts/plot/plot.py ...[
+ --title "time to create trusted setups for certain curves"
+ --x-label "degree"
+ --y-label "time (in ms)"
+ (
+ open setup.ndjson
+ | update times { each { $in / 1_000_000 } }
+ | insert mean {|it| $it.times | math avg}
+ | insert stddev {|it| $it.times | into float | math stddev}
+ | insert degree { get label | parse "degree {d}" | into record | get d | into int}
+ | insert curve {|it| if ($it.name | str starts-with "ARK") {
+ let c = $it.name | parse "ARK setup on {curve}" | into record | get curve
+ $"($c)-ark"
+ } else {
+ $it.name | parse "setup on {curve}" | into record | get curve
+ }}
+ | rename --column { degree: "x", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+]
+```
-### commit
+## commit
```nushell
let degrees = seq 0 15 | each { 2 ** $in }
-let res = cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
+cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
| lines
| each { from nuon }
- | update times { into duration }
- | insert mean {|it| $it.times | math avg}
- | insert stddev {|it| $it.times | into int | into float | math stddev | into int | into duration}
- | update label { parse "degree {d}" | into record | get d | into int }
- | rename --column { label: "degree", name: "curve" }
-
-python scripts/plot/bench_commit.py (
- $res | group-by curve --to-table | update items { reject curve } | to json
-)
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
+ | save --force commit.ndjson
+```
+```nushell
+python scripts/plot/plot.py ...[
+ --title "time to commit polynomials for certain curves"
+ --x-label "degree"
+ --y-label "time (in ms)"
+ (
+ open commit.ndjson
+ | update times { each { $in / 1_000_000 } }
+ | insert mean {|it| $it.times | math avg}
+ | insert stddev {|it| $it.times | into float | math stddev}
+ | update label { parse "degree {d}" | into record | get d | into int }
+ | rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+]
```
diff --git a/benches/commit.rs b/benches/commit.rs
deleted file mode 100644
index 21f59cbf582577552c11ec4065b008ef86b133e4..0000000000000000000000000000000000000000
--- a/benches/commit.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ec::{pairing::Pairing, CurveGroup};
-use ark_ff::PrimeField;
-use ark_poly::{univariate::DensePolynomial, DenseUVPolynomial};
-use ark_poly_commit::kzg10::{Powers, KZG10};
-use ark_std::ops::Div;
-
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use komodo::zk;
-
-fn commit_template<F, G, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- F: PrimeField,
- G: CurveGroup<ScalarField = F>,
- P: DenseUVPolynomial<F>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- let setup = zk::setup::<F, G>(degree, rng).unwrap();
- let polynomial = P::rand(degree, rng);
-
- c.bench_function(&format!("commit (komodo) {} on {}", degree, curve), |b| {
- b.iter(|| zk::commit(&setup, &polynomial))
- });
-}
-
-fn ark_commit_template<E, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- E: Pairing,
- P: DenseUVPolynomial<E::ScalarField>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- let setup = KZG10::<E, P>::setup(degree, false, rng).unwrap();
- let powers_of_g = setup.powers_of_g[..=degree].to_vec();
- let powers_of_gamma_g = (0..=degree).map(|i| setup.powers_of_gamma_g[&i]).collect();
- let powers = Powers::<E> {
- powers_of_g: ark_std::borrow::Cow::Owned(powers_of_g),
- powers_of_gamma_g: ark_std::borrow::Cow::Owned(powers_of_gamma_g),
- };
- let polynomial = P::rand(degree, rng);
-
- c.bench_function(&format!("commit (arkworks) {} on {}", degree, curve), |b| {
- b.iter(|| KZG10::commit(&powers, &polynomial, None, None))
- });
-}
-
-fn commit(c: &mut Criterion) {
- fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- degree: usize,
- curve: &str,
- ) {
- commit_template::<F, G, DensePolynomial<F>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(c, n, "BLS12-381");
- aux::<ark_bn254::Fr, ark_bn254::G1Projective>(c, n, "BN-254");
- aux::<ark_pallas::Fr, ark_pallas::Projective>(c, n, "PALLAS");
- }
-}
-
-fn ark_commit(c: &mut Criterion) {
- fn aux<E: Pairing>(c: &mut Criterion, degree: usize, curve: &str) {
- ark_commit_template::<E, DensePolynomial<E::ScalarField>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Bls12_381>(c, n, "BLS12-381");
- aux::<ark_bn254::Bn254>(c, n, "BN-254");
- }
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = commit, ark_commit
-);
-criterion_main!(benches);
diff --git a/benches/linalg.rs b/benches/linalg.rs
deleted file mode 100644
index 547f71a11aba87f2df90fbf845f6936eeb20a4c7..0000000000000000000000000000000000000000
--- a/benches/linalg.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ff::PrimeField;
-
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use komodo::linalg::Matrix;
-
-fn inverse_template<F: PrimeField>(c: &mut Criterion, n: usize, curve: &str) {
- let mut rng = rand::thread_rng();
- let matrix = Matrix::<F>::random(n, n, &mut rng);
-
- c.bench_function(&format!("inverse {}x{} on {}", n, n, curve), |b| {
- b.iter(|| matrix.invert().unwrap())
- });
-}
-
-fn inverse(c: &mut Criterion) {
- for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
- inverse_template::<ark_bls12_381::Fr>(c, black_box(n), "BLS12-381");
- inverse_template::<ark_bn254::Fr>(c, black_box(n), "BN-254");
- inverse_template::<ark_pallas::Fr>(c, black_box(n), "PALLAS");
- }
-}
-
-fn transpose_template<F: PrimeField>(c: &mut Criterion, n: usize, curve: &str) {
- let mut rng = rand::thread_rng();
- let matrix = Matrix::<F>::random(n, n, &mut rng);
-
- c.bench_function(&format!("transpose {}x{} on {}", n, n, curve), |b| {
- b.iter(|| matrix.transpose())
- });
-}
-
-fn transpose(c: &mut Criterion) {
- for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
- transpose_template::<ark_bls12_381::Fr>(c, black_box(n), "BLS-12-381");
- transpose_template::<ark_bn254::Fr>(c, black_box(n), "BN-254");
- transpose_template::<ark_pallas::Fr>(c, black_box(n), "PALLAS");
- }
-}
-
-fn mul_template<F: PrimeField>(c: &mut Criterion, n: usize, curve: &str) {
- let mut rng = rand::thread_rng();
- let mat_a = Matrix::<F>::random(n, n, &mut rng);
- let mat_b = Matrix::<F>::random(n, n, &mut rng);
-
- c.bench_function(&format!("mul {}x{} on {}", n, n, curve), |b| {
- b.iter(|| mat_a.mul(&mat_b))
- });
-}
-
-fn mul(c: &mut Criterion) {
- for n in [10, 15, 20, 30, 40, 60, 80, 120] {
- mul_template::<ark_bls12_381::Fr>(c, black_box(n), "BLS-12-381");
- mul_template::<ark_bn254::Fr>(c, black_box(n), "BN-254");
- mul_template::<ark_pallas::Fr>(c, black_box(n), "PALLAS");
- }
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = inverse, transpose, mul
-);
-criterion_main!(benches);
diff --git a/benches/setup.rs b/benches/setup.rs
deleted file mode 100644
index 15aaa211c4b5b0083002988cc6d74a3821c00c82..0000000000000000000000000000000000000000
--- a/benches/setup.rs
+++ /dev/null
@@ -1,191 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ec::{pairing::Pairing, CurveGroup};
-use ark_ff::PrimeField;
-use ark_poly::{univariate::DensePolynomial, DenseUVPolynomial};
-use ark_poly_commit::kzg10::{self, KZG10};
-use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate};
-use ark_std::ops::Div;
-
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use komodo::zk::{self, Powers};
-
-fn setup_template<F, G, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- F: PrimeField,
- G: CurveGroup<ScalarField = F>,
- P: DenseUVPolynomial<F>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- c.bench_function(&format!("setup (komodo) {} on {}", degree, curve), |b| {
- b.iter(|| zk::setup::<F, G>(degree, rng).unwrap())
- });
-}
-
-fn ark_setup_template<E, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- E: Pairing,
- P: DenseUVPolynomial<E::ScalarField>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- c.bench_function(
- &format!("setup (arkworks) {} bytes on {}", degree, curve),
- |b| {
- b.iter(|| {
- let setup = KZG10::<E, P>::setup(degree, false, rng).unwrap();
- let powers_of_g = setup.powers_of_g[..=degree].to_vec();
- let powers_of_gamma_g = (0..=degree).map(|i| setup.powers_of_gamma_g[&i]).collect();
- kzg10::Powers::<E> {
- powers_of_g: ark_std::borrow::Cow::Owned(powers_of_g),
- powers_of_gamma_g: ark_std::borrow::Cow::Owned(powers_of_gamma_g),
- }
- })
- },
- );
-}
-
-fn serde_template<F, G, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- F: PrimeField,
- G: CurveGroup<ScalarField = F>,
- P: DenseUVPolynomial<F>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let mut group = c.benchmark_group("setup");
-
- let rng = &mut rand::thread_rng();
-
- let setup = zk::setup::<F, G>(degree, rng).unwrap();
-
- group.bench_function(
- &format!("serializing with compression {} on {}", degree, curve),
- |b| {
- b.iter(|| {
- let mut serialized = vec![0; setup.serialized_size(Compress::Yes)];
- setup
- .serialize_with_mode(&mut serialized[..], Compress::Yes)
- .unwrap();
- })
- },
- );
-
- group.bench_function(
- &format!("serializing with no compression {} on {}", degree, curve),
- |b| {
- b.iter(|| {
- let mut serialized = vec![0; setup.serialized_size(Compress::No)];
- setup
- .serialize_with_mode(&mut serialized[..], Compress::No)
- .unwrap();
- })
- },
- );
-
- for (compress, validate) in [
- (Compress::Yes, Validate::Yes),
- (Compress::Yes, Validate::No),
- (Compress::No, Validate::Yes),
- (Compress::No, Validate::No),
- ] {
- let mut serialized = vec![0; setup.serialized_size(compress)];
- setup
- .serialize_with_mode(&mut serialized[..], compress)
- .unwrap();
-
- println!(
- r#"["id": "{} degree serialized with {} and {} on {}", "size": {}"#,
- degree,
- match compress {
- Compress::Yes => "compression",
- Compress::No => "no compression",
- },
- match validate {
- Validate::Yes => "validation",
- Validate::No => "no validation",
- },
- curve,
- serialized.len(),
- );
-
- group.bench_function(
- &format!(
- "deserializing with {} and {} {} on {}",
- match compress {
- Compress::Yes => "compression",
- Compress::No => "no compression",
- },
- match validate {
- Validate::Yes => "validation",
- Validate::No => "no validation",
- },
- degree,
- curve
- ),
- |b| {
- b.iter(|| {
- Powers::<F, G>::deserialize_with_mode(&serialized[..], compress, validate)
- })
- },
- );
- }
-
- group.finish();
-}
-
-fn setup(c: &mut Criterion) {
- fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- degree: usize,
- curve: &str,
- ) {
- setup_template::<F, G, DensePolynomial<F>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(c, n, "BLS-12-381");
- aux::<ark_bn254::Fr, ark_bn254::G1Projective>(c, n, "BN-254");
- aux::<ark_pallas::Fr, ark_pallas::Projective>(c, n, "PALLAS");
- }
-}
-
-fn serde(c: &mut Criterion) {
- fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- degree: usize,
- curve: &str,
- ) {
- serde_template::<F, G, DensePolynomial<F>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(c, n, "BLS-12-381");
- aux::<ark_bn254::Fr, ark_bn254::G1Projective>(c, n, "BN-254");
- aux::<ark_pallas::Fr, ark_pallas::Projective>(c, n, "PALLAS");
- }
-}
-
-fn ark_setup(c: &mut Criterion) {
- fn aux<E: Pairing>(c: &mut Criterion, degree: usize, curve: &str) {
- ark_setup_template::<E, DensePolynomial<E::ScalarField>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Bls12_381>(c, n, "BLS-12-381");
- aux::<ark_bn254::Bn254>(c, n, "BN-254");
- }
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = setup, ark_setup, serde
-);
-criterion_main!(benches);
diff --git a/examples/benches/linalg.rs b/examples/benches/linalg.rs
new file mode 100644
index 0000000000000000000000000000000000000000..62fffc90655a2a438d344a12f7d231e71df488c7
--- /dev/null
+++ b/examples/benches/linalg.rs
@@ -0,0 +1,67 @@
+// see `benches/README.md`
+use ark_ff::PrimeField;
+
+use clap::{arg, command, Parser};
+use komodo::linalg::Matrix;
+use plnk::Bencher;
+
+fn inverse_template<F: PrimeField>(b: &Bencher, n: usize) {
+ let mut rng = rand::thread_rng();
+ let matrix = Matrix::<F>::random(n, n, &mut rng);
+
+ plnk::bench(b, &format!("inverse {}", n), || {
+ plnk::timeit(|| matrix.invert().unwrap())
+ });
+}
+
+fn transpose_template<F: PrimeField>(b: &Bencher, n: usize) {
+ let mut rng = rand::thread_rng();
+ let matrix = Matrix::<F>::random(n, n, &mut rng);
+
+ plnk::bench(b, &format!("transpose {}", n), || {
+ plnk::timeit(|| matrix.transpose())
+ });
+}
+
+fn mul_template<F: PrimeField>(b: &Bencher, n: usize) {
+ let mut rng = rand::thread_rng();
+ let mat_a = Matrix::<F>::random(n, n, &mut rng);
+ let mat_b = Matrix::<F>::random(n, n, &mut rng);
+
+ plnk::bench(b, &format!("mul {}", n), || {
+ plnk::timeit(|| mat_a.mul(&mat_b))
+ });
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// the sizes of the matrices to consider
+ #[arg(num_args = 1.., value_delimiter = ' ')]
+ sizes: Vec<usize>,
+
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long)]
+ nb_measurements: usize,
+}
+
+fn main() {
+ let cli = Cli::parse();
+
+ let b = plnk::Bencher::new(cli.nb_measurements);
+
+ for n in cli.sizes {
+ inverse_template::<ark_bls12_381::Fr>(&b.with_name("BLS12-381"), n);
+ inverse_template::<ark_bn254::Fr>(&b.with_name("BN-254"), n);
+ inverse_template::<ark_pallas::Fr>(&b.with_name("PALLAS"), n);
+
+ transpose_template::<ark_bls12_381::Fr>(&b.with_name("BLS12-381"), n);
+ transpose_template::<ark_bn254::Fr>(&b.with_name("BN-254"), n);
+ transpose_template::<ark_pallas::Fr>(&b.with_name("PALLAS"), n);
+
+ mul_template::<ark_bls12_381::Fr>(&b.with_name("BLS12-381"), n);
+ mul_template::<ark_bn254::Fr>(&b.with_name("BN-254"), n);
+ mul_template::<ark_pallas::Fr>(&b.with_name("PALLAS"), n);
+ }
+}
diff --git a/examples/benches/setup.rs b/examples/benches/setup.rs
new file mode 100644
index 0000000000000000000000000000000000000000..231466e6386886c69b7af31a75cfc68f65b4e894
--- /dev/null
+++ b/examples/benches/setup.rs
@@ -0,0 +1,188 @@
+// see `benches/README.md`
+use ark_ec::{pairing::Pairing, CurveGroup};
+use ark_ff::PrimeField;
+use ark_poly::{univariate::DensePolynomial, DenseUVPolynomial};
+use ark_poly_commit::kzg10::{self, KZG10};
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate};
+use ark_std::ops::Div;
+
+use clap::{command, Parser};
+use komodo::zk::{self, Powers};
+use plnk::Bencher;
+
+fn setup_template<F, G, P>(b: &Bencher, degree: usize)
+where
+ F: PrimeField,
+ G: CurveGroup<ScalarField = F>,
+ P: DenseUVPolynomial<F>,
+ for<'a, 'b> &'a P: Div<&'b P, Output = P>,
+{
+ let rng = &mut rand::thread_rng();
+
+ plnk::bench(b, &format!("degree {}", degree), || {
+ plnk::timeit(|| zk::setup::<F, G>(degree, rng))
+ });
+}
+
+fn ark_setup_template<E, P>(b: &Bencher, degree: usize)
+where
+ E: Pairing,
+ P: DenseUVPolynomial<E::ScalarField>,
+ for<'a, 'b> &'a P: Div<&'b P, Output = P>,
+{
+ let rng = &mut rand::thread_rng();
+
+ plnk::bench(b, &format!("degree {}", degree), || {
+ plnk::timeit(|| {
+ let setup = KZG10::<E, P>::setup(degree, false, rng).unwrap();
+ let powers_of_g = setup.powers_of_g[..=degree].to_vec();
+ let powers_of_gamma_g = (0..=degree).map(|i| setup.powers_of_gamma_g[&i]).collect();
+ kzg10::Powers::<E> {
+ powers_of_g: ark_std::borrow::Cow::Owned(powers_of_g),
+ powers_of_gamma_g: ark_std::borrow::Cow::Owned(powers_of_gamma_g),
+ }
+ })
+ });
+}
+
+#[allow(dead_code)]
+fn serde_template<F, G, P>(b: &Bencher, degree: usize)
+where
+ F: PrimeField,
+ G: CurveGroup<ScalarField = F>,
+ P: DenseUVPolynomial<F>,
+ for<'a, 'b> &'a P: Div<&'b P, Output = P>,
+{
+ let rng = &mut rand::thread_rng();
+
+ let setup = zk::setup::<F, G>(degree, rng).unwrap();
+
+ plnk::bench(
+ b,
+ &format!("serializing with compression {}", degree),
+ || {
+ plnk::timeit(|| {
+ let mut serialized = vec![0; setup.serialized_size(Compress::Yes)];
+ setup
+ .serialize_with_mode(&mut serialized[..], Compress::Yes)
+ .unwrap();
+ })
+ },
+ );
+
+ plnk::bench(
+ b,
+ &format!("serializing with no compression {}", degree),
+ || {
+ plnk::timeit(|| {
+ let mut serialized = vec![0; setup.serialized_size(Compress::No)];
+ setup
+ .serialize_with_mode(&mut serialized[..], Compress::No)
+ .unwrap();
+ })
+ },
+ );
+
+ for (compress, validate) in [
+ (Compress::Yes, Validate::Yes),
+ (Compress::Yes, Validate::No),
+ (Compress::No, Validate::Yes),
+ (Compress::No, Validate::No),
+ ] {
+ let mut serialized = vec![0; setup.serialized_size(compress)];
+ setup
+ .serialize_with_mode(&mut serialized[..], compress)
+ .unwrap();
+
+ plnk::bench(
+ b,
+ &format!(
+ "deserializing with {} and {} {}",
+ match compress {
+ Compress::Yes => "compression",
+ Compress::No => "no compression",
+ },
+ match validate {
+ Validate::Yes => "validation",
+ Validate::No => "no validation",
+ },
+ degree,
+ ),
+ || {
+ plnk::timeit(|| {
+ Powers::<F, G>::deserialize_with_mode(&serialized[..], compress, validate)
+ })
+ },
+ );
+ }
+}
+
+fn setup(degrees: &[usize], nb_measurements: usize) {
+ fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
+ degree: usize,
+ curve: &str,
+ nb_measurements: usize,
+ ) {
+ let b = plnk::Bencher::new(nb_measurements).with_name(format!("setup on {}", curve));
+ setup_template::<F, G, DensePolynomial<F>>(&b, degree);
+ }
+
+ for d in degrees {
+ aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(*d, "BLS-12-381", nb_measurements);
+ aux::<ark_bn254::Fr, ark_bn254::G1Projective>(*d, "BN-254", nb_measurements);
+ aux::<ark_pallas::Fr, ark_pallas::Projective>(*d, "PALLAS", nb_measurements);
+ }
+}
+
+#[allow(dead_code)]
+fn serde(degrees: &[usize], nb_measurements: usize) {
+ fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
+ degree: usize,
+ curve: &str,
+ nb_measurements: usize,
+ ) {
+ let b =
+ plnk::Bencher::new(nb_measurements).with_name(format!("serialization on {}", curve));
+ serde_template::<F, G, DensePolynomial<F>>(&b, degree);
+ }
+
+ for d in degrees {
+ aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(*d, "BLS-12-381", nb_measurements);
+ aux::<ark_bn254::Fr, ark_bn254::G1Projective>(*d, "BN-254", nb_measurements);
+ aux::<ark_pallas::Fr, ark_pallas::Projective>(*d, "PALLAS", nb_measurements);
+ }
+}
+
+fn ark_setup(degrees: &[usize], nb_measurements: usize) {
+ fn aux<E: Pairing>(degree: usize, curve: &str, nb_measurements: usize) {
+ let b = plnk::Bencher::new(nb_measurements).with_name(format!("ARK setup on {}", curve));
+ ark_setup_template::<E, DensePolynomial<E::ScalarField>>(&b, degree);
+ }
+
+ for d in degrees {
+ aux::<ark_bls12_381::Bls12_381>(*d, "BLS-12-381", nb_measurements);
+ aux::<ark_bn254::Bn254>(*d, "BN-254", nb_measurements);
+ }
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// the polynomial degrees to measure the commit time on
+ #[arg(num_args = 1.., value_delimiter = ' ')]
+ degrees: Vec<usize>,
+
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long)]
+ nb_measurements: usize,
+}
+
+fn main() {
+ let cli = Cli::parse();
+
+ setup(&cli.degrees, cli.nb_measurements);
+ ark_setup(&cli.degrees, cli.nb_measurements);
+ // NOTE: this is disabled for now because it takes so much time...
+ // serde(&cli.degrees, cli.nb_measurements);
+}
diff --git a/scripts/parse.nu b/scripts/parse.nu
new file mode 100644
index 0000000000000000000000000000000000000000..3fb4b86990fe33ba3e0065c2f0acb79190584365
--- /dev/null
+++ b/scripts/parse.nu
@@ -0,0 +1,22 @@
+export def read-atomic-ops [
+ --include: list<string> = [], --exclude: list<string> = []
+]: list -> record {
+ let raw = $in
+ | insert t {|it| $it.times |math avg}
+ | reject times
+ | rename --column { label: "group", name: "species", t: "measurement" }
+
+ let included = if $include != [] {
+ $raw | where group in $include
+ } else {
+ $raw
+ }
+
+ $included
+ | where group not-in $exclude
+ | group-by group --to-table
+ | reject items.group
+ | update items { transpose -r | into record }
+ | transpose -r
+ | into record
+}
diff --git a/scripts/plot/bench_commit.py b/scripts/plot/bench_commit.py
deleted file mode 100644
index ba9f5033b29251f8e7ed5c2cd1798a59d95527b2..0000000000000000000000000000000000000000
--- a/scripts/plot/bench_commit.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# see `benches/README.md`
-import json
-import sys
-import matplotlib.pyplot as plt
-
-NB_NS_IN_MS = 1e6
-
-
-if __name__ == "__main__":
- data = json.loads(sys.argv[1])
-
- for group in data:
- xs = [x["degree"] for x in group["items"]]
- ys = [x["mean"] / NB_NS_IN_MS for x in group["items"]]
- zs = [x["stddev"] / NB_NS_IN_MS for x in group["items"]]
-
- down = [y - z for (y, z) in zip(ys, zs)]
- up = [y + z for (y, z) in zip(ys, zs)]
-
- style = "dashed" if group["group"].endswith("-ark") else "solid"
- plt.plot(xs, ys, label=group["group"], marker='o', linestyle=style)
- plt.fill_between(xs, down, up, alpha=0.3)
-
- plt.xlabel("degree")
- plt.ylabel("time (in ms)")
-
- plt.title("time to commit polynomials for certain curves")
-
- plt.legend()
- plt.grid(True)
- plt.show()
diff --git a/scripts/plot/benches.py b/scripts/plot/benches.py
deleted file mode 100644
index a525783f39cc87474e721e36869a56f22c7f3085..0000000000000000000000000000000000000000
--- a/scripts/plot/benches.py
+++ /dev/null
@@ -1,242 +0,0 @@
-# see `benches/README.md`
-import matplotlib.pyplot as plt
-import json
-import sys
-import os
-import argparse
-from typing import Any, Dict, List
-
-NB_NS_IN_MS = 1e6
-NB_BYTES_IN_KB = 1_024
-
-FULLSCREEN_DPI = 300
-
-# represents a full NDJSON dataset, i.e. directly generated by `cargo criterion`,
-# filtered to remove invalid lines, e.g. whose `$.reason` is not
-# `benchmark-complete`
-Data = List[Dict[str, Any]]
-
-
-# k1: namely `mean` or `median`
-# k2: namely `estimation`, `upper_bound`, `lower_bound` or None
-def extract(data: Data, k1: str, k2: str) -> List[float]:
- return [line[k1][k2] if k2 is not None else line[k1] for line in data]
-
-
-# convert a list of times in nanoseconds to the same list in milliseconds
-def ns_to_ms(times: List[float]) -> List[float]:
- return [t / NB_NS_IN_MS for t in times]
-
-
-# convert a list of sizes in bytes to the same list in kilobytes
-def b_to_kb(sizes: List[int]) -> List[float]:
- return [s / NB_BYTES_IN_KB for s in sizes]
-
-
-# read a result dataset from an NDJSON file and filter out invalid lines
-#
-# here, invalid lines are all the lines with `$.reason` not equal to
-# `benchmark-complete` that are generated by `cargo criterion` but useless.
-def read_data(data_file: str) -> Data:
- if not os.path.exists(data_file):
- print(f"no such file: `{data_file}`")
- exit(1)
-
- with open(data_file, "r") as file:
- data = list(filter(
- lambda line: line["reason"] == "benchmark-complete",
- map(
- json.loads,
- file.readlines()
- )
- ))
-
- return data
-
-
-def plot_linalg(data: Data, save: bool = False):
- # key: the start of the `$.id` field
- def plot(data: Data, key: str, curve: str, color: str, ax):
- filtered_data = list(filter(
- lambda line: line["id"].startswith(key) and line["id"].endswith(f" on {curve}"),
- data
- ))
- if len(filtered_data) == 0:
- return
-
- sizes = [
- int(line["id"].split(' ')[1].split('x')[0]) for line in filtered_data
- ]
-
- means = ns_to_ms(extract(filtered_data, "mean", "estimate"))
- up = ns_to_ms(extract(filtered_data, "mean", "upper_bound"))
- down = ns_to_ms(extract(filtered_data, "mean", "lower_bound"))
-
- ax.plot(sizes, means, label=curve, color=color)
- ax.fill_between(sizes, down, up, color=color, alpha=0.3)
-
- keys = ["transpose", "mul", "inverse"]
-
- fig, axs = plt.subplots(len(keys), 1, figsize=(16, 9))
-
- for key, ax in zip(keys, axs):
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, key=key, curve=curve, color=color, ax=ax)
- ax.set_title(key)
- ax.set_yscale("log")
- ax.set_ylabel("time in ms")
- ax.legend()
- ax.grid()
-
- if save:
- output = "linalg.png"
- plt.savefig(output, dpi=FULLSCREEN_DPI)
- print(f"figure saved as {output}")
- else:
- plt.show()
-
-
-def plot_setup(data: Data, save: bool = False):
- fig, axs = plt.subplots(4, 1, sharex=True, figsize=(16, 9))
-
- # key: the start of the `$.id` field
- def plot(data: Data, key: str, curve: str, label: str, color: str, style: str, error_bar: bool, ax):
- filtered_data = list(filter(
- lambda line: line["id"].startswith(key) and line["id"].endswith(f" on {curve}"),
- data
- ))
- if len(filtered_data) == 0:
- return
-
- sizes = [int(line["id"].lstrip(key).split(' ')[0]) for line in filtered_data]
-
- if error_bar:
- means = ns_to_ms(extract(filtered_data, "mean", "estimate"))
- up = ns_to_ms(extract(filtered_data, "mean", "upper_bound"))
- down = ns_to_ms(extract(filtered_data, "mean", "lower_bound"))
- else:
- means = b_to_kb(extract(filtered_data, "mean", None))
-
- ax.plot(sizes, means, label=f"{label} on {curve}", color=color, linestyle=style)
-
- if error_bar:
- ax.fill_between(sizes, down, up, color=color, alpha=0.3)
-
- # setup
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "setup/setup (komodo)", curve, "komodo", color, "solid", True, axs[0])
- plot(data, "setup (arkworks)", curve, "arkworks", color, "dashed", True, axs[0])
- axs[0].set_title("time to generate a random trusted setup")
- axs[0].set_ylabel("time (in ms)")
- axs[0].legend()
- axs[0].grid()
-
- # serialization
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "setup/serializing with compression", curve, "compressed", color, "solid", True, axs[1])
- plot(data, "setup/serializing with no compression", curve, "uncompressed", color, "dashed", True, axs[1])
- axs[1].set_title("serialization")
- axs[1].set_ylabel("time (in ms)")
- axs[1].legend()
- axs[1].grid()
-
- # deserialization
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "setup/deserializing with no compression and no validation", curve, "uncompressed unvalidated", color, "dotted", True, axs[2])
- plot(data, "setup/deserializing with compression and no validation", curve, "compressed unvalidated", color, "dashed", True, axs[2])
- plot(data, "setup/deserializing with no compression and validation", curve, "uncompressed validated", color, "dashdot", True, axs[2])
- plot(data, "setup/deserializing with compression and validation", curve, "compressed validated", color, "solid", True, axs[2])
- axs[2].set_title("deserialization")
- axs[2].set_ylabel("time (in ms)")
- axs[2].legend()
- axs[2].grid()
-
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "serialized size with no compression", curve, "uncompressed", color, "dashed", False, axs[3])
- plot(data, "serialized size with compression", curve, "compressed", color, "solid", False, axs[3])
- axs[3].set_title("size")
- axs[3].set_xlabel("degree")
- axs[3].set_ylabel("size (in kb)")
- axs[3].legend()
- axs[3].grid()
-
- if save:
- output = "setup.png"
- plt.savefig(output, dpi=FULLSCREEN_DPI)
- print(f"figure saved as {output}")
- else:
- plt.show()
-
-
-def plot_commit(data: Data, save: bool = False):
- fig, ax = plt.subplots(1, 1, figsize=(16, 9))
-
- # key: the start of the `$.id` field
- def plot(data: Data, key: str, curve: str, style: str, color: str, ax):
- filtered_data = list(filter(
- lambda line: line["id"].startswith(key) and line["id"].endswith(f" on {curve}"),
- data
- ))
- if len(filtered_data) == 0:
- return
-
- sizes = [
- int(line["id"].lstrip(key).split(' ')[0]) for line in filtered_data
- ]
-
- means = ns_to_ms(extract(filtered_data, "mean", "estimate"))
- up = ns_to_ms(extract(filtered_data, "mean", "upper_bound"))
- down = ns_to_ms(extract(filtered_data, "mean", "lower_bound"))
-
- ax.plot(sizes, means, label=f"{key} on {curve}", color=color, linestyle=style)
- ax.fill_between(sizes, down, up, color=color, linestyle=style, alpha=0.3)
-
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, key="commit (komodo)", curve=curve, style="solid", color=color, ax=ax)
- plot(data, key="commit (arkworks)", curve=curve, style="dashed", color=color, ax=ax)
-
- ax.set_title("commit times")
- ax.set_ylabel("time (in ms)")
- ax.set_xlabel("degree")
- ax.legend()
- ax.grid(True)
-
- if save:
- output = "commit.png"
- plt.savefig(output, dpi=FULLSCREEN_DPI)
- print(f"figure saved as {output}")
- else:
- plt.show()
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("filename", type=str)
- parser.add_argument(
- "--bench", "-b", type=str, choices=["linalg", "setup", "commit"],
- )
- parser.add_argument(
- "--save", "-s", action="store_true", default=False,
- )
- parser.add_argument(
- "--all", "-a", action="store_true", default=False,
- )
- args = parser.parse_args()
-
- data = read_data(args.filename)
-
- if args.all:
- plot_linalg(data, save=args.save)
- plot_setup(data, save=args.save)
- plot_commit(data, save=args.save)
- exit(0)
-
- match args.bench:
- case "linalg":
- plot_linalg(data, save=args.save)
- case "setup":
- plot_setup(data, save=args.save)
- case "commit":
- plot_commit(data, save=args.save)
- case _:
- print("nothing to do: you might want to use `--bench <bench>` or `--all`")
diff --git a/scripts/plot/plot.py b/scripts/plot/plot.py
new file mode 100644
index 0000000000000000000000000000000000000000..339fd9092ea16f1fc5b0a8c89f21b856f1d01fbd
--- /dev/null
+++ b/scripts/plot/plot.py
@@ -0,0 +1,104 @@
+# see `benches/README.md`
+import json
+import sys
+import matplotlib.pyplot as plt
+import argparse
+
+
+# # Example
+# ```nuon
+# [
+# {
+# group: "Alice",
+# items: [
+# [ x, measurement, error ];
+# [ 1, 1143, 120 ],
+# [ 2, 1310, 248 ],
+# [ 4, 1609, 258 ],
+# [ 8, 1953, 343 ],
+# [ 16, 2145, 270 ],
+# [ 32, 3427, 301 ]
+# ]
+# },
+# {
+# group: "Bob",
+# items: [
+# [ x, measurement, error ];
+# [ 1, 2388, 374 ],
+# [ 2, 2738, 355 ],
+# [ 4, 3191, 470 ],
+# [ 8, 3932, 671 ],
+# [ 16, 4571, 334 ],
+# [ 32, 4929, 1094 ]
+# ]
+# },
+# ]
+# ```
+def plot(data, title: str, x_label: str, y_label: str, save: str = None):
+ for group in data:
+ xs = [x["x"] for x in group["items"]]
+ ys = [x["measurement"] for x in group["items"]]
+ zs = [x["error"] for x in group["items"]]
+
+ down = [y - z for (y, z) in zip(ys, zs)]
+ up = [y + z for (y, z) in zip(ys, zs)]
+
+ style = "dashed" if group["group"].endswith("-ark") else "solid"
+ plt.plot(xs, ys, label=group["group"], marker='o', linestyle=style)
+ plt.fill_between(xs, down, up, alpha=0.3)
+
+ plt.xlabel(x_label)
+ plt.ylabel(y_label)
+
+ plt.title(title)
+
+ plt.legend()
+ plt.grid(True)
+
+ if save is not None:
+ fig.set_size_inches((16, 9), forward=False)
+ fig.savefig(save, dpi=500)
+
+ print(f"plot saved as `{save}`")
+ else:
+ plt.show()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("data", type=str, help="""
+ the actual data to show in a multibar plot, here is an example:
+ [
+ {
+ group: "Alice",
+ items: [
+ [ x, measurement, error ];
+ [ 1, 1143, 120 ],
+ [ 2, 1310, 248 ],
+ [ 4, 1609, 258 ],
+ [ 8, 1953, 343 ],
+ [ 16, 2145, 270 ],
+ [ 32, 3427, 301 ]
+ ]
+ },
+ {
+ group: "Bob",
+ items: [
+ [ x, measurement, error ];
+ [ 1, 2388, 374 ],
+ [ 2, 2738, 355 ],
+ [ 4, 3191, 470 ],
+ [ 8, 3932, 671 ],
+ [ 16, 4571, 334 ],
+ [ 32, 4929, 1094 ]
+ ]
+ },
+ ]
+ """)
+ parser.add_argument("--title", "-t", type=str, help="the title of the plot")
+ parser.add_argument("--x-label", "-x", type=str, help="the x label of the plot")
+ parser.add_argument("--y-label", "-y", type=str, help="the y label of the plot")
+ parser.add_argument("--save", "-s", type=str, help="a path to save the figure to")
+ args = parser.parse_args()
+
+ plot(json.loads(args.data), args.title, args.x_label, args.y_label, save=args.save)