From daa2d42a2dfac2fe3e5b8f59c9b65e394c51ff67 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 11:05:56 +0200
Subject: [PATCH 01/25] remove "commit" criterion benchmark
---
Cargo.toml | 4 ---
benches/commit.rs | 87 -----------------------------------------------
2 files changed, 91 deletions(-)
delete mode 100644 benches/commit.rs
diff --git a/Cargo.toml b/Cargo.toml
index 31e8a733..b2444329 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -63,10 +63,6 @@ harness = false
name = "setup"
harness = false
-[[bench]]
-name = "commit"
-harness = false
-
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
diff --git a/benches/commit.rs b/benches/commit.rs
deleted file mode 100644
index 21f59cbf..00000000
--- a/benches/commit.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ec::{pairing::Pairing, CurveGroup};
-use ark_ff::PrimeField;
-use ark_poly::{univariate::DensePolynomial, DenseUVPolynomial};
-use ark_poly_commit::kzg10::{Powers, KZG10};
-use ark_std::ops::Div;
-
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use komodo::zk;
-
-fn commit_template<F, G, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- F: PrimeField,
- G: CurveGroup<ScalarField = F>,
- P: DenseUVPolynomial<F>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- let setup = zk::setup::<F, G>(degree, rng).unwrap();
- let polynomial = P::rand(degree, rng);
-
- c.bench_function(&format!("commit (komodo) {} on {}", degree, curve), |b| {
- b.iter(|| zk::commit(&setup, &polynomial))
- });
-}
-
-fn ark_commit_template<E, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- E: Pairing,
- P: DenseUVPolynomial<E::ScalarField>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- let setup = KZG10::<E, P>::setup(degree, false, rng).unwrap();
- let powers_of_g = setup.powers_of_g[..=degree].to_vec();
- let powers_of_gamma_g = (0..=degree).map(|i| setup.powers_of_gamma_g[&i]).collect();
- let powers = Powers::<E> {
- powers_of_g: ark_std::borrow::Cow::Owned(powers_of_g),
- powers_of_gamma_g: ark_std::borrow::Cow::Owned(powers_of_gamma_g),
- };
- let polynomial = P::rand(degree, rng);
-
- c.bench_function(&format!("commit (arkworks) {} on {}", degree, curve), |b| {
- b.iter(|| KZG10::commit(&powers, &polynomial, None, None))
- });
-}
-
-fn commit(c: &mut Criterion) {
- fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- degree: usize,
- curve: &str,
- ) {
- commit_template::<F, G, DensePolynomial<F>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(c, n, "BLS12-381");
- aux::<ark_bn254::Fr, ark_bn254::G1Projective>(c, n, "BN-254");
- aux::<ark_pallas::Fr, ark_pallas::Projective>(c, n, "PALLAS");
- }
-}
-
-fn ark_commit(c: &mut Criterion) {
- fn aux<E: Pairing>(c: &mut Criterion, degree: usize, curve: &str) {
- ark_commit_template::<E, DensePolynomial<E::ScalarField>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Bls12_381>(c, n, "BLS12-381");
- aux::<ark_bn254::Bn254>(c, n, "BN-254");
- }
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = commit, ark_commit
-);
-criterion_main!(benches);
--
GitLab
From fe61c4683db1254efe97d2d0a81b4694519dc147 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 11:28:07 +0200
Subject: [PATCH 02/25] migrate "setup" benchmark to PLNK example
---
Cargo.toml | 9 +-
benches/setup.rs | 191 --------------------------------------
examples/benches/setup.rs | 184 ++++++++++++++++++++++++++++++++++++
3 files changed, 189 insertions(+), 195 deletions(-)
delete mode 100644 benches/setup.rs
create mode 100644 examples/benches/setup.rs
diff --git a/Cargo.toml b/Cargo.toml
index b2444329..4ec0aa98 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -59,10 +59,6 @@ harness = false
name = "linalg"
harness = false
-[[bench]]
-name = "setup"
-harness = false
-
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
@@ -80,3 +76,8 @@ harness = false
name = "bench_curve_group_operations"
path = "examples/benches/operations/curve_group.rs"
harness = false
+
+[[example]]
+name = "bench_setup"
+path = "examples/benches/setup.rs"
+harness = false
diff --git a/benches/setup.rs b/benches/setup.rs
deleted file mode 100644
index 15aaa211..00000000
--- a/benches/setup.rs
+++ /dev/null
@@ -1,191 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ec::{pairing::Pairing, CurveGroup};
-use ark_ff::PrimeField;
-use ark_poly::{univariate::DensePolynomial, DenseUVPolynomial};
-use ark_poly_commit::kzg10::{self, KZG10};
-use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate};
-use ark_std::ops::Div;
-
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use komodo::zk::{self, Powers};
-
-fn setup_template<F, G, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- F: PrimeField,
- G: CurveGroup<ScalarField = F>,
- P: DenseUVPolynomial<F>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- c.bench_function(&format!("setup (komodo) {} on {}", degree, curve), |b| {
- b.iter(|| zk::setup::<F, G>(degree, rng).unwrap())
- });
-}
-
-fn ark_setup_template<E, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- E: Pairing,
- P: DenseUVPolynomial<E::ScalarField>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let rng = &mut rand::thread_rng();
-
- c.bench_function(
- &format!("setup (arkworks) {} bytes on {}", degree, curve),
- |b| {
- b.iter(|| {
- let setup = KZG10::<E, P>::setup(degree, false, rng).unwrap();
- let powers_of_g = setup.powers_of_g[..=degree].to_vec();
- let powers_of_gamma_g = (0..=degree).map(|i| setup.powers_of_gamma_g[&i]).collect();
- kzg10::Powers::<E> {
- powers_of_g: ark_std::borrow::Cow::Owned(powers_of_g),
- powers_of_gamma_g: ark_std::borrow::Cow::Owned(powers_of_gamma_g),
- }
- })
- },
- );
-}
-
-fn serde_template<F, G, P>(c: &mut Criterion, degree: usize, curve: &str)
-where
- F: PrimeField,
- G: CurveGroup<ScalarField = F>,
- P: DenseUVPolynomial<F>,
- for<'a, 'b> &'a P: Div<&'b P, Output = P>,
-{
- let mut group = c.benchmark_group("setup");
-
- let rng = &mut rand::thread_rng();
-
- let setup = zk::setup::<F, G>(degree, rng).unwrap();
-
- group.bench_function(
- &format!("serializing with compression {} on {}", degree, curve),
- |b| {
- b.iter(|| {
- let mut serialized = vec![0; setup.serialized_size(Compress::Yes)];
- setup
- .serialize_with_mode(&mut serialized[..], Compress::Yes)
- .unwrap();
- })
- },
- );
-
- group.bench_function(
- &format!("serializing with no compression {} on {}", degree, curve),
- |b| {
- b.iter(|| {
- let mut serialized = vec![0; setup.serialized_size(Compress::No)];
- setup
- .serialize_with_mode(&mut serialized[..], Compress::No)
- .unwrap();
- })
- },
- );
-
- for (compress, validate) in [
- (Compress::Yes, Validate::Yes),
- (Compress::Yes, Validate::No),
- (Compress::No, Validate::Yes),
- (Compress::No, Validate::No),
- ] {
- let mut serialized = vec![0; setup.serialized_size(compress)];
- setup
- .serialize_with_mode(&mut serialized[..], compress)
- .unwrap();
-
- println!(
- r#"["id": "{} degree serialized with {} and {} on {}", "size": {}"#,
- degree,
- match compress {
- Compress::Yes => "compression",
- Compress::No => "no compression",
- },
- match validate {
- Validate::Yes => "validation",
- Validate::No => "no validation",
- },
- curve,
- serialized.len(),
- );
-
- group.bench_function(
- &format!(
- "deserializing with {} and {} {} on {}",
- match compress {
- Compress::Yes => "compression",
- Compress::No => "no compression",
- },
- match validate {
- Validate::Yes => "validation",
- Validate::No => "no validation",
- },
- degree,
- curve
- ),
- |b| {
- b.iter(|| {
- Powers::<F, G>::deserialize_with_mode(&serialized[..], compress, validate)
- })
- },
- );
- }
-
- group.finish();
-}
-
-fn setup(c: &mut Criterion) {
- fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- degree: usize,
- curve: &str,
- ) {
- setup_template::<F, G, DensePolynomial<F>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(c, n, "BLS-12-381");
- aux::<ark_bn254::Fr, ark_bn254::G1Projective>(c, n, "BN-254");
- aux::<ark_pallas::Fr, ark_pallas::Projective>(c, n, "PALLAS");
- }
-}
-
-fn serde(c: &mut Criterion) {
- fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
- c: &mut Criterion,
- degree: usize,
- curve: &str,
- ) {
- serde_template::<F, G, DensePolynomial<F>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(c, n, "BLS-12-381");
- aux::<ark_bn254::Fr, ark_bn254::G1Projective>(c, n, "BN-254");
- aux::<ark_pallas::Fr, ark_pallas::Projective>(c, n, "PALLAS");
- }
-}
-
-fn ark_setup(c: &mut Criterion) {
- fn aux<E: Pairing>(c: &mut Criterion, degree: usize, curve: &str) {
- ark_setup_template::<E, DensePolynomial<E::ScalarField>>(c, black_box(degree), curve);
- }
-
- for n in [1, 2, 4, 8, 16] {
- aux::<ark_bls12_381::Bls12_381>(c, n, "BLS-12-381");
- aux::<ark_bn254::Bn254>(c, n, "BN-254");
- }
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = setup, ark_setup, serde
-);
-criterion_main!(benches);
diff --git a/examples/benches/setup.rs b/examples/benches/setup.rs
new file mode 100644
index 00000000..18f96d0b
--- /dev/null
+++ b/examples/benches/setup.rs
@@ -0,0 +1,184 @@
+// see `benches/README.md`
+use ark_ec::{pairing::Pairing, CurveGroup};
+use ark_ff::PrimeField;
+use ark_poly::{univariate::DensePolynomial, DenseUVPolynomial};
+use ark_poly_commit::kzg10::{self, KZG10};
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Compress, Validate};
+use ark_std::ops::Div;
+
+use clap::{command, Parser};
+use komodo::zk::{self, Powers};
+use plnk::Bencher;
+
+fn setup_template<F, G, P>(b: &Bencher, degree: usize)
+where
+ F: PrimeField,
+ G: CurveGroup<ScalarField = F>,
+ P: DenseUVPolynomial<F>,
+ for<'a, 'b> &'a P: Div<&'b P, Output = P>,
+{
+ let rng = &mut rand::thread_rng();
+
+ plnk::bench(b, &format!("degree {}", degree), || {
+ plnk::timeit(|| zk::setup::<F, G>(degree, rng))
+ });
+}
+
+fn ark_setup_template<E, P>(b: &Bencher, degree: usize)
+where
+ E: Pairing,
+ P: DenseUVPolynomial<E::ScalarField>,
+ for<'a, 'b> &'a P: Div<&'b P, Output = P>,
+{
+ let rng = &mut rand::thread_rng();
+
+ plnk::bench(b, &format!("degree {}", degree), || {
+ plnk::timeit(|| {
+ let setup = KZG10::<E, P>::setup(degree, false, rng).unwrap();
+ let powers_of_g = setup.powers_of_g[..=degree].to_vec();
+ let powers_of_gamma_g = (0..=degree).map(|i| setup.powers_of_gamma_g[&i]).collect();
+ kzg10::Powers::<E> {
+ powers_of_g: ark_std::borrow::Cow::Owned(powers_of_g),
+ powers_of_gamma_g: ark_std::borrow::Cow::Owned(powers_of_gamma_g),
+ }
+ })
+ });
+}
+
+fn serde_template<F, G, P>(b: &Bencher, degree: usize)
+where
+ F: PrimeField,
+ G: CurveGroup<ScalarField = F>,
+ P: DenseUVPolynomial<F>,
+ for<'a, 'b> &'a P: Div<&'b P, Output = P>,
+{
+ let rng = &mut rand::thread_rng();
+
+ let setup = zk::setup::<F, G>(degree, rng).unwrap();
+
+ plnk::bench(
+ b,
+ &format!("serializing with compression {}", degree),
+ || {
+ plnk::timeit(|| {
+ let mut serialized = vec![0; setup.serialized_size(Compress::Yes)];
+ setup
+ .serialize_with_mode(&mut serialized[..], Compress::Yes)
+ .unwrap();
+ })
+ },
+ );
+
+ plnk::bench(
+ b,
+ &format!("serializing with no compression {}", degree),
+ || {
+ plnk::timeit(|| {
+ let mut serialized = vec![0; setup.serialized_size(Compress::No)];
+ setup
+ .serialize_with_mode(&mut serialized[..], Compress::No)
+ .unwrap();
+ })
+ },
+ );
+
+ for (compress, validate) in [
+ (Compress::Yes, Validate::Yes),
+ (Compress::Yes, Validate::No),
+ (Compress::No, Validate::Yes),
+ (Compress::No, Validate::No),
+ ] {
+ let mut serialized = vec![0; setup.serialized_size(compress)];
+ setup
+ .serialize_with_mode(&mut serialized[..], compress)
+ .unwrap();
+
+ plnk::bench(
+ b,
+ &format!(
+ "deserializing with {} and {} {}",
+ match compress {
+ Compress::Yes => "compression",
+ Compress::No => "no compression",
+ },
+ match validate {
+ Validate::Yes => "validation",
+ Validate::No => "no validation",
+ },
+ degree,
+ ),
+ || {
+ plnk::timeit(|| {
+ Powers::<F, G>::deserialize_with_mode(&serialized[..], compress, validate)
+ })
+ },
+ );
+ }
+}
+
+fn setup(degrees: &[usize], nb_measurements: usize) {
+ fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
+ degree: usize,
+ curve: &str,
+ nb_measurements: usize,
+ ) {
+ let b = plnk::Bencher::new(nb_measurements).with_name(format!("setup on {}", curve));
+ setup_template::<F, G, DensePolynomial<F>>(&b, degree);
+ }
+
+ for d in degrees {
+ aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(*d, "BLS-12-381", nb_measurements);
+ aux::<ark_bn254::Fr, ark_bn254::G1Projective>(*d, "BN-254", nb_measurements);
+ aux::<ark_pallas::Fr, ark_pallas::Projective>(*d, "PALLAS", nb_measurements);
+ }
+}
+
+fn serde(degrees: &[usize], nb_measurements: usize) {
+ fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
+ degree: usize,
+ curve: &str,
+ nb_measurements: usize,
+ ) {
+ let b = plnk::Bencher::new(nb_measurements).with_name(format!("serialization on {}", curve));
+ serde_template::<F, G, DensePolynomial<F>>(&b, degree);
+ }
+
+ for d in degrees {
+ aux::<ark_bls12_381::Fr, ark_bls12_381::G1Projective>(*d, "BLS-12-381", nb_measurements);
+ aux::<ark_bn254::Fr, ark_bn254::G1Projective>(*d, "BN-254", nb_measurements);
+ aux::<ark_pallas::Fr, ark_pallas::Projective>(*d, "PALLAS", nb_measurements);
+ }
+}
+
+fn ark_setup(degrees: &[usize], nb_measurements: usize) {
+ fn aux<E: Pairing>(degree: usize, curve: &str, nb_measurements: usize) {
+ let b = plnk::Bencher::new(nb_measurements).with_name(format!("ARK setup on {}", curve));
+ ark_setup_template::<E, DensePolynomial<E::ScalarField>>(&b, degree);
+ }
+
+ for d in degrees {
+ aux::<ark_bls12_381::Bls12_381>(*d, "BLS-12-381", nb_measurements);
+ aux::<ark_bn254::Bn254>(*d, "BN-254", nb_measurements);
+ }
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// the polynomial degrees to measure the commit time on
+ #[arg(num_args = 1.., value_delimiter = ' ')]
+ degrees: Vec<usize>,
+
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long)]
+ nb_measurements: usize,
+}
+
+fn main() {
+ let cli = Cli::parse();
+
+ setup(&cli.degrees, cli.nb_measurements);
+ ark_setup(&cli.degrees, cli.nb_measurements);
+ serde(&cli.degrees, cli.nb_measurements);
+}
--
GitLab
From f1388379b12d75bee4f22f09393343169f80ed94 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:23:20 +0200
Subject: [PATCH 03/25] make the "bench commit" plot script more general
---
benches/README.md | 15 +++++++----
scripts/plot/bench_commit.py | 31 ----------------------
scripts/plot/plot.py | 50 ++++++++++++++++++++++++++++++++++++
3 files changed, 60 insertions(+), 36 deletions(-)
delete mode 100644 scripts/plot/bench_commit.py
create mode 100644 scripts/plot/plot.py
diff --git a/benches/README.md b/benches/README.md
index b9df21b9..57c2493b 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -84,13 +84,18 @@ let degrees = seq 0 15 | each { 2 ** $in }
let res = cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
| lines
| each { from nuon }
- | update times { into duration }
+ | update times { each { $in / 1_000_000 * 1ms } }
| insert mean {|it| $it.times | math avg}
| insert stddev {|it| $it.times | into int | into float | math stddev | into int | into duration}
| update label { parse "degree {d}" | into record | get d | into int }
- | rename --column { label: "degree", name: "curve" }
+ | rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
-python scripts/plot/bench_commit.py (
- $res | group-by curve --to-table | update items { reject curve } | to json
-)
+python scripts/plot/plot.py ...[
+ --title "time to commit polynomials for certain curves"
+ --x-label "degree"
+ --y-label "time (in ms)"
+ (
+ $res | group-by curve --to-table | update items { reject curve } | to json
+ )
+]
```
diff --git a/scripts/plot/bench_commit.py b/scripts/plot/bench_commit.py
deleted file mode 100644
index ba9f5033..00000000
--- a/scripts/plot/bench_commit.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# see `benches/README.md`
-import json
-import sys
-import matplotlib.pyplot as plt
-
-NB_NS_IN_MS = 1e6
-
-
-if __name__ == "__main__":
- data = json.loads(sys.argv[1])
-
- for group in data:
- xs = [x["degree"] for x in group["items"]]
- ys = [x["mean"] / NB_NS_IN_MS for x in group["items"]]
- zs = [x["stddev"] / NB_NS_IN_MS for x in group["items"]]
-
- down = [y - z for (y, z) in zip(ys, zs)]
- up = [y + z for (y, z) in zip(ys, zs)]
-
- style = "dashed" if group["group"].endswith("-ark") else "solid"
- plt.plot(xs, ys, label=group["group"], marker='o', linestyle=style)
- plt.fill_between(xs, down, up, alpha=0.3)
-
- plt.xlabel("degree")
- plt.ylabel("time (in ms)")
-
- plt.title("time to commit polynomials for certain curves")
-
- plt.legend()
- plt.grid(True)
- plt.show()
diff --git a/scripts/plot/plot.py b/scripts/plot/plot.py
new file mode 100644
index 00000000..24d50169
--- /dev/null
+++ b/scripts/plot/plot.py
@@ -0,0 +1,50 @@
+# see `benches/README.md`
+import json
+import sys
+import matplotlib.pyplot as plt
+import argparse
+
+
+def plot(data, title: str, x_label: str, y_label: str, save: str = None):
+ for group in data:
+ xs = [x["x"] for x in group["items"]]
+ ys = [x["measurement"] for x in group["items"]]
+ zs = [x["error"] for x in group["items"]]
+
+ down = [y - z for (y, z) in zip(ys, zs)]
+ up = [y + z for (y, z) in zip(ys, zs)]
+
+ style = "dashed" if group["group"].endswith("-ark") else "solid"
+ plt.plot(xs, ys, label=group["group"], marker='o', linestyle=style)
+ plt.fill_between(xs, down, up, alpha=0.3)
+
+ plt.xlabel(x_label)
+ plt.ylabel(y_label)
+
+ plt.title(title)
+
+ plt.legend()
+ plt.grid(True)
+
+ if save is not None:
+ fig.set_size_inches((16, 9), forward=False)
+ fig.savefig(save, dpi=500)
+
+ print(f"plot saved as `{save}`")
+ else:
+ plt.show()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("data", type=str, help="""
+ the actual data to show in a multibar plot, here is an example:
+ ...
+ """)
+ parser.add_argument("--title", "-t", type=str, help="the title of the plot")
+ parser.add_argument("--x-label", "-x", type=str, help="the x label of the plot")
+ parser.add_argument("--y-label", "-y", type=str, help="the y label of the plot")
+ parser.add_argument("--save", "-s", type=str, help="a path to save the figure to")
+ args = parser.parse_args()
+
+ plot(json.loads(args.data), args.title, args.x_label, args.y_label, save=args.save)
--
GitLab
From caf6dc07fc813300619092ee0185f7e9e9d52543 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:30:38 +0200
Subject: [PATCH 04/25] add a NOTE about a bug in Nushell
---
benches/README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index 57c2493b..8f9e2bc2 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -19,12 +19,12 @@ python scripts/plot/benches.py results.ndjson --bench setup
cargo run --example bench_field_operations -- --nb-measurements 1000
| lines
| each { from json }
- | to ndjson
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
| save --force field.ndjson
cargo run --example bench_curve_group_operations -- --nb-measurements 1000
| lines
| each { from json }
- | to ndjson
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
| save --force curve_group.ndjson
```
```nushell
--
GitLab
From 63da3c600768172d9be7f31de2f403b54031e578 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:32:27 +0200
Subject: [PATCH 05/25] move "read atomic ops" to `scripts/parse.nu`
---
benches/README.md | 24 +-----------------------
scripts/parse.nu | 22 ++++++++++++++++++++++
2 files changed, 23 insertions(+), 23 deletions(-)
create mode 100644 scripts/parse.nu
diff --git a/benches/README.md b/benches/README.md
index 8f9e2bc2..71eff0c5 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -28,30 +28,8 @@ cargo run --example bench_curve_group_operations -- --nb-measurements 1000
| save --force curve_group.ndjson
```
```nushell
-def read-atomic-ops [
- --include: list<string> = [], --exclude: list<string> = []
-]: list -> record {
- let raw = $in
- | insert t {|it| $it.times |math avg}
- | reject times
- | rename --column { label: "group", name: "species", t: "measurement" }
+use scripts/parse.nu read-atomic-ops
- let included = if $include != [] {
- $raw | where group in $include
- } else {
- $raw
- }
-
- $included
- | where group not-in $exclude
- | group-by group --to-table
- | reject items.group
- | update items { transpose -r | into record }
- | transpose -r
- | into record
-}
-```
-```nushell
python scripts/plot/multi_bar.py --title "simple field operations" -l "time (in ns)" (
open field.ndjson
| read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
diff --git a/scripts/parse.nu b/scripts/parse.nu
new file mode 100644
index 00000000..3fb4b869
--- /dev/null
+++ b/scripts/parse.nu
@@ -0,0 +1,22 @@
+export def read-atomic-ops [
+ --include: list<string> = [], --exclude: list<string> = []
+]: list -> record {
+ let raw = $in
+ | insert t {|it| $it.times |math avg}
+ | reject times
+ | rename --column { label: "group", name: "species", t: "measurement" }
+
+ let included = if $include != [] {
+ $raw | where group in $include
+ } else {
+ $raw
+ }
+
+ $included
+ | where group not-in $exclude
+ | group-by group --to-table
+ | reject items.group
+ | update items { transpose -r | into record }
+ | transpose -r
+ | into record
+}
--
GitLab
From df7cb5fe2b2ec09153bb3b4660c980bc6b17c1f9 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:33:32 +0200
Subject: [PATCH 06/25] split bench and plot for the commit
---
benches/README.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/benches/README.md b/benches/README.md
index 71eff0c5..b0a3bdca 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -67,7 +67,8 @@ let res = cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
| insert stddev {|it| $it.times | into int | into float | math stddev | into int | into duration}
| update label { parse "degree {d}" | into record | get d | into int }
| rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
-
+```
+```nushell
python scripts/plot/plot.py ...[
--title "time to commit polynomials for certain curves"
--x-label "degree"
--
GitLab
From 7b91d3ad027a97d05f2bb0206e5e59a1c42e8e13 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:41:12 +0200
Subject: [PATCH 07/25] refactor README further
---
benches/README.md | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index b0a3bdca..b6d99bcd 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -56,17 +56,28 @@ python scripts/plot/multi_bar.py --title "complex curve group operations" -l "ti
these are benchmarks that run a single measurement, implemented as _examples_ in
`examples/benches/`.
+### trusted setup
+```nushell
+let degrees = seq 0 15 | each { 2 ** $in }
+cargo run --example bench_setup -- --nb-measurements 10 ...$degrees
+ | lines
+ | each { from json }
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
+ | save --force setup.ndjson
+```
+
### commit
```nushell
let degrees = seq 0 15 | each { 2 ** $in }
-let res = cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
+cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
| lines
| each { from nuon }
| update times { each { $in / 1_000_000 * 1ms } }
| insert mean {|it| $it.times | math avg}
| insert stddev {|it| $it.times | into int | into float | math stddev | into int | into duration}
| update label { parse "degree {d}" | into record | get d | into int }
- | rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
+ | save --force commit.ndjson
```
```nushell
python scripts/plot/plot.py ...[
@@ -74,7 +85,7 @@ python scripts/plot/plot.py ...[
--x-label "degree"
--y-label "time (in ms)"
(
- $res | group-by curve --to-table | update items { reject curve } | to json
+ open commit.ndjson | group-by curve --to-table | update items { reject curve } | to json
)
]
```
--
GitLab
From 04702d8da4a8fb1e68935fd0cd11300abaa3011b Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:43:22 +0200
Subject: [PATCH 08/25] move linalg bench to `examples/benches/`
---
Cargo.toml | 9 +++++----
{benches => examples/benches}/linalg.rs | 0
2 files changed, 5 insertions(+), 4 deletions(-)
rename {benches => examples/benches}/linalg.rs (100%)
diff --git a/Cargo.toml b/Cargo.toml
index 4ec0aa98..7d413659 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,10 +55,6 @@ criterion = "0.3"
name = "recoding"
harness = false
-[[bench]]
-name = "linalg"
-harness = false
-
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
@@ -81,3 +77,8 @@ harness = false
name = "bench_setup"
path = "examples/benches/setup.rs"
harness = false
+
+[[example]]
+name = "bench_linalg"
+path = "examples/benches/linalg.rs"
+harness = false
diff --git a/benches/linalg.rs b/examples/benches/linalg.rs
similarity index 100%
rename from benches/linalg.rs
rename to examples/benches/linalg.rs
--
GitLab
From be008b6921c3aa11ac434902c1fc80336c99df54 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:43:39 +0200
Subject: [PATCH 09/25] remove useless "harness" on examples
---
Cargo.toml | 4 ----
1 file changed, 4 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index 7d413659..df73a393 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,19 +66,15 @@ path = "examples/benches/setup_size.rs"
[[example]]
name = "bench_field_operations"
path = "examples/benches/operations/field.rs"
-harness = false
[[example]]
name = "bench_curve_group_operations"
path = "examples/benches/operations/curve_group.rs"
-harness = false
[[example]]
name = "bench_setup"
path = "examples/benches/setup.rs"
-harness = false
[[example]]
name = "bench_linalg"
path = "examples/benches/linalg.rs"
-harness = false
--
GitLab
From 0dd67a120ced15edb6cb77722dd6e45b2a7ae090 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 12:57:11 +0200
Subject: [PATCH 10/25] migrate "linalg" bench to PLNK
---
examples/benches/linalg.rs | 84 +++++++++++++++++++-------------------
1 file changed, 41 insertions(+), 43 deletions(-)
diff --git a/examples/benches/linalg.rs b/examples/benches/linalg.rs
index 547f71a1..62fffc90 100644
--- a/examples/benches/linalg.rs
+++ b/examples/benches/linalg.rs
@@ -1,69 +1,67 @@
// see `benches/README.md`
-use std::time::Duration;
-
use ark_ff::PrimeField;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
+use clap::{arg, command, Parser};
use komodo::linalg::Matrix;
+use plnk::Bencher;
-fn inverse_template<F: PrimeField>(c: &mut Criterion, n: usize, curve: &str) {
+fn inverse_template<F: PrimeField>(b: &Bencher, n: usize) {
let mut rng = rand::thread_rng();
let matrix = Matrix::<F>::random(n, n, &mut rng);
- c.bench_function(&format!("inverse {}x{} on {}", n, n, curve), |b| {
- b.iter(|| matrix.invert().unwrap())
+ plnk::bench(b, &format!("inverse {}", n), || {
+ plnk::timeit(|| matrix.invert().unwrap())
});
}
-fn inverse(c: &mut Criterion) {
- for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
- inverse_template::<ark_bls12_381::Fr>(c, black_box(n), "BLS12-381");
- inverse_template::<ark_bn254::Fr>(c, black_box(n), "BN-254");
- inverse_template::<ark_pallas::Fr>(c, black_box(n), "PALLAS");
- }
-}
-
-fn transpose_template<F: PrimeField>(c: &mut Criterion, n: usize, curve: &str) {
+fn transpose_template<F: PrimeField>(b: &Bencher, n: usize) {
let mut rng = rand::thread_rng();
let matrix = Matrix::<F>::random(n, n, &mut rng);
- c.bench_function(&format!("transpose {}x{} on {}", n, n, curve), |b| {
- b.iter(|| matrix.transpose())
+ plnk::bench(b, &format!("transpose {}", n), || {
+ plnk::timeit(|| matrix.transpose())
});
}
-fn transpose(c: &mut Criterion) {
- for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
- transpose_template::<ark_bls12_381::Fr>(c, black_box(n), "BLS-12-381");
- transpose_template::<ark_bn254::Fr>(c, black_box(n), "BN-254");
- transpose_template::<ark_pallas::Fr>(c, black_box(n), "PALLAS");
- }
-}
-
-fn mul_template<F: PrimeField>(c: &mut Criterion, n: usize, curve: &str) {
+fn mul_template<F: PrimeField>(b: &Bencher, n: usize) {
let mut rng = rand::thread_rng();
let mat_a = Matrix::<F>::random(n, n, &mut rng);
let mat_b = Matrix::<F>::random(n, n, &mut rng);
- c.bench_function(&format!("mul {}x{} on {}", n, n, curve), |b| {
- b.iter(|| mat_a.mul(&mat_b))
+ plnk::bench(b, &format!("mul {}", n), || {
+ plnk::timeit(|| mat_a.mul(&mat_b))
});
}
-fn mul(c: &mut Criterion) {
- for n in [10, 15, 20, 30, 40, 60, 80, 120] {
- mul_template::<ark_bls12_381::Fr>(c, black_box(n), "BLS-12-381");
- mul_template::<ark_bn254::Fr>(c, black_box(n), "BN-254");
- mul_template::<ark_pallas::Fr>(c, black_box(n), "PALLAS");
- }
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// the sizes of the matrices to consider
+ #[arg(num_args = 1.., value_delimiter = ' ')]
+ sizes: Vec<usize>,
+
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long)]
+ nb_measurements: usize,
}
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = inverse, transpose, mul
-);
-criterion_main!(benches);
+fn main() {
+ let cli = Cli::parse();
+
+ let b = plnk::Bencher::new(cli.nb_measurements);
+
+ for n in cli.sizes {
+ inverse_template::<ark_bls12_381::Fr>(&b.with_name("BLS12-381"), n);
+ inverse_template::<ark_bn254::Fr>(&b.with_name("BN-254"), n);
+ inverse_template::<ark_pallas::Fr>(&b.with_name("PALLAS"), n);
+
+ transpose_template::<ark_bls12_381::Fr>(&b.with_name("BLS12-381"), n);
+ transpose_template::<ark_bn254::Fr>(&b.with_name("BN-254"), n);
+ transpose_template::<ark_pallas::Fr>(&b.with_name("PALLAS"), n);
+
+ mul_template::<ark_bls12_381::Fr>(&b.with_name("BLS12-381"), n);
+ mul_template::<ark_bn254::Fr>(&b.with_name("BN-254"), n);
+ mul_template::<ark_pallas::Fr>(&b.with_name("PALLAS"), n);
+ }
+}
--
GitLab
From df94350b49e0deacfbfe81be75610305eda5c51f Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:01:36 +0200
Subject: [PATCH 11/25] reduce the number of loops for the setup bench
---
benches/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/benches/README.md b/benches/README.md
index b6d99bcd..ba35d5bd 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -58,7 +58,7 @@ these are benchmarks that run a single measurement, implemented as _examples_ in
### trusted setup
```nushell
-let degrees = seq 0 15 | each { 2 ** $in }
+let degrees = seq 0 13 | each { 2 ** $in }
cargo run --example bench_setup -- --nb-measurements 10 ...$degrees
| lines
| each { from json }
--
GitLab
From 4df8dd8ae6b6ece3e5a51453aca8b88098f6be3c Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:10:43 +0200
Subject: [PATCH 12/25] disable "serde" bench for the setup
it takes too much time
---
examples/benches/setup.rs | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/examples/benches/setup.rs b/examples/benches/setup.rs
index 18f96d0b..ee3578d0 100644
--- a/examples/benches/setup.rs
+++ b/examples/benches/setup.rs
@@ -45,6 +45,7 @@ where
});
}
+#[allow(dead_code)]
fn serde_template<F, G, P>(b: &Bencher, degree: usize)
where
F: PrimeField,
@@ -133,6 +134,7 @@ fn setup(degrees: &[usize], nb_measurements: usize) {
}
}
+#[allow(dead_code)]
fn serde(degrees: &[usize], nb_measurements: usize) {
fn aux<F: PrimeField, G: CurveGroup<ScalarField = F>>(
degree: usize,
@@ -180,5 +182,6 @@ fn main() {
setup(&cli.degrees, cli.nb_measurements);
ark_setup(&cli.degrees, cli.nb_measurements);
- serde(&cli.degrees, cli.nb_measurements);
+ // NOTE: this is disabled for now because it takes so much time...
+ // serde(&cli.degrees, cli.nb_measurements);
}
--
GitLab
From 789dbac09af3e9a6265880af6059fa7edb7bb89c Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:13:52 +0200
Subject: [PATCH 13/25] fix commit plot call
---
benches/README.md | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/benches/README.md b/benches/README.md
index ba35d5bd..919b7121 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -85,7 +85,11 @@ python scripts/plot/plot.py ...[
--x-label "degree"
--y-label "time (in ms)"
(
- open commit.ndjson | group-by curve --to-table | update items { reject curve } | to json
+ open commit.ndjson
+ | rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
)
]
```
--
GitLab
From e6b46dd3e07d31654034faa0f6f4bec914a6f5fd Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:28:09 +0200
Subject: [PATCH 14/25] plot the setup results
---
benches/README.md | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/benches/README.md b/benches/README.md
index 919b7121..6120e5de 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -65,6 +65,30 @@ cargo run --example bench_setup -- --nb-measurements 10 ...$degrees
| to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
| save --force setup.ndjson
```
+```nushell
+python scripts/plot/plot.py ...[
+ --title "time to commit polynomials for certain curves"
+ --x-label "degree"
+ --y-label "time (in ms)"
+ (
+ open setup.ndjson
+ | update times { each { $in / 1_000_000 } }
+ | insert mean {|it| $it.times | math avg}
+ | insert stddev {|it| $it.times | into int | into float | math stddev | into int}
+ | insert degree { get label | parse "degree {d}" | into record | get d | into int}
+ | insert curve {|it| if ($it.name | str starts-with "ARK") {
+ let c = $it.name | parse "ARK setup on {curve}" | into record | get curve
+ $"($c)-ark"
+ } else {
+ $it.name | parse "setup on {curve}" | into record | get curve
+ }}
+ | rename --column { degree: "x", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+]
+```
### commit
```nushell
--
GitLab
From 0ed6c9dac5aae1dbdd8805483a3665f374371a22 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:28:51 +0200
Subject: [PATCH 15/25] fix the commit plot
---
benches/README.md | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index 6120e5de..6f14d94d 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -96,10 +96,6 @@ let degrees = seq 0 15 | each { 2 ** $in }
cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
| lines
| each { from nuon }
- | update times { each { $in / 1_000_000 * 1ms } }
- | insert mean {|it| $it.times | math avg}
- | insert stddev {|it| $it.times | into int | into float | math stddev | into int | into duration}
- | update label { parse "degree {d}" | into record | get d | into int }
| to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
| save --force commit.ndjson
```
@@ -110,6 +106,10 @@ python scripts/plot/plot.py ...[
--y-label "time (in ms)"
(
open commit.ndjson
+ | update times { each { $in / 1_000_000 } }
+ | insert mean {|it| $it.times | math avg}
+ | insert stddev {|it| $it.times | into int | into float | math stddev | into int}
+ | update label { parse "degree {d}" | into record | get d | into int }
| rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
| group-by curve --to-table
| update items { reject curve }
--
GitLab
From a1c76baeb1d0ca4f07f631752ebbbf6f3c044b64 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:32:34 +0200
Subject: [PATCH 16/25] fix the title of the setup plot
---
benches/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/benches/README.md b/benches/README.md
index 6f14d94d..59026760 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -67,7 +67,7 @@ cargo run --example bench_setup -- --nb-measurements 10 ...$degrees
```
```nushell
python scripts/plot/plot.py ...[
- --title "time to commit polynomials for certain curves"
+ --title "time to create trusted setups for certain curves"
--x-label "degree"
--y-label "time (in ms)"
(
--
GitLab
From e71ee63df949c911e987922411ab837fcf6195d9 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:35:11 +0200
Subject: [PATCH 17/25] simplify stddev computation
---
benches/README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index 59026760..2fa62d44 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -74,7 +74,7 @@ python scripts/plot/plot.py ...[
open setup.ndjson
| update times { each { $in / 1_000_000 } }
| insert mean {|it| $it.times | math avg}
- | insert stddev {|it| $it.times | into int | into float | math stddev | into int}
+ | insert stddev {|it| $it.times | into float | math stddev}
| insert degree { get label | parse "degree {d}" | into record | get d | into int}
| insert curve {|it| if ($it.name | str starts-with "ARK") {
let c = $it.name | parse "ARK setup on {curve}" | into record | get curve
@@ -108,7 +108,7 @@ python scripts/plot/plot.py ...[
open commit.ndjson
| update times { each { $in / 1_000_000 } }
| insert mean {|it| $it.times | math avg}
- | insert stddev {|it| $it.times | into int | into float | math stddev | into int}
+ | insert stddev {|it| $it.times | into float | math stddev}
| update label { parse "degree {d}" | into record | get d | into int }
| rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
| group-by curve --to-table
--
GitLab
From 1156125cf2647d25396057acd6a23ce2a1a64f8a Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:42:46 +0200
Subject: [PATCH 18/25] add a section about linear algebra
---
benches/README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 61 insertions(+)
diff --git a/benches/README.md b/benches/README.md
index 2fa62d44..a0dda43e 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -52,6 +52,67 @@ python scripts/plot/multi_bar.py --title "complex curve group operations" -l "ti
)
```
+## linear algebra
+```nushell
+let sizes = seq 0 10 | each { 2 ** $in }
+cargo run --example bench_linalg -- --nb-measurements 10 ...$sizes
+ | lines
+ | each { from json }
+ | to ndjson # NOTE: see https://github.com/nushell/nushell/issues/12655
+ | save --force linalg.ndjson
+```
+```nushell
+let linalg = open linalg.ndjson
+ | update times { each { $in / 1_000_000 } }
+ | insert mean {|it| $it.times | math avg}
+ | insert stddev {|it| $it.times | into float | math stddev}
+ | update label { parse "{op} {n}"}
+ | flatten --all label
+ | into int n
+
+python scripts/plot/plot.py ...[
+ --title "time to inverse an nxn matrix on certain curves"
+ --x-label "size"
+ --y-label "time (in ms)"
+ (
+ $linalg
+ | where op == inverse
+ | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+]
+
+python scripts/plot/plot.py ...[
+ --title "time to transpose an nxn matrix on certain curves"
+ --x-label "size"
+ --y-label "time (in ms)"
+ (
+ $linalg
+ | where op == transpose
+ | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+]
+
+python scripts/plot/plot.py ...[
+ --title "time to multiply two nxn matrices on certain curves"
+ --x-label "size"
+ --y-label "time (in ms)"
+ (
+ $linalg
+ | where op == mul
+ | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+]
+```
+
## oneshot benchmarks
these are benchmarks that run a single measurement, implemented as _examples_ in
`examples/benches/`.
--
GitLab
From 75068e412f91c741f19d3257e835134f7817273c Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:47:21 +0200
Subject: [PATCH 19/25] measure smaller matrices
---
benches/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/benches/README.md b/benches/README.md
index a0dda43e..29033aa9 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -54,7 +54,7 @@ python scripts/plot/multi_bar.py --title "complex curve group operations" -l "ti
## linear algebra
```nushell
-let sizes = seq 0 10 | each { 2 ** $in }
+let sizes = seq 0 7 | each { 2 ** $in }
cargo run --example bench_linalg -- --nb-measurements 10 ...$sizes
| lines
| each { from json }
--
GitLab
From 4451afbbf7f002d3e08991360f2d5657a7a40cba Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 13:47:35 +0200
Subject: [PATCH 20/25] refactor "linalg" plot
---
benches/README.md | 60 ++++++++++++++++-------------------------------
1 file changed, 20 insertions(+), 40 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index 29033aa9..7ad0e9ec 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -70,47 +70,27 @@ let linalg = open linalg.ndjson
| flatten --all label
| into int n
-python scripts/plot/plot.py ...[
- --title "time to inverse an nxn matrix on certain curves"
- --x-label "size"
- --y-label "time (in ms)"
- (
- $linalg
- | where op == inverse
- | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
- | group-by curve --to-table
- | update items { reject curve }
- | to json
- )
-]
+for it in [
+ [op, title];
-python scripts/plot/plot.py ...[
- --title "time to transpose an nxn matrix on certain curves"
- --x-label "size"
- --y-label "time (in ms)"
- (
- $linalg
- | where op == transpose
- | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
- | group-by curve --to-table
- | update items { reject curve }
- | to json
- )
-]
-
-python scripts/plot/plot.py ...[
- --title "time to multiply two nxn matrices on certain curves"
- --x-label "size"
- --y-label "time (in ms)"
- (
- $linalg
- | where op == mul
- | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
- | group-by curve --to-table
- | update items { reject curve }
- | to json
- )
-]
+ ["inverse", "time to inverse an nxn matrix on certain curves"],
+ ["transpose", "time to transpose an nxn matrix on certain curves"],
+ ["mul", "time to multiply two nxn matrices on certain curves"]
+] {
+ python scripts/plot/plot.py ...[
+ --title $it.title
+ --x-label "size"
+ --y-label "time (in ms)"
+ (
+ $linalg
+ | where op == $it.op
+ | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
+ | group-by curve --to-table
+ | update items { reject curve }
+ | to json
+ )
+ ]
+}
```
## oneshot benchmarks
--
GitLab
From da5073561273083d88df5f043e9d68a73f230eae Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 16:04:56 +0200
Subject: [PATCH 21/25] refactor README
---
benches/README.md | 24 ++----------------------
1 file changed, 2 insertions(+), 22 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index 7ad0e9ec..2b695f93 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -1,19 +1,3 @@
-## run the benchmarks
-```shell
-nushell> cargo criterion --output-format verbose --message-format json out> results.ndjson
-```
-
-## add the _trusted setup_ sizes
-```shell
-nushell> cargo run --example bench_setup_size out>> results.ndjson
-```
-
-## plot the results
-```shell
-python scripts/plot/benches.py results.ndjson --bench linalg
-python scripts/plot/benches.py results.ndjson --bench setup
-```
-
## atomic operations
```nushell
cargo run --example bench_field_operations -- --nb-measurements 1000
@@ -93,11 +77,7 @@ for it in [
}
```
-## oneshot benchmarks
-these are benchmarks that run a single measurement, implemented as _examples_ in
-`examples/benches/`.
-
-### trusted setup
+## trusted setup
```nushell
let degrees = seq 0 13 | each { 2 ** $in }
cargo run --example bench_setup -- --nb-measurements 10 ...$degrees
@@ -131,7 +111,7 @@ python scripts/plot/plot.py ...[
]
```
-### commit
+## commit
```nushell
let degrees = seq 0 15 | each { 2 ** $in }
cargo run --example bench_commit -- --nb-measurements 10 ...$degrees
--
GitLab
From 8cc7119e91bf89c2459646bb9c8aea5fbc7567a4 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 16:05:04 +0200
Subject: [PATCH 22/25] fix bug in README scripts
---
benches/README.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/benches/README.md b/benches/README.md
index 2b695f93..93be8223 100644
--- a/benches/README.md
+++ b/benches/README.md
@@ -54,7 +54,7 @@ let linalg = open linalg.ndjson
| flatten --all label
| into int n
-for it in [
+for graph in [
[op, title];
["inverse", "time to inverse an nxn matrix on certain curves"],
@@ -62,12 +62,12 @@ for it in [
["mul", "time to multiply two nxn matrices on certain curves"]
] {
python scripts/plot/plot.py ...[
- --title $it.title
+ --title $graph.title
--x-label "size"
--y-label "time (in ms)"
(
$linalg
- | where op == $it.op
+ | where op == $graph.op
| rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
| group-by curve --to-table
| update items { reject curve }
--
GitLab
From b0ad358cd4565786f9aadceb3c26d1937cd9c611 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 16:05:26 +0200
Subject: [PATCH 23/25] remove unused "benches.py" plot script
this is not used anymore, in favor of `plot.py` and `multi_bar.py`
---
scripts/plot/benches.py | 242 ----------------------------------------
1 file changed, 242 deletions(-)
delete mode 100644 scripts/plot/benches.py
diff --git a/scripts/plot/benches.py b/scripts/plot/benches.py
deleted file mode 100644
index a525783f..00000000
--- a/scripts/plot/benches.py
+++ /dev/null
@@ -1,242 +0,0 @@
-# see `benches/README.md`
-import matplotlib.pyplot as plt
-import json
-import sys
-import os
-import argparse
-from typing import Any, Dict, List
-
-NB_NS_IN_MS = 1e6
-NB_BYTES_IN_KB = 1_024
-
-FULLSCREEN_DPI = 300
-
-# represents a full NDJSON dataset, i.e. directly generated by `cargo criterion`,
-# filtered to remove invalid lines, e.g. whose `$.reason` is not
-# `benchmark-complete`
-Data = List[Dict[str, Any]]
-
-
-# k1: namely `mean` or `median`
-# k2: namely `estimation`, `upper_bound`, `lower_bound` or None
-def extract(data: Data, k1: str, k2: str) -> List[float]:
- return [line[k1][k2] if k2 is not None else line[k1] for line in data]
-
-
-# convert a list of times in nanoseconds to the same list in milliseconds
-def ns_to_ms(times: List[float]) -> List[float]:
- return [t / NB_NS_IN_MS for t in times]
-
-
-# convert a list of sizes in bytes to the same list in kilobytes
-def b_to_kb(sizes: List[int]) -> List[float]:
- return [s / NB_BYTES_IN_KB for s in sizes]
-
-
-# read a result dataset from an NDJSON file and filter out invalid lines
-#
-# here, invalid lines are all the lines with `$.reason` not equal to
-# `benchmark-complete` that are generated by `cargo criterion` but useless.
-def read_data(data_file: str) -> Data:
- if not os.path.exists(data_file):
- print(f"no such file: `{data_file}`")
- exit(1)
-
- with open(data_file, "r") as file:
- data = list(filter(
- lambda line: line["reason"] == "benchmark-complete",
- map(
- json.loads,
- file.readlines()
- )
- ))
-
- return data
-
-
-def plot_linalg(data: Data, save: bool = False):
- # key: the start of the `$.id` field
- def plot(data: Data, key: str, curve: str, color: str, ax):
- filtered_data = list(filter(
- lambda line: line["id"].startswith(key) and line["id"].endswith(f" on {curve}"),
- data
- ))
- if len(filtered_data) == 0:
- return
-
- sizes = [
- int(line["id"].split(' ')[1].split('x')[0]) for line in filtered_data
- ]
-
- means = ns_to_ms(extract(filtered_data, "mean", "estimate"))
- up = ns_to_ms(extract(filtered_data, "mean", "upper_bound"))
- down = ns_to_ms(extract(filtered_data, "mean", "lower_bound"))
-
- ax.plot(sizes, means, label=curve, color=color)
- ax.fill_between(sizes, down, up, color=color, alpha=0.3)
-
- keys = ["transpose", "mul", "inverse"]
-
- fig, axs = plt.subplots(len(keys), 1, figsize=(16, 9))
-
- for key, ax in zip(keys, axs):
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, key=key, curve=curve, color=color, ax=ax)
- ax.set_title(key)
- ax.set_yscale("log")
- ax.set_ylabel("time in ms")
- ax.legend()
- ax.grid()
-
- if save:
- output = "linalg.png"
- plt.savefig(output, dpi=FULLSCREEN_DPI)
- print(f"figure saved as {output}")
- else:
- plt.show()
-
-
-def plot_setup(data: Data, save: bool = False):
- fig, axs = plt.subplots(4, 1, sharex=True, figsize=(16, 9))
-
- # key: the start of the `$.id` field
- def plot(data: Data, key: str, curve: str, label: str, color: str, style: str, error_bar: bool, ax):
- filtered_data = list(filter(
- lambda line: line["id"].startswith(key) and line["id"].endswith(f" on {curve}"),
- data
- ))
- if len(filtered_data) == 0:
- return
-
- sizes = [int(line["id"].lstrip(key).split(' ')[0]) for line in filtered_data]
-
- if error_bar:
- means = ns_to_ms(extract(filtered_data, "mean", "estimate"))
- up = ns_to_ms(extract(filtered_data, "mean", "upper_bound"))
- down = ns_to_ms(extract(filtered_data, "mean", "lower_bound"))
- else:
- means = b_to_kb(extract(filtered_data, "mean", None))
-
- ax.plot(sizes, means, label=f"{label} on {curve}", color=color, linestyle=style)
-
- if error_bar:
- ax.fill_between(sizes, down, up, color=color, alpha=0.3)
-
- # setup
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "setup/setup (komodo)", curve, "komodo", color, "solid", True, axs[0])
- plot(data, "setup (arkworks)", curve, "arkworks", color, "dashed", True, axs[0])
- axs[0].set_title("time to generate a random trusted setup")
- axs[0].set_ylabel("time (in ms)")
- axs[0].legend()
- axs[0].grid()
-
- # serialization
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "setup/serializing with compression", curve, "compressed", color, "solid", True, axs[1])
- plot(data, "setup/serializing with no compression", curve, "uncompressed", color, "dashed", True, axs[1])
- axs[1].set_title("serialization")
- axs[1].set_ylabel("time (in ms)")
- axs[1].legend()
- axs[1].grid()
-
- # deserialization
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "setup/deserializing with no compression and no validation", curve, "uncompressed unvalidated", color, "dotted", True, axs[2])
- plot(data, "setup/deserializing with compression and no validation", curve, "compressed unvalidated", color, "dashed", True, axs[2])
- plot(data, "setup/deserializing with no compression and validation", curve, "uncompressed validated", color, "dashdot", True, axs[2])
- plot(data, "setup/deserializing with compression and validation", curve, "compressed validated", color, "solid", True, axs[2])
- axs[2].set_title("deserialization")
- axs[2].set_ylabel("time (in ms)")
- axs[2].legend()
- axs[2].grid()
-
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, "serialized size with no compression", curve, "uncompressed", color, "dashed", False, axs[3])
- plot(data, "serialized size with compression", curve, "compressed", color, "solid", False, axs[3])
- axs[3].set_title("size")
- axs[3].set_xlabel("degree")
- axs[3].set_ylabel("size (in kb)")
- axs[3].legend()
- axs[3].grid()
-
- if save:
- output = "setup.png"
- plt.savefig(output, dpi=FULLSCREEN_DPI)
- print(f"figure saved as {output}")
- else:
- plt.show()
-
-
-def plot_commit(data: Data, save: bool = False):
- fig, ax = plt.subplots(1, 1, figsize=(16, 9))
-
- # key: the start of the `$.id` field
- def plot(data: Data, key: str, curve: str, style: str, color: str, ax):
- filtered_data = list(filter(
- lambda line: line["id"].startswith(key) and line["id"].endswith(f" on {curve}"),
- data
- ))
- if len(filtered_data) == 0:
- return
-
- sizes = [
- int(line["id"].lstrip(key).split(' ')[0]) for line in filtered_data
- ]
-
- means = ns_to_ms(extract(filtered_data, "mean", "estimate"))
- up = ns_to_ms(extract(filtered_data, "mean", "upper_bound"))
- down = ns_to_ms(extract(filtered_data, "mean", "lower_bound"))
-
- ax.plot(sizes, means, label=f"{key} on {curve}", color=color, linestyle=style)
- ax.fill_between(sizes, down, up, color=color, linestyle=style, alpha=0.3)
-
- for (curve, color) in [("BLS12-381", "blue"), ("BN-254", "orange"), ("PALLAS", "green")]:
- plot(data, key="commit (komodo)", curve=curve, style="solid", color=color, ax=ax)
- plot(data, key="commit (arkworks)", curve=curve, style="dashed", color=color, ax=ax)
-
- ax.set_title("commit times")
- ax.set_ylabel("time (in ms)")
- ax.set_xlabel("degree")
- ax.legend()
- ax.grid(True)
-
- if save:
- output = "commit.png"
- plt.savefig(output, dpi=FULLSCREEN_DPI)
- print(f"figure saved as {output}")
- else:
- plt.show()
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("filename", type=str)
- parser.add_argument(
- "--bench", "-b", type=str, choices=["linalg", "setup", "commit"],
- )
- parser.add_argument(
- "--save", "-s", action="store_true", default=False,
- )
- parser.add_argument(
- "--all", "-a", action="store_true", default=False,
- )
- args = parser.parse_args()
-
- data = read_data(args.filename)
-
- if args.all:
- plot_linalg(data, save=args.save)
- plot_setup(data, save=args.save)
- plot_commit(data, save=args.save)
- exit(0)
-
- match args.bench:
- case "linalg":
- plot_linalg(data, save=args.save)
- case "setup":
- plot_setup(data, save=args.save)
- case "commit":
- plot_commit(data, save=args.save)
- case _:
- print("nothing to do: you might want to use `--bench <bench>` or `--all`")
--
GitLab
From 4a0659bc645ee6af5556fd9758c4be62dd9a0afb Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 16:19:21 +0200
Subject: [PATCH 24/25] add example to `plot.py`
---
scripts/plot/plot.py | 56 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 55 insertions(+), 1 deletion(-)
diff --git a/scripts/plot/plot.py b/scripts/plot/plot.py
index 24d50169..339fd909 100644
--- a/scripts/plot/plot.py
+++ b/scripts/plot/plot.py
@@ -5,6 +5,35 @@ import matplotlib.pyplot as plt
import argparse
+# # Example
+# ```nuon
+# [
+# {
+# group: "Alice",
+# items: [
+# [ x, measurement, error ];
+# [ 1, 1143, 120 ],
+# [ 2, 1310, 248 ],
+# [ 4, 1609, 258 ],
+# [ 8, 1953, 343 ],
+# [ 16, 2145, 270 ],
+# [ 32, 3427, 301 ]
+# ]
+# },
+# {
+# group: "Bob",
+# items: [
+# [ x, measurement, error ];
+# [ 1, 2388, 374 ],
+# [ 2, 2738, 355 ],
+# [ 4, 3191, 470 ],
+# [ 8, 3932, 671 ],
+# [ 16, 4571, 334 ],
+# [ 32, 4929, 1094 ]
+# ]
+# },
+# ]
+# ```
def plot(data, title: str, x_label: str, y_label: str, save: str = None):
for group in data:
xs = [x["x"] for x in group["items"]]
@@ -39,7 +68,32 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("data", type=str, help="""
the actual data to show in a multibar plot, here is an example:
- ...
+ [
+ {
+ group: "Alice",
+ items: [
+ [ x, measurement, error ];
+ [ 1, 1143, 120 ],
+ [ 2, 1310, 248 ],
+ [ 4, 1609, 258 ],
+ [ 8, 1953, 343 ],
+ [ 16, 2145, 270 ],
+ [ 32, 3427, 301 ]
+ ]
+ },
+ {
+ group: "Bob",
+ items: [
+ [ x, measurement, error ];
+ [ 1, 2388, 374 ],
+ [ 2, 2738, 355 ],
+ [ 4, 3191, 470 ],
+ [ 8, 3932, 671 ],
+ [ 16, 4571, 334 ],
+ [ 32, 4929, 1094 ]
+ ]
+ },
+ ]
""")
parser.add_argument("--title", "-t", type=str, help="the title of the plot")
parser.add_argument("--x-label", "-x", type=str, help="the x label of the plot")
--
GitLab
From 065f851bfcd5ca1df99e5508e00f640973122d8d Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Thu, 25 Apr 2024 16:21:37 +0200
Subject: [PATCH 25/25] format
---
examples/benches/setup.rs | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/examples/benches/setup.rs b/examples/benches/setup.rs
index ee3578d0..231466e6 100644
--- a/examples/benches/setup.rs
+++ b/examples/benches/setup.rs
@@ -141,7 +141,8 @@ fn serde(degrees: &[usize], nb_measurements: usize) {
curve: &str,
nb_measurements: usize,
) {
- let b = plnk::Bencher::new(nb_measurements).with_name(format!("serialization on {}", curve));
+ let b =
+ plnk::Bencher::new(nb_measurements).with_name(format!("serialization on {}", curve));
serde_template::<F, G, DensePolynomial<F>>(&b, degree);
}
--
GitLab