migrate recoding benchmark to PLNK (dragoon/komodo!89)

this MR
- moves the last "recoding" benchmark to `examples/benches/`
- moves the README, which is now all alone, to `examples/benches/`
- adds a mention to `examples/benches/README.md` in `README.md`
- some minor improvements to the bench README

## TODO
- [x] find a way to plot the "recoding" results (thanks to !90)

Cargo.toml

@@ -51,10 +51,6 @@ ark-secq256k1 = "0.4.0"
 ark-vesta = "0.4.0"
 criterion = "0.3"
 
-[[bench]]
-name = "recoding"
-harness = false
-
 [[example]]
 name = "bench_commit"
 path = "examples/benches/commit.rs"
@@ -78,3 +74,7 @@ path = "examples/benches/setup.rs"
 [[example]]
 name = "bench_linalg"
 path = "examples/benches/linalg.rs"
+
+[[example]]
+name = "bench_recoding"
+path = "examples/benches/recoding.rs"

README.md

@@ -19,3 +19,6 @@ tests for the binary application can also be run with
 ```bash
 nu tests/cli.nu
 ```
+
+## the benchmarks
+see [the `README`](examples/benches/README.md)

benches/README.md → examples/benches/README.md

 ```nushell
 use scripts/math.nu *
+use scripts/formats.nu *
 ```
 
 ## atomic operations
@@ -62,9 +63,8 @@ for graph in [
         (
             $linalg
                 | where op == $graph.op
-                | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
-                | group-by curve --to-table
-                | update items { reject curve }
+                | rename --column { n: "x", mean: "measurement", stddev: "error" }
+                | group-by name --to-table
                 | to json
         )
     ]
@@ -86,15 +86,14 @@ python scripts/plot/plot.py ...[
             | ns-to-ms $.times
             | compute-stats $.times
             | insert degree { get label | parse "degree {d}" | into record | get d | into int}
-            | insert curve {|it| if ($it.name | str starts-with  "ARK") {
+            | update name {|it| if ($it.name | str starts-with  "ARK") {
                 let c = $it.name | parse "ARK setup on {curve}" | into record | get curve
                 $"($c)-ark"
             } else {
                 $it.name | parse "setup on {curve}" | into record | get curve
             }}
             | rename --column { degree: "x", mean: "measurement", stddev: "error" }
-            | group-by curve --to-table
-            | update items { reject curve }
+            | group-by name --to-table
             | to json
     )
 ]
@@ -114,11 +113,52 @@ python scripts/plot/plot.py ...[
         open commit.ndjson
             | ns-to-ms $.times
             | compute-stats $.times
-            | update label { parse "degree {d}" | into record | get d | into int }
-            | rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
-            | group-by curve --to-table
-            | update items { reject curve }
+            | insert degree { get label | parse "degree {d}" | into record | get d | into int }
+            | rename --column { degree: "x", mean: "measurement", stddev: "error" }
+            | group-by name --to-table
             | to json
     )
 ]
 ```
+
+## end-to-end benchmarks
+### recoding
+```nushell
+cargo run --example bench_recoding -- ...[
+    --nb-measurements 10
+    ...[1, 1_024, (1_024 * 1_024)]
+    --shards ...[2, 4, 8, 16]
+    --ks ...[2, 4, 8, 16]
+] | from ndnuon | to ndjson out> recoding.ndjson
+```
+```nushell
+python scripts/plot/plot.py --title "recoding with k = 4" (
+    open recoding.ndjson
+        | ns-to-ms $.times
+        | compute-stats $.times
+        | update label { from nuon }
+        | flatten --all label
+        | insert case { $"($in.name) / ($in.shards)" }
+        | where k == 4  # $k$ has a negligible influence on _recoding_
+        | rename --column { bytes: "x", mean: "measurement", stddev: "error" }
+        | group-by case --to-table
+        | insert style {|it|
+            let g = $it.group | parse "{c} / {s}" | into record | into int s
+            let c = match $g.c {
+                "BLS-12-381" => "blue"
+                "BN-254" => "orange"
+                "PALLAS" => "green"
+                _ => "gray"
+            }
+            let t = match $g.s {
+                2 => "dotted"
+                4 => "dashdot"
+                8 => "dashed"
+                16 => "solid"
+                _ => { color: "loosely dotted" }
+            }
+            { color: $c, line: { type: $t } }
+        }
+        | to json
+)
+```

benches/recoding.rs → examples/benches/recoding.rs

 // see `benches/README.md`
-use std::time::Duration;
-
 use ark_ff::PrimeField;
 use ark_std::rand::Rng;
 
-use criterion::{criterion_group, criterion_main, Criterion};
-
+use clap::{arg, command, Parser};
 use komodo::{
     fec::{recode_with_coeffs, Shard},
     field,
 };
+use plnk::Bencher;
 
 fn to_curve<F: PrimeField>(n: u128) -> F {
     F::from_le_bytes_mod_order(&n.to_le_bytes())
@@ -30,13 +28,7 @@ fn create_fake_shard<F: PrimeField>(nb_bytes: usize, k: usize) -> Shard<F> {
     }
 }
 
-fn bench_template<F: PrimeField>(
-    c: &mut Criterion,
-    nb_bytes: usize,
-    k: usize,
-    nb_shards: usize,
-    curve: &str,
-) {
+fn bench_template<F: PrimeField>(b: &Bencher, nb_bytes: usize, k: usize, nb_shards: usize) {
     let shards: Vec<Shard<F>> = (0..nb_shards)
         .map(|_| create_fake_shard(nb_bytes, k))
         .collect();
@@ -46,32 +38,46 @@ fn bench_template<F: PrimeField>(
         .map(|_| to_curve::<F>(rng.gen::<u128>()))
         .collect();
 
-    c.bench_function(
+    plnk::bench(
+        b,
         &format!(
-            "recoding {} bytes and {} shards with k = {} on {}",
-            nb_bytes, nb_shards, k, curve
+            r#"{{"bytes": {}, "shards": {}, "k": {}}}"#,
+            nb_bytes, nb_shards, k
         ),
-        |b| b.iter(|| recode_with_coeffs(&shards, &coeffs)),
+        || plnk::timeit(|| recode_with_coeffs(&shards, &coeffs)),
     );
 }
 
-fn criterion_benchmark(c: &mut Criterion) {
-    for nb_bytes in [1, 1_024, 1_024 * 1_024] {
-        for nb_shards in [2, 4, 8, 16] {
-            for k in [2, 4, 8, 16] {
-                bench_template::<ark_bls12_381::Fr>(c, nb_bytes, k, nb_shards, "BLS-12-381");
-                bench_template::<ark_bn254::Fr>(c, nb_bytes, k, nb_shards, "BN-254");
-                bench_template::<ark_pallas::Fr>(c, nb_bytes, k, nb_shards, "PALLAS");
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    #[arg(num_args = 1.., value_delimiter = ' ')]
+    bytes: Vec<usize>,
+
+    #[arg(short, long, num_args = 1.., value_delimiter = ' ')]
+    shards: Vec<usize>,
+
+    #[arg(short, long, num_args = 1.., value_delimiter = ' ')]
+    ks: Vec<usize>,
+
+    /// the number of measurements to repeat each case, larger values will reduce the variance of
+    /// the measurements
+    #[arg(short, long)]
+    nb_measurements: usize,
+}
+
+fn main() {
+    let cli = Cli::parse();
+
+    let bencher = plnk::Bencher::new(cli.nb_measurements);
+
+    for b in cli.bytes {
+        for s in &cli.shards {
+            for k in &cli.ks {
+                bench_template::<ark_bls12_381::Fr>(&bencher.with_name("BLS-12-381"), b, *k, *s);
+                bench_template::<ark_bn254::Fr>(&bencher.with_name("BN-254"), b, *k, *s);
+                bench_template::<ark_pallas::Fr>(&bencher.with_name("PALLAS"), b, *k, *s);
             }
         }
     }
 }
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default()
-        .warm_up_time(Duration::from_secs_f32(0.5))
-        .sample_size(10);
-    targets = criterion_benchmark
-);
-criterion_main!(benches);

scripts/formats.nu

+export def "from ndnuon" []: [string -> any] {
+    lines | each { from nuon }
+}
+
+export def "to ndnuon" []: [any -> string] {
+    each { to nuon --raw } | to text
+}
+
+export def "from nuonl" []: [string -> any] {
+    from ndnuon
+}
+
+export def "to nuonl" []: [any -> string] {
+    to ndnuon
+}