From ab0d9a8ba69e4873dea2945d9a49b6d67ffe5b8b Mon Sep 17 00:00:00 2001
From: STEVAN Antoine <antoine.stevan@isae-supaero.fr>
Date: Mon, 3 Feb 2025 12:15:17 +0000
Subject: [PATCH] rework benchmarks: script and output format
(dragoon/komodo!193)
this is an attempt at making benchmarks easier to work with
- `benchmarks run` will read benchmarks from NUON data and run them
- `benchmarks plot` will plot benchmark results
- the output format will be something
- a directory whose name is the hash of the CPU spec and the Komodo commit hash
- contains `cpu.json` with the CPU info
- contains `komodo.txt` with the Komodo commit hash
- contains NDJSON result files
> :bulb: **Note**
>
> results will typically be uploaded to https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results
> :bulb: **Note**
>
> this MR goes alongside the [`komodo-benchmark-results@restart`](https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results/-/compare/main...restart) branch
## changelog
- bump Nushell to 0.101.0
- parallel `$in` => remove useless `let input = $in` when possible, e.g. still required when using the function's `$in` in a `for` loop
- `group-by` changed => `group-by x --to-table` will now produce a table with columns `x` and `items` instead of `group` and `items` as in 0.95.0
- add link to results repo
- the Nushell benchmarks lib
- rename `--force` to `--no-confirm (-y)`
- add `--append`
- reject columns that GPLT will complain about, e.g. `$.points.k` for the FRI plots
- add `--save` to the FRI plot
- move the "field" and "curve group" benchmarks from `benchmarks/src/bin/operations/` to `benchmarks/src/bin/`
- remove `benchmarks/params/fri.nu` because it's been uniformized with the other methods
- rewrite the README
- add main function to `benchmarks/` that runs the benchmarks from a NUON record specification
- simplify the output of FRI run
## TODO
- [x] fix "_atomic operations_" (done in 4f69a1d)
- [x] check that _plotting_ still works
## images
### Field
### Linear algebra
### FEC
### ZK
### FRI
---
Makefile | 2 +-
README.md | 2 +
benchmarks/README.md | 231 ++++++++--------
benchmarks/mod.nu | 258 ++++++++++++++++++
benchmarks/nu-lib/commit.nu | 24 +-
benchmarks/nu-lib/fec/plot.nu | 18 +-
benchmarks/nu-lib/fec/run.nu | 25 +-
benchmarks/nu-lib/fri/plot.nu | 4 +-
benchmarks/nu-lib/fri/run.nu | 2 +-
benchmarks/nu-lib/linalg.nu | 25 +-
benchmarks/nu-lib/recoding.nu | 27 +-
benchmarks/nu-lib/setup.nu | 24 +-
benchmarks/nu-lib/utils/plot.nu | 6 +-
benchmarks/params/fri.nu | 16 --
.../src/bin/{operations => }/curve_group.rs | 0
benchmarks/src/bin/{operations => }/field.rs | 0
16 files changed, 475 insertions(+), 189 deletions(-)
delete mode 100644 benchmarks/params/fri.nu
rename benchmarks/src/bin/{operations => }/curve_group.rs (100%)
rename benchmarks/src/bin/{operations => }/field.rs (100%)
diff --git a/Makefile b/Makefile
index 2a89ea42..e0a254ab 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ NU="nu"
NU_FLAGS="--no-config-file"
NU_ARCH="x86_64-unknown-linux-musl"
-NU_VERSION="0.95.0"
+NU_VERSION="0.101.0"
NU_BUILD="nu-${NU_VERSION}-${NU_ARCH}"
NU_DEST="/tmp/"
diff --git a/README.md b/README.md
index 6c099078..d29178c0 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,8 @@ Other examples that showcase the Komodo API are available in [`examples/`](examp
## the benchmarks
see [`benchmarks/`](benchmarks/README.md)
+the results can be found in [`dragoon/komodo-benchmark-results`](https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results).
+
## contributors
Because the code for this project has been originally extracted from
diff --git a/benchmarks/README.md b/benchmarks/README.md
index e0870187..47f2b428 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -1,10 +1,9 @@
# Table of contents
- [Requirements](#requirements)
-- [Atomic operations](#atomic-operations)
-- [Linear algebra](#linear-algebra)
-- [Trusted setup and commit](#trusted-setup-and-commit)
-- [End to end benchmarks](#end-to-end-benchmarks)
-- [FRI](#fri)
+- [Run the benchmarks](#run-the-benchmarks)
+ - [define them](#define-them)
+ - [run them](#run-them)
+- [Plot the benchmarks](#plot-the-benchmarks)
## requirements
> :bulb: **Note**
@@ -33,130 +32,126 @@ use benchmarks
> i personally use the [`nuenv` hook](https://github.com/nushell/nu_scripts/blob/main/nu-hooks/nu-hooks/nuenv/hook.nu)
> that reads [`.env.nu`](../.env.nu).
-## atomic operations
-```nushell
-cargo run --release --package benchmarks --bin field_operations -- --nb-measurements 1000 out> field.ndjson
-cargo run --release --package benchmarks --bin curve_group_operations -- --nb-measurements 1000 out> curve_group.ndjson
-```
-```nushell
-use benchmarks/nu-lib/utils/parse.nu read-atomic-ops
-
-gplt multi_bar --title "simple field operations" -l "time (in ns)" (
- open field.ndjson
- | read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
- | to json
-)
-gplt multi_bar --title "complex field operations" -l "time (in ns)" (
- open field.ndjson
- | read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ]
- | to json
-)
-gplt multi_bar --title "simple curve group operations" -l "time (in ns)" (
- open curve_group.ndjson
- | read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
- | to json
-)
-gplt multi_bar --title "complex curve group operations" -l "time (in ns)" (
- open curve_group.ndjson
- | read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
- | to json
-)
-```
-
-## linear algebra
-```nushell
-let sizes = seq 0 7 | each { 2 ** $in }
-
-let out_linalg = $sizes | benchmarks linalg run
-
-benchmarks linalg plot $out_linalg inverse
-```
-
-## trusted setup and commit
-```nushell
-let degrees = seq 0 13 | each { 2 ** $in }
-let curves = [ bls12381, pallas, bn254 ]
-
-let out_setup = $degrees | benchmarks setup run --curves $curves
-let out_commit = $degrees | benchmarks commit run --curves $curves
-
-benchmarks setup plot $out_setup
-benchmarks commit plot $out_commit
-```
-
-## end-to-end benchmarks
-```nushell
-let sizes = seq 0 18 | each { 512 * 2 ** $in }
-let ks = [2, 4, 8, 16]
-let curves = [ bls12381 ]
-```
+## Run the benchmarks
+### define them
-### run
-```nushell
-let out_recoding = $sizes | benchmarks recoding run --ks $ks --curves $curves
-let out_fec = $sizes | benchmarks fec run --ks $ks --curves $curves
-```
-
-### plot
-```nushell
-benchmarks recoding plot $out_recoding
-benchmarks fec plot encoding $out_fec
-benchmarks fec plot decoding $out_fec
-benchmarks fec plot e2e $out_fec
-benchmarks fec plot combined $out_fec --recoding $out_recoding
-benchmarks fec plot ratio $out_fec --recoding $out_recoding
-```
-
-## FRI
> :bulb: **Note**
>
> the FRI benchmarks don't use a module from [src/bin/](src/bin/) with PLNK but rather an
> [example](../examples/fri.rs)
-- modify [benchmarks/params/fri.nu](benchmarks/params/fri.nu)
-- source it
-```nushell
-source benchmarks/params/fri.nu
+```bash
+const RESULTS_DIR = "/path/to/komodo-benchmark-results/"
+
+let benchmarks = {
+ linalg: {
+ enabled: true,
+ sizes: (seq 0 7 | each { 2 ** $in }),
+ output: "linalg.ndjson",
+ append: true,
+ },
+ setup: {
+ enabled: true,
+ degrees: (seq 0 13 | each { 2 ** $in }),
+ curves: [ bls12381, pallas, bn254 ],
+ output: "setup.ndjson",
+ append: true,
+ },
+ commit: {
+ enabled: true,
+ degrees: (seq 0 13 | each { 2 ** $in }),
+ curves: [ bls12381, pallas, bn254 ],
+ output: "commit.ndjson",
+ append: true,
+ },
+ recoding: {
+ enabled: true,
+ sizes: (seq 0 18 | each { 512 * 2 ** $in }),
+ ks: [2, 4, 8, 16],
+ curves: [ bls12381 ],
+ output: "recoding.ndjson",
+ append: true,
+ },
+ fec: {
+ enabled: true,
+ sizes: (seq 0 18 | each { 512 * 2 ** $in }),
+ ks: [2, 4, 8, 16],
+ curves: [ bls12381 ],
+ output: "fec.ndjson",
+ append: true,
+ },
+ fri: {
+ enabled: true,
+ sizes: (seq 0 15 | each { 2 ** $in * 4096b }),
+ ks: [8, 128, 1024, 4096],
+ blowup_factors: [2, 4],
+ ns: [2],
+ remainder_plus_ones: [1],
+ nb_queries: [50],
+ hashes: ["sha3-512"],
+ ffs: ["fp128", "bls12-381"],
+ output: "fri.ndjson",
+ append: true,
+ },
+ field: {
+ enabled: true,
+ nb_measurements: 1000,
+ output: "field.ndjson",
+ append: true,
+ },
+ curve_group: {
+ enabled: true,
+ nb_measurements: 1000,
+ output: "curve_group.ndjson",
+ append: true,
+ },
+}
```
-- run the benchmarks
-```nushell
-use std formats "to ndjson"
-(benchmarks fri run
- --data-sizes $DATA_SIZES
- --ks $KS
- --blowup-factors $BFS
- --nb-queries $QS
- --hashes $HS
- --finite-fields $FFS
- --remainders $RPOS
- --folding-factors $NS
-) | to ndjson out> $DATA
+### run them
+```bash
+benchmarks run --output-dir $RESULTS_DIR $benchmarks
```
-> the following `watch` call can be used to see the results as they are dumped to `$DATA`
-> ```nushell
-> use std formats "from ndjson"
->
-> watch . {
-> open --raw $DATA
-> | lines
-> | last
-> | from ndjson
-> | into int evaluating encoding proving verifying decoding
-> | into duration evaluating encoding proving verifying decoding
-> | into filesize proofs commits d
-> | into record
+> the following `watch` can be used to see the results as they are dumped to `$RESULTS_DIR`
+> ```bash
+> watch $RESULTS_DIR { |op, path|
+> $"($op) ($path)"
> }
> ```
-```nushell
-benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA evaluating encoding proving decoding --y-type "duration"
-benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA verifying --y-type "duration" --single
-
-benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA proofs --y-type "filesize" --identity --normalize
-benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA commits --y-type "filesize" --single --identity --normalize
-
-benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA proofs --y-type "filesize" --identity
-benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA commits --y-type "filesize" --single --identity
+## Plot the benchmarks
+```bash
+let plots = {
+ linalg: { file: "linalg.ndjson" },
+ setup: { file: "setup.ndjson" },
+ commit: { file: "commit.ndjson" },
+ fec: { file: "fec.ndjson" },
+ recoding: { file: "recoding.ndjson" },
+ fri: [
+ [name, y_type, single, identity, normalize];
+ [evaluating, duration, false, false, false ],
+ [encoding, duration, false, false, false ],
+ [proving, duration, false, false, false ],
+ [decoding, duration, false, false, false ],
+ [verifying, duration, true, false, false ],
+ [proofs, filesize, false, true, true ],
+ [commits, filesize, true, true, true ],
+ [proofs, filesize, false, true, false ],
+ [commits, filesize, true, true, false ],
+ ],
+ field: {
+ title: "field operations",
+ file: field.ndjson,
+ simple_operations: [ "exponentiation", "legendre", "inverse", "sqrt" ],
+ },
+ curve_group: {
+ title: "curve group operations",
+ file: curve_group.ndjson,
+ simple_operations: [ "random sampling", "scalar multiplication", "affine scalar multiplication" ],
+ },
+}
+```
+```bash
+benchmarks plot $plots --input-dir "/path/to/komodo-benchmark-results/<hash>" --output-dir "./figures/"
```
diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu
index 838b6183..92e02052 100644
--- a/benchmarks/mod.nu
+++ b/benchmarks/mod.nu
@@ -4,3 +4,261 @@ export module nu-lib/fec/
export module nu-lib/recoding.nu
export module nu-lib/linalg.nu
export module nu-lib/fri/
+
+use nu-lib/linalg.nu
+use nu-lib/setup.nu
+use nu-lib/commit.nu
+use nu-lib/recoding.nu
+use nu-lib/fec/
+use nu-lib/fri/
+
+use nu-lib/utils/log.nu
+use nu-lib/utils/parse.nu read-atomic-ops
+
+const CPU_FIELDS = [
+ "Architecture",
+ "CPU op-mode(s)",
+ "Address sizes",
+ "Byte Order",
+ "CPU(s)",
+ "On-line CPU(s) list",
+ "Model name",
+ "CPU family",
+ "Model",
+ "Thread(s) per core",
+ "Core(s) per socket",
+ "Socket(s)",
+ "Stepping",
+ "CPU max MHz",
+ "CPU min MHz",
+ "BogoMIPS",
+ "Virtualization",
+ "L1d cache",
+ "L1i cache",
+ "L2 cache",
+ "L3 cache",
+ "NUMA node(s)",
+ "NUMA node0 CPU(s)",
+]
+
+export def run [
+ benchmarks: record<
+ linalg: record<
+ enabled: bool,
+ sizes: list<int>,
+ output: string,
+ append: bool,
+ >,
+ setup: record<
+ enabled: bool,
+ degrees: list<int>,
+ curves: list<string>,
+ output: string,
+ append: bool,
+ >,
+ commit: record<
+ enabled: bool,
+ degrees: list<int>,
+ curves: list<string>,
+ output: string,
+ append: bool,
+ >,
+ recoding: record<
+ enabled: bool,
+ sizes: list<int>,
+ ks: list<int>,
+ curves: list<string>,
+ output: string,
+ append: bool,
+ >,
+ fec: record<
+ enabled: bool,
+ sizes: list<int>,
+ ks: list<int>,
+ curves: list<string>,
+ output: string,
+ append: bool,
+ >,
+ fri: record<
+ enabled: bool,
+ sizes: list<filesize>,
+ ks: list<int>,
+ blowup_factors: list<int>,
+ ns: list<int>,
+ remainder_plus_ones: list<int>,
+ nb_queries: list<int>,
+ hashes: list<string>,
+ ffs: list<string>,
+ output: string,
+ append: bool,
+ >,
+ field: record<enabled: bool, nb_measurements: int, output: string, append: bool>,
+ curve_group: record<enabled: bool, nb_measurements: int, output: string, append: bool>,
+ >,
+ --output-dir: path = ".",
+] {
+ let cpu = lscpu --json
+ | from json
+ | get lscpu
+ | update field { str trim --right --char ":" }
+ | transpose --header-row
+ | into record
+ | select ...$CPU_FIELDS
+
+ let commit = git rev-parse HEAD
+ let hash = $cpu | to json | $in + $commit | hash sha256
+
+ let target = $output_dir | path join $hash
+ mkdir $target
+
+ $cpu | to json | save --force ($target | path join "cpu.json")
+ $commit | save --force ($target | path join "komodo.txt")
+
+ let benchmarks = $benchmarks
+ | insert linalg.run {{ |it|
+ let output = $target | path join $it.output
+ $it.sizes | linalg run --no-confirm --output $output --append=$it.append
+ }}
+ | insert setup.run {{ |it|
+ let output = $target | path join $it.output
+ $it.degrees | setup run --curves $it.curves --no-confirm --output $output --append=$it.append
+ }}
+ | insert commit.run {{ |it|
+ let output = $target | path join $it.output
+ $it.degrees | commit run --curves $it.curves --no-confirm --output $output --append=$it.append
+ }}
+ | insert recoding.run {{ |it|
+ let output = $target | path join $it.output
+ $it.sizes | recoding run --ks $it.ks --curves $it.curves --no-confirm --output $output --append=$it.append
+ }}
+ | insert fec.run {{ |it|
+ let output = $target | path join $it.output
+ $it.sizes | fec run --ks $it.ks --curves $it.curves --no-confirm --output $output --append=$it.append
+ }}
+ | insert fri.run {{ |it|
+ # FIXME: refactor this
+ if $it.append {
+ (
+ fri run
+ --data-sizes $it.sizes
+ --ks $it.ks
+ --blowup-factors $it.blowup_factors
+ --nb-queries $it.nb_queries
+ --hashes $it.hashes
+ --finite-fields $it.ffs
+ --remainders $it.remainder_plus_ones
+ --folding-factors $it.ns
+ ) | to ndjson out>> ($target | path join $it.output)
+ } else {
+ (
+ fri run
+ --data-sizes $it.sizes
+ --ks $it.ks
+ --blowup-factors $it.blowup_factors
+ --nb-queries $it.nb_queries
+ --hashes $it.hashes
+ --finite-fields $it.ffs
+ --remainders $it.remainder_plus_ones
+ --folding-factors $it.ns
+ ) | to ndjson out> ($target | path join $it.output)
+ }
+ }}
+ | insert field.run {{ |it|
+ let options = [
+ --bin field
+ --release
+ --package benchmarks
+ --
+ --nb-measurements $it.nb_measurements
+ ]
+ # FIXME: refactor this
+ if $it.append {
+ cargo run ...$options out>> ($target | path join $it.output)
+ } else {
+ cargo run ...$options out> ($target | path join $it.output)
+ }
+ }}
+ | insert curve_group.run {{ |it|
+ let options = [
+ --bin curve_group
+ --release
+ --package benchmarks
+ --
+ --nb-measurements $it.nb_measurements
+ ]
+ # FIXME: refactor this
+ if $it.append {
+ cargo run ...$options out>> ($target | path join $it.output)
+ } else {
+ cargo run ...$options out> ($target | path join $it.output)
+ }
+ }}
+
+ let _ = $benchmarks | items { |k, b|
+ if ($b.enabled? | default true) {
+ log info $"running (ansi cyan)($k)(ansi reset)"
+ do $b.run $b
+ } else {
+ log warning $"skipping (ansi cyan)($k)(ansi reset)"
+ }
+ }
+}
+
+export def plot [plots: record, --input-dir: path, --output-dir: path = "./figures/"] {
+ mkdir $output_dir
+
+ let linalg_file = $input_dir | path join $plots.linalg.file
+ let fec_file = $input_dir | path join $plots.fec.file
+ let recoding_file = $input_dir | path join $plots.recoding.file
+
+ for op in [ "mul", "transpose", "inverse" ] {
+ linalg plot $linalg_file $op --save ($output_dir | path join $"linalg-($op).png")
+ }
+
+ setup plot ($input_dir | path join $plots.setup.file) --save ($output_dir | path join setup.png)
+ commit plot ($input_dir | path join $plots.commit.file) --save ($output_dir | path join commit.png)
+
+ recoding plot $recoding_file --save ($output_dir | path join recoding.png)
+
+ fec plot encoding $fec_file --save ($output_dir | path join encoding.png)
+ fec plot decoding $fec_file --save ($output_dir | path join decoding.png)
+ fec plot e2e $fec_file --save ($output_dir | path join end_to_end.png)
+ fec plot combined $fec_file --recoding $recoding_file --save ($output_dir | path join combined.png)
+ fec plot ratio $fec_file --recoding $recoding_file --save ($output_dir | path join ratio.png)
+
+ for plot in $plots.fri {(
+ fri plot
+ --dump-dir $output_dir
+ --file ($input_dir | path join fri.ndjson)
+ $plot.name
+ --y-type $plot.y_type
+ --single=$plot.single
+ --identity=$plot.identity
+ --normalize=$plot.normalize
+ --save
+ )}
+
+ for plot in ($plots | select field curve_group | values) {
+ def output [prefix: string]: [ nothing -> record<path: path, title: string> ] {
+ let title_tokens = $plot.title | split row " " | prepend $prefix
+ {
+ path: ({
+ parent: $output_dir,
+ stem: ($title_tokens | str join "_"),
+ extension: "png",
+ } | path join),
+ title: ($title_tokens | str join " "),
+ }
+ }
+
+ let data = open ($input_dir | path join $plot.file)
+
+ output "simple" | gplt multi-bar --title $in.title -l "time (in ns)" (
+ $data | read-atomic-ops --include $plot.simple_operations | to json
+ ) --save $in.path
+
+ output "complex" | gplt multi-bar --title $in.title -l "time (in ns)" (
+ $data | read-atomic-ops --exclude $plot.simple_operations | to json
+ ) --save $in.path
+ }
+}
diff --git a/benchmarks/nu-lib/commit.nu b/benchmarks/nu-lib/commit.nu
index 0d9cd4c5..ddaefa17 100644
--- a/benchmarks/nu-lib/commit.nu
+++ b/benchmarks/nu-lib/commit.nu
@@ -13,11 +13,10 @@ use std formats *
export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--curves: list<string>, # the curves to benchmark
- --force, # does not ask for confirmation if the output file already exists, it will be overwritten
+ --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
+ --append, # append to the output path instead of overwritting
]: list<int> -> path {
- let input = $in
-
$curves | check-list-arg --cmd "commit run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "commit run" --arg "pipeline input"
@@ -26,7 +25,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
- if not $force {
+ if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
@@ -36,11 +35,20 @@ export def run [
}
}
- cargo run --release --package benchmarks --bin commit -- ...[
+ let options = [
+ --release
+ --package benchmarks
+ --bin commit
+ --
--nb-measurements $nb_measurements
- ...$input
+ ...$in
--curves ...$curves
- ] out> $output
+ ]
+ if $append {
+ cargo run ...$options out>> $output
+ } else {
+ cargo run ...$options out> $output
+ }
log info $"results saved to ($pretty_output)"
$output
@@ -64,7 +72,7 @@ export def plot [
| select name x y e
| group-by name --to-table
| reject items.name
- | rename --column { group: "name", items: "points" }
+ | rename --column { name: "name", items: "points" }
| insert style.color {|it|
match $it.name {
"BLS12-381" => "tab:blue"
diff --git a/benchmarks/nu-lib/fec/plot.nu b/benchmarks/nu-lib/fec/plot.nu
index da3c0dc2..a8024518 100644
--- a/benchmarks/nu-lib/fec/plot.nu
+++ b/benchmarks/nu-lib/fec/plot.nu
@@ -24,7 +24,7 @@ export def encoding [
| sort-by x
| group-by k --to-table
| reject items.k
- | rename --column { group: "name", items: "points" }
+ | rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
@@ -56,7 +56,7 @@ export def decoding [
| sort-by x
| group-by k --to-table
| reject items.k
- | rename --column { group: "name", items: "points" }
+ | rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
@@ -89,7 +89,7 @@ export def e2e [
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
- | reject group foo
+ | reject foo
| ns-to-ms times
| compute-stats times
| reject times
@@ -99,7 +99,7 @@ export def e2e [
| sort-by x
| group-by k --to-table
| reject items.k
- | rename --column { group: "name", items: "points" }
+ | rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
@@ -144,7 +144,7 @@ export def combined [
}
| reject items.shards
| insert style.line.type "solid"
- | rename --column { group: "name", items: "points" }
+ | rename --column { shards: "name", items: "points" }
| update name { $"$k = ($in)$" }
let re_encoding_graphs = open --raw $data
@@ -159,7 +159,7 @@ export def combined [
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
- | reject group key
+ | reject key
| ns-to-ms times
| compute-stats times
| reject times
@@ -179,7 +179,7 @@ export def combined [
}
| insert style.line.type "dashed"
| reject items.k
- | rename --column { group: "name", items: "points" }
+ | rename --column { k: "name", items: "points" }
| reject name
let graphs = $recoding_graphs
@@ -254,7 +254,7 @@ export def ratio [
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
- | reject group key
+ | reject key
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381"
@@ -281,7 +281,7 @@ export def ratio [
}
}
| reject items.k
- | rename --column { group: "name", items: "points" }
+ | rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
diff --git a/benchmarks/nu-lib/fec/run.nu b/benchmarks/nu-lib/fec/run.nu
index 4210ffc5..adee4274 100644
--- a/benchmarks/nu-lib/fec/run.nu
+++ b/benchmarks/nu-lib/fec/run.nu
@@ -12,11 +12,10 @@ export def main [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--ks: list<int>, # the values of $k$ to benchmark
--curves: list<string>, # the curves to benchmark
- --force, # does not ask for confirmation if the output file already exists, it will be overwritten
+ --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
+ --append, # append to the output path instead of overwritting
]: list<int> -> path {
- let input = $in
-
$ks | check-list-arg --cmd "fec run" --arg "--ks" --span (metadata $ks).span
$curves | check-list-arg --cmd "fec run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "fec run" --arg "pipeline input"
@@ -26,7 +25,7 @@ export def main [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
- if not $force {
+ if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
@@ -36,17 +35,29 @@ export def main [
}
}
- "" out> $output
+ if not $append {
+ "" out> $output
+ }
+ let input = $in
for k in $ks {
- cargo run --release --package benchmarks --bin fec -- ...[
+ let options = [
+ --release
+ --package benchmarks
+ --bin fec
+ --
--nb-measurements $nb_measurements
...$input
--encoding vandermonde
-k $k
-n 1
--curves ...$curves
- ] | from ndnuon | to ndjson out>> $output
+ ]
+ if $append {
+ cargo run ...$options | from ndnuon | to ndjson out>> $output
+ } else {
+ cargo run ...$options | from ndnuon | to ndjson out> $output
+ }
}
log info $"results saved to ($pretty_output)"
diff --git a/benchmarks/nu-lib/fri/plot.nu b/benchmarks/nu-lib/fri/plot.nu
index eba787a6..66ecaae0 100644
--- a/benchmarks/nu-lib/fri/plot.nu
+++ b/benchmarks/nu-lib/fri/plot.nu
@@ -54,6 +54,7 @@ def plot [
points: ($ds | wrap x | merge ($ds | wrap y) | if $normalize { update y { |it| $it.y / $it.x } } else { $in }),
style: { color: "black", line: { type: "dotted" } },
} } else { $in }
+ | reject points.k? points.bf? points.ff?
let title = [
$name,
@@ -86,6 +87,7 @@ export def main [
--identity,
--normalize,
--dump-dir: path = "./",
+ --save,
] {
if ($x | is-empty) {
error make --unspanned { msg: "nothing to do, x is empty" }
@@ -101,6 +103,6 @@ export def main [
let data = open $file | where h == "sha3-512" and q == 50
for i in $x {
- $data | plot --save $i --y-type=$y_type --single=$single --identity=$identity --normalize=$normalize --dump-dir=$dump_dir
+ $data | plot --save=$save $i --y-type=$y_type --single=$single --identity=$identity --normalize=$normalize --dump-dir=$dump_dir
}
}
diff --git a/benchmarks/nu-lib/fri/run.nu b/benchmarks/nu-lib/fri/run.nu
index 8f6b2162..e20cc046 100644
--- a/benchmarks/nu-lib/fri/run.nu
+++ b/benchmarks/nu-lib/fri/run.nu
@@ -72,7 +72,7 @@ export def main [
}
$params | each { |p|
- print $p
+ print ($p | to nuon --raw)
run $p
}
}
diff --git a/benchmarks/nu-lib/linalg.nu b/benchmarks/nu-lib/linalg.nu
index a31085a8..25cc8451 100644
--- a/benchmarks/nu-lib/linalg.nu
+++ b/benchmarks/nu-lib/linalg.nu
@@ -12,11 +12,10 @@ use std formats *
# - output: the output path, as NDJSON
export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
- --force, # does not ask for confirmation if the output file already exists, it will be overwritten
+ --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
+ --append, # append to the output path instead of overwritting
]: list<int> -> path {
- let input = $in
-
$in | check-list-arg --cmd "linalg run" --arg "pipeline input"
let new_file = $output == null
@@ -24,7 +23,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
- if not $force {
+ if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
@@ -34,10 +33,19 @@ export def run [
}
}
- cargo run --release --package benchmarks --bin linalg -- ...[
+ let options = [
+ --release
+ --package benchmarks
+ --bin linalg
+ --
--nb-measurements $nb_measurements
- ...$input
- ] out> $output
+ ...$in
+ ]
+ if $append {
+ cargo run ...$options out>> $output
+ } else {
+ cargo run ...$options out> $output
+ }
log info $"results saved to ($pretty_output)"
$output
@@ -87,7 +95,8 @@ export def plot [
| where op == $op
| rename --column { n: "x", mean: "y", stddev: "e" }
| group-by name --to-table
- | rename --column { group: "name", items: "points" }
+ | reject items.name items.op items.times
+ | rename --column { name: "name", items: "points" }
| insert style.color {|it|
match $it.name {
"BLS12-381" => "tab:blue"
diff --git a/benchmarks/nu-lib/recoding.nu b/benchmarks/nu-lib/recoding.nu
index 3c0f2390..6357baf4 100644
--- a/benchmarks/nu-lib/recoding.nu
+++ b/benchmarks/nu-lib/recoding.nu
@@ -15,11 +15,10 @@ export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--ks: list<int>, # the values of $k$ to benchmark
--curves: list<string>, # the curves to benchmark
- --force, # does not ask for confirmation if the output file already exists, it will be overwritten
+ --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
+ --append, # append to the output path instead of overwritting
]: list<int> -> path {
- let input = $in
-
$ks | check-list-arg --cmd "recoding run" --arg "--ks" --span (metadata $ks).span
$curves | check-list-arg --cmd "recoding run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "recoding run" --arg "pipeline input"
@@ -29,7 +28,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
- if not $force {
+ if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
@@ -39,16 +38,28 @@ export def run [
}
}
- "" out> $output
+ if not $append {
+ "" out> $output
+ }
+ let input = $in
for k in $ks {
- cargo run --release --package benchmarks --bin recoding -- ...[
+ let options = [
+ --release
+ --package benchmarks
+ --bin recoding
+ --
--nb-measurements $nb_measurements
...$input
--shards $k
--ks $k
--curves ...$curves
- ] | from ndnuon | to ndjson out>> $output
+ ]
+ if $append {
+ cargo run ...$options | from ndnuon | to ndjson out>> $output
+ } else {
+ cargo run ...$options | from ndnuon | to ndjson out> $output
+ }
}
log info $"results saved to ($pretty_output)"
@@ -73,7 +84,7 @@ export def plot [
| select shards x y e
| group-by shards --to-table
| reject items.shards
- | rename --column { group: "name", items: "points" }
+ | rename --column { shards: "name", items: "points" }
| update name { $"$k = ($in)$"}
let options = [
diff --git a/benchmarks/nu-lib/setup.nu b/benchmarks/nu-lib/setup.nu
index 37f105ad..cac6b650 100644
--- a/benchmarks/nu-lib/setup.nu
+++ b/benchmarks/nu-lib/setup.nu
@@ -13,11 +13,10 @@ use std formats *
export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--curves: list<string>, # the curves to benchmark
- --force, # does not ask for confirmation if the output file already exists, it will be overwritten
+ --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
+ --append, # append to the output path instead of overwritting
]: list<int> -> path {
- let input = $in
-
$curves | check-list-arg --cmd "setup run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "setup run" --arg "pipeline input"
@@ -26,7 +25,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
- if not $force {
+ if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
@@ -36,11 +35,20 @@ export def run [
}
}
- cargo run --release --package benchmarks --bin setup -- ...[
+ let options = [
+ --release
+ --package benchmarks
+ --bin setup
+ --
--nb-measurements $nb_measurements
- ...$input
+ ...$in
--curves ...$curves
- ] out> $output
+ ]
+ if $append {
+ cargo run ...$options out>> $output
+ } else {
+ cargo run ...$options out> $output
+ }
log info $"results saved to ($pretty_output)"
$output
@@ -69,7 +77,7 @@ export def plot [
| select name x y e
| group-by name --to-table
| reject items.name
- | rename --column { group: "name", items: "points" }
+ | rename --column { name: "name", items: "points" }
| insert style.color {|it|
match $it.name {
"BLS12-381" => "tab:blue"
diff --git a/benchmarks/nu-lib/utils/plot.nu b/benchmarks/nu-lib/utils/plot.nu
index 72c753c5..c001e3d8 100644
--- a/benchmarks/nu-lib/utils/plot.nu
+++ b/benchmarks/nu-lib/utils/plot.nu
@@ -29,9 +29,7 @@ export def into-filesize-tick-labels []: list<int> -> list<string> {
}
export def into-axis-options [-x: string, -y: string]: table<x: float, y: float> -> list<string> {
- let input = $in
-
- let xs = $input | flatten | get x | uniq
+ let xs = $in | flatten | get x | uniq
let x_tick_labels = match $x {
"filesize" => ($xs | into-filesize-tick-labels),
@@ -47,7 +45,7 @@ export def into-axis-options [-x: string, -y: string]: table<x: float, y: float>
--x-tick-labels ...$x_tick_labels
]
- let ys = $input | flatten | get y
+ let ys = $in | flatten | get y
let y_ticks = seq ($ys | math min | math log 10 | math ceil | $in - 1) ($ys | math max | math log 10 | math floor)
| into float
| each { 10 ** $in }
diff --git a/benchmarks/params/fri.nu b/benchmarks/params/fri.nu
deleted file mode 100644
index 24fe8100..00000000
--- a/benchmarks/params/fri.nu
+++ /dev/null
@@ -1,16 +0,0 @@
-let DATA_SIZES = seq 0 15 | each { 2 ** $in * 4096b }
-const KS = [8, 128, 1024, 4096]
-const BFS = [2, 4]
-const NS = [2]
-const RPOS = [1]
-const QS = [50]
-const HS = ["sha3-512"]
-const FFS = ["fp128", "bls12-381"]
-
-const DATA = "benchmarks/results/fri.ndjson"
-const OUTPUT_DIR = "benchmarks/results/figures/"
-
-if not ($DATA | path dirname | path exists) {
- print $"creating directory for (ansi purple)($DATA)(ansi reset)"
- $DATA | path dirname | mkdir $in
-}
diff --git a/benchmarks/src/bin/operations/curve_group.rs b/benchmarks/src/bin/curve_group.rs
similarity index 100%
rename from benchmarks/src/bin/operations/curve_group.rs
rename to benchmarks/src/bin/curve_group.rs
diff --git a/benchmarks/src/bin/operations/field.rs b/benchmarks/src/bin/field.rs
similarity index 100%
rename from benchmarks/src/bin/operations/field.rs
rename to benchmarks/src/bin/field.rs
--
GitLab