From 100d6ce0d079e34cf5b3e7bd660bbf01136e89df Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 17:39:57 +0100 Subject: [PATCH 01/27] use better script to run the benchmarks --- benchmarks/README.md | 97 +++++++++++++++++++++++++------------------- benchmarks/mod.nu | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 42 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index e0870187..1a300b45 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,9 +1,6 @@ # Table of contents - [Requirements](#requirements) -- [Atomic operations](#atomic-operations) -- [Linear algebra](#linear-algebra) -- [Trusted setup and commit](#trusted-setup-and-commit) -- [End to end benchmarks](#end-to-end-benchmarks) +- [The rest](#the-rest) - [FRI](#fri) ## requirements @@ -63,48 +60,64 @@ gplt multi_bar --title "complex curve group operations" -l "time (in ns)" ( ) ``` -## linear algebra -```nushell -let sizes = seq 0 7 | each { 2 ** $in } - -let out_linalg = $sizes | benchmarks linalg run - -benchmarks linalg plot $out_linalg inverse -``` - -## trusted setup and commit -```nushell -let degrees = seq 0 13 | each { 2 ** $in } -let curves = [ bls12381, pallas, bn254 ] - -let out_setup = $degrees | benchmarks setup run --curves $curves -let out_commit = $degrees | benchmarks commit run --curves $curves - -benchmarks setup plot $out_setup -benchmarks commit plot $out_commit -``` - -## end-to-end benchmarks -```nushell -let sizes = seq 0 18 | each { 512 * 2 ** $in } -let ks = [2, 4, 8, 16] -let curves = [ bls12381 ] +## The rest +### define the benchmarks +```bash +const OUTPUT_DIR = "/path/to/komodo-benchmarks-results/" + +let benchmarks = { + linalg: { + enabled: true, + sizes: (seq 0 7 | each { 2 ** $in }), + output: "linalg.ndjson", + }, + setup: { + enabled: true, + degrees: (seq 0 13 | each { 2 ** $in }), + curves: [ bls12381, pallas, bn254 ], + output: "setup.ndjson", + }, + commit: { + enabled: true, + degrees: (seq 0 13 | each { 2 ** $in }), + curves: [ bls12381, pallas, bn254 ], + output: "commit.ndjson", + }, + recoding: { + enabled: true, + sizes: (seq 0 18 | each { 512 * 2 ** $in }), + ks: [2, 4, 8, 16], + curves: [ bls12381 ], + output: "recoding.ndjson", + }, + fec: { + enabled: true, + sizes: (seq 0 18 | each { 512 * 2 ** $in }), + ks: [2, 4, 8, 16], + curves: [ bls12381 ], + output: "fec.ndjson", + }, +} ``` -### run -```nushell -let out_recoding = $sizes | benchmarks recoding run --ks $ks --curves $curves -let out_fec = $sizes | benchmarks fec run --ks $ks --curves $curves +### run them +```bash +benchmarks --output-dir $OUTPUT_DIR ``` -### plot -```nushell -benchmarks recoding plot $out_recoding -benchmarks fec plot encoding $out_fec -benchmarks fec plot decoding $out_fec -benchmarks fec plot e2e $out_fec -benchmarks fec plot combined $out_fec --recoding $out_recoding -benchmarks fec plot ratio $out_fec --recoding $out_recoding +### plot them +```bash +benchmarks linalg plot <linalg> mul +benchmarks linalg plot <linalg> transpose +benchmarks linalg plot <linalg> inverse +benchmarks setup plot <setup> +benchmarks commit plot <commit> +benchmarks recoding plot <recoding> +benchmarks fec plot encoding <fec> +benchmarks fec plot decoding <fec> +benchmarks fec plot e2e <fec> +benchmarks fec plot combined <fec> --recoding <recoding> +benchmarks fec plot ratio <fec> --recoding <recoding> ``` ## FRI diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 838b6183..7459fb38 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -4,3 +4,78 @@ export module nu-lib/fec/ export module nu-lib/recoding.nu export module nu-lib/linalg.nu export module nu-lib/fri/ + +use nu-lib/linalg.nu +use nu-lib/setup.nu +use nu-lib/commit.nu +use nu-lib/recoding.nu +use nu-lib/fec/ + +use nu-lib/utils/log.nu + +const CPU_FIELDS = [ + "Architecture", + "CPU op-mode(s)", + "Address sizes", + "Byte Order", + "CPU(s)", + "On-line CPU(s) list", + "Model name", + "CPU family", + "Model", + "Thread(s) per core", + "Core(s) per socket", + "Socket(s)", + "Stepping", + "CPU max MHz", + "CPU min MHz", + "BogoMIPS", + "Virtualization", + "L1d cache", + "L1i cache", + "L2 cache", + "L3 cache", + "NUMA node(s)", + "NUMA node0 CPU(s)", +] + +export def main [benchmarks: record, --output-dir: path = "."] { + let cpu = lscpu --json + | from json + | get lscpu + | update field { str trim --right --char ":" } + | transpose --header-row + | into record + | select ...$CPU_FIELDS + + let hash = $cpu | to json | hash sha256 + + let target = $output_dir | path join $hash + mkdir $target + + $cpu | to json | save --force ($target | path join "cpu.json") + + let benchmarks = $benchmarks + | insert linalg.run {{ |it| $it.sizes | linalg run }} + | insert setup.run {{ |it| $it.degrees | setup run --curves $it.curves }} + | insert commit.run {{ |it| $it.degrees | commit run --curves $it.curves }} + | insert recoding.run {{ |it| $it.sizes | recoding run --ks $it.ks --curves $it.curves }} + | insert fec.run {{ |it| $it.sizes | fec run --ks $it.ks --curves $it.curves }} + + for b in ($benchmarks | values) { + if ($b.enabled? | default true) { + let output = $target | path join $b.output + $b | do $b.run $in | open $in | save --append $output + log info $"results saved to (ansi purple)($output)(ansi reset)" + } + } + + # const OPTIONS = [ + # --release + # --package benchmarks + # -- + # --nb-measurements 1000 + # ] + # cargo run --bin field_operations ...$OPTIONS out> ($target | path join field.ndjson) + # cargo run --bin curve_group_operations ...$OPTIONS out> ($target | path join curve_group.ndjson) +} -- GitLab From 08cc3f24eb4710fc43d2100dcaeaa7783fa63af1 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 17:59:02 +0100 Subject: [PATCH 02/27] add FRI as well --- benchmarks/README.md | 101 ++++++++++++++++++--------------------- benchmarks/mod.nu | 12 +++++ benchmarks/params/fri.nu | 16 ------- 3 files changed, 58 insertions(+), 71 deletions(-) delete mode 100644 benchmarks/params/fri.nu diff --git a/benchmarks/README.md b/benchmarks/README.md index 1a300b45..674c7ecf 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,7 +1,9 @@ # Table of contents - [Requirements](#requirements) -- [The rest](#the-rest) -- [FRI](#fri) +- [Run the benchmarks](#run-the-benchmarks) + - [define them](#define-them) + - [run them](#run-them) +- [Plot the benchmarks](#plot-the-benchmarks) ## requirements > :bulb: **Note** @@ -60,8 +62,14 @@ gplt multi_bar --title "complex curve group operations" -l "time (in ns)" ( ) ``` -## The rest -### define the benchmarks +## Run the benchmarks +### define them + +> :bulb: **Note** +> +> the FRI benchmarks don't use a module from [src/bin/](src/bin/) with PLNK but rather an +> [example](../examples/fri.rs) + ```bash const OUTPUT_DIR = "/path/to/komodo-benchmarks-results/" @@ -97,54 +105,24 @@ let benchmarks = { curves: [ bls12381 ], output: "fec.ndjson", }, + fri: { + enabled: true, + sizes: (seq 0 15 | each { 2 ** $in * 4096b }), + ks: [8, 128, 1024, 4096], + blowup_factors: [2, 4], + ns: [2], + remainder_plus_ones: [1], + nb_queries: [50], + hashes: ["sha3-512"], + ffs: ["fp128", "bls12-381"], + output: "fri.ndjson", + }, } ``` ### run them ```bash -benchmarks --output-dir $OUTPUT_DIR -``` - -### plot them -```bash -benchmarks linalg plot <linalg> mul -benchmarks linalg plot <linalg> transpose -benchmarks linalg plot <linalg> inverse -benchmarks setup plot <setup> -benchmarks commit plot <commit> -benchmarks recoding plot <recoding> -benchmarks fec plot encoding <fec> -benchmarks fec plot decoding <fec> -benchmarks fec plot e2e <fec> -benchmarks fec plot combined <fec> --recoding <recoding> -benchmarks fec plot ratio <fec> --recoding <recoding> -``` - -## FRI -> :bulb: **Note** -> -> the FRI benchmarks don't use a module from [src/bin/](src/bin/) with PLNK but rather an -> [example](../examples/fri.rs) - -- modify [benchmarks/params/fri.nu](benchmarks/params/fri.nu) -- source it -```nushell -source benchmarks/params/fri.nu -``` -- run the benchmarks -```nushell -use std formats "to ndjson" - -(benchmarks fri run - --data-sizes $DATA_SIZES - --ks $KS - --blowup-factors $BFS - --nb-queries $QS - --hashes $HS - --finite-fields $FFS - --remainders $RPOS - --folding-factors $NS -) | to ndjson out> $DATA +benchmarks --output-dir $OUTPUT_DIR $benchmarks ``` > the following `watch` call can be used to see the results as they are dumped to `$DATA` @@ -152,7 +130,7 @@ use std formats "to ndjson" > use std formats "from ndjson" > > watch . { -> open --raw $DATA +> open --raw /path/to/komodo-benchmarks-results/ > | lines > | last > | from ndjson @@ -163,13 +141,26 @@ use std formats "to ndjson" > } > ``` -```nushell -benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA evaluating encoding proving decoding --y-type "duration" -benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA verifying --y-type "duration" --single +## Plot the benchmarks +```bash +benchmarks linalg plot <linalg> mul +benchmarks linalg plot <linalg> transpose +benchmarks linalg plot <linalg> inverse +benchmarks setup plot <setup> +benchmarks commit plot <commit> +benchmarks recoding plot <recoding> +benchmarks fec plot encoding <fec> +benchmarks fec plot decoding <fec> +benchmarks fec plot e2e <fec> +benchmarks fec plot combined <fec> --recoding <recoding> +benchmarks fec plot ratio <fec> --recoding <recoding> + +benchmarks fri plot --dump-dir <out> --file <fri> evaluating encoding proving decoding --y-type "duration" +benchmarks fri plot --dump-dir <out> --file <fri> verifying --y-type "duration" --single -benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA proofs --y-type "filesize" --identity --normalize -benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA commits --y-type "filesize" --single --identity --normalize +benchmarks fri plot --dump-dir <out> --file <fri> proofs --y-type "filesize" --identity --normalize +benchmarks fri plot --dump-dir <out> --file <fri> commits --y-type "filesize" --single --identity --normalize -benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA proofs --y-type "filesize" --identity -benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA commits --y-type "filesize" --single --identity +benchmarks fri plot --dump-dir <out> --file <fri> proofs --y-type "filesize" --identity +benchmarks fri plot --dump-dir <out> --file <fri> commits --y-type "filesize" --single --identity ``` diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 7459fb38..0fac8a14 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -10,6 +10,7 @@ use nu-lib/setup.nu use nu-lib/commit.nu use nu-lib/recoding.nu use nu-lib/fec/ +use nu-lib/fri/ use nu-lib/utils/log.nu @@ -61,6 +62,17 @@ export def main [benchmarks: record, --output-dir: path = "."] { | insert commit.run {{ |it| $it.degrees | commit run --curves $it.curves }} | insert recoding.run {{ |it| $it.sizes | recoding run --ks $it.ks --curves $it.curves }} | insert fec.run {{ |it| $it.sizes | fec run --ks $it.ks --curves $it.curves }} + | insert fri.run {{ |it| ( + fri run + --data-sizes $it.sizes + --ks $it.ks + --blowup-factors $it.blowup_factors + --nb-queries $it.nb_queries + --hashes $it.hashes + --finite-fields $it.ffs + --remainders $it.remainder_plus_ones + --folding-factors $it.ns + )}} for b in ($benchmarks | values) { if ($b.enabled? | default true) { diff --git a/benchmarks/params/fri.nu b/benchmarks/params/fri.nu deleted file mode 100644 index 24fe8100..00000000 --- a/benchmarks/params/fri.nu +++ /dev/null @@ -1,16 +0,0 @@ -let DATA_SIZES = seq 0 15 | each { 2 ** $in * 4096b } -const KS = [8, 128, 1024, 4096] -const BFS = [2, 4] -const NS = [2] -const RPOS = [1] -const QS = [50] -const HS = ["sha3-512"] -const FFS = ["fp128", "bls12-381"] - -const DATA = "benchmarks/results/fri.ndjson" -const OUTPUT_DIR = "benchmarks/results/figures/" - -if not ($DATA | path dirname | path exists) { - print $"creating directory for (ansi purple)($DATA)(ansi reset)" - $DATA | path dirname | mkdir $in -} -- GitLab From efc6a46384254cb6be244a3e39e5f0acb24f781c Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 17:59:26 +0100 Subject: [PATCH 03/27] use better logs in the benchmark loop --- benchmarks/mod.nu | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 0fac8a14..ad725591 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -74,11 +74,14 @@ export def main [benchmarks: record, --output-dir: path = "."] { --folding-factors $it.ns )}} - for b in ($benchmarks | values) { + let _ = $benchmarks | items { |k, b| if ($b.enabled? | default true) { + log info $"running (ansi cyan)($k)(ansi reset)" let output = $target | path join $b.output - $b | do $b.run $in | open $in | save --append $output + $b | do $b.run $in | tee { print $in } | open $in | save --append $output log info $"results saved to (ansi purple)($output)(ansi reset)" + } else { + log warning $"skipping (ansi cyan)($k)(ansi reset)" } } -- GitLab From a7cd561e686d1d9c6a2e49e26af4ccae37970770 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 18:05:24 +0100 Subject: [PATCH 04/27] fix FRI command --- benchmarks/mod.nu | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index ad725591..558184a1 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -62,17 +62,21 @@ export def main [benchmarks: record, --output-dir: path = "."] { | insert commit.run {{ |it| $it.degrees | commit run --curves $it.curves }} | insert recoding.run {{ |it| $it.sizes | recoding run --ks $it.ks --curves $it.curves }} | insert fec.run {{ |it| $it.sizes | fec run --ks $it.ks --curves $it.curves }} - | insert fri.run {{ |it| ( - fri run - --data-sizes $it.sizes - --ks $it.ks - --blowup-factors $it.blowup_factors - --nb-queries $it.nb_queries - --hashes $it.hashes - --finite-fields $it.ffs - --remainders $it.remainder_plus_ones - --folding-factors $it.ns - )}} + | insert fri.run {{ |it| + let output = mktemp --tmpdir komodo_commit.XXXXXX + ( + fri run + --data-sizes $it.sizes + --ks $it.ks + --blowup-factors $it.blowup_factors + --nb-queries $it.nb_queries + --hashes $it.hashes + --finite-fields $it.ffs + --remainders $it.remainder_plus_ones + --folding-factors $it.ns + ) | to ndjson out> $output + $output + }} let _ = $benchmarks | items { |k, b| if ($b.enabled? | default true) { -- GitLab From 11a6bd17db2062ff7272606b83cb5b2344524e3a Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 18:12:14 +0100 Subject: [PATCH 05/27] output results directly to file --- benchmarks/mod.nu | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 558184a1..7950cb39 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -57,13 +57,12 @@ export def main [benchmarks: record, --output-dir: path = "."] { $cpu | to json | save --force ($target | path join "cpu.json") let benchmarks = $benchmarks - | insert linalg.run {{ |it| $it.sizes | linalg run }} - | insert setup.run {{ |it| $it.degrees | setup run --curves $it.curves }} - | insert commit.run {{ |it| $it.degrees | commit run --curves $it.curves }} - | insert recoding.run {{ |it| $it.sizes | recoding run --ks $it.ks --curves $it.curves }} - | insert fec.run {{ |it| $it.sizes | fec run --ks $it.ks --curves $it.curves }} + | insert linalg.run {{ |it| $it.sizes | linalg run --force --output ($target | path join $it.output) }} + | insert setup.run {{ |it| $it.degrees | setup run --curves $it.curves --force --output ($target | path join $it.output) }} + | insert commit.run {{ |it| $it.degrees | commit run --curves $it.curves --force --output ($target | path join $it.output) }} + | insert recoding.run {{ |it| $it.sizes | recoding run --ks $it.ks --curves $it.curves --force --output ($target | path join $it.output) }} + | insert fec.run {{ |it| $it.sizes | fec run --ks $it.ks --curves $it.curves --force --output ($target | path join $it.output) }} | insert fri.run {{ |it| - let output = mktemp --tmpdir komodo_commit.XXXXXX ( fri run --data-sizes $it.sizes @@ -74,16 +73,13 @@ export def main [benchmarks: record, --output-dir: path = "."] { --finite-fields $it.ffs --remainders $it.remainder_plus_ones --folding-factors $it.ns - ) | to ndjson out> $output - $output + ) | to ndjson out> $it.output }} let _ = $benchmarks | items { |k, b| if ($b.enabled? | default true) { log info $"running (ansi cyan)($k)(ansi reset)" - let output = $target | path join $b.output - $b | do $b.run $in | tee { print $in } | open $in | save --append $output - log info $"results saved to (ansi purple)($output)(ansi reset)" + do $b.run $b } else { log warning $"skipping (ansi cyan)($k)(ansi reset)" } -- GitLab From 8e250f1943e2c676809acf952fb2d14170367f93 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 18:13:33 +0100 Subject: [PATCH 06/27] fix watch snippet --- benchmarks/README.md | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 674c7ecf..0cc8484c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -125,19 +125,10 @@ let benchmarks = { benchmarks --output-dir $OUTPUT_DIR $benchmarks ``` -> the following `watch` call can be used to see the results as they are dumped to `$DATA` -> ```nushell -> use std formats "from ndjson" -> -> watch . { -> open --raw /path/to/komodo-benchmarks-results/ -> | lines -> | last -> | from ndjson -> | into int evaluating encoding proving verifying decoding -> | into duration evaluating encoding proving verifying decoding -> | into filesize proofs commits d -> | into record +> the following `watch` can be used to see the results as they are dumped to `$OUTPUT_DIR` +> ```bash +> watch $OUTPUT_DIR { |op, path| +> $"($op) ($path)" > } > ``` -- GitLab From d585190281021cc378bda2923fe6a52b061d1988 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Wed, 29 Jan 2025 18:14:53 +0100 Subject: [PATCH 07/27] add link to results repo --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6c099078..d29178c0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ Other examples that showcase the Komodo API are available in [`examples/`](examp ## the benchmarks see [`benchmarks/`](benchmarks/README.md) +the results can be found in [`dragoon/komodo-benchmark-results`](https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results). + ## contributors Because the code for this project has been originally extracted from -- GitLab From d2e037a5fd346acd34034f6eadbc1f72b25c6434 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 10:46:21 +0100 Subject: [PATCH 08/27] add annotations to benchmarks --- benchmarks/mod.nu | 49 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 7950cb39..79567142 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -40,7 +40,54 @@ const CPU_FIELDS = [ "NUMA node0 CPU(s)", ] -export def main [benchmarks: record, --output-dir: path = "."] { +export def main [ + benchmarks: record< + linalg: record< + enabled?: bool, + sizes: list<int>, + output: string, + >, + setup: record< + enabled?: bool, + degrees: list<int>, + curves: list<string>, + output: string, + >, + commit: record< + enabled?: bool, + degrees: list<int>, + curves: list<string>, + output: string, + >, + recoding: record< + enabled?: bool, + sizes: list<int>, + ks: list<int>, + curves: list<string>, + output: string, + >, + fec: record< + enabled?: bool, + sizes: list<int>, + ks: list<int>, + curves: list<string>, + output: string, + >, + fri: record< + enabled?: bool, + sizes: list<filesize>, + ks: list<int>, + blowup_factors: list<int>, + ns: list<int>, + remainder_plus_ones: list<int>, + nb_queries: list<int>, + hashes: list<string>, + ffs: list<string>, + output: string, + >, + >, + --output-dir: path = ".", +] { let cpu = lscpu --json | from json | get lscpu -- GitLab From d6c200f3fa7a3391644f60d77d570ec31cab0318 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 10:53:20 +0100 Subject: [PATCH 09/27] fix benchmarks annotation --- benchmarks/mod.nu | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 79567142..cec02143 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -43,38 +43,38 @@ const CPU_FIELDS = [ export def main [ benchmarks: record< linalg: record< - enabled?: bool, + enabled: bool, sizes: list<int>, output: string, >, setup: record< - enabled?: bool, + enabled: bool, degrees: list<int>, curves: list<string>, output: string, >, commit: record< - enabled?: bool, + enabled: bool, degrees: list<int>, curves: list<string>, output: string, >, recoding: record< - enabled?: bool, + enabled: bool, sizes: list<int>, ks: list<int>, curves: list<string>, output: string, >, fec: record< - enabled?: bool, + enabled: bool, sizes: list<int>, ks: list<int>, curves: list<string>, output: string, >, fri: record< - enabled?: bool, + enabled: bool, sizes: list<filesize>, ks: list<int>, blowup_factors: list<int>, @@ -141,3 +141,4 @@ export def main [ # cargo run --bin field_operations ...$OPTIONS out> ($target | path join field.ndjson) # cargo run --bin curve_group_operations ...$OPTIONS out> ($target | path join curve_group.ndjson) } + -- GitLab From 4f69a1d6c70e56e60517d55a85f87e08caa6ccdb Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 11:04:35 +0100 Subject: [PATCH 10/27] fix "atomic operations" --- benchmarks/README.md | 10 +++++++ benchmarks/mod.nu | 30 ++++++++++++------- .../src/bin/{operations => }/curve_group.rs | 0 benchmarks/src/bin/{operations => }/field.rs | 0 4 files changed, 30 insertions(+), 10 deletions(-) rename benchmarks/src/bin/{operations => }/curve_group.rs (100%) rename benchmarks/src/bin/{operations => }/field.rs (100%) diff --git a/benchmarks/README.md b/benchmarks/README.md index 0cc8484c..dbaba968 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -117,6 +117,16 @@ let benchmarks = { ffs: ["fp128", "bls12-381"], output: "fri.ndjson", }, + field: { + enabled: true, + nb_measurements: 1000, + output: "field.ndjson", + }, + curve_group: { + enabled: true, + nb_measurements: 1000, + output: "curve_group.ndjson", + }, } ``` diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index cec02143..559ce8f3 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -85,6 +85,8 @@ export def main [ ffs: list<string>, output: string, >, + field: record<enabled: bool, nb_measurements: int, output: string>, + curve_group: record<enabled: bool, nb_measurements: int, output: string>, >, --output-dir: path = ".", ] { @@ -122,6 +124,24 @@ export def main [ --folding-factors $it.ns ) | to ndjson out> $it.output }} + | insert field.run {{ |it| + cargo run ...[ + --bin field + --release + --package benchmarks + -- + --nb-measurements $it.nb_measurements + ] out>> ($target | path join $it.output) + }} + | insert curve_group.run {{ |it| + cargo run ...[ + --bin curve_group + --release + --package benchmarks + -- + --nb-measurements $it.nb_measurements + ] out>> ($target | path join $it.output) + }} let _ = $benchmarks | items { |k, b| if ($b.enabled? | default true) { @@ -131,14 +151,4 @@ export def main [ log warning $"skipping (ansi cyan)($k)(ansi reset)" } } - - # const OPTIONS = [ - # --release - # --package benchmarks - # -- - # --nb-measurements 1000 - # ] - # cargo run --bin field_operations ...$OPTIONS out> ($target | path join field.ndjson) - # cargo run --bin curve_group_operations ...$OPTIONS out> ($target | path join curve_group.ndjson) } - diff --git a/benchmarks/src/bin/operations/curve_group.rs b/benchmarks/src/bin/curve_group.rs similarity index 100% rename from benchmarks/src/bin/operations/curve_group.rs rename to benchmarks/src/bin/curve_group.rs diff --git a/benchmarks/src/bin/operations/field.rs b/benchmarks/src/bin/field.rs similarity index 100% rename from benchmarks/src/bin/operations/field.rs rename to benchmarks/src/bin/field.rs -- GitLab From ca4cd6bfa8dcb51dfbc1150e7c2e17500cdac9e7 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 11:05:40 +0100 Subject: [PATCH 11/27] make it more readable --- benchmarks/mod.nu | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 559ce8f3..84a0900e 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -106,11 +106,26 @@ export def main [ $cpu | to json | save --force ($target | path join "cpu.json") let benchmarks = $benchmarks - | insert linalg.run {{ |it| $it.sizes | linalg run --force --output ($target | path join $it.output) }} - | insert setup.run {{ |it| $it.degrees | setup run --curves $it.curves --force --output ($target | path join $it.output) }} - | insert commit.run {{ |it| $it.degrees | commit run --curves $it.curves --force --output ($target | path join $it.output) }} - | insert recoding.run {{ |it| $it.sizes | recoding run --ks $it.ks --curves $it.curves --force --output ($target | path join $it.output) }} - | insert fec.run {{ |it| $it.sizes | fec run --ks $it.ks --curves $it.curves --force --output ($target | path join $it.output) }} + | insert linalg.run {{ |it| + let output = $target | path join $it.output + $it.sizes | linalg run --force --output $output + }} + | insert setup.run {{ |it| + let output = $target | path join $it.output + $it.degrees | setup run --curves $it.curves --force --output $output + }} + | insert commit.run {{ |it| + let output = $target | path join $it.output + $it.degrees | commit run --curves $it.curves --force --output $output + }} + | insert recoding.run {{ |it| + let output = $target | path join $it.output + $it.sizes | recoding run --ks $it.ks --curves $it.curves --force --output $output + }} + | insert fec.run {{ |it| + let output = $target | path join $it.output + $it.sizes | fec run --ks $it.ks --curves $it.curves --force --output $output + }} | insert fri.run {{ |it| ( fri run -- GitLab From 2a5a323642351ffcefc0677c013b1f7d872d5eca Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 11:22:44 +0100 Subject: [PATCH 12/27] fix FRI output --- benchmarks/mod.nu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 84a0900e..f21f68ff 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -137,7 +137,7 @@ export def main [ --finite-fields $it.ffs --remainders $it.remainder_plus_ones --folding-factors $it.ns - ) | to ndjson out> $it.output + ) | to ndjson out> ($target | path join $it.output) }} | insert field.run {{ |it| cargo run ...[ -- GitLab From 4bf1c9e0e775eb7f16a6acd3e1ffdd602856c1ba Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 11:27:53 +0100 Subject: [PATCH 13/27] bump to Nushell 0.101.0 - parallel $in - group-by --- Makefile | 2 +- benchmarks/nu-lib/commit.nu | 6 ++---- benchmarks/nu-lib/fec/plot.nu | 18 +++++++++--------- benchmarks/nu-lib/fec/run.nu | 3 +-- benchmarks/nu-lib/linalg.nu | 6 ++---- benchmarks/nu-lib/recoding.nu | 5 ++--- benchmarks/nu-lib/setup.nu | 6 ++---- benchmarks/nu-lib/utils/plot.nu | 6 ++---- 8 files changed, 21 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 2a89ea42..e0a254ab 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ NU="nu" NU_FLAGS="--no-config-file" NU_ARCH="x86_64-unknown-linux-musl" -NU_VERSION="0.95.0" +NU_VERSION="0.101.0" NU_BUILD="nu-${NU_VERSION}-${NU_ARCH}" NU_DEST="/tmp/" diff --git a/benchmarks/nu-lib/commit.nu b/benchmarks/nu-lib/commit.nu index 0d9cd4c5..1d7c4d41 100644 --- a/benchmarks/nu-lib/commit.nu +++ b/benchmarks/nu-lib/commit.nu @@ -16,8 +16,6 @@ export def run [ --force, # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { - let input = $in - $curves | check-list-arg --cmd "commit run" --arg "--curves" --span (metadata $curves).span $in | check-list-arg --cmd "commit run" --arg "pipeline input" @@ -38,7 +36,7 @@ export def run [ cargo run --release --package benchmarks --bin commit -- ...[ --nb-measurements $nb_measurements - ...$input + ...$in --curves ...$curves ] out> $output @@ -64,7 +62,7 @@ export def plot [ | select name x y e | group-by name --to-table | reject items.name - | rename --column { group: "name", items: "points" } + | rename --column { name: "name", items: "points" } | insert style.color {|it| match $it.name { "BLS12-381" => "tab:blue" diff --git a/benchmarks/nu-lib/fec/plot.nu b/benchmarks/nu-lib/fec/plot.nu index da3c0dc2..a8024518 100644 --- a/benchmarks/nu-lib/fec/plot.nu +++ b/benchmarks/nu-lib/fec/plot.nu @@ -24,7 +24,7 @@ export def encoding [ | sort-by x | group-by k --to-table | reject items.k - | rename --column { group: "name", items: "points" } + | rename --column { k: "name", items: "points" } | update name { $"$k = ($in)$" } let options = [ @@ -56,7 +56,7 @@ export def decoding [ | sort-by x | group-by k --to-table | reject items.k - | rename --column { group: "name", items: "points" } + | rename --column { k: "name", items: "points" } | update name { $"$k = ($in)$" } let options = [ @@ -89,7 +89,7 @@ export def e2e [ | update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } } } | flatten --all - | reject group foo + | reject foo | ns-to-ms times | compute-stats times | reject times @@ -99,7 +99,7 @@ export def e2e [ | sort-by x | group-by k --to-table | reject items.k - | rename --column { group: "name", items: "points" } + | rename --column { k: "name", items: "points" } | update name { $"$k = ($in)$" } let options = [ @@ -144,7 +144,7 @@ export def combined [ } | reject items.shards | insert style.line.type "solid" - | rename --column { group: "name", items: "points" } + | rename --column { shards: "name", items: "points" } | update name { $"$k = ($in)$" } let re_encoding_graphs = open --raw $data @@ -159,7 +159,7 @@ export def combined [ | update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } } } | flatten --all - | reject group key + | reject key | ns-to-ms times | compute-stats times | reject times @@ -179,7 +179,7 @@ export def combined [ } | insert style.line.type "dashed" | reject items.k - | rename --column { group: "name", items: "points" } + | rename --column { k: "name", items: "points" } | reject name let graphs = $recoding_graphs @@ -254,7 +254,7 @@ export def ratio [ | update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } } } | flatten --all - | reject group key + | reject key | ns-to-ms times | compute-stats times | where name == "BLS12-381" @@ -281,7 +281,7 @@ export def ratio [ } } | reject items.k - | rename --column { group: "name", items: "points" } + | rename --column { k: "name", items: "points" } | update name { $"$k = ($in)$" } let options = [ diff --git a/benchmarks/nu-lib/fec/run.nu b/benchmarks/nu-lib/fec/run.nu index 4210ffc5..6874823d 100644 --- a/benchmarks/nu-lib/fec/run.nu +++ b/benchmarks/nu-lib/fec/run.nu @@ -15,8 +15,6 @@ export def main [ --force, # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { - let input = $in - $ks | check-list-arg --cmd "fec run" --arg "--ks" --span (metadata $ks).span $curves | check-list-arg --cmd "fec run" --arg "--curves" --span (metadata $curves).span $in | check-list-arg --cmd "fec run" --arg "pipeline input" @@ -38,6 +36,7 @@ export def main [ "" out> $output + let input = $in for k in $ks { cargo run --release --package benchmarks --bin fec -- ...[ --nb-measurements $nb_measurements diff --git a/benchmarks/nu-lib/linalg.nu b/benchmarks/nu-lib/linalg.nu index a31085a8..d0adff09 100644 --- a/benchmarks/nu-lib/linalg.nu +++ b/benchmarks/nu-lib/linalg.nu @@ -15,8 +15,6 @@ export def run [ --force, # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { - let input = $in - $in | check-list-arg --cmd "linalg run" --arg "pipeline input" let new_file = $output == null @@ -36,7 +34,7 @@ export def run [ cargo run --release --package benchmarks --bin linalg -- ...[ --nb-measurements $nb_measurements - ...$input + ...$in ] out> $output log info $"results saved to ($pretty_output)" @@ -87,7 +85,7 @@ export def plot [ | where op == $op | rename --column { n: "x", mean: "y", stddev: "e" } | group-by name --to-table - | rename --column { group: "name", items: "points" } + | rename --column { name: "name", items: "points" } | insert style.color {|it| match $it.name { "BLS12-381" => "tab:blue" diff --git a/benchmarks/nu-lib/recoding.nu b/benchmarks/nu-lib/recoding.nu index 3c0f2390..1638adff 100644 --- a/benchmarks/nu-lib/recoding.nu +++ b/benchmarks/nu-lib/recoding.nu @@ -18,8 +18,6 @@ export def run [ --force, # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { - let input = $in - $ks | check-list-arg --cmd "recoding run" --arg "--ks" --span (metadata $ks).span $curves | check-list-arg --cmd "recoding run" --arg "--curves" --span (metadata $curves).span $in | check-list-arg --cmd "recoding run" --arg "pipeline input" @@ -41,6 +39,7 @@ export def run [ "" out> $output + let input = $in for k in $ks { cargo run --release --package benchmarks --bin recoding -- ...[ --nb-measurements $nb_measurements @@ -73,7 +72,7 @@ export def plot [ | select shards x y e | group-by shards --to-table | reject items.shards - | rename --column { group: "name", items: "points" } + | rename --column { shards: "name", items: "points" } | update name { $"$k = ($in)$"} let options = [ diff --git a/benchmarks/nu-lib/setup.nu b/benchmarks/nu-lib/setup.nu index 37f105ad..311852c0 100644 --- a/benchmarks/nu-lib/setup.nu +++ b/benchmarks/nu-lib/setup.nu @@ -16,8 +16,6 @@ export def run [ --force, # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { - let input = $in - $curves | check-list-arg --cmd "setup run" --arg "--curves" --span (metadata $curves).span $in | check-list-arg --cmd "setup run" --arg "pipeline input" @@ -38,7 +36,7 @@ export def run [ cargo run --release --package benchmarks --bin setup -- ...[ --nb-measurements $nb_measurements - ...$input + ...$in --curves ...$curves ] out> $output @@ -69,7 +67,7 @@ export def plot [ | select name x y e | group-by name --to-table | reject items.name - | rename --column { group: "name", items: "points" } + | rename --column { name: "name", items: "points" } | insert style.color {|it| match $it.name { "BLS12-381" => "tab:blue" diff --git a/benchmarks/nu-lib/utils/plot.nu b/benchmarks/nu-lib/utils/plot.nu index 72c753c5..c001e3d8 100644 --- a/benchmarks/nu-lib/utils/plot.nu +++ b/benchmarks/nu-lib/utils/plot.nu @@ -29,9 +29,7 @@ export def into-filesize-tick-labels []: list<int> -> list<string> { } export def into-axis-options [-x: string, -y: string]: table<x: float, y: float> -> list<string> { - let input = $in - - let xs = $input | flatten | get x | uniq + let xs = $in | flatten | get x | uniq let x_tick_labels = match $x { "filesize" => ($xs | into-filesize-tick-labels), @@ -47,7 +45,7 @@ export def into-axis-options [-x: string, -y: string]: table<x: float, y: float> --x-tick-labels ...$x_tick_labels ] - let ys = $input | flatten | get y + let ys = $in | flatten | get y let y_ticks = seq ($ys | math min | math log 10 | math ceil | $in - 1) ($ys | math max | math log 10 | math floor) | into float | each { 10 ** $in } -- GitLab From 09a9eee7178f1ec344d924c44c959fb6a7c2bf21 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 11:53:33 +0100 Subject: [PATCH 14/27] add --save to FRI plot --- benchmarks/nu-lib/fri/plot.nu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/nu-lib/fri/plot.nu b/benchmarks/nu-lib/fri/plot.nu index eba787a6..f1af2ed3 100644 --- a/benchmarks/nu-lib/fri/plot.nu +++ b/benchmarks/nu-lib/fri/plot.nu @@ -86,6 +86,7 @@ export def main [ --identity, --normalize, --dump-dir: path = "./", + --save, ] { if ($x | is-empty) { error make --unspanned { msg: "nothing to do, x is empty" } @@ -101,6 +102,6 @@ export def main [ let data = open $file | where h == "sha3-512" and q == 50 for i in $x { - $data | plot --save $i --y-type=$y_type --single=$single --identity=$identity --normalize=$normalize --dump-dir=$dump_dir + $data | plot --save=$save $i --y-type=$y_type --single=$single --identity=$identity --normalize=$normalize --dump-dir=$dump_dir } } -- GitLab From f2b4799844819281b9e59cf199c859c945bf8528 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 12:01:45 +0100 Subject: [PATCH 15/27] reject useless columns for plotting --- benchmarks/nu-lib/fri/plot.nu | 1 + benchmarks/nu-lib/linalg.nu | 1 + 2 files changed, 2 insertions(+) diff --git a/benchmarks/nu-lib/fri/plot.nu b/benchmarks/nu-lib/fri/plot.nu index f1af2ed3..66ecaae0 100644 --- a/benchmarks/nu-lib/fri/plot.nu +++ b/benchmarks/nu-lib/fri/plot.nu @@ -54,6 +54,7 @@ def plot [ points: ($ds | wrap x | merge ($ds | wrap y) | if $normalize { update y { |it| $it.y / $it.x } } else { $in }), style: { color: "black", line: { type: "dotted" } }, } } else { $in } + | reject points.k? points.bf? points.ff? let title = [ $name, diff --git a/benchmarks/nu-lib/linalg.nu b/benchmarks/nu-lib/linalg.nu index d0adff09..b9e740f3 100644 --- a/benchmarks/nu-lib/linalg.nu +++ b/benchmarks/nu-lib/linalg.nu @@ -85,6 +85,7 @@ export def plot [ | where op == $op | rename --column { n: "x", mean: "y", stddev: "e" } | group-by name --to-table + | reject items.name items.op items.times | rename --column { name: "name", items: "points" } | insert style.color {|it| match $it.name { -- GitLab From 81e847e38d1d49b554c7089c78f1411afa251449 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 12:17:58 +0100 Subject: [PATCH 16/27] use --no-confirm instead of --force --- benchmarks/mod.nu | 10 +++++----- benchmarks/nu-lib/commit.nu | 4 ++-- benchmarks/nu-lib/fec/run.nu | 4 ++-- benchmarks/nu-lib/linalg.nu | 4 ++-- benchmarks/nu-lib/recoding.nu | 4 ++-- benchmarks/nu-lib/setup.nu | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index f21f68ff..e48dd39b 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -108,23 +108,23 @@ export def main [ let benchmarks = $benchmarks | insert linalg.run {{ |it| let output = $target | path join $it.output - $it.sizes | linalg run --force --output $output + $it.sizes | linalg run --no-confirm --output $output }} | insert setup.run {{ |it| let output = $target | path join $it.output - $it.degrees | setup run --curves $it.curves --force --output $output + $it.degrees | setup run --curves $it.curves --no-confirm --output $output }} | insert commit.run {{ |it| let output = $target | path join $it.output - $it.degrees | commit run --curves $it.curves --force --output $output + $it.degrees | commit run --curves $it.curves --no-confirm --output $output }} | insert recoding.run {{ |it| let output = $target | path join $it.output - $it.sizes | recoding run --ks $it.ks --curves $it.curves --force --output $output + $it.sizes | recoding run --ks $it.ks --curves $it.curves --no-confirm --output $output }} | insert fec.run {{ |it| let output = $target | path join $it.output - $it.sizes | fec run --ks $it.ks --curves $it.curves --force --output $output + $it.sizes | fec run --ks $it.ks --curves $it.curves --no-confirm --output $output }} | insert fri.run {{ |it| ( diff --git a/benchmarks/nu-lib/commit.nu b/benchmarks/nu-lib/commit.nu index 1d7c4d41..5460e838 100644 --- a/benchmarks/nu-lib/commit.nu +++ b/benchmarks/nu-lib/commit.nu @@ -13,7 +13,7 @@ use std formats * export def run [ --output: path, # the output path (defaults to a random file in $nu.temp-path) --curves: list<string>, # the curves to benchmark - --force, # does not ask for confirmation if the output file already exists, it will be overwritten + --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { $curves | check-list-arg --cmd "commit run" --arg "--curves" --span (metadata $curves).span @@ -24,7 +24,7 @@ export def run [ let pretty_output = $"(ansi purple)($output)(ansi reset)" if ($output | path exists) and not $new_file { log warning $"($pretty_output) already exists" - if not $force { + if not $no_confirm { let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?" if $res == null or $res == "no" { log info "aborting" diff --git a/benchmarks/nu-lib/fec/run.nu b/benchmarks/nu-lib/fec/run.nu index 6874823d..7fee6789 100644 --- a/benchmarks/nu-lib/fec/run.nu +++ b/benchmarks/nu-lib/fec/run.nu @@ -12,7 +12,7 @@ export def main [ --output: path, # the output path (defaults to a random file in $nu.temp-path) --ks: list<int>, # the values of $k$ to benchmark --curves: list<string>, # the curves to benchmark - --force, # does not ask for confirmation if the output file already exists, it will be overwritten + --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { $ks | check-list-arg --cmd "fec run" --arg "--ks" --span (metadata $ks).span @@ -24,7 +24,7 @@ export def main [ let pretty_output = $"(ansi purple)($output)(ansi reset)" if ($output | path exists) and not $new_file { log warning $"($pretty_output) already exists" - if not $force { + if not $no_confirm { let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?" if $res == null or $res == "no" { log info "aborting" diff --git a/benchmarks/nu-lib/linalg.nu b/benchmarks/nu-lib/linalg.nu index b9e740f3..104fd7fd 100644 --- a/benchmarks/nu-lib/linalg.nu +++ b/benchmarks/nu-lib/linalg.nu @@ -12,7 +12,7 @@ use std formats * # - output: the output path, as NDJSON export def run [ --output: path, # the output path (defaults to a random file in $nu.temp-path) - --force, # does not ask for confirmation if the output file already exists, it will be overwritten + --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { $in | check-list-arg --cmd "linalg run" --arg "pipeline input" @@ -22,7 +22,7 @@ export def run [ let pretty_output = $"(ansi purple)($output)(ansi reset)" if ($output | path exists) and not $new_file { log warning $"($pretty_output) already exists" - if not $force { + if not $no_confirm { let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?" if $res == null or $res == "no" { log info "aborting" diff --git a/benchmarks/nu-lib/recoding.nu b/benchmarks/nu-lib/recoding.nu index 1638adff..6ab24c98 100644 --- a/benchmarks/nu-lib/recoding.nu +++ b/benchmarks/nu-lib/recoding.nu @@ -15,7 +15,7 @@ export def run [ --output: path, # the output path (defaults to a random file in $nu.temp-path) --ks: list<int>, # the values of $k$ to benchmark --curves: list<string>, # the curves to benchmark - --force, # does not ask for confirmation if the output file already exists, it will be overwritten + --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { $ks | check-list-arg --cmd "recoding run" --arg "--ks" --span (metadata $ks).span @@ -27,7 +27,7 @@ export def run [ let pretty_output = $"(ansi purple)($output)(ansi reset)" if ($output | path exists) and not $new_file { log warning $"($pretty_output) already exists" - if not $force { + if not $no_confirm { let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?" if $res == null or $res == "no" { log info "aborting" diff --git a/benchmarks/nu-lib/setup.nu b/benchmarks/nu-lib/setup.nu index 311852c0..9a43103a 100644 --- a/benchmarks/nu-lib/setup.nu +++ b/benchmarks/nu-lib/setup.nu @@ -13,7 +13,7 @@ use std formats * export def run [ --output: path, # the output path (defaults to a random file in $nu.temp-path) --curves: list<string>, # the curves to benchmark - --force, # does not ask for confirmation if the output file already exists, it will be overwritten + --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run ]: list<int> -> path { $curves | check-list-arg --cmd "setup run" --arg "--curves" --span (metadata $curves).span @@ -24,7 +24,7 @@ export def run [ let pretty_output = $"(ansi purple)($output)(ansi reset)" if ($output | path exists) and not $new_file { log warning $"($pretty_output) already exists" - if not $force { + if not $no_confirm { let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?" if $res == null or $res == "no" { log info "aborting" -- GitLab From 38f29de18b9d7f5e849ded54f4c3dc82cbf6e3cd Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 12:25:41 +0100 Subject: [PATCH 17/27] add append support --- benchmarks/README.md | 8 ++++ benchmarks/mod.nu | 77 +++++++++++++++++++++++++---------- benchmarks/nu-lib/commit.nu | 14 ++++++- benchmarks/nu-lib/fec/run.nu | 18 ++++++-- benchmarks/nu-lib/linalg.nu | 14 ++++++- benchmarks/nu-lib/recoding.nu | 18 ++++++-- benchmarks/nu-lib/setup.nu | 14 ++++++- 7 files changed, 129 insertions(+), 34 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index dbaba968..1f19e25e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -78,18 +78,21 @@ let benchmarks = { enabled: true, sizes: (seq 0 7 | each { 2 ** $in }), output: "linalg.ndjson", + append: true, }, setup: { enabled: true, degrees: (seq 0 13 | each { 2 ** $in }), curves: [ bls12381, pallas, bn254 ], output: "setup.ndjson", + append: true, }, commit: { enabled: true, degrees: (seq 0 13 | each { 2 ** $in }), curves: [ bls12381, pallas, bn254 ], output: "commit.ndjson", + append: true, }, recoding: { enabled: true, @@ -97,6 +100,7 @@ let benchmarks = { ks: [2, 4, 8, 16], curves: [ bls12381 ], output: "recoding.ndjson", + append: true, }, fec: { enabled: true, @@ -104,6 +108,7 @@ let benchmarks = { ks: [2, 4, 8, 16], curves: [ bls12381 ], output: "fec.ndjson", + append: true, }, fri: { enabled: true, @@ -116,16 +121,19 @@ let benchmarks = { hashes: ["sha3-512"], ffs: ["fp128", "bls12-381"], output: "fri.ndjson", + append: true, }, field: { enabled: true, nb_measurements: 1000, output: "field.ndjson", + append: true, }, curve_group: { enabled: true, nb_measurements: 1000, output: "curve_group.ndjson", + append: true, }, } ``` diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index e48dd39b..0e5967e1 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -46,18 +46,21 @@ export def main [ enabled: bool, sizes: list<int>, output: string, + append: bool, >, setup: record< enabled: bool, degrees: list<int>, curves: list<string>, output: string, + append: bool, >, commit: record< enabled: bool, degrees: list<int>, curves: list<string>, output: string, + append: bool, >, recoding: record< enabled: bool, @@ -65,6 +68,7 @@ export def main [ ks: list<int>, curves: list<string>, output: string, + append: bool, >, fec: record< enabled: bool, @@ -72,6 +76,7 @@ export def main [ ks: list<int>, curves: list<string>, output: string, + append: bool, >, fri: record< enabled: bool, @@ -84,9 +89,10 @@ export def main [ hashes: list<string>, ffs: list<string>, output: string, + append: bool, >, - field: record<enabled: bool, nb_measurements: int, output: string>, - curve_group: record<enabled: bool, nb_measurements: int, output: string>, + field: record<enabled: bool, nb_measurements: int, output: string, append: bool>, + curve_group: record<enabled: bool, nb_measurements: int, output: string, append: bool>, >, --output-dir: path = ".", ] { @@ -108,54 +114,81 @@ export def main [ let benchmarks = $benchmarks | insert linalg.run {{ |it| let output = $target | path join $it.output - $it.sizes | linalg run --no-confirm --output $output + $it.sizes | linalg run --no-confirm --output $output --append=$it.append }} | insert setup.run {{ |it| let output = $target | path join $it.output - $it.degrees | setup run --curves $it.curves --no-confirm --output $output + $it.degrees | setup run --curves $it.curves --no-confirm --output $output --append=$it.append }} | insert commit.run {{ |it| let output = $target | path join $it.output - $it.degrees | commit run --curves $it.curves --no-confirm --output $output + $it.degrees | commit run --curves $it.curves --no-confirm --output $output --append=$it.append }} | insert recoding.run {{ |it| let output = $target | path join $it.output - $it.sizes | recoding run --ks $it.ks --curves $it.curves --no-confirm --output $output + $it.sizes | recoding run --ks $it.ks --curves $it.curves --no-confirm --output $output --append=$it.append }} | insert fec.run {{ |it| let output = $target | path join $it.output - $it.sizes | fec run --ks $it.ks --curves $it.curves --no-confirm --output $output + $it.sizes | fec run --ks $it.ks --curves $it.curves --no-confirm --output $output --append=$it.append }} | insert fri.run {{ |it| - ( - fri run - --data-sizes $it.sizes - --ks $it.ks - --blowup-factors $it.blowup_factors - --nb-queries $it.nb_queries - --hashes $it.hashes - --finite-fields $it.ffs - --remainders $it.remainder_plus_ones - --folding-factors $it.ns - ) | to ndjson out> ($target | path join $it.output) + # FIXME: refactor this + if $it.append { + ( + fri run + --data-sizes $it.sizes + --ks $it.ks + --blowup-factors $it.blowup_factors + --nb-queries $it.nb_queries + --hashes $it.hashes + --finite-fields $it.ffs + --remainders $it.remainder_plus_ones + --folding-factors $it.ns + ) | to ndjson out>> ($target | path join $it.output) + } else { + ( + fri run + --data-sizes $it.sizes + --ks $it.ks + --blowup-factors $it.blowup_factors + --nb-queries $it.nb_queries + --hashes $it.hashes + --finite-fields $it.ffs + --remainders $it.remainder_plus_ones + --folding-factors $it.ns + ) | to ndjson out> ($target | path join $it.output) + } }} | insert field.run {{ |it| - cargo run ...[ + let options = [ --bin field --release --package benchmarks -- --nb-measurements $it.nb_measurements - ] out>> ($target | path join $it.output) + ] + # FIXME: refactor this + if $it.append { + cargo run ...$options out>> ($target | path join $it.output) + } else { + cargo run ...$options out> ($target | path join $it.output) + } }} | insert curve_group.run {{ |it| - cargo run ...[ + let options = [ --bin curve_group --release --package benchmarks -- --nb-measurements $it.nb_measurements - ] out>> ($target | path join $it.output) + ] + # FIXME: refactor this + if $it.append { + cargo run ...$options out>> ($target | path join $it.output) + } else { + cargo run ...$options out> ($target | path join $it.output) + } }} let _ = $benchmarks | items { |k, b| diff --git a/benchmarks/nu-lib/commit.nu b/benchmarks/nu-lib/commit.nu index 5460e838..ddaefa17 100644 --- a/benchmarks/nu-lib/commit.nu +++ b/benchmarks/nu-lib/commit.nu @@ -15,6 +15,7 @@ export def run [ --curves: list<string>, # the curves to benchmark --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run + --append, # append to the output path instead of overwritting ]: list<int> -> path { $curves | check-list-arg --cmd "commit run" --arg "--curves" --span (metadata $curves).span $in | check-list-arg --cmd "commit run" --arg "pipeline input" @@ -34,11 +35,20 @@ export def run [ } } - cargo run --release --package benchmarks --bin commit -- ...[ + let options = [ + --release + --package benchmarks + --bin commit + -- --nb-measurements $nb_measurements ...$in --curves ...$curves - ] out> $output + ] + if $append { + cargo run ...$options out>> $output + } else { + cargo run ...$options out> $output + } log info $"results saved to ($pretty_output)" $output diff --git a/benchmarks/nu-lib/fec/run.nu b/benchmarks/nu-lib/fec/run.nu index 7fee6789..adee4274 100644 --- a/benchmarks/nu-lib/fec/run.nu +++ b/benchmarks/nu-lib/fec/run.nu @@ -14,6 +14,7 @@ export def main [ --curves: list<string>, # the curves to benchmark --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run + --append, # append to the output path instead of overwritting ]: list<int> -> path { $ks | check-list-arg --cmd "fec run" --arg "--ks" --span (metadata $ks).span $curves | check-list-arg --cmd "fec run" --arg "--curves" --span (metadata $curves).span @@ -34,18 +35,29 @@ export def main [ } } - "" out> $output + if not $append { + "" out> $output + } let input = $in for k in $ks { - cargo run --release --package benchmarks --bin fec -- ...[ + let options = [ + --release + --package benchmarks + --bin fec + -- --nb-measurements $nb_measurements ...$input --encoding vandermonde -k $k -n 1 --curves ...$curves - ] | from ndnuon | to ndjson out>> $output + ] + if $append { + cargo run ...$options | from ndnuon | to ndjson out>> $output + } else { + cargo run ...$options | from ndnuon | to ndjson out> $output + } } log info $"results saved to ($pretty_output)" diff --git a/benchmarks/nu-lib/linalg.nu b/benchmarks/nu-lib/linalg.nu index 104fd7fd..25cc8451 100644 --- a/benchmarks/nu-lib/linalg.nu +++ b/benchmarks/nu-lib/linalg.nu @@ -14,6 +14,7 @@ export def run [ --output: path, # the output path (defaults to a random file in $nu.temp-path) --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run + --append, # append to the output path instead of overwritting ]: list<int> -> path { $in | check-list-arg --cmd "linalg run" --arg "pipeline input" @@ -32,10 +33,19 @@ export def run [ } } - cargo run --release --package benchmarks --bin linalg -- ...[ + let options = [ + --release + --package benchmarks + --bin linalg + -- --nb-measurements $nb_measurements ...$in - ] out> $output + ] + if $append { + cargo run ...$options out>> $output + } else { + cargo run ...$options out> $output + } log info $"results saved to ($pretty_output)" $output diff --git a/benchmarks/nu-lib/recoding.nu b/benchmarks/nu-lib/recoding.nu index 6ab24c98..6357baf4 100644 --- a/benchmarks/nu-lib/recoding.nu +++ b/benchmarks/nu-lib/recoding.nu @@ -17,6 +17,7 @@ export def run [ --curves: list<string>, # the curves to benchmark --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run + --append, # append to the output path instead of overwritting ]: list<int> -> path { $ks | check-list-arg --cmd "recoding run" --arg "--ks" --span (metadata $ks).span $curves | check-list-arg --cmd "recoding run" --arg "--curves" --span (metadata $curves).span @@ -37,17 +38,28 @@ export def run [ } } - "" out> $output + if not $append { + "" out> $output + } let input = $in for k in $ks { - cargo run --release --package benchmarks --bin recoding -- ...[ + let options = [ + --release + --package benchmarks + --bin recoding + -- --nb-measurements $nb_measurements ...$input --shards $k --ks $k --curves ...$curves - ] | from ndnuon | to ndjson out>> $output + ] + if $append { + cargo run ...$options | from ndnuon | to ndjson out>> $output + } else { + cargo run ...$options | from ndnuon | to ndjson out> $output + } } log info $"results saved to ($pretty_output)" diff --git a/benchmarks/nu-lib/setup.nu b/benchmarks/nu-lib/setup.nu index 9a43103a..cac6b650 100644 --- a/benchmarks/nu-lib/setup.nu +++ b/benchmarks/nu-lib/setup.nu @@ -15,6 +15,7 @@ export def run [ --curves: list<string>, # the curves to benchmark --no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten --nb-measurements: int = 10, # the number of measurements per benchmark run + --append, # append to the output path instead of overwritting ]: list<int> -> path { $curves | check-list-arg --cmd "setup run" --arg "--curves" --span (metadata $curves).span $in | check-list-arg --cmd "setup run" --arg "pipeline input" @@ -34,11 +35,20 @@ export def run [ } } - cargo run --release --package benchmarks --bin setup -- ...[ + let options = [ + --release + --package benchmarks + --bin setup + -- --nb-measurements $nb_measurements ...$in --curves ...$curves - ] out> $output + ] + if $append { + cargo run ...$options out>> $output + } else { + cargo run ...$options out> $output + } log info $"results saved to ($pretty_output)" $output -- GitLab From 5714c0c466ede9713090627bdf9639c0a4d599ce Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 12:31:18 +0100 Subject: [PATCH 18/27] remove "atomic operations" section --- benchmarks/README.md | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 1f19e25e..2b1689fb 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -32,36 +32,6 @@ use benchmarks > i personally use the [`nuenv` hook](https://github.com/nushell/nu_scripts/blob/main/nu-hooks/nu-hooks/nuenv/hook.nu) > that reads [`.env.nu`](../.env.nu). -## atomic operations -```nushell -cargo run --release --package benchmarks --bin field_operations -- --nb-measurements 1000 out> field.ndjson -cargo run --release --package benchmarks --bin curve_group_operations -- --nb-measurements 1000 out> curve_group.ndjson -``` -```nushell -use benchmarks/nu-lib/utils/parse.nu read-atomic-ops - -gplt multi_bar --title "simple field operations" -l "time (in ns)" ( - open field.ndjson - | read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ] - | to json -) -gplt multi_bar --title "complex field operations" -l "time (in ns)" ( - open field.ndjson - | read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ] - | to json -) -gplt multi_bar --title "simple curve group operations" -l "time (in ns)" ( - open curve_group.ndjson - | read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ] - | to json -) -gplt multi_bar --title "complex curve group operations" -l "time (in ns)" ( - open curve_group.ndjson - | read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ] - | to json -) -``` - ## Run the benchmarks ### define them -- GitLab From 81c916b66cf95159a44dd68f2c555ef95bced7cc Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 13:16:23 +0100 Subject: [PATCH 19/27] update README --- benchmarks/README.md | 76 ++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 2b1689fb..1eafb6d6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,7 +41,7 @@ use benchmarks > [example](../examples/fri.rs) ```bash -const OUTPUT_DIR = "/path/to/komodo-benchmarks-results/" +const RESULTS_DIR = "/path/to/komodo-benchmarks-results/" let benchmarks = { linalg: { @@ -110,36 +110,64 @@ let benchmarks = { ### run them ```bash -benchmarks --output-dir $OUTPUT_DIR $benchmarks +benchmarks --output-dir $RESULTS_DIR $benchmarks ``` -> the following `watch` can be used to see the results as they are dumped to `$OUTPUT_DIR` +> the following `watch` can be used to see the results as they are dumped to `$RESULTS_DIR` > ```bash -> watch $OUTPUT_DIR { |op, path| +> watch $RESULTS_DIR { |op, path| > $"($op) ($path)" > } > ``` ## Plot the benchmarks ```bash -benchmarks linalg plot <linalg> mul -benchmarks linalg plot <linalg> transpose -benchmarks linalg plot <linalg> inverse -benchmarks setup plot <setup> -benchmarks commit plot <commit> -benchmarks recoding plot <recoding> -benchmarks fec plot encoding <fec> -benchmarks fec plot decoding <fec> -benchmarks fec plot e2e <fec> -benchmarks fec plot combined <fec> --recoding <recoding> -benchmarks fec plot ratio <fec> --recoding <recoding> - -benchmarks fri plot --dump-dir <out> --file <fri> evaluating encoding proving decoding --y-type "duration" -benchmarks fri plot --dump-dir <out> --file <fri> verifying --y-type "duration" --single - -benchmarks fri plot --dump-dir <out> --file <fri> proofs --y-type "filesize" --identity --normalize -benchmarks fri plot --dump-dir <out> --file <fri> commits --y-type "filesize" --single --identity --normalize - -benchmarks fri plot --dump-dir <out> --file <fri> proofs --y-type "filesize" --identity -benchmarks fri plot --dump-dir <out> --file <fri> commits --y-type "filesize" --single --identity +const RESULTS_DIR = "/path/to/komodo-benchmarks-results/<cpu-hash>" +const FIGURES_DIR = "./figures/" + +mkdir $FIGURES_DIR + +benchmarks linalg plot ($RESULTS_DIR | path join linalg.ndjson) mul --save ($FIGURES_DIR | path join linalg-mul.png) +benchmarks linalg plot ($RESULTS_DIR | path join linalg.ndjson) transpose --save ($FIGURES_DIR | path join linalg-transpose.png) +benchmarks linalg plot ($RESULTS_DIR | path join linalg.ndjson) inverse --save ($FIGURES_DIR | path join linalg-inverse.png) +benchmarks setup plot ($RESULTS_DIR | path join setup.ndjson) --save ($FIGURES_DIR | path join setup.png) +benchmarks commit plot ($RESULTS_DIR | path join commit.ndjson) --save ($FIGURES_DIR | path join commit.png) + +benchmarks recoding plot ($RESULTS_DIR | path join recoding.ndjson) --save ($FIGURES_DIR | path join recoding.png) + +benchmarks fec plot encoding ($RESULTS_DIR | path join fec.ndjson) --save ($FIGURES_DIR | path join encoding.png) +benchmarks fec plot decoding ($RESULTS_DIR | path join fec.ndjson) --save ($FIGURES_DIR | path join decoding.png) +benchmarks fec plot e2e ($RESULTS_DIR | path join fec.ndjson) --save ($FIGURES_DIR | path join end_to_end.png) +benchmarks fec plot combined ($RESULTS_DIR | path join fec.ndjson) --recoding ($RESULTS_DIR | path join recoding.ndjson) --save ($FIGURES_DIR | path join combined.png) +benchmarks fec plot ratio ($RESULTS_DIR | path join fec.ndjson) --recoding ($RESULTS_DIR | path join recoding.ndjson) --save ($FIGURES_DIR | path join ratio.png) + +benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) evaluating encoding proving decoding --y-type "duration" --save +benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) verifying --y-type "duration" --single --save +benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) proofs --y-type "filesize" --identity --normalize --save +benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) commits --y-type "filesize" --single --identity --normalize --save +benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) proofs --y-type "filesize" --identity --save +benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) commits --y-type "filesize" --single --identity --save + +use benchmarks/nu-lib/utils/parse.nu read-atomic-ops + +gplt multi-bar --title "simple field operations" -l "time (in ns)" ( + open ($RESULTS_DIR | path join field.ndjson) + | read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ] + | to json +) --save ($FIGURES_DIR | path join simple_field_operations.png) +gplt multi-bar --title "complex field operations" -l "time (in ns)" ( + open ($RESULTS_DIR | path join field.ndjson) + | read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ] + | to json +) --save ($FIGURES_DIR | path join complex_field_operations.png) +gplt multi-bar --title "simple curve group operations" -l "time (in ns)" ( + open ($RESULTS_DIR | path join curve_group.ndjson) + | read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ] + | to json +) --save ($FIGURES_DIR | path join simple_curve_group_operations.png) +gplt multi-bar --title "complex curve group operations" -l "time (in ns)" ( + open ($RESULTS_DIR | path join curve_group.ndjson) + | read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ] + | to json +) --save ($FIGURES_DIR | path join complex_curve_group_operations.png) ``` -- GitLab From 9104945504ca0b20e8fa54b942b0aeaa5deeb422 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 13:22:45 +0100 Subject: [PATCH 20/27] fix path to results repo --- benchmarks/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 1eafb6d6..7582dc69 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,7 +41,7 @@ use benchmarks > [example](../examples/fri.rs) ```bash -const RESULTS_DIR = "/path/to/komodo-benchmarks-results/" +const RESULTS_DIR = "/path/to/komodo-benchmark-results/" let benchmarks = { linalg: { @@ -122,7 +122,7 @@ benchmarks --output-dir $RESULTS_DIR $benchmarks ## Plot the benchmarks ```bash -const RESULTS_DIR = "/path/to/komodo-benchmarks-results/<cpu-hash>" +const RESULTS_DIR = "/path/to/komodo-benchmark-results/<cpu-hash>" const FIGURES_DIR = "./figures/" mkdir $FIGURES_DIR -- GitLab From ae620051f99761e53ce1bd11535890083d173ec0 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 14:24:32 +0100 Subject: [PATCH 21/27] store the Komodo commit and add to bench hash --- benchmarks/mod.nu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 0e5967e1..40599cdc 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -104,12 +104,14 @@ export def main [ | into record | select ...$CPU_FIELDS - let hash = $cpu | to json | hash sha256 + let commit = git rev-parse HEAD + let hash = $cpu | to json | $in + $commit | hash sha256 let target = $output_dir | path join $hash mkdir $target $cpu | to json | save --force ($target | path join "cpu.json") + $commit | save --force ($target | path join "komodo.txt") let benchmarks = $benchmarks | insert linalg.run {{ |it| -- GitLab From d04bf8a98ec2e2900d527feda77f7f575eb9ec27 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 14:27:06 +0100 Subject: [PATCH 22/27] print FRI params as oneline NUON --- benchmarks/nu-lib/fri/run.nu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/nu-lib/fri/run.nu b/benchmarks/nu-lib/fri/run.nu index 8f6b2162..e20cc046 100644 --- a/benchmarks/nu-lib/fri/run.nu +++ b/benchmarks/nu-lib/fri/run.nu @@ -72,7 +72,7 @@ export def main [ } $params | each { |p| - print $p + print ($p | to nuon --raw) run $p } } -- GitLab From 36eede9878ab47d435e0b980dfb0b9c297aeebd4 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 15:26:48 +0100 Subject: [PATCH 23/27] rename function to `benchmarks run` --- benchmarks/README.md | 2 +- benchmarks/mod.nu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 7582dc69..4ae9a548 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -110,7 +110,7 @@ let benchmarks = { ### run them ```bash -benchmarks --output-dir $RESULTS_DIR $benchmarks +benchmarks run --output-dir $RESULTS_DIR $benchmarks ``` > the following `watch` can be used to see the results as they are dumped to `$RESULTS_DIR` diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 40599cdc..a7a2fd40 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -40,7 +40,7 @@ const CPU_FIELDS = [ "NUMA node0 CPU(s)", ] -export def main [ +export def run [ benchmarks: record< linalg: record< enabled: bool, -- GitLab From ac235713623b1a11a608d737dc255981f369715c Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 15:40:42 +0100 Subject: [PATCH 24/27] move "plotting" to `benchmarks plot` --- benchmarks/README.md | 80 ++++++++++++++++++-------------------------- benchmarks/mod.nu | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 48 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 4ae9a548..26a84f49 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -122,52 +122,36 @@ benchmarks run --output-dir $RESULTS_DIR $benchmarks ## Plot the benchmarks ```bash -const RESULTS_DIR = "/path/to/komodo-benchmark-results/<cpu-hash>" -const FIGURES_DIR = "./figures/" - -mkdir $FIGURES_DIR - -benchmarks linalg plot ($RESULTS_DIR | path join linalg.ndjson) mul --save ($FIGURES_DIR | path join linalg-mul.png) -benchmarks linalg plot ($RESULTS_DIR | path join linalg.ndjson) transpose --save ($FIGURES_DIR | path join linalg-transpose.png) -benchmarks linalg plot ($RESULTS_DIR | path join linalg.ndjson) inverse --save ($FIGURES_DIR | path join linalg-inverse.png) -benchmarks setup plot ($RESULTS_DIR | path join setup.ndjson) --save ($FIGURES_DIR | path join setup.png) -benchmarks commit plot ($RESULTS_DIR | path join commit.ndjson) --save ($FIGURES_DIR | path join commit.png) - -benchmarks recoding plot ($RESULTS_DIR | path join recoding.ndjson) --save ($FIGURES_DIR | path join recoding.png) - -benchmarks fec plot encoding ($RESULTS_DIR | path join fec.ndjson) --save ($FIGURES_DIR | path join encoding.png) -benchmarks fec plot decoding ($RESULTS_DIR | path join fec.ndjson) --save ($FIGURES_DIR | path join decoding.png) -benchmarks fec plot e2e ($RESULTS_DIR | path join fec.ndjson) --save ($FIGURES_DIR | path join end_to_end.png) -benchmarks fec plot combined ($RESULTS_DIR | path join fec.ndjson) --recoding ($RESULTS_DIR | path join recoding.ndjson) --save ($FIGURES_DIR | path join combined.png) -benchmarks fec plot ratio ($RESULTS_DIR | path join fec.ndjson) --recoding ($RESULTS_DIR | path join recoding.ndjson) --save ($FIGURES_DIR | path join ratio.png) - -benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) evaluating encoding proving decoding --y-type "duration" --save -benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) verifying --y-type "duration" --single --save -benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) proofs --y-type "filesize" --identity --normalize --save -benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) commits --y-type "filesize" --single --identity --normalize --save -benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) proofs --y-type "filesize" --identity --save -benchmarks fri plot --dump-dir $FIGURES_DIR --file ($RESULTS_DIR | path join fri.ndjson) commits --y-type "filesize" --single --identity --save - -use benchmarks/nu-lib/utils/parse.nu read-atomic-ops - -gplt multi-bar --title "simple field operations" -l "time (in ns)" ( - open ($RESULTS_DIR | path join field.ndjson) - | read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ] - | to json -) --save ($FIGURES_DIR | path join simple_field_operations.png) -gplt multi-bar --title "complex field operations" -l "time (in ns)" ( - open ($RESULTS_DIR | path join field.ndjson) - | read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ] - | to json -) --save ($FIGURES_DIR | path join complex_field_operations.png) -gplt multi-bar --title "simple curve group operations" -l "time (in ns)" ( - open ($RESULTS_DIR | path join curve_group.ndjson) - | read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ] - | to json -) --save ($FIGURES_DIR | path join simple_curve_group_operations.png) -gplt multi-bar --title "complex curve group operations" -l "time (in ns)" ( - open ($RESULTS_DIR | path join curve_group.ndjson) - | read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ] - | to json -) --save ($FIGURES_DIR | path join complex_curve_group_operations.png) +let plots = { + linalg: { file: "linalg.ndjson" }, + setup: { file: "setup.ndjson" }, + commit: { file: "commit.ndjson" }, + fec: { file: "fec.ndjson" }, + recoding: { file: "recoding.ndjson" }, + fri: [ + [name, y_type, single, identity, normalize]; + [evaluating, duration, false, false, false ], + [encoding, duration, false, false, false ], + [proving, duration, false, false, false ], + [decoding, duration, false, false, false ], + [verifying, duration, true, false, false ], + [proofs, filesize, false, true, true ], + [commits, filesize, true, true, true ], + [proofs, filesize, false, true, false ], + [commits, filesize, true, true, false ], + ], + field: { + title: "field operations", + file: field.ndjson, + simple_operations: [ "exponentiation", "legendre", "inverse", "sqrt" ], + }, + curve_group: { + title: "curve group operations", + file: curve_group.ndjson, + simple_operations: [ "random sampling", "scalar multiplication", "affine scalar multiplication" ], + }, +] +``` +```bash +benchmarks plot $plots --input-dir "/path/to/komodo-benchmark-results/<hash>" --output-dir "./figures/" ``` diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index a7a2fd40..20da9354 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -13,6 +13,7 @@ use nu-lib/fec/ use nu-lib/fri/ use nu-lib/utils/log.nu +use nu-lib/utils/parse.nu read-atomic-ops const CPU_FIELDS = [ "Architecture", @@ -202,3 +203,62 @@ export def run [ } } } + +export def plot [plots: record, --input-dir: path, --output-dir: path = "./figures/"] { + mkdir $output_dir + + let linalg_file = $input_dir | path join $plots.linalg.file + let fec_file = $input_dir | path join $plots.fec.file + let recoding_file = $input_dir | path join $plots.recoding.file + + for op in [ "mul", "transpose", "inverse" ] { + benchmarks linalg plot $linalg_file $op --save ($output_dir | path join $"linalg-($op).png") + } + + benchmarks setup plot ($input_dir | path join $plots.setup.file) --save ($output_dir | path join setup.png) + benchmarks commit plot ($input_dir | path join $plots.commit.file) --save ($output_dir | path join commit.png) + + benchmarks recoding plot $recoding_file --save ($output_dir | path join recoding.png) + + benchmarks fec plot encoding $fec_file --save ($output_dir | path join encoding.png) + benchmarks fec plot decoding $fec_file --save ($output_dir | path join decoding.png) + benchmarks fec plot e2e $fec_file --save ($output_dir | path join end_to_end.png) + benchmarks fec plot combined $fec_file --recoding $recoding_file --save ($output_dir | path join combined.png) + benchmarks fec plot ratio $fec_file --recoding $recoding_file --save ($output_dir | path join ratio.png) + + for plot in $plots.fri {( + fri plot + --dump-dir $output_dir + --file ($input_dir | path join fri.ndjson) + $plot.name + --y-type $plot.y_type + --single=$plot.single + --identity=$plot.identity + --normalize=$plot.normalize + --save + )} + + for plot in ($plots | select field curve_group) { + def output [prefix: string]: [ nothing -> record<path: path, title: string> ] { + let title_tokens = $plot.title | split row " " | prepend $prefix + { + path: ({ + parent: $output_dir, + stem: ($title_tokens | str join "_"), + extension: "png", + } | path join), + title: ($title_tokens | str join " "), + } + } + + let data = open ($input_dir | path join $plot.file) + + output "simple" | gplt multi-bar --title $in.title -l "time (in ns)" ( + $data | read-atomic-ops --include $plot.simple_operations | to json + ) --save $in.path + + output "complex" | gplt multi-bar --title $in.title -l "time (in ns)" ( + $data | read-atomic-ops --exclude $plot.simple_operations | to json + ) --save $in.path + } +} -- GitLab From b54381feb21cb2eafa60de9f5f17ddee0ace4042 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 15:56:13 +0100 Subject: [PATCH 25/27] fix inner calls to plotting commands --- benchmarks/mod.nu | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 20da9354..178ba432 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -212,19 +212,19 @@ export def plot [plots: record, --input-dir: path, --output-dir: path = "./figur let recoding_file = $input_dir | path join $plots.recoding.file for op in [ "mul", "transpose", "inverse" ] { - benchmarks linalg plot $linalg_file $op --save ($output_dir | path join $"linalg-($op).png") + linalg plot $linalg_file $op --save ($output_dir | path join $"linalg-($op).png") } - benchmarks setup plot ($input_dir | path join $plots.setup.file) --save ($output_dir | path join setup.png) - benchmarks commit plot ($input_dir | path join $plots.commit.file) --save ($output_dir | path join commit.png) + setup plot ($input_dir | path join $plots.setup.file) --save ($output_dir | path join setup.png) + commit plot ($input_dir | path join $plots.commit.file) --save ($output_dir | path join commit.png) - benchmarks recoding plot $recoding_file --save ($output_dir | path join recoding.png) + recoding plot $recoding_file --save ($output_dir | path join recoding.png) - benchmarks fec plot encoding $fec_file --save ($output_dir | path join encoding.png) - benchmarks fec plot decoding $fec_file --save ($output_dir | path join decoding.png) - benchmarks fec plot e2e $fec_file --save ($output_dir | path join end_to_end.png) - benchmarks fec plot combined $fec_file --recoding $recoding_file --save ($output_dir | path join combined.png) - benchmarks fec plot ratio $fec_file --recoding $recoding_file --save ($output_dir | path join ratio.png) + fec plot encoding $fec_file --save ($output_dir | path join encoding.png) + fec plot decoding $fec_file --save ($output_dir | path join decoding.png) + fec plot e2e $fec_file --save ($output_dir | path join end_to_end.png) + fec plot combined $fec_file --recoding $recoding_file --save ($output_dir | path join combined.png) + fec plot ratio $fec_file --recoding $recoding_file --save ($output_dir | path join ratio.png) for plot in $plots.fri {( fri plot -- GitLab From 65838d56da9b8ea507c83eab0044e9e1d3428a24 Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 15:56:28 +0100 Subject: [PATCH 26/27] fix README snippet --- benchmarks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 26a84f49..47f2b428 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -150,7 +150,7 @@ let plots = { file: curve_group.ndjson, simple_operations: [ "random sampling", "scalar multiplication", "affine scalar multiplication" ], }, -] +} ``` ```bash benchmarks plot $plots --input-dir "/path/to/komodo-benchmark-results/<hash>" --output-dir "./figures/" -- GitLab From a98f93e37731a6940bf24a56b6c199e4cc7f874c Mon Sep 17 00:00:00 2001 From: "a.stevan" <antoine.stevan@isae-supaero.fr> Date: Fri, 31 Jan 2025 15:58:07 +0100 Subject: [PATCH 27/27] fix `benchmarks plot` --- benchmarks/mod.nu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/mod.nu b/benchmarks/mod.nu index 178ba432..92e02052 100644 --- a/benchmarks/mod.nu +++ b/benchmarks/mod.nu @@ -238,7 +238,7 @@ export def plot [plots: record, --input-dir: path, --output-dir: path = "./figur --save )} - for plot in ($plots | select field curve_group) { + for plot in ($plots | select field curve_group | values) { def output [prefix: string]: [ nothing -> record<path: path, title: string> ] { let title_tokens = $plot.title | split row " " | prepend $prefix { -- GitLab