Skip to content
Snippets Groups Projects
Commit ab0d9a8b authored by STEVAN Antoine's avatar STEVAN Antoine :crab:
Browse files

rework benchmarks: script and output format (!193) 

this is an attempt at making benchmarks easier to work with
- `benchmarks run` will read benchmarks from NUON data and run them
- `benchmarks plot` will plot benchmark results
- the output format will be something 
  - a directory whose name is the hash of the CPU spec and the Komodo commit hash
  - contains `cpu.json` with the CPU info
  - contains `komodo.txt` with the Komodo commit hash
  - contains NDJSON result files

> :bulb: **Note**
>
> results will typically be uploaded to https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results

> :bulb: **Note**
>
> this MR goes alongside the [`komodo-benchmark-results@restart`](https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results/-/compare/main...restart) branch

## changelog
- bump Nushell to 0.101.0
  - parallel `$in` => remove useless `let input = $in` when possible, e.g. still required when using the function's `$in` in a `for` loop
  - `group-by` changed => `group-by x --to...
parent 5511404f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 475 additions and 189 deletions
Makefile
+ 1
− 1
View file @ ab0d9a8b
......@@ -2,7 +2,7 @@ NU="nu"
NU_FLAGS="--no-config-file"
NU_ARCH="x86_64-unknown-linux-musl"
NU_VERSION="0.95.0"
NU_VERSION="0.101.0"
NU_BUILD="nu-${NU_VERSION}-${NU_ARCH}"
NU_DEST="/tmp/"
......
README.md
+ 2
− 0
View file @ ab0d9a8b
......@@ -39,6 +39,8 @@ Other examples that showcase the Komodo API are available in [`examples/`](examp
## the benchmarks
see [`benchmarks/`](benchmarks/README.md)
the results can be found in [`dragoon/komodo-benchmark-results`](https://gitlab.isae-supaero.fr/dragoon/komodo-benchmark-results).
## contributors
Because the code for this project has been originally extracted from
......
benchmarks/README.md
+ 113
− 118
View file @ ab0d9a8b
# Table of contents
- [Requirements](#requirements)
- [Atomic operations](#atomic-operations)
- [Linear algebra](#linear-algebra)
- [Trusted setup and commit](#trusted-setup-and-commit)
- [End to end benchmarks](#end-to-end-benchmarks)
- [FRI](#fri)
- [Run the benchmarks](#run-the-benchmarks)
- [define them](#define-them)
- [run them](#run-them)
- [Plot the benchmarks](#plot-the-benchmarks)
## requirements
> :bulb: **Note**
......@@ -33,130 +32,126 @@ use benchmarks
> i personally use the [`nuenv` hook](https://github.com/nushell/nu_scripts/blob/main/nu-hooks/nu-hooks/nuenv/hook.nu)
> that reads [`.env.nu`](../.env.nu).
## atomic operations
```nushell
cargo run --release --package benchmarks --bin field_operations -- --nb-measurements 1000 out> field.ndjson
cargo run --release --package benchmarks --bin curve_group_operations -- --nb-measurements 1000 out> curve_group.ndjson
```
```nushell
use benchmarks/nu-lib/utils/parse.nu read-atomic-ops
gplt multi_bar --title "simple field operations" -l "time (in ns)" (
open field.ndjson
| read-atomic-ops --exclude [ "exponentiation", "legendre", "inverse", "sqrt" ]
| to json
)
gplt multi_bar --title "complex field operations" -l "time (in ns)" (
open field.ndjson
| read-atomic-ops --include [ "exponentiation", "legendre", "inverse", "sqrt" ]
| to json
)
gplt multi_bar --title "simple curve group operations" -l "time (in ns)" (
open curve_group.ndjson
| read-atomic-ops --exclude [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
| to json
)
gplt multi_bar --title "complex curve group operations" -l "time (in ns)" (
open curve_group.ndjson
| read-atomic-ops --include [ "random sampling", "scalar multiplication", "affine scalar multiplication" ]
| to json
)
```
## linear algebra
```nushell
let sizes = seq 0 7 | each { 2 ** $in }
let out_linalg = $sizes | benchmarks linalg run
benchmarks linalg plot $out_linalg inverse
```
## trusted setup and commit
```nushell
let degrees = seq 0 13 | each { 2 ** $in }
let curves = [ bls12381, pallas, bn254 ]
let out_setup = $degrees | benchmarks setup run --curves $curves
let out_commit = $degrees | benchmarks commit run --curves $curves
benchmarks setup plot $out_setup
benchmarks commit plot $out_commit
```
## end-to-end benchmarks
```nushell
let sizes = seq 0 18 | each { 512 * 2 ** $in }
let ks = [2, 4, 8, 16]
let curves = [ bls12381 ]
```
## Run the benchmarks
### define them
### run
```nushell
let out_recoding = $sizes | benchmarks recoding run --ks $ks --curves $curves
let out_fec = $sizes | benchmarks fec run --ks $ks --curves $curves
```
### plot
```nushell
benchmarks recoding plot $out_recoding
benchmarks fec plot encoding $out_fec
benchmarks fec plot decoding $out_fec
benchmarks fec plot e2e $out_fec
benchmarks fec plot combined $out_fec --recoding $out_recoding
benchmarks fec plot ratio $out_fec --recoding $out_recoding
```
## FRI
> :bulb: **Note**
>
> the FRI benchmarks don't use a module from [src/bin/](src/bin/) with PLNK but rather an
> [example](../examples/fri.rs)
- modify [benchmarks/params/fri.nu](benchmarks/params/fri.nu)
- source it
```nushell
source benchmarks/params/fri.nu
```bash
const RESULTS_DIR = "/path/to/komodo-benchmark-results/"
let benchmarks = {
linalg: {
enabled: true,
sizes: (seq 0 7 | each { 2 ** $in }),
output: "linalg.ndjson",
append: true,
},
setup: {
enabled: true,
degrees: (seq 0 13 | each { 2 ** $in }),
curves: [ bls12381, pallas, bn254 ],
output: "setup.ndjson",
append: true,
},
commit: {
enabled: true,
degrees: (seq 0 13 | each { 2 ** $in }),
curves: [ bls12381, pallas, bn254 ],
output: "commit.ndjson",
append: true,
},
recoding: {
enabled: true,
sizes: (seq 0 18 | each { 512 * 2 ** $in }),
ks: [2, 4, 8, 16],
curves: [ bls12381 ],
output: "recoding.ndjson",
append: true,
},
fec: {
enabled: true,
sizes: (seq 0 18 | each { 512 * 2 ** $in }),
ks: [2, 4, 8, 16],
curves: [ bls12381 ],
output: "fec.ndjson",
append: true,
},
fri: {
enabled: true,
sizes: (seq 0 15 | each { 2 ** $in * 4096b }),
ks: [8, 128, 1024, 4096],
blowup_factors: [2, 4],
ns: [2],
remainder_plus_ones: [1],
nb_queries: [50],
hashes: ["sha3-512"],
ffs: ["fp128", "bls12-381"],
output: "fri.ndjson",
append: true,
},
field: {
enabled: true,
nb_measurements: 1000,
output: "field.ndjson",
append: true,
},
curve_group: {
enabled: true,
nb_measurements: 1000,
output: "curve_group.ndjson",
append: true,
},
}
```
- run the benchmarks
```nushell
use std formats "to ndjson"
(benchmarks fri run
--data-sizes $DATA_SIZES
--ks $KS
--blowup-factors $BFS
--nb-queries $QS
--hashes $HS
--finite-fields $FFS
--remainders $RPOS
--folding-factors $NS
) | to ndjson out> $DATA
### run them
```bash
benchmarks run --output-dir $RESULTS_DIR $benchmarks
```
> the following `watch` call can be used to see the results as they are dumped to `$DATA`
> ```nushell
> use std formats "from ndjson"
>
> watch . {
> open --raw $DATA
> | lines
> | last
> | from ndjson
> | into int evaluating encoding proving verifying decoding
> | into duration evaluating encoding proving verifying decoding
> | into filesize proofs commits d
> | into record
> the following `watch` can be used to see the results as they are dumped to `$RESULTS_DIR`
> ```bash
> watch $RESULTS_DIR { |op, path|
> $"($op) ($path)"
> }
> ```
```nushell
benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA evaluating encoding proving decoding --y-type "duration"
benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA verifying --y-type "duration" --single
benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA proofs --y-type "filesize" --identity --normalize
benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA commits --y-type "filesize" --single --identity --normalize
benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA proofs --y-type "filesize" --identity
benchmarks fri plot --dump-dir $OUTPUT_DIR --file $DATA commits --y-type "filesize" --single --identity
## Plot the benchmarks
```bash
let plots = {
linalg: { file: "linalg.ndjson" },
setup: { file: "setup.ndjson" },
commit: { file: "commit.ndjson" },
fec: { file: "fec.ndjson" },
recoding: { file: "recoding.ndjson" },
fri: [
[name, y_type, single, identity, normalize];
[evaluating, duration, false, false, false ],
[encoding, duration, false, false, false ],
[proving, duration, false, false, false ],
[decoding, duration, false, false, false ],
[verifying, duration, true, false, false ],
[proofs, filesize, false, true, true ],
[commits, filesize, true, true, true ],
[proofs, filesize, false, true, false ],
[commits, filesize, true, true, false ],
],
field: {
title: "field operations",
file: field.ndjson,
simple_operations: [ "exponentiation", "legendre", "inverse", "sqrt" ],
},
curve_group: {
title: "curve group operations",
file: curve_group.ndjson,
simple_operations: [ "random sampling", "scalar multiplication", "affine scalar multiplication" ],
},
}
```
```bash
benchmarks plot $plots --input-dir "/path/to/komodo-benchmark-results/<hash>" --output-dir "./figures/"
```
benchmarks/mod.nu
+ 258
− 0
View file @ ab0d9a8b
......@@ -4,3 +4,261 @@ export module nu-lib/fec/
export module nu-lib/recoding.nu
export module nu-lib/linalg.nu
export module nu-lib/fri/
use nu-lib/linalg.nu
use nu-lib/setup.nu
use nu-lib/commit.nu
use nu-lib/recoding.nu
use nu-lib/fec/
use nu-lib/fri/
use nu-lib/utils/log.nu
use nu-lib/utils/parse.nu read-atomic-ops
const CPU_FIELDS = [
"Architecture",
"CPU op-mode(s)",
"Address sizes",
"Byte Order",
"CPU(s)",
"On-line CPU(s) list",
"Model name",
"CPU family",
"Model",
"Thread(s) per core",
"Core(s) per socket",
"Socket(s)",
"Stepping",
"CPU max MHz",
"CPU min MHz",
"BogoMIPS",
"Virtualization",
"L1d cache",
"L1i cache",
"L2 cache",
"L3 cache",
"NUMA node(s)",
"NUMA node0 CPU(s)",
]
export def run [
benchmarks: record<
linalg: record<
enabled: bool,
sizes: list<int>,
output: string,
append: bool,
>,
setup: record<
enabled: bool,
degrees: list<int>,
curves: list<string>,
output: string,
append: bool,
>,
commit: record<
enabled: bool,
degrees: list<int>,
curves: list<string>,
output: string,
append: bool,
>,
recoding: record<
enabled: bool,
sizes: list<int>,
ks: list<int>,
curves: list<string>,
output: string,
append: bool,
>,
fec: record<
enabled: bool,
sizes: list<int>,
ks: list<int>,
curves: list<string>,
output: string,
append: bool,
>,
fri: record<
enabled: bool,
sizes: list<filesize>,
ks: list<int>,
blowup_factors: list<int>,
ns: list<int>,
remainder_plus_ones: list<int>,
nb_queries: list<int>,
hashes: list<string>,
ffs: list<string>,
output: string,
append: bool,
>,
field: record<enabled: bool, nb_measurements: int, output: string, append: bool>,
curve_group: record<enabled: bool, nb_measurements: int, output: string, append: bool>,
>,
--output-dir: path = ".",
] {
let cpu = lscpu --json
| from json
| get lscpu
| update field { str trim --right --char ":" }
| transpose --header-row
| into record
| select ...$CPU_FIELDS
let commit = git rev-parse HEAD
let hash = $cpu | to json | $in + $commit | hash sha256
let target = $output_dir | path join $hash
mkdir $target
$cpu | to json | save --force ($target | path join "cpu.json")
$commit | save --force ($target | path join "komodo.txt")
let benchmarks = $benchmarks
| insert linalg.run {{ |it|
let output = $target | path join $it.output
$it.sizes | linalg run --no-confirm --output $output --append=$it.append
}}
| insert setup.run {{ |it|
let output = $target | path join $it.output
$it.degrees | setup run --curves $it.curves --no-confirm --output $output --append=$it.append
}}
| insert commit.run {{ |it|
let output = $target | path join $it.output
$it.degrees | commit run --curves $it.curves --no-confirm --output $output --append=$it.append
}}
| insert recoding.run {{ |it|
let output = $target | path join $it.output
$it.sizes | recoding run --ks $it.ks --curves $it.curves --no-confirm --output $output --append=$it.append
}}
| insert fec.run {{ |it|
let output = $target | path join $it.output
$it.sizes | fec run --ks $it.ks --curves $it.curves --no-confirm --output $output --append=$it.append
}}
| insert fri.run {{ |it|
# FIXME: refactor this
if $it.append {
(
fri run
--data-sizes $it.sizes
--ks $it.ks
--blowup-factors $it.blowup_factors
--nb-queries $it.nb_queries
--hashes $it.hashes
--finite-fields $it.ffs
--remainders $it.remainder_plus_ones
--folding-factors $it.ns
) | to ndjson out>> ($target | path join $it.output)
} else {
(
fri run
--data-sizes $it.sizes
--ks $it.ks
--blowup-factors $it.blowup_factors
--nb-queries $it.nb_queries
--hashes $it.hashes
--finite-fields $it.ffs
--remainders $it.remainder_plus_ones
--folding-factors $it.ns
) | to ndjson out> ($target | path join $it.output)
}
}}
| insert field.run {{ |it|
let options = [
--bin field
--release
--package benchmarks
--
--nb-measurements $it.nb_measurements
]
# FIXME: refactor this
if $it.append {
cargo run ...$options out>> ($target | path join $it.output)
} else {
cargo run ...$options out> ($target | path join $it.output)
}
}}
| insert curve_group.run {{ |it|
let options = [
--bin curve_group
--release
--package benchmarks
--
--nb-measurements $it.nb_measurements
]
# FIXME: refactor this
if $it.append {
cargo run ...$options out>> ($target | path join $it.output)
} else {
cargo run ...$options out> ($target | path join $it.output)
}
}}
let _ = $benchmarks | items { |k, b|
if ($b.enabled? | default true) {
log info $"running (ansi cyan)($k)(ansi reset)"
do $b.run $b
} else {
log warning $"skipping (ansi cyan)($k)(ansi reset)"
}
}
}
export def plot [plots: record, --input-dir: path, --output-dir: path = "./figures/"] {
mkdir $output_dir
let linalg_file = $input_dir | path join $plots.linalg.file
let fec_file = $input_dir | path join $plots.fec.file
let recoding_file = $input_dir | path join $plots.recoding.file
for op in [ "mul", "transpose", "inverse" ] {
linalg plot $linalg_file $op --save ($output_dir | path join $"linalg-($op).png")
}
setup plot ($input_dir | path join $plots.setup.file) --save ($output_dir | path join setup.png)
commit plot ($input_dir | path join $plots.commit.file) --save ($output_dir | path join commit.png)
recoding plot $recoding_file --save ($output_dir | path join recoding.png)
fec plot encoding $fec_file --save ($output_dir | path join encoding.png)
fec plot decoding $fec_file --save ($output_dir | path join decoding.png)
fec plot e2e $fec_file --save ($output_dir | path join end_to_end.png)
fec plot combined $fec_file --recoding $recoding_file --save ($output_dir | path join combined.png)
fec plot ratio $fec_file --recoding $recoding_file --save ($output_dir | path join ratio.png)
for plot in $plots.fri {(
fri plot
--dump-dir $output_dir
--file ($input_dir | path join fri.ndjson)
$plot.name
--y-type $plot.y_type
--single=$plot.single
--identity=$plot.identity
--normalize=$plot.normalize
--save
)}
for plot in ($plots | select field curve_group | values) {
def output [prefix: string]: [ nothing -> record<path: path, title: string> ] {
let title_tokens = $plot.title | split row " " | prepend $prefix
{
path: ({
parent: $output_dir,
stem: ($title_tokens | str join "_"),
extension: "png",
} | path join),
title: ($title_tokens | str join " "),
}
}
let data = open ($input_dir | path join $plot.file)
output "simple" | gplt multi-bar --title $in.title -l "time (in ns)" (
$data | read-atomic-ops --include $plot.simple_operations | to json
) --save $in.path
output "complex" | gplt multi-bar --title $in.title -l "time (in ns)" (
$data | read-atomic-ops --exclude $plot.simple_operations | to json
) --save $in.path
}
}
benchmarks/nu-lib/commit.nu
+ 16
− 8
View file @ ab0d9a8b
......@@ -13,11 +13,10 @@ use std formats *
export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--curves: list<string>, # the curves to benchmark
--force, # does not ask for confirmation if the output file already exists, it will be overwritten
--no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
--append, # append to the output path instead of overwritting
]: list<int> -> path {
let input = $in
$curves | check-list-arg --cmd "commit run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "commit run" --arg "pipeline input"
......@@ -26,7 +25,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
if not $force {
if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
......@@ -36,11 +35,20 @@ export def run [
}
}
cargo run --release --package benchmarks --bin commit -- ...[
let options = [
--release
--package benchmarks
--bin commit
--
--nb-measurements $nb_measurements
...$input
...$in
--curves ...$curves
] out> $output
]
if $append {
cargo run ...$options out>> $output
} else {
cargo run ...$options out> $output
}
log info $"results saved to ($pretty_output)"
$output
......@@ -64,7 +72,7 @@ export def plot [
| select name x y e
| group-by name --to-table
| reject items.name
| rename --column { group: "name", items: "points" }
| rename --column { name: "name", items: "points" }
| insert style.color {|it|
match $it.name {
"BLS12-381" => "tab:blue"
......
benchmarks/nu-lib/fec/plot.nu
+ 9
− 9
View file @ ab0d9a8b
......@@ -24,7 +24,7 @@ export def encoding [
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
......@@ -56,7 +56,7 @@ export def decoding [
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
......@@ -89,7 +89,7 @@ export def e2e [
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group foo
| reject foo
| ns-to-ms times
| compute-stats times
| reject times
......@@ -99,7 +99,7 @@ export def e2e [
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
......@@ -144,7 +144,7 @@ export def combined [
}
| reject items.shards
| insert style.line.type "solid"
| rename --column { group: "name", items: "points" }
| rename --column { shards: "name", items: "points" }
| update name { $"$k = ($in)$" }
let re_encoding_graphs = open --raw $data
......@@ -159,7 +159,7 @@ export def combined [
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group key
| reject key
| ns-to-ms times
| compute-stats times
| reject times
......@@ -179,7 +179,7 @@ export def combined [
}
| insert style.line.type "dashed"
| reject items.k
| rename --column { group: "name", items: "points" }
| rename --column { k: "name", items: "points" }
| reject name
let graphs = $recoding_graphs
......@@ -254,7 +254,7 @@ export def ratio [
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group key
| reject key
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381"
......@@ -281,7 +281,7 @@ export def ratio [
}
}
| reject items.k
| rename --column { group: "name", items: "points" }
| rename --column { k: "name", items: "points" }
| update name { $"$k = ($in)$" }
let options = [
......
benchmarks/nu-lib/fec/run.nu
+ 18
− 7
View file @ ab0d9a8b
......@@ -12,11 +12,10 @@ export def main [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--ks: list<int>, # the values of $k$ to benchmark
--curves: list<string>, # the curves to benchmark
--force, # does not ask for confirmation if the output file already exists, it will be overwritten
--no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
--append, # append to the output path instead of overwritting
]: list<int> -> path {
let input = $in
$ks | check-list-arg --cmd "fec run" --arg "--ks" --span (metadata $ks).span
$curves | check-list-arg --cmd "fec run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "fec run" --arg "pipeline input"
......@@ -26,7 +25,7 @@ export def main [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
if not $force {
if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
......@@ -36,17 +35,29 @@ export def main [
}
}
"" out> $output
if not $append {
"" out> $output
}
let input = $in
for k in $ks {
cargo run --release --package benchmarks --bin fec -- ...[
let options = [
--release
--package benchmarks
--bin fec
--
--nb-measurements $nb_measurements
...$input
--encoding vandermonde
-k $k
-n 1
--curves ...$curves
] | from ndnuon | to ndjson out>> $output
]
if $append {
cargo run ...$options | from ndnuon | to ndjson out>> $output
} else {
cargo run ...$options | from ndnuon | to ndjson out> $output
}
}
log info $"results saved to ($pretty_output)"
......
benchmarks/nu-lib/fri/plot.nu
+ 3
− 1
View file @ ab0d9a8b
......@@ -54,6 +54,7 @@ def plot [
points: ($ds | wrap x | merge ($ds | wrap y) | if $normalize { update y { |it| $it.y / $it.x } } else { $in }),
style: { color: "black", line: { type: "dotted" } },
} } else { $in }
| reject points.k? points.bf? points.ff?
let title = [
$name,
......@@ -86,6 +87,7 @@ export def main [
--identity,
--normalize,
--dump-dir: path = "./",
--save,
] {
if ($x | is-empty) {
error make --unspanned { msg: "nothing to do, x is empty" }
......@@ -101,6 +103,6 @@ export def main [
let data = open $file | where h == "sha3-512" and q == 50
for i in $x {
$data | plot --save $i --y-type=$y_type --single=$single --identity=$identity --normalize=$normalize --dump-dir=$dump_dir
$data | plot --save=$save $i --y-type=$y_type --single=$single --identity=$identity --normalize=$normalize --dump-dir=$dump_dir
}
}
benchmarks/nu-lib/fri/run.nu
+ 1
− 1
View file @ ab0d9a8b
......@@ -72,7 +72,7 @@ export def main [
}
$params | each { |p|
print $p
print ($p | to nuon --raw)
run $p
}
}
benchmarks/nu-lib/linalg.nu
+ 17
− 8
View file @ ab0d9a8b
......@@ -12,11 +12,10 @@ use std formats *
# - output: the output path, as NDJSON
export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--force, # does not ask for confirmation if the output file already exists, it will be overwritten
--no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
--append, # append to the output path instead of overwritting
]: list<int> -> path {
let input = $in
$in | check-list-arg --cmd "linalg run" --arg "pipeline input"
let new_file = $output == null
......@@ -24,7 +23,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
if not $force {
if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
......@@ -34,10 +33,19 @@ export def run [
}
}
cargo run --release --package benchmarks --bin linalg -- ...[
let options = [
--release
--package benchmarks
--bin linalg
--
--nb-measurements $nb_measurements
...$input
] out> $output
...$in
]
if $append {
cargo run ...$options out>> $output
} else {
cargo run ...$options out> $output
}
log info $"results saved to ($pretty_output)"
$output
......@@ -87,7 +95,8 @@ export def plot [
| where op == $op
| rename --column { n: "x", mean: "y", stddev: "e" }
| group-by name --to-table
| rename --column { group: "name", items: "points" }
| reject items.name items.op items.times
| rename --column { name: "name", items: "points" }
| insert style.color {|it|
match $it.name {
"BLS12-381" => "tab:blue"
......
benchmarks/nu-lib/recoding.nu
+ 19
− 8
View file @ ab0d9a8b
......@@ -15,11 +15,10 @@ export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--ks: list<int>, # the values of $k$ to benchmark
--curves: list<string>, # the curves to benchmark
--force, # does not ask for confirmation if the output file already exists, it will be overwritten
--no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
--append, # append to the output path instead of overwritting
]: list<int> -> path {
let input = $in
$ks | check-list-arg --cmd "recoding run" --arg "--ks" --span (metadata $ks).span
$curves | check-list-arg --cmd "recoding run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "recoding run" --arg "pipeline input"
......@@ -29,7 +28,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
if not $force {
if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
......@@ -39,16 +38,28 @@ export def run [
}
}
"" out> $output
if not $append {
"" out> $output
}
let input = $in
for k in $ks {
cargo run --release --package benchmarks --bin recoding -- ...[
let options = [
--release
--package benchmarks
--bin recoding
--
--nb-measurements $nb_measurements
...$input
--shards $k
--ks $k
--curves ...$curves
] | from ndnuon | to ndjson out>> $output
]
if $append {
cargo run ...$options | from ndnuon | to ndjson out>> $output
} else {
cargo run ...$options | from ndnuon | to ndjson out> $output
}
}
log info $"results saved to ($pretty_output)"
......@@ -73,7 +84,7 @@ export def plot [
| select shards x y e
| group-by shards --to-table
| reject items.shards
| rename --column { group: "name", items: "points" }
| rename --column { shards: "name", items: "points" }
| update name { $"$k = ($in)$"}
let options = [
......
benchmarks/nu-lib/setup.nu
+ 16
− 8
View file @ ab0d9a8b
......@@ -13,11 +13,10 @@ use std formats *
export def run [
--output: path, # the output path (defaults to a random file in $nu.temp-path)
--curves: list<string>, # the curves to benchmark
--force, # does not ask for confirmation if the output file already exists, it will be overwritten
--no-confirm (-y), # does not ask for confirmation if the output file already exists, it will be overwritten
--nb-measurements: int = 10, # the number of measurements per benchmark run
--append, # append to the output path instead of overwritting
]: list<int> -> path {
let input = $in
$curves | check-list-arg --cmd "setup run" --arg "--curves" --span (metadata $curves).span
$in | check-list-arg --cmd "setup run" --arg "pipeline input"
......@@ -26,7 +25,7 @@ export def run [
let pretty_output = $"(ansi purple)($output)(ansi reset)"
if ($output | path exists) and not $new_file {
log warning $"($pretty_output) already exists"
if not $force {
if not $no_confirm {
let res = ["no", "yes"] | input list $"Do you want to overwrite ($pretty_output)?"
if $res == null or $res == "no" {
log info "aborting"
......@@ -36,11 +35,20 @@ export def run [
}
}
cargo run --release --package benchmarks --bin setup -- ...[
let options = [
--release
--package benchmarks
--bin setup
--
--nb-measurements $nb_measurements
...$input
...$in
--curves ...$curves
] out> $output
]
if $append {
cargo run ...$options out>> $output
} else {
cargo run ...$options out> $output
}
log info $"results saved to ($pretty_output)"
$output
......@@ -69,7 +77,7 @@ export def plot [
| select name x y e
| group-by name --to-table
| reject items.name
| rename --column { group: "name", items: "points" }
| rename --column { name: "name", items: "points" }
| insert style.color {|it|
match $it.name {
"BLS12-381" => "tab:blue"
......
benchmarks/nu-lib/utils/plot.nu
+ 2
− 4
View file @ ab0d9a8b
......@@ -29,9 +29,7 @@ export def into-filesize-tick-labels []: list<int> -> list<string> {
}
export def into-axis-options [-x: string, -y: string]: table<x: float, y: float> -> list<string> {
let input = $in
let xs = $input | flatten | get x | uniq
let xs = $in | flatten | get x | uniq
let x_tick_labels = match $x {
"filesize" => ($xs | into-filesize-tick-labels),
......@@ -47,7 +45,7 @@ export def into-axis-options [-x: string, -y: string]: table<x: float, y: float>
--x-tick-labels ...$x_tick_labels
]
let ys = $input | flatten | get y
let ys = $in | flatten | get y
let y_ticks = seq ($ys | math min | math log 10 | math ceil | $in - 1) ($ys | math max | math log 10 | math floor)
| into float
| each { 10 ** $in }
......
benchmarks/params/fri.nu deleted 100644 → 0
+ 0
− 16
View file @ 5511404f
let DATA_SIZES = seq 0 15 | each { 2 ** $in * 4096b }
const KS = [8, 128, 1024, 4096]
const BFS = [2, 4]
const NS = [2]
const RPOS = [1]
const QS = [50]
const HS = ["sha3-512"]
const FFS = ["fp128", "bls12-381"]
const DATA = "benchmarks/results/fri.ndjson"
const OUTPUT_DIR = "benchmarks/results/figures/"
if not ($DATA | path dirname | path exists) {
print $"creating directory for (ansi purple)($DATA)(ansi reset)"
$DATA | path dirname | mkdir $in
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment