Skip to content
Snippets Groups Projects

dont' hardcode curves in the benchmarks

Merged dont' hardcode curves in the benchmarks
fix-hardcoded-curve-benchmarks into main
Merged STEVAN Antoine requested to merge fix-hardcoded-curve-benchmarks into main
Viewing commit 4552d2af
Prev
Show latest version
21 files
+ 1022
− 451
Compare changes
  • Side-by-side
  • Inline
Files
21
examples/benches/README.md
+ 16
− 338
@@ -77,369 +77,47 @@ for graph in [
@@ -77,369 +77,47 @@ for graph in [
## trusted setup
## trusted setup
```nushell
```nushell
let degrees = seq 0 18 | each { 2 ** $in }
use scripts/setup/run.nu; seq 0 13 | each { 2 ** $in } | run --output setup.ndjson --curves [ bls12381, pallas, bn254 ]
cargo run --release --example bench_setup -- ...[
--nb-measurements 10
...$degrees
--curves bls12381,pallas,bn254
] out> setup.ndjson
```
```
```nushell
```nushell
open setup.ndjson
use ./scripts/setup/plot.nu; plot setup.ndjson
| ns-to-ms $.times
| compute-stats $.times
| insert degree { get label | parse "degree {d}" | into record | get d | into int}
| update name {|it| if ($it.name | str starts-with "ARK") {
let c = $it.name | parse "ARK setup on {curve}" | into record | get curve
$"($c)-ark"
} else {
$it.name | parse "setup on {curve}" | into record | get curve
}}
| rename --column { degree: "x", mean: "y", stddev: "e" }
| select name x y e
| group-by name --to-table
| reject items.name
| rename --column { group: "name", items: "points" }
| sort-by name
| gplt plot ...[
# --title "time to create trusted setups for certain curves"
--x-label "degree"
--y-label "time (in ms)"
($in | to json)
...($in.points | flatten | into-axis-options -x "plain" -y "duration")
...$COMMON_OPTIONS
# --save setup.pdf
]
```
```
## commit
## commit
```nushell
```nushell
let degrees = seq 0 18 | each { 2 ** $in }
use scripts/commit/run.nu; seq 0 13 | each { 2 ** $in } | run --output commit.ndjson --curves [bls12381, pallas, bn254 ]
cargo run --release --example bench_commit -- ...[
--nb-measurements 10
...$degrees
--curves bls12381,pallas,bn254
] out> commit.ndjson
```
```
```nushell
```nushell
open commit.ndjson
use ./scripts/commit/plot.nu; plot commit.ndjson
| where name !~ '^SEC'
| ns-to-ms $.times
| compute-stats $.times
| insert degree { get label | parse "degree {d}" | into record | get d | into int }
| rename --column { degree: "x", mean: "y", stddev: "e" }
| select name x y e
| group-by name --to-table
| reject items.name
| rename --column { group: "name", items: "points" }
| sort-by name
| gplt plot ...[
# --title "time to commit polynomials for certain curves"
--x-label "degree"
--y-label "time (in ms)"
($in | to json)
...($in.points | flatten | into-axis-options -x "plain" -y "duration")
...$COMMON_OPTIONS
# --save commit.pdf
]
```
```
## end-to-end benchmarks
## end-to-end benchmarks
```nushell
use scripts/plot.nu [
into-duration-tick-labels,
into-filesize-tick-labels,
into-axis-options,
COMMON_OPTIONS,
]
```
### recoding
### recoding
```nushell
```nushell
"" out> recoding.ndjson
use scripts/recoding/run.nu
seq 0 18 | each { 512 * 2 ** $in } | run --ks [2, 4, 8, 16] --output recoding.ndjson --curves [ bls12381 ]
[2, 4, 8, 16] | each { |k|
cargo run --release --example bench_recoding -- ...[
--nb-measurements 10
...(seq 0 18 | each { 512 * 2 ** $in })
--shards $k
--ks $k
--curves bls12381
] | from ndnuon | to ndjson out>> recoding.ndjson
}
```
```
```nushell
```nushell
let graphs = open recoding.ndjson
use ./scripts/recoding/plot.nu; plot recoding.ndjson
| ns-to-ms $.times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select shards x y e
| group-by shards --to-table
| reject items.shards
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$"}
gplt plot ($graphs | to json) ...[
# --y-label "time (in ms)"
...($graphs.points | flatten | into-axis-options -x "filesize" -y "duration")
--no-legend
...$COMMON_OPTIONS
# --save recoding.pdf
]
```
```
### FEC
### FEC
```nushell
```nushell
"" out> fec.ndjson
use scripts/fec/run.nu
seq 0 18 | each { 512 * 2 ** $in } | run --ks [2, 4, 8, 16] --output fec.ndjson --curves [ bls12381 ]
[2, 4, 8, 16] | each { |k|
cargo run --release --example bench_fec -- ...[
...(seq 0 18 | each { 512 * 2 ** $in })
--encoding vandermonde
-k $k
-n 1
--nb-measurements 10
--curves bls12381
] | from ndnuon | to ndjson out>> fec.ndjson
}
```
```
```nushell
```nushell
let graphs = open fec.ndjson
use ./scripts/fec/plot.nu; plot encoding fec.ndjson
| update label { from json }
use ./scripts/fec/plot.nu; plot decoding fec.ndjson
| flatten label
use ./scripts/fec/plot.nu; plot e2e fec.ndjson
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381" and step == "encode"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
# --title "1-encoding"
...($graphs.points | flatten | into-axis-options -x "filesize" -y "duration")
...$COMMON_OPTIONS
# --save encoding.pdf
]
let graphs = open fec.ndjson
| update label { from json }
| flatten label
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381" and step == "decode"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
# --title "k-decoding"
--y-label "time (in ms)"
...($graphs.points | flatten | into-axis-options -x "filesize" -y "duration")
--no-legend
...$COMMON_OPTIONS
# --save decoding.pdf
]
let graphs = open fec.ndjson
| update label { from json }
| flatten label
| insert foo { $"($in.name) / ($in.k) / ($in.bytes)" }
| group-by foo --to-table
| update items {|it|
$it.items
| update step e2e
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group foo
| ns-to-ms times
| compute-stats times
| reject times
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
# --title "e2e: k-decoding + 1-encoding"
--y-label "time (in ms)"
...($graphs.points | flatten | into-axis-options -x "filesize" -y "duration")
--no-legend
...$COMMON_OPTIONS
# --save e2e.pdf
]
```
```
combined graph
## combined graph
```nushell
```nushell
let true_recoding_graphs = open recoding.ndjson
use ./scripts/fec/plot.nu; plot combined fec.ndjson --recoding recoding.ndjson
| ns-to-ms $.times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select shards x y e
| sort-by x
| group-by shards --to-table
| insert style.color {|it|
match $it.items.shards.0 {
2 => "tab:blue"
4 => "tab:orange"
8 => "tab:green"
16 => "tab:red"
_ => "tab:grey"
}
}
| reject items.shards
| insert style.line.type "solid"
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
let naive_recoding_graphs = open fec.ndjson
| update label { from json }
| flatten label
| insert key { $"($in.name) / ($in.k) / ($in.bytes)" }
| group-by key --to-table
| update items {|it|
$it.items
| update step e2e
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group key
| ns-to-ms times
| compute-stats times
| reject times
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| insert style.color {|it|
match $it.items.k.0 {
2 => "tab:blue"
4 => "tab:orange"
8 => "tab:green"
16 => "tab:red"
_ => "tab:grey"
}
}
| insert style.line.type "dashed"
| reject items.k
| rename --column { group: "name", items: "points" }
| reject name
let graphs = $true_recoding_graphs
| append $naive_recoding_graphs
| append {
name: "naive recoding ($k$-decoding + $1$-encoding)",
legend: "second",
points: [],
style: {
color: "grey",
line: {
type: "dashed",
marker: {
size: 0,
},
},
},
}
| append {
name: "true recoding ($k$-recoding)",
legend: "second",
points: [],
style: {
color: "grey",
line: {
type: "solid",
marker: {
size: 0,
},
},
},
}
gplt plot ($graphs | to json) ...[
...($graphs.points | flatten | into-axis-options -x "filesize" -y "duration")
...$COMMON_OPTIONS
--legend-loc "upper left" "lower right"
# --save comparison.png
]
```
```
ratio graph
## ratio graph
```nushell
```nushell
let true_recoding_graphs = open recoding.ndjson
use ./scripts/fec/plot.nu; plot ratio fec.ndjson --recoding recoding.ndjson
| ns-to-ms times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| select shards bytes mean
| rename --column { shards: "k" }
let naive_recoding_graphs = open fec.ndjson
| update label { from json }
| flatten label
| insert key { $"($in.name) / ($in.k) / ($in.bytes)" }
| group-by key --to-table
| update items {|it|
$it.items
| update step e2e
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group key
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381"
| select k bytes mean
| uniq
let graphs = $true_recoding_graphs
| rename --column { mean: "true" }
| insert key { $"($in.k) ($in.bytes)" }
| join ($naive_recoding_graphs | rename --column { mean: "naive" } | insert key { $"($in.k) ($in.bytes)" }) key
| select k bytes $.true naive
| sort-by k bytes
| insert cmp { $in.naive / $in.true }
| rename --column { bytes: "x", cmp: "y" }
| select k x y
| group-by k --to-table
| insert style.color {|it|
match $it.items.k.0 {
2 => "tab:blue"
4 => "tab:orange"
8 => "tab:green"
16 => "tab:red"
_ => "tab:grey"
}
}
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
...($graphs.points | flatten | into-axis-options -x "filesize" -y "plain")
...$COMMON_OPTIONS
--legend-loc "upper right"
# --save ratio.png
]
```
```