Skip to content
Snippets Groups Projects
Commit 9c3cd0a7 authored by STEVAN Antoine's avatar STEVAN Antoine 🦀
Browse files

better plots (dragoon/komodo!102) 

- new `scripts/plot.nu` with common tools and options
- better sets of parameters
- better commands in `benches/README.md`
parent c9f4b174
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
examples/benches/README.md
+ 241
− 127
View file @ 9c3cd0a7
......@@ -147,6 +147,15 @@ gplt plot ...[
```
## end-to-end benchmarks
```nushell
use scripts/plot.nu [
into-duration-tick-labels,
into-filesize-tick-labels,
into-axis-options,
COMMON_OPTIONS,
]
```
### recoding
```nushell
"" out> recoding.ndjson
......@@ -154,69 +163,31 @@ gplt plot ...[
[2, 4, 8, 16] | each { |k|
cargo run --example bench_recoding -- ...[
--nb-measurements 10
...[100, 1_000, 10_000]
...(seq 0 18 | each { 512 * 2 ** $in })
--shards $k
--ks $k
] | from ndnuon | to ndjson out>> recoding.ndjson
}
```
```nushell
gplt plot --title "k-recoding with k = #shards" --x-label '\#bytes' --y-label "time (in ms)" (
open recoding.ndjson
| ns-to-ms $.times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| insert case { $"($in.name) / ($in.shards)" }
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| group-by case --to-table
| rename --column { group: "name", items: "points" }
| insert style {|it|
let g = $it.name | parse "{c} / {s}" | into record | into int s
let c = match $g.c {
"BLS12-381" => "blue"
"BN-254" => "orange"
"PALLAS" => "green"
_ => "gray"
}
let t = match $g.s {
2 => "dotted"
4 => "dashdot"
8 => "dashed"
16 => "solid"
_ => "loosely dotted"
}
{ color: $c, line: { type: $t } }
}
| to json
)
```
an alternate plot
```nushell
gplt plot ...[
# --title "k-recoding with k = #shards"
--x-label '\#bytes'
--y-label "time (in ms)"
(
open recoding.ndjson
| ns-to-ms $.times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select shards x y e
| group-by shards --to-table
| reject items.shards
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$"}
| to json
)
--fullscreen
--dpi 150
--fig-size ...[16, 9]
--font ({ size: 30, family: serif, sans-serif: Helvetica } | to json)
--use-tex
let graphs = open recoding.ndjson
| ns-to-ms $.times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select shards x y e
| group-by shards --to-table
| reject items.shards
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$"}
gplt plot ($graphs | to json) ...[
# --y-label "time (in ms)"
...($graphs.points | flatten | into-axis-options)
--no-legend
...$COMMON_OPTIONS
# --save recoding.pdf
]
```
......@@ -227,70 +198,60 @@ gplt plot ...[
[2, 4, 8, 16] | each { |k|
cargo run --example bench_fec -- ...[
...[100, 1_000, 10_000]
...(seq 0 18 | each { 512 * 2 ** $in })
--encoding vandermonde
-k $k
-n 1
--nb-measurements 100
--nb-measurements 10
] | from ndnuon | to ndjson out>> fec.ndjson
}
```
```nushell
gplt plot ...[
let graphs = open fec.ndjson
| update label { from json }
| flatten label
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381" and step == "encode"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
# --title "1-encoding"
--x-label '\#bytes'
--y-label "time (in ms)"
(
open fec.ndjson
| update label { from json }
| flatten label
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381" and step == "encode"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
| to json
)
--fullscreen
--dpi 150
--fig-size ...[16, 9]
--font ({ size: 30, family: serif, sans-serif: Helvetica } | to json)
--use-tex
...($graphs.points | flatten | into-axis-options -y "duration")
...$COMMON_OPTIONS
# --save encoding.pdf
]
gplt plot ...[
# --title "k-encoding"
--x-label '\#bytes'
let graphs = open fec.ndjson
| update label { from json }
| flatten label
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381" and step == "decode"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
# --title "k-decoding"
--y-label "time (in ms)"
(
open fec.ndjson
| update label { from json }
| flatten label
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381" and step == "decode"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
| to json
)
--fullscreen
--dpi 150
--fig-size ...[16, 9]
--font ({ size: 30, family: serif, sans-serif: Helvetica } | to json)
--use-tex
...($graphs.points | flatten | into-axis-options -y "duration")
--no-legend
...$COMMON_OPTIONS
# --save decoding.pdf
]
let x = open fec.ndjson
let graphs = open fec.ndjson
| update label { from json }
| flatten label
| insert foo { $"($in.name) / ($in.k) / ($in.bytes)" }
......@@ -302,30 +263,183 @@ let x = open fec.ndjson
}
| flatten --all
| reject group foo
| ns-to-ms times
| compute-stats times
| reject times
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ...[
gplt plot ($graphs | to json) ...[
# --title "e2e: k-decoding + 1-encoding"
--x-label '\#bytes'
--y-label "time (in ms)"
(
$x
| ns-to-ms times
| compute-stats times
| reject times
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| group-by k --to-table
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
| to json
)
--fullscreen
--dpi 150
--fig-size ...[16, 9]
--font ({ size: 30, family: serif, sans-serif: Helvetica } | to json)
--use-tex
...($graphs.points | flatten | into-axis-options -y "duration")
--no-legend
...$COMMON_OPTIONS
# --save e2e.pdf
]
```
combined graph
```nushell
let true_recoding_graphs = open recoding.ndjson
| ns-to-ms $.times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select shards x y e
| sort-by x
| group-by shards --to-table
| insert style.color {|it|
match $it.items.shards.0 {
2 => "tab:blue"
4 => "tab:orange"
8 => "tab:green"
16 => "tab:red"
_ => "tab:grey"
}
}
| reject items.shards
| insert style.line.type "solid"
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
let naive_recoding_graphs = open fec.ndjson
| update label { from json }
| flatten label
| insert key { $"($in.name) / ($in.k) / ($in.bytes)" }
| group-by key --to-table
| update items {|it|
$it.items
| update step e2e
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group key
| ns-to-ms times
| compute-stats times
| reject times
| where name == "BLS12-381"
| rename --column { bytes: "x", mean: "y", stddev: "e" }
| select k x y e
| sort-by x
| group-by k --to-table
| insert style.color {|it|
match $it.items.k.0 {
2 => "tab:blue"
4 => "tab:orange"
8 => "tab:green"
16 => "tab:red"
_ => "tab:grey"
}
}
| insert style.line.type "dashed"
| reject items.k
| rename --column { group: "name", items: "points" }
| reject name
let graphs = $true_recoding_graphs
| append $naive_recoding_graphs
| append {
name: "naive recoding ($k$-decoding + $1$-encoding)",
legend: "second",
points: [],
style: {
color: "grey",
line: {
type: "dashed",
marker: {
size: 0,
},
},
},
}
| append {
name: "true recoding ($k$-recoding)",
legend: "second",
points: [],
style: {
color: "grey",
line: {
type: "solid",
marker: {
size: 0,
},
},
},
}
gplt plot ($graphs | to json) ...[
...($graphs.points | flatten | into-axis-options -y "duration")
...$COMMON_OPTIONS
--legend-loc "upper left" "lower right"
# --save comparison.png
]
```
ratio graph
```nushell
let true_recoding_graphs = open recoding.ndjson
| ns-to-ms times
| compute-stats $.times
| update label { from nuon }
| flatten --all label
| where name == "BLS12-381"
| select shards bytes mean
| rename --column { shards: "k" }
let naive_recoding_graphs = open fec.ndjson
| update label { from json }
| flatten label
| insert key { $"($in.name) / ($in.k) / ($in.bytes)" }
| group-by key --to-table
| update items {|it|
$it.items
| update step e2e
| update times { $it.items.0.times | zip $it.items.1.times | each { $in.0 + $in.1 } }
}
| flatten --all
| reject group key
| ns-to-ms times
| compute-stats times
| where name == "BLS12-381"
| select k bytes mean
| uniq
let graphs = $true_recoding_graphs
| rename --column { mean: "true" }
| insert key { $"($in.k) ($in.bytes)" }
| join ($naive_recoding_graphs | rename --column { mean: "naive" } | insert key { $"($in.k) ($in.bytes)" }) key
| select k bytes $.true naive
| sort-by k bytes
| insert cmp { $in.naive / $in.true }
| rename --column { bytes: "x", cmp: "y" }
| select k x y
| group-by k --to-table
| insert style.color {|it|
match $it.items.k.0 {
2 => "tab:blue"
4 => "tab:orange"
8 => "tab:green"
16 => "tab:red"
_ => "tab:grey"
}
}
| reject items.k
| rename --column { group: "name", items: "points" }
| update name { $"$k = ($in)$" }
gplt plot ($graphs | to json) ...[
...($graphs.points | flatten | into-axis-options)
...$COMMON_OPTIONS
--legend-loc "upper right"
# --save ratio.png
]
```
examples/curves.rs
+ 1
− 1
View file @ 9c3cd0a7
......@@ -17,6 +17,7 @@
//! | ark_bw6_761 | 761 | 377 | 50.5 |
//! | ark_cp6_782 | 782 | 377 | 51.8 |
//! | ark_curve25519 | 255 | 253 | 0.8 |
//! | ark_ed25519 | 255 | 253 | 0.8 |
//! | ark_ed_on_bls12_377 | 253 | 251 | 0.8 |
//! | ark_ed_on_bls12_381 | 255 | 252 | 1.2 |
//! | ark_ed_on_bls12_381_bandersnatch | 255 | 253 | 0.8 |
......@@ -25,7 +26,6 @@
//! | ark_ed_on_cp6_782 | 377 | 374 | 0.8 |
//! | ark_ed_on_mnt4_298 | 298 | 296 | 0.7 |
//! | ark_ed_on_mnt4_753 | 753 | 750 | 0.4 |
//! | ark_ed25519 | 255 | 253 | 0.8 |
//! | ark_mnt4_298 | 298 | 298 | 0 |
//! | ark_mnt4_753 | 753 | 753 | 0 |
//! | ark_mnt6_298 | 298 | 298 | 0 |
......
examples/inbreeding.rs
+ 14
− 14
View file @ 9c3cd0a7
......@@ -18,7 +18,7 @@
/// --test-case end-to-end
/// ] | lines | into float | save --force baseline.nuon
///
/// seq 2 $K | reverse | each {|r|
/// seq 1 $K | reverse | each {|r|
/// let inbreeding = cargo run --example inbreeding -- ...[
/// $NB_BYTES,
/// -k $K
......@@ -52,26 +52,26 @@
/// $smooth
/// | insert name {|it|
/// let r = if $it.r == $k { "k" } else { $"k - ($k - $it.r)" }
/// $"$r = ($r)$"
/// $"$\\sigma = ($r)$"
/// }
/// # | append ($raw | insert name null | insert style { line: { alpha: 0.1 } })
/// | update inbreeding {|it|
/// let l = $it.inbreeding | length
/// $it.inbreeding | wrap y | merge (seq 1 $l | wrap x) | insert e 0
/// $it.inbreeding | wrap y | merge (seq 0 $l | wrap x) | insert e 0
/// }
/// | rename --column { inbreeding: "points" }
/// | insert style.color {|it|
/// match $it.r {
/// 10 => "red",
/// 9 => "orange",
/// 8 => "yellow",
/// 7 => "blue",
/// 6 => "purple",
/// 5 => "green",
/// 4 => "cyan",
/// 3 => "black",
/// 2 => "magenta",
/// _ => "gray",
/// 10 => "tab:red",
/// 9 => "tab:orange",
/// 8 => "tab:olive",
/// 7 => "tab:blue",
/// 6 => "tab:purple",
/// 5 => "tab:green",
/// 4 => "tab:cyan",
/// 3 => "tab:brown",
/// 2 => "tab:pink",
/// _ => "tab:gray",
/// }
/// }
/// | reject r
......@@ -88,7 +88,7 @@
/// --fullscreen
/// # --title "diversity over time when recoding shards $r$ shards"
/// --x-label "time (in nb of steps)"
/// --y-label "diversity"
/// --y-label "diversity $\\delta$"
/// --dpi 150
/// --fig-size ...[16, 5]
/// --font ({ size: 15, family: serif, sans-serif: Helvetica } | to json)
......
scripts/plot.nu 0 → 100644
+ 60
− 0
View file @ 9c3cd0a7
export const COMMON_OPTIONS = [
--x-scale log
--x-scale-base 2
--y-scale log
--x-ticks-rotation 45
--fullscreen
--dpi 150
--fig-size ...[16, 9]
--font '{ "size": 30, "family": "serif", "sans-serif": "Helvetica" }'
--use-tex
]
export def into-duration-tick-labels []: list<int> -> list<string> {
each { $in * 1ms } | each { to text }
}
export def into-filesize-tick-labels []: list<int> -> list<string> {
into filesize
| each {
if $in < 1000b {
format filesize iB
} else if $in < 1000000b {
format filesize KiB
} else {
format filesize MiB
}
}
| each { to text | str replace ".0 " " " }
}
export def into-axis-options [-y: string]: table<x: float, y: float> -> list<string> {
let input = $in
let xs = $input | flatten | get x | uniq
let options = [
--x-lim ($xs | first) ($xs | last)
--x-ticks ...$xs
--x-tick-labels ...($xs | into-filesize-tick-labels)
]
let ys = $input | flatten | get y
let y_ticks = seq ($ys | math min | math log 10 | math ceil | $in - 1) ($ys | math max | math log 10 | math floor)
| into float
| each { 10 ** $in }
let y_tick_labels = match $y {
"duration" => ($y_ticks | into-duration-tick-labels),
"plain" => $y_ticks,
_ => {
print $"warning: ($y) option is unknown for -y"
$y_ticks
},
}
$options | append [
--y-ticks ...$y_ticks
--y-tick-labels ...$y_tick_labels
]
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment