Skip to content

N
neo
Commit 7a97206e authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X timings as of Sep 7 morning

parent 1fce1043
Show whitespace changes
Inline Side-by-side
Showing with 183 additions and 66 deletions
go/neo/t/t.sh
View file @ 7a97206e
......@@ -48,6 +48,8 @@ kill $j' EXIT
# M{py,go} ... - spawn master
Mpy() {
# XXX run via relative path to neomaster? (../../neo/neomaster) so we do not need to `pip install -e` ?
# XXX --autostart=1 ?
exec -a Mpy \
neomaster --cluster=$cluster --bind=$Mbind --masters=$Mbind -r 1 -p 1 --logfile=$log/Mpy.log $@ &
......@@ -69,6 +71,8 @@ Spy() {
# Sgo <data.fs> - spawn NEO/go storage
Sgo() {
# XXX use `go run ...` so it does not need go install?
# -alsologtostderr
exec -a Sgo \
neo -log_dir=$log storage -cluster=$cluster -bind=$Sbind -masters=$Mbind $@ &
......@@ -225,8 +229,8 @@ GENsql() {
# ---- main driver ----
GENfs
#GENsqlite
#GENsql
GENsqlite
GENsql
wait
sync
......@@ -238,7 +242,7 @@ N=`seq 2` # XXX repeat benchmarks N time
bench1() {
url=$1
# time demo-zbigarray read $url
# ./zsha1.py $url
./zsha1.py $url
if [[ $url == zeo://* ]]; then
echo "(skipping zsha1.go on ZEO -- Cgo does not support zeo:// protocol)"
return
......@@ -252,38 +256,38 @@ for i in $N; do
bench1 $fs1/data.fs
done
# echo -e "\n*** ZEO"
# Zpy $fs1/data.fs
# for i in $N; do
# bench1 zeo://$Zbind
# done
# killall runzeo
# wait
# echo -e "\n*** NEO/py sqlite"
# NEOpylite
# for i in $N; do
# bench1 neo://$cluster@$Mbind
# done
# xneoctl set cluster stopping
# wait
# echo -e "\n*** NEO/py sql"
# NEOpysql
# for i in $N; do
# bench1 neo://$cluster@$Mbind
# done
# xneoctl set cluster stopping
# xmysql -e "SHUTDOWN"
# wait
# echo -e "\n*** NEO/go"
# NEOgo
# for i in $N; do
# bench1 neo://$cluster@$Mbind
# done
# xneoctl set cluster stopping
# wait
echo -e "\n*** ZEO"
Zpy $fs1/data.fs
for i in $N; do
bench1 zeo://$Zbind
done
killall runzeo
wait
echo -e "\n*** NEO/py sqlite"
NEOpylite
for i in $N; do
bench1 neo://$cluster@$Mbind
done
xneoctl set cluster stopping
wait
echo -e "\n*** NEO/py sql"
NEOpysql
for i in $N; do
bench1 neo://$cluster@$Mbind
done
xneoctl set cluster stopping
xmysql -e "SHUTDOWN"
wait
echo -e "\n*** NEO/go"
NEOgo
for i in $N; do
bench1 neo://$cluster@$Mbind
done
xneoctl set cluster stopping
wait
# all ok
trap - EXIT
......
go/neo/t/zsha1.go
View file @ 7a97206e
......@@ -100,7 +100,7 @@ func zsha1(ctx context.Context, url string, useprefetch bool) (err error) {
}
before := lastTid + 1 // XXX overflow ?
if true {
if false {
defer profile.Start(profile.TraceProfile).Stop()
}
......
go/time-sep7.txt 0 → 100644
View file @ 7a97206e
(@ 7 Sep 2017)
(neo) (z-dev) (g.env) kirr@deco:~/src/neo/src/lab.nexedi.com/kirr/neo/go/neo/t$ time ./t.sh
*** generating fs1 data...
I: RAM: 7.48GB
I: WORK: 0.06GB
gen signal t=0...8.39e+06 float64 (= 0.06GB)
gen signal blk [0:4194304] (50.0%)
gen signal blk [4194304:8388608] (100.0%)
VIRT: 344 MB RSS: 104MB
*** generating sqlite data...
I: RAM: 7.48GB
I: WORK: 0.06GB
gen signal t=0...8.39e+06 float64 (= 0.06GB)
gen signal blk [0:4194304] (50.0%)
gen signal blk [4194304:8388608] (100.0%)
VIRT: 471 MB RSS: 149MB
2017-09-07 10:15:12.4942 ERROR NEO [ app: 91] primary master is down
Cluster state changed
*** generating sql data...
2017-09-07 10:15:12 140377545237376 [Note] /usr/sbin/mysqld (mysqld 10.1.26-MariaDB-1) starting as process 10026 ...
2017-09-07 10:15:16 139835902292864 [Note] /usr/sbin/mysqld (mysqld 10.1.26-MariaDB-1) starting as process 10055 ...
2017-09-07 10:15:19 140350144623488 [Note] /usr/sbin/mysqld (mysqld 10.1.26-MariaDB-1) starting as process 10085 ...
2017-09-07 10:15:22 139936065475456 [Note] mysqld (mysqld 10.1.26-MariaDB-1) starting as process 10112 ...
I: RAM: 7.48GB
I: WORK: 0.06GB
gen signal t=0...8.39e+06 float64 (= 0.06GB)
gen signal blk [0:4194304] (50.0%)
gen signal blk [4194304:8388608] (100.0%)
VIRT: 485 MB RSS: 162MB
2017-09-07 10:15:33.8986 ERROR NEO [ app: 91] primary master is down
Cluster state changed
*** FileStorage
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=0.292s (17.2μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=130.700046ms (7.69µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=159.584996ms (9.389µs / object) x=zsha1.go +prefetch
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=0.289s (17.0μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=129.699956ms (7.631µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=157.831648ms (9.286µs / object) x=zsha1.go +prefetch
*** ZEO
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=5.532s (325.5μs / object) x=zsha1.py
(skipping zsha1.go on ZEO -- Cgo does not support zeo:// protocol)
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=5.295s (311.6μs / object) x=zsha1.py
(skipping zsha1.go on ZEO -- Cgo does not support zeo:// protocol)
*** NEO/py sqlite
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=5.470s (321.8μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=3.701967602s (217.814µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=2.347308074s (138.109µs / object) x=zsha1.go +prefetch
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=5.512s (324.3μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=3.862412698s (227.254µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=2.35216104s (138.394µs / object) x=zsha1.go +prefetch
2017-09-07 10:16:17.7122 ERROR NEO [ app: 91] primary master is down
Cluster state changed
*** NEO/py sql
2017-09-07 10:16:17 140308575191936 [Note] mysqld (mysqld 10.1.26-MariaDB-1) starting as process 10486 ...
2017-09-07 10:16:19.1711 ERROR NEO [ app:233] Connection to None lost
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=11.605s (682.8μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=8.211994021s (483.172µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=6.879179752s (404.752µs / object) x=zsha1.go +prefetch
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=10.474s (616.3μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=7.878703192s (463.562µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=6.645186736s (390.985µs / object) x=zsha1.go +prefetch
2017-09-07 10:17:13.4350 ERROR NEO [ app: 91] primary master is down
Cluster state changed
*** NEO/go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=3.271s (192.4μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=1.042710741s (61.35µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=607.029828ms (35.716µs / object) x=zsha1.go +prefetch
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=3.140s (184.8μs / object) x=zsha1.py
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=1.03721356s (61.026µs / object) x=zsha1.go
50aa58a7d47d168ec550762c25d7aaaed3f4177b ; oid=0..16995 nread=68269354 t=634.242335ms (37.317µs / object) x=zsha1.go +prefetch
2017-09-07 10:17:28.5996 ERROR NEO [ app: 91] primary master is down
Cluster state changed
real 2m24,089s
user 2m29,765s
sys 0m22,275s
--------
t(getobject) client client client
zsha1.py zsha1.go zsha1.go + prefetch
server
fs1 17.0μs 7.6µs 9.3µs (*)
zeo 311.6μs - -
neo/sqlite 321.8μs 217.8µs 138.2µs
neo/sql 616.3μs 463.5µs 391.0µs
neo/go 184.8μs 61.0µs 37.3µs (+)
(*) prefetch is not working good yet
(+) Sgo (storage in neo/go) is not working good yet
go/todo.dot
View file @ 7a97206e
......@@ -12,7 +12,9 @@ digraph {
Sgo_speed -> Mmap_go;
Sgo_speed -> sha1_index;
Sgo_speed -> B_link_recv;
//Sgo_speed -> B_link_recv;
Sgo_speed -> garbage;
Sgo_speed -> gosched;
Cpy_r_Sgo -> Sgo;
//Cpy_r_Sgo -> Protog_fix;
......@@ -44,8 +46,9 @@ digraph {
load_serial_next [label=<? GetObject return += serial<sub>next</sub>>];
sha1_index [label="? sha1 index"];
B_link_recv [label="Bench link.Recv1/Ask1"];
//B_link_recv [label="Bench link.Recv1/Ask1" style=filled fillcolor=lightyellow];
garbage [label="garbage"];
gosched [label="? go scheduling"];
//Cgo -> Cgo_tasks;
//Cgo_tasks [label="- talkMaster\n- update nodeTab/partTab\n- dial S on demand\n..." style=filled fillcolor=lightyellow];
......
go/todo.svg
View file @ 7a97206e
......@@ -4,11 +4,11 @@
<!-- Generated by graphviz version 2.38.0 (20140413.2041)
-->
<!-- Title: %3 Pages: 1 -->
<svg width="822pt" height="404pt"
viewBox="0.00 0.00 822.13 404.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<svg width="805pt" height="404pt"
viewBox="0.00 0.00 804.54 404.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 400)">
<title>%3</title>
<polygon fill="white" stroke="none" points="-4,4 -4,-400 818.133,-400 818.133,4 -4,4"/>
<polygon fill="white" stroke="none" points="-4,4 -4,-400 800.536,-400 800.536,4 -4,4"/>
<!-- Cgo -->
<g id="node1" class="node"><title>Cgo</title>
<ellipse fill="lightyellow" stroke="black" cx="416.242" cy="-306" rx="27" ry="18"/>
......@@ -96,36 +96,46 @@
</g>
<!-- Mmap_go -->
<g id="node8" class="node"><title>Mmap_go</title>
<ellipse fill="none" stroke="black" cx="385.242" cy="-18" rx="92.0841" ry="18"/>
<text text-anchor="middle" x="385.242" y="-14.3" font-family="Times,serif" font-size="14.00">io.ReaderAt via mmap</text>
<ellipse fill="none" stroke="black" cx="329.242" cy="-18" rx="92.0841" ry="18"/>
<text text-anchor="middle" x="329.242" y="-14.3" font-family="Times,serif" font-size="14.00">io.ReaderAt via mmap</text>
</g>
<!-- Sgo_speed&#45;&gt;Mmap_go -->
<g id="edge9" class="edge"><title>Sgo_speed&#45;&gt;Mmap_go</title>
<path fill="none" stroke="black" d="M515.63,-73.9832C491.11,-63.6432 458.291,-49.8038 431.714,-38.5966"/>
<polygon fill="black" stroke="black" points="432.806,-35.2587 422.231,-34.5981 430.086,-41.7087 432.806,-35.2587"/>
<path fill="none" stroke="black" d="M507.382,-75.1703C472.467,-64.161 423.355,-48.6754 385.539,-36.7513"/>
<polygon fill="black" stroke="black" points="386.279,-33.3149 375.689,-33.6457 384.174,-39.9909 386.279,-33.3149"/>
</g>
<!-- sha1_index -->
<g id="node9" class="node"><title>sha1_index</title>
<ellipse fill="none" stroke="black" cx="551.242" cy="-18" rx="55.4913" ry="18"/>
<text text-anchor="middle" x="551.242" y="-14.3" font-family="Times,serif" font-size="14.00">? sha1 index</text>
<ellipse fill="none" stroke="black" cx="495.242" cy="-18" rx="55.4913" ry="18"/>
<text text-anchor="middle" x="495.242" y="-14.3" font-family="Times,serif" font-size="14.00">? sha1 index</text>
</g>
<!-- Sgo_speed&#45;&gt;sha1_index -->
<g id="edge10" class="edge"><title>Sgo_speed&#45;&gt;sha1_index</title>
<path fill="none" stroke="black" d="M551.242,-71.6966C551.242,-63.9827 551.242,-54.7125 551.242,-46.1124"/>
<polygon fill="black" stroke="black" points="554.742,-46.1043 551.242,-36.1043 547.742,-46.1044 554.742,-46.1043"/>
<path fill="none" stroke="black" d="M537.686,-72.055C530.876,-63.5427 522.496,-53.0684 514.987,-43.6815"/>
<polygon fill="black" stroke="black" points="517.653,-41.4113 508.673,-35.789 512.187,-45.7842 517.653,-41.4113"/>
</g>
<!-- B_link_recv -->
<g id="node10" class="node"><title>B_link_recv</title>
<ellipse fill="none" stroke="black" cx="719.242" cy="-18" rx="94.7833" ry="18"/>
<text text-anchor="middle" x="719.242" y="-14.3" font-family="Times,serif" font-size="14.00">Bench link.Recv1/Ask1</text>
<!-- garbage -->
<g id="node10" class="node"><title>garbage</title>
<ellipse fill="none" stroke="black" cx="607.242" cy="-18" rx="38.1938" ry="18"/>
<text text-anchor="middle" x="607.242" y="-14.3" font-family="Times,serif" font-size="14.00">garbage</text>
</g>
<!-- Sgo_speed&#45;&gt;B_link_recv -->
<g id="edge11" class="edge"><title>Sgo_speed&#45;&gt;B_link_recv</title>
<path fill="none" stroke="black" d="M587.282,-73.9832C612.207,-63.5979 645.604,-49.6824 672.563,-38.4493"/>
<polygon fill="black" stroke="black" points="673.922,-41.6751 681.806,-34.5981 671.229,-35.2135 673.922,-41.6751"/>
<!-- Sgo_speed&#45;&gt;garbage -->
<g id="edge11" class="edge"><title>Sgo_speed&#45;&gt;garbage</title>
<path fill="none" stroke="black" d="M564.798,-72.055C571.708,-63.4175 580.234,-52.7599 587.827,-43.2681"/>
<polygon fill="black" stroke="black" points="590.682,-45.3022 594.196,-35.307 585.216,-40.9293 590.682,-45.3022"/>
</g>
<!-- gosched -->
<g id="node11" class="node"><title>gosched</title>
<ellipse fill="none" stroke="black" cx="730.242" cy="-18" rx="66.0889" ry="18"/>
<text text-anchor="middle" x="730.242" y="-14.3" font-family="Times,serif" font-size="14.00">? go scheduling</text>
</g>
<!-- Sgo_speed&#45;&gt;gosched -->
<g id="edge12" class="edge"><title>Sgo_speed&#45;&gt;gosched</title>
<path fill="none" stroke="black" d="M589.203,-74.1548C617.048,-63.2657 654.989,-48.4284 684.506,-36.8854"/>
<polygon fill="black" stroke="black" points="686.128,-40.0093 694.166,-33.1076 683.578,-33.4901 686.128,-40.0093"/>
</g>
<!-- Cpy_r_Sgo -->
<g id="node11" class="node"><title>Cpy_r_Sgo</title>
<g id="node12" class="node"><title>Cpy_r_Sgo</title>
<ellipse fill="none" stroke="black" cx="228.242" cy="-234" rx="61.99" ry="18"/>
<text text-anchor="start" x="188.742" y="-231.3" font-family="Times,serif" font-size="14.00">C</text>
<text text-anchor="start" x="198.742" y="-231.3" font-family="Times,serif" baseline-shift="sub" font-size="14.00">py</text>
......@@ -133,12 +143,12 @@
<text text-anchor="start" x="254.742" y="-231.3" font-family="Times,serif" baseline-shift="sub" font-size="14.00">go</text>
</g>
<!-- Cpy_r_Sgo&#45;&gt;Sgo -->
<g id="edge12" class="edge"><title>Cpy_r_Sgo&#45;&gt;Sgo</title>
<g id="edge13" class="edge"><title>Cpy_r_Sgo&#45;&gt;Sgo</title>
<path fill="none" stroke="black" d="M251.357,-217.291C267.071,-206.611 287.912,-192.447 304.367,-181.264"/>
<polygon fill="black" stroke="black" points="306.693,-183.915 312.997,-175.399 302.759,-178.126 306.693,-183.915"/>
</g>
<!-- Cgo_r_Sgo -->
<g id="node12" class="node"><title>Cgo_r_Sgo</title>
<g id="node13" class="node"><title>Cgo_r_Sgo</title>
<ellipse fill="none" stroke="black" cx="318.242" cy="-378" rx="61.99" ry="18"/>
<text text-anchor="start" x="278.742" y="-375.3" font-family="Times,serif" font-size="14.00">C</text>
<text text-anchor="start" x="288.742" y="-375.3" font-family="Times,serif" baseline-shift="sub" font-size="14.00">go</text>
......@@ -146,17 +156,17 @@
<text text-anchor="start" x="344.742" y="-375.3" font-family="Times,serif" baseline-shift="sub" font-size="14.00">go</text>
</g>
<!-- Cgo_r_Sgo&#45;&gt;Cgo -->
<g id="edge13" class="edge"><title>Cgo_r_Sgo&#45;&gt;Cgo</title>
<g id="edge14" class="edge"><title>Cgo_r_Sgo&#45;&gt;Cgo</title>
<path fill="none" stroke="black" d="M340.479,-361.116C355.251,-350.565 374.702,-336.671 390.197,-325.603"/>
<polygon fill="black" stroke="black" points="392.241,-328.445 398.344,-319.784 388.172,-322.749 392.241,-328.445"/>
</g>
<!-- Cgo_r_Sgo&#45;&gt;Sgo -->
<g id="edge14" class="edge"><title>Cgo_r_Sgo&#45;&gt;Sgo</title>
<g id="edge15" class="edge"><title>Cgo_r_Sgo&#45;&gt;Sgo</title>
<path fill="none" stroke="black" d="M319.284,-359.849C321.532,-322.832 326.857,-235.181 329.578,-190.386"/>
<polygon fill="black" stroke="black" points="333.082,-190.426 330.195,-180.232 326.095,-190.002 333.082,-190.426"/>
</g>
<!-- Cgo_r_Spy -->
<g id="node13" class="node"><title>Cgo_r_Spy</title>
<g id="node14" class="node"><title>Cgo_r_Spy</title>
<ellipse fill="none" stroke="black" cx="459.242" cy="-378" rx="61.99" ry="18"/>
<text text-anchor="start" x="419.742" y="-375.3" font-family="Times,serif" font-size="14.00">C</text>
<text text-anchor="start" x="429.742" y="-375.3" font-family="Times,serif" baseline-shift="sub" font-size="14.00">go</text>
......@@ -164,12 +174,12 @@
<text text-anchor="start" x="485.742" y="-375.3" font-family="Times,serif" baseline-shift="sub" font-size="14.00">py</text>
</g>
<!-- Cgo_r_Spy&#45;&gt;Cgo -->
<g id="edge15" class="edge"><title>Cgo_r_Spy&#45;&gt;Cgo</title>
<g id="edge16" class="edge"><title>Cgo_r_Spy&#45;&gt;Cgo</title>
<path fill="none" stroke="black" d="M448.833,-360.055C443.559,-351.469 437.058,-340.888 431.254,-331.439"/>
<polygon fill="black" stroke="black" points="434.182,-329.518 425.965,-322.829 428.217,-333.182 434.182,-329.518"/>
</g>
<!-- Cleanup -->
<g id="node14" class="node"><title>Cleanup</title>
<g id="node15" class="node"><title>Cleanup</title>
<ellipse fill="none" stroke="black" cx="606.242" cy="-378" rx="66.8882" ry="18"/>
<text text-anchor="middle" x="606.242" y="-374.3" font-family="Times,serif" font-size="14.00">Cleanup history</text>
</g>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7