Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
neoppod
Commits
c470505a
Commit
c470505a
authored
Mar 14, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
aa5fb3f0
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
126 additions
and
38 deletions
+126
-38
t/neo/zodb/cmd/zodbdump/zodbdump.go
t/neo/zodb/cmd/zodbdump/zodbdump.go
+126
-38
No files found.
t/neo/zodb/cmd/zodbdump/zodbdump.go
View file @
c470505a
...
@@ -13,6 +13,8 @@
...
@@ -13,6 +13,8 @@
/*
/*
Zodbdump - Tool to dump content of a ZODB database
Zodbdump - Tool to dump content of a ZODB database
TODO sync text with zodbdump/py
Format
Format
------
------
...
@@ -30,7 +32,6 @@ txn ...
...
@@ -30,7 +32,6 @@ txn ...
package
main
package
main
import
(
import
(
"bytes"
"crypto/sha1"
"crypto/sha1"
"flag"
"flag"
"fmt"
"fmt"
...
@@ -42,54 +43,136 @@ import (
...
@@ -42,54 +43,136 @@ import (
"../../../storage/fs1"
"../../../storage/fs1"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/mem"
//"lab.nexedi.com/kirr/go123/xio"
pickle
"github.com/kisielk/og-rek"
)
)
// dumpb pickles an object to []byte
// object must be picklable (i.e. no func, chan, unsafe.Pointer, ... inside)
// dumper dumps zodb record to a writer
// objects created by pickle.Decoder are always picklable
type
dumper
struct
{
func
dumpb
(
obj
interface
{})
[]
byte
{
W
io
.
Writer
buf
:=
bytes
.
Buffer
{}
HashOnly
bool
// whether to dump only hashes of data without content
p
:=
pickle
.
NewEncoder
(
&
buf
)
err
:=
p
.
Encode
(
obj
)
afterFirst
bool
// true after first transaction has been dumped
// as bytes.Buffer.Write will never return an error (it panics on oom)
}
// the only case when we can get error here is due to non-picklable object
// DumpData dumps one data record
func
(
d
*
dumper
)
DumpData
(
datai
*
zodb
.
StorageRecordInformation
)
error
{
entry
:=
"obj "
+
datai
.
Oid
.
String
()
+
" "
writeData
:=
false
switch
{
case
datai
.
Data
==
nil
:
entry
+=
"delete"
case
datai
.
Tid
!=
datai
.
DataTid
:
entry
+=
"from "
+
datai
.
DataTid
.
String
()
default
:
entry
+=
fmt
.
Sprintf
(
"%d sha1:%x"
,
len
(
datai
.
Data
),
sha1
.
Sum
(
datai
.
Data
))
writeData
=
true
// XXX write data here
}
entry
+=
"
\n
"
_
,
err
:=
d
.
W
.
Write
(
mem
.
Bytes
(
entry
))
if
err
!=
nil
{
if
err
!=
nil
{
panic
(
fmt
.
Errorf
(
"dumpb: Non-picklable object %#v: %v"
,
obj
,
err
))
return
err
}
}
return
buf
.
Bytes
()
if
writeData
&&
!
d
.
HashOnly
{
_
,
err
=
d
.
W
.
Write
(
datai
.
Data
)
if
err
!=
nil
{
return
err
}
_
,
err
=
d
.
W
.
Write
([]
byte
(
"
\n
"
))
if
err
!=
nil
{
return
err
}
}
// XXX ^^^ add oid: %v as prefix for err
return
nil
}
}
/*
// DumpTxn dumps one transaction record
// normalizeExtPy normalizes ZODB extension to the form zodbdump/py would print it.
func
(
d
*
dumper
)
DumpTxn
(
txni
*
zodb
.
TxnInfo
,
dataIter
zodb
.
IStorageRecordIterator
)
error
{
// specifically the dictionary pickle inside is analyzed and then ... XXX
// LF in-between txn records
func normalizeExtPy(ext []byte) []byte {
vskip
:=
"
\n
"
// unpickle ext
if
!
d
.
afterFirst
{
r := bytes.NewBuffer(ext)
vskip
=
""
p := pickle.NewDecoder(r)
d
.
afterFirst
=
true
xv, _ := p.Decode()
}
v, ok := xv.(map[interface{}]interface{})
_
,
err
:=
fmt
.
Fprintf
(
d
.
W
,
"%stxn %s (%c)
\n
user %q
\n
description %q
\n
extension %q
\n
"
,
vskip
,
txni
.
Tid
,
txni
.
Status
,
txni
.
User
,
txni
.
Description
,
txni
.
Extension
)
if
err
!=
nil
{
return
err
}
// data records
for
{
datai
,
err
:=
dataIter
.
NextData
()
if
err
!=
nil
{
if
err
==
io
.
EOF
{
err
=
nil
// XXX -> okEOF ?
}
break
}
err
=
d
.
DumpData
(
datai
)
if
err
!=
nil
{
break
}
}
// on any error (e.g. ext is not pickle at all) or if it was not dict return original
if
err
!=
nil
{
if !ok {
return
fmt
.
Errorf
(
"%v: %v"
,
txni
.
Tid
,
err
)
return ext
}
}
keyv := make([]*struct{key interface{}; kpickle []byte}, len(v))
return
nil
for i, key := range v {
}
keyv[i].key = key
// NOTE key was created by pickle.Decoder - it must be picklable
// Dump dumps transaction records in between tidMin..tidMax
keyv[i].kpickle = dumpb(key)
func
(
d
*
dumper
)
Dump
(
stor
zodb
.
IStorage
,
tidMin
,
tidMax
zodb
.
Tid
)
error
{
var
txni
*
zodb
.
TxnInfo
var
dataIter
zodb
.
IStorageRecordIterator
var
err
error
iter
:=
stor
.
Iterate
(
tidMin
,
tidMax
)
// transactions
for
{
txni
,
dataIter
,
err
=
iter
.
NextTxn
()
if
err
!=
nil
{
if
err
==
io
.
EOF
{
err
=
nil
// XXX -> okEOF ?
}
break
}
err
=
d
.
DumpTxn
(
txni
,
dataIter
)
if
err
!=
nil
{
break
}
}
}
if
err
!=
nil
{
return
fmt
.
Errorf
(
"%s: dumping %v..%v: %v"
,
stor
,
tidMin
,
tidMax
,
err
)
}
return
nil
}
}
*/
// zodbDump dumps contents of a storage in between tidMin..tidMax range to a writer.
// zodbDump dumps contents of a storage in between tidMin..tidMax range to a writer.
// see top-level documentation for the dump format.
// see top-level documentation for the dump format.
func
zodbDump
(
w
io
.
Writer
,
stor
zodb
.
IStorage
,
tidMin
,
tidMax
zodb
.
Tid
,
hashOnly
bool
)
error
{
d
:=
dumper
{
W
:
w
,
HashOnly
:
hashOnly
}
return
d
.
Dump
(
stor
,
tidMin
,
tidMax
)
}
/*
func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly bool) error {
func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly bool) error {
var retErr error
var retErr error
iter := stor.Iterate(tidMin, tidMax)
iter := stor.Iterate(tidMin, tidMax)
...
@@ -106,7 +189,7 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
...
@@ -106,7 +189,7 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
}
}
retErr = err
retErr = err
goto
out
break
}
}
// LF in-between txn records
// LF in-between txn records
...
@@ -119,6 +202,7 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
...
@@ -119,6 +202,7 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
_, err = fmt.Fprintf(w, "%stxn %s (%c)\nuser %q\ndescription %q\nextension %q\n",
_, err = fmt.Fprintf(w, "%stxn %s (%c)\nuser %q\ndescription %q\nextension %q\n",
vskip, txni.Tid, txni.Status, txni.User, txni.Description, txni.Extension)
vskip, txni.Tid, txni.Status, txni.User, txni.Description, txni.Extension)
if err != nil {
if err != nil {
retErr = err
break
break
}
}
...
@@ -153,18 +237,21 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
...
@@ -153,18 +237,21 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
entry += "\n"
entry += "\n"
_, err = w.Write(mem.Bytes(entry))
_, err = w.Write(mem.Bytes(entry))
if err != nil {
if err != nil {
break
retErr = err
goto out
}
}
if !hashOnly && writeData {
if !hashOnly && writeData {
_, err = w.Write(datai.Data)
_, err = w.Write(datai.Data)
if err != nil {
if err != nil {
break
retErr = err
goto out
}
}
_, err = w.Write([]byte("\n"))
_, err = w.Write([]byte("\n"))
if err != nil {
if err != nil {
break
retErr = err
goto out
}
}
}
}
}
}
...
@@ -172,11 +259,12 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
...
@@ -172,11 +259,12 @@ func zodbDump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly
out:
out:
if retErr != nil {
if retErr != nil {
return
fmt
.
Errorf
(
"%s: dump %v..%v: %v"
,
stor
,
tidMin
,
tidMax
,
retErr
)
return fmt.Errorf("%s: dump
ing
%v..%v: %v", stor, tidMin, tidMax, retErr)
}
}
return nil
return nil
}
}
*/
func
usage
()
{
func
usage
()
{
fmt
.
Fprintf
(
os
.
Stderr
,
fmt
.
Fprintf
(
os
.
Stderr
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment