Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
neoppod
Commits
c38ab95d
Commit
c38ab95d
authored
7 years ago
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
d1866974
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
69 additions
and
18 deletions
+69
-18
t/neo/storage/fs1/cmd/fstail/fstail.go
t/neo/storage/fs1/cmd/fstail/fstail.go
+58
-17
t/neo/storage/fs1/filestorage.go
t/neo/storage/fs1/filestorage.go
+11
-1
No files found.
t/neo/storage/fs1/cmd/fstail/fstail.go
View file @
c38ab95d
...
@@ -24,55 +24,96 @@ package main
...
@@ -24,55 +24,96 @@ package main
import
(
import
(
"crypto/sha1"
"crypto/sha1"
"flag"
"flag"
"fmt"
"io"
"log"
"os"
"../../../../storage/fs1"
)
)
fsDump
(
w
io
.
Writer
,
path
string
,
ntxn
int
)
error
{
func
fsDump
(
w
io
.
Writer
,
path
string
,
ntxn
int
)
(
err
error
)
{
// path & fsdump on error context
defer
func
()
{
if
err
!=
nil
{
err
=
fmt
.
Errorf
(
"%s: fsdump: %v"
,
err
)
}
}()
// we are not using fs1.Open here, since the file could be e.g. corrupt
// we are not using fs1.Open here, since the file could be e.g. corrupt
// (same logic as in fstail/py)
// (same logic as in fstail/py)
f
,
err
:=
os
.
Open
(
path
)
f
,
err
:=
os
.
Open
(
path
)
// XXX err
if
err
!=
nil
{
return
err
}
// TODO get fSize
// get file size as topPos
fi
,
err
:=
f
.
Stat
()
if
err
!=
nil
{
return
err
}
topPos
:=
fi
.
Size
()
// txn header & data buffer for iterating
txnh
:=
fs1
.
TxnHeader
{}
txnh
:=
fs1
.
TxnHeader
{}
data
:=
[]
byte
{}
// start iterating at tail.
// start iterating at tail.
// this should get EOF but read txnh.LenPrev ok.
// this should get EOF but read txnh.LenPrev ok.
err
=
txnh
.
Load
(
f
,
fSize
,
LoadAll
)
err
=
txnh
.
Load
(
f
,
topPos
,
fs1
.
LoadAll
)
if
err
!=
io
.
EOF
{
if
err
!=
io
.
EOF
{
if
err
==
nil
{
if
err
==
nil
{
// XXX or allow this?
// XXX or allow this?
// though then, if we start not from a txn boundary - there are
// though then, if we start not from a txn boundary - there are
// high chances further iteration will go wrong.
// high chances further iteration will go wrong.
err
=
fmt
.
Errorf
(
"%s @%v: reading past file end was unexpectedly successful: probably the file is being modified simultaneously"
,
err
=
fmt
.
Errorf
(
"@%v: reading past file end was unexpectedly successful: probably the file is being modified simultaneously"
,
topPos
)
path
,
fSize
)
}
}
// otherwise err already has the context
// otherwise err already has the context
return
err
return
err
}
}
if
txnh
.
LenPrev
<=
0
{
if
txnh
.
LenPrev
<=
0
{
return
fmt
.
Errorf
(
"
%s @%v: previous record could not be read"
,
path
,
fSize
)
return
fmt
.
Errorf
(
"
@%v: previous record could not be read"
,
topPos
)
}
}
// now loop loading previous txn until EOF / ntxn limit
// now loop loading previous txn until EOF / ntxn limit
for
{
for
i
:=
ntxn
;
i
>
0
;
i
--
{
err
=
txnh
.
LoadPrev
(
fs1
.
LoadAll
)
err
=
txnh
.
LoadPrev
(
f
,
f
s1
.
LoadAll
)
if
err
!=
nil
{
if
err
!=
nil
{
if
err
==
io
.
EOF
{
if
err
==
io
.
EOF
{
err
=
nil
err
=
nil
// XXX -> okEOF(err)
}
}
break
break
}
}
// TODO read raw data (get_raw_data) -> sha1
// read raw data inside transaction record
// txnh.Tid.TimeStamp() ? XXX -> PerTimeStamp ? (get_timestamp)
dataLen
:=
txnh
.
DataLen
()
if
int64
(
cap
(
data
))
<
dataLen
{
data
=
make
([]
byte
,
dataLen
)
}
else
{
data
=
data
[
:
dataLen
]
}
//"user=%'q description=%'q length=%d offset=%d (+%d)", txnh.User, txnh.Description, // .Len, .offset ...
_
,
err
=
f
.
ReadAt
(
data
,
txnh
.
DataPos
())
if
err
!=
nil
{
// XXX -> txnh.Err(...) ?
// XXX err = noEOF(err)
err
=
&
fs1
.
ErrTxnRecord
{
txnh
.
Pos
,
"read data payload"
,
err
}
break
}
}
// print information about read txn record
_
,
err
=
fmt
.
Fprintf
(
w
,
"%s: hash=%x
\n
user=%q description=%q length=%d offset=%d (+%d)
\n\n
"
,
txnh
.
Tid
.
Time
(),
sha1
.
Sum
(
data
),
txnh
.
User
,
txnh
.
Description
,
// NOTE in zodb/py .length is len - 8, in zodb/go - whole txn record length
txnh
.
Len
-
8
,
txnh
.
Pos
,
txnh
.
HeaderLen
())
if
err
!=
nil
{
if
err
!=
nil
{
// TODO
break
}
}
}
return
err
}
}
func
usage
()
{
func
usage
()
{
...
@@ -101,7 +142,7 @@ func main() {
...
@@ -101,7 +142,7 @@ func main() {
}
}
storPath
:=
argv
[
0
]
storPath
:=
argv
[
0
]
err
=
fsDump
(
os
.
Stdout
,
storPath
,
n
)
err
:
=
fsDump
(
os
.
Stdout
,
storPath
,
n
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Fatal
(
err
)
log
.
Fatal
(
err
)
}
}
...
...
This diff is collapsed.
Click to expand it.
t/neo/storage/fs1/filestorage.go
View file @
c38ab95d
...
@@ -174,6 +174,16 @@ func (txnh *TxnHeader) HeaderLen() int64 {
...
@@ -174,6 +174,16 @@ func (txnh *TxnHeader) HeaderLen() int64 {
return
TxnHeaderFixSize
+
int64
(
len
(
txnh
.
workMem
))
return
TxnHeaderFixSize
+
int64
(
len
(
txnh
.
workMem
))
}
}
// DataPos returns start position of data inside transaction record
func
(
txnh
*
TxnHeader
)
DataPos
()
int64
{
return
txnh
.
Pos
+
txnh
.
HeaderLen
()
}
// DataLen returns length of all data inside transaction record container
func
(
txnh
*
TxnHeader
)
DataLen
()
int64
{
return
txnh
.
Len
-
txnh
.
HeaderLen
()
-
8
/* trailer redundant length */
}
// CloneFrom copies txnh2 to txnh making sure underlying slices (.workMem .User
// CloneFrom copies txnh2 to txnh making sure underlying slices (.workMem .User
// .Desc ...) are not shared.
// .Desc ...) are not shared.
func
(
txnh
*
TxnHeader
)
CloneFrom
(
txnh2
*
TxnHeader
)
{
func
(
txnh
*
TxnHeader
)
CloneFrom
(
txnh2
*
TxnHeader
)
{
...
@@ -899,7 +909,7 @@ func (fsi *iterator) NextTxn() (*zodb.TxnInfo, zodb.IStorageRecordIterator, erro
...
@@ -899,7 +909,7 @@ func (fsi *iterator) NextTxn() (*zodb.TxnInfo, zodb.IStorageRecordIterator, erro
}
}
// set .dataIter to iterate over .txnIter.Txnh
// set .dataIter to iterate over .txnIter.Txnh
fsi
.
dataIter
.
Datah
.
Pos
=
fsi
.
txnIter
.
Txnh
.
Pos
+
fsi
.
txnIter
.
Txnh
.
HeaderLen
()
fsi
.
dataIter
.
Datah
.
Pos
=
fsi
.
txnIter
.
Txnh
.
DataPos
()
fsi
.
dataIter
.
Datah
.
DataLen
=
-
DataHeaderSize
// first iteration will go to first data record
fsi
.
dataIter
.
Datah
.
DataLen
=
-
DataHeaderSize
// first iteration will go to first data record
return
&
fsi
.
txnIter
.
Txnh
.
TxnInfo
,
&
fsi
.
dataIter
,
nil
return
&
fsi
.
txnIter
.
Txnh
.
TxnInfo
,
&
fsi
.
dataIter
,
nil
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment