Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
3
Merge Requests
3
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
821efb55
Commit
821efb55
authored
Mar 12, 2024
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
wip
parent
f8e306b6
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
50 additions
and
24 deletions
+50
-24
tools/reflink
tools/reflink
+50
-24
No files found.
tools/reflink
View file @
821efb55
...
@@ -15,7 +15,7 @@ from msgpack import dumps, loads
...
@@ -15,7 +15,7 @@ from msgpack import dumps, loads
from
pkg_resources
import
iter_entry_points
from
pkg_resources
import
iter_entry_points
import
ZODB
import
ZODB
from
persistent.TimeStamp
import
TimeStamp
from
persistent.TimeStamp
import
TimeStamp
from
ZODB._compat
import
dumps
,
loads
,
HIGHEST_PROTOCOL
,
PersistentUnpickler
from
ZODB._compat
import
PersistentUnpickler
from
ZODB.POSException
import
ConflictError
,
POSKeyError
from
ZODB.POSException
import
ConflictError
,
POSKeyError
from
ZODB.serialize
import
referencesf
from
ZODB.serialize
import
referencesf
from
ZODB.utils
import
p64
,
u64
,
z64
from
ZODB.utils
import
p64
,
u64
,
z64
...
@@ -53,6 +53,9 @@ def openStorage(uri):
...
@@ -53,6 +53,9 @@ def openStorage(uri):
def
inc64
(
tid
):
def
inc64
(
tid
):
return
p64
(
u64
(
tid
)
+
1
)
return
p64
(
u64
(
tid
)
+
1
)
def
tidFromTime
(
t
):
return
TimeStamp
(
*
gmtime
(
t
)[:
5
]
+
(
t
%
60
,)).
raw
()
class
InvalidationListener
(
object
):
class
InvalidationListener
(
object
):
...
@@ -145,12 +148,19 @@ class Changeset(object):
...
@@ -145,12 +148,19 @@ class Changeset(object):
checkAPI
(
storage
)
checkAPI
(
storage
)
self
.
bootstrap
=
bootstrap
,
0
self
.
bootstrap
=
bootstrap
,
0
self
.
_last_gc
=
bucket
.
get
(
'__reflink_last_gc__'
,
z64
)
self
.
_last_gc
=
bucket
.
get
(
'__reflink_last_gc__'
,
z64
)
self
.
_last_pack
=
bucket
.
get
(
'__reflink_last_pack__'
,
z64
)
self
.
_pack
=
None
self
.
check_orphan
=
{}
self
.
check_orphan
=
{}
@
partial
(
property
,
attrgetter
(
'_last_gc'
))
@
partial
(
property
,
attrgetter
(
'_last_gc'
))
def
last_gc
(
self
,
value
):
def
last_gc
(
self
,
value
):
self
.
_last_gc
=
self
.
_get
(
z64
)[
'__reflink_last_gc__'
]
=
value
self
.
_last_gc
=
self
.
_get
(
z64
)[
'__reflink_last_gc__'
]
=
value
def
pack
(
self
,
tid
):
self
.
storage
.
app
.
setPackOrder
.
__call__
# check non-standard API
if
self
.
_last_pack
<
tid
:
self
.
_pack
=
tid
@
partial
(
property
,
attrgetter
(
'_bootstrap'
))
@
partial
(
property
,
attrgetter
(
'_bootstrap'
))
def
bootstrap
(
self
,
value
):
def
bootstrap
(
self
,
value
):
self
.
_bootstrap
=
value
self
.
_bootstrap
=
value
...
@@ -205,6 +215,11 @@ class Changeset(object):
...
@@ -205,6 +215,11 @@ class Changeset(object):
del
v
[
-
1
]
del
v
[
-
1
]
return
True
return
True
bootstrap
=
self
.
_bootstrap
bootstrap
=
self
.
_bootstrap
pack
=
self
.
_pack
if
pack
and
not
bootstrap
:
self
.
_pack
=
None
self
.
_last_pack
=
self
.
_get
(
z64
)[
'__reflink_last_pack__'
]
=
pack
storage
.
app
.
setPackOrder
(
txn
,
pack
)
for
oid
,
(
orig
,
serial
,
bucket
)
in
buckets
.
iteritems
():
for
oid
,
(
orig
,
serial
,
bucket
)
in
buckets
.
iteritems
():
base_oid
=
u64
(
oid
)
<<
8
base_oid
=
u64
(
oid
)
<<
8
data
=
{}
data
=
{}
...
@@ -453,24 +468,22 @@ def main(args=None):
...
@@ -453,24 +468,22 @@ def main(args=None):
" argument is not 0, a GC calculates twice the list of OIDs to"
" argument is not 0, a GC calculates twice the list of OIDs to"
" delete, now and in the past, to exclude those that should be"
" delete, now and in the past, to exclude those that should be"
" kept."
%
extra_help
)
" kept."
%
extra_help
)
def
tid
(
**
kw
):
tid_arg
=
dict
(
metavar
=
"TID"
,
type
=
eval
,
help
=
"TID as Python integer."
)
s
.
add_argument
(
'tid'
,
metavar
=
"TID"
,
type
=
eval
,
help
=
"TID as Python integer."
,
**
kw
)
s
=
parsers
.
add_parser
(
'bootstrap'
,
_
=
parsers
.
add_parser
(
'bootstrap'
,
help
=
"By default, all transactions since the creation of main DB are"
help
=
"By default, all transactions since the creation of main DB are"
" scanned. In the case where such history would take too long to"
" scanned. In the case where such history would take too long to"
" process, you can try this command so that the 'run' command"
" process, you can try this command so that the 'run' command"
" starts with 2 extra steps: scan the whole main DB at the given"
" starts with 2 extra steps: scan the whole main DB at the given"
" TID and do a full GC. Both DB must provide a non-standard API"
" TID and do a full GC. Both DB must provide a non-standard API"
" that only NEO is known to have."
,
" that only NEO is known to have."
,
**
kw
)
**
kw
)
.
add_argument
tid
(
)
_
(
'tid'
,
**
tid_arg
)
s
=
parsers
.
add_parser
(
'dump'
,
_
=
parsers
.
add_parser
(
'dump'
,
help
=
"Dump the whole reflink DB to stdout."
,
help
=
"Dump the whole reflink DB to stdout."
,
**
kw
)
**
kw
)
.
add_argument
tid
(
nargs
=
'?'
)
_
(
'tid'
,
nargs
=
'?'
,
**
tid_arg
)
s
=
parsers
.
add_parser
(
'gc'
,
s
=
parsers
.
add_parser
(
'gc'
,
help
=
"Do a GC immediately, ignoring new transactions, and exit."
,
help
=
"Do a GC immediately, ignoring new transactions, and exit."
,
...
@@ -481,6 +494,7 @@ def main(args=None):
...
@@ -481,6 +494,7 @@ def main(args=None):
_
=
parsers
.
add_parser
(
'path'
,
_
=
parsers
.
add_parser
(
'path'
,
help
=
"Get ancestors of an OID."
,
help
=
"Get ancestors of an OID."
,
**
kw
).
add_argument
**
kw
).
add_argument
_
(
'--tid'
,
**
tid_arg
)
_
(
'oid'
,
metavar
=
"OID"
,
type
=
eval
,
_
(
'oid'
,
metavar
=
"OID"
,
type
=
eval
,
help
=
"OID as Python integer."
)
help
=
"OID as Python integer."
)
_
(
'main'
,
metavar
=
"MAIN_URI"
,
nargs
=
'?'
,
_
(
'main'
,
metavar
=
"MAIN_URI"
,
nargs
=
'?'
,
...
@@ -499,6 +513,12 @@ def main(args=None):
...
@@ -499,6 +513,12 @@ def main(args=None):
_
=
s
.
add_argument
_
=
s
.
add_argument
_
(
'-i'
,
'--commit-interval'
,
type
=
float
,
metavar
=
"SECONDS"
,
default
=
10
,
_
(
'-i'
,
'--commit-interval'
,
type
=
float
,
metavar
=
"SECONDS"
,
default
=
10
,
help
=
"Commit every SECONDS of work."
)
help
=
"Commit every SECONDS of work."
)
_
(
'--pack-neo'
,
type
=
float
,
metavar
=
"EPOCH"
,
help
=
"Pack time in seconds since the epoch. This argument is ignored"
" during bootstrap and it is only to pack the refs DB when it is"
" run by NEO. Other IStorage implementations don't store pack"
" commands in transactions and pack() can be used as long as it's"
" done without GC."
)
period
(
86400
,
period
(
86400
,
" For performance reasons, this revision won't be older than the"
" For performance reasons, this revision won't be older than the"
" previous GC commit so GCs may be delayed this number of seconds."
)
" previous GC commit so GCs may be delayed this number of seconds."
)
...
@@ -518,7 +538,7 @@ def main(args=None):
...
@@ -518,7 +538,7 @@ def main(args=None):
if
bootstrap
:
if
bootstrap
:
changeset
.
commit
(
inc64
(
tid
))
changeset
.
commit
(
inc64
(
tid
))
print
(
"Bootstrap at %s. You can now use the 'run' command."
print
(
"Bootstrap at %s
UTC
. You can now use the 'run' command."
%
TimeStamp
(
p64
(
args
.
tid
)))
%
TimeStamp
(
p64
(
args
.
tid
)))
return
return
...
@@ -530,9 +550,15 @@ def main(args=None):
...
@@ -530,9 +550,15 @@ def main(args=None):
return
"Main storage shall implement "
+
iface
.
__name__
return
"Main storage shall implement "
+
iface
.
__name__
if
command
==
"path"
:
if
command
==
"path"
:
t
=
args
.
tid
if
t
is
None
:
tid
=
inc64
(
tid
)
tid
=
inc64
(
tid
)
else
:
tid
=
p64
(
t
+
1
)
t
=
p64
(
t
)
def
find_global
(
*
args
):
def
find_global
(
*
args
):
x
.
extend
(
args
)
x
.
extend
(
args
)
with
changeset
.
historical
(
t
):
for
oid
in
reversed
(
changeset
.
path
(
p64
(
args
.
oid
))):
for
oid
in
reversed
(
changeset
.
path
(
p64
(
args
.
oid
))):
x
=
[
hex
(
u64
(
oid
))]
x
=
[
hex
(
u64
(
oid
))]
if
args
.
main
:
if
args
.
main
:
...
@@ -555,6 +581,9 @@ def main(args=None):
...
@@ -555,6 +581,9 @@ def main(args=None):
parser
.
error
(
"--commit-interval must be strictly positive."
)
parser
.
error
(
"--commit-interval must be strictly positive."
)
exit_before_gc
=
args
.
exit_before_gc
exit_before_gc
=
args
.
exit_before_gc
exit_after_gc
=
args
.
exit_after_gc
exit_after_gc
=
args
.
exit_after_gc
t
=
args
.
pack_neo
if
t
:
changeset
.
pack
(
tidFromTime
(
t
))
period
=
args
.
period
period
=
args
.
period
if
period
<
0
:
if
period
<
0
:
parser
.
error
(
"--period must be positive."
)
parser
.
error
(
"--period must be positive."
)
...
@@ -742,11 +771,8 @@ def main(args=None):
...
@@ -742,11 +771,8 @@ def main(args=None):
break
break
print
(
'gc'
)
print
(
'gc'
)
gc
=
changeset
.
full
if
full
or
bootstrap
else
changeset
.
orphans
gc
=
changeset
.
full
if
full
or
bootstrap
else
changeset
.
orphans
if
bootstrap
or
not
period
:
gc_tid
=
tid
if
bootstrap
or
not
period
else
\
gc_tid
=
tid
tidFromTime
(
time
()
-
period
)
else
:
t
=
time
()
-
period
gc_tid
=
TimeStamp
(
*
gmtime
(
t
)[:
5
]
+
(
t
%
60
,)).
raw
()
if
gc_tid
<
tid
:
if
gc_tid
<
tid
:
if
gc_tid
<
changeset
.
last_gc
:
if
gc_tid
<
changeset
.
last_gc
:
logging
.
warning
(
logging
.
warning
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment