Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZODB
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
ZODB
Commits
4128d7ae
Commit
4128d7ae
authored
Jan 16, 2004
by
Jeremy Hylton
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add test and fix for the redundant pack bug.
Also remove some unused imports and a somewhat irrelevant comment.
parent
31fa209f
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
76 additions
and
11 deletions
+76
-11
src/ZODB/FileStorage/fspack.py
src/ZODB/FileStorage/fspack.py
+29
-8
src/ZODB/tests/PackableStorage.py
src/ZODB/tests/PackableStorage.py
+47
-3
No files found.
src/ZODB/FileStorage/fspack.py
View file @
4128d7ae
...
@@ -24,17 +24,10 @@ from the revision of the root at that time or if it is reachable from
...
@@ -24,17 +24,10 @@ from the revision of the root at that time or if it is reachable from
a backpointer after that time.
a backpointer after that time.
"""
"""
# This module contains code backported from ZODB4 from the
# zodb.storage.file package. It's been edited heavily to work with
# ZODB3 code and storage layout.
import
os
import
os
import
struct
from
types
import
StringType
from
ZODB.referencesf
import
referencesf
from
ZODB.referencesf
import
referencesf
from
ZODB.utils
import
p64
,
u64
,
z64
,
oid_repr
from
ZODB.utils
import
p64
,
u64
,
z64
from
zLOG
import
LOG
,
BLATHER
,
WARNING
,
ERROR
,
PANIC
from
ZODB.fsIndex
import
fsIndex
from
ZODB.fsIndex
import
fsIndex
from
ZODB.FileStorage.format
\
from
ZODB.FileStorage.format
\
...
@@ -232,11 +225,19 @@ class GC(FileStorageFormatter):
...
@@ -232,11 +225,19 @@ class GC(FileStorageFormatter):
def
buildPackIndex
(
self
):
def
buildPackIndex
(
self
):
pos
=
4L
pos
=
4L
# We make the initial assumption that the database has been
# packed before and set unpacked to True only after seeing the
# first record with a status == " ". If we get to the packtime
# and unpacked is still False, we need to watch for a redundant
# pack.
unpacked
=
False
while
pos
<
self
.
eof
:
while
pos
<
self
.
eof
:
th
=
self
.
_read_txn_header
(
pos
)
th
=
self
.
_read_txn_header
(
pos
)
if
th
.
tid
>
self
.
packtime
:
if
th
.
tid
>
self
.
packtime
:
break
break
self
.
checkTxn
(
th
,
pos
)
self
.
checkTxn
(
th
,
pos
)
if
th
.
status
!=
"p"
:
unpacked
=
True
tpos
=
pos
tpos
=
pos
end
=
pos
+
th
.
tlen
end
=
pos
+
th
.
tlen
...
@@ -260,6 +261,25 @@ class GC(FileStorageFormatter):
...
@@ -260,6 +261,25 @@ class GC(FileStorageFormatter):
self
.
packpos
=
pos
self
.
packpos
=
pos
if
unpacked
:
return
# check for a redundant pack. If the first record following
# the newly computed packpos has status 'p', then it was
# packed earlier and the current pack is redudant.
try
:
th
=
self
.
_read_txn_header
(
pos
)
except
CorruptedDataError
,
err
:
if
err
.
buf
!=
""
:
raise
if
th
.
status
==
'p'
:
# Delay import to code with circular imports.
# XXX put exceptions in a separate module
from
ZODB.FileStorage.FileStorage
import
FileStorageError
print
"Yow!"
raise
FileStorageError
(
"The database has already been packed to a later time"
" or no changes have been made since the last pack"
)
def
findReachableAtPacktime
(
self
,
roots
):
def
findReachableAtPacktime
(
self
,
roots
):
"""Mark all objects reachable from the oids in roots as reachable."""
"""Mark all objects reachable from the oids in roots as reachable."""
todo
=
list
(
roots
)
todo
=
list
(
roots
)
...
@@ -645,3 +665,4 @@ class FileStoragePacker(FileStorageFormatter):
...
@@ -645,3 +665,4 @@ class FileStoragePacker(FileStorageFormatter):
if
self
.
_lock_counter
%
20
==
0
:
if
self
.
_lock_counter
%
20
==
0
:
self
.
_commit_lock_acquire
()
self
.
_commit_lock_acquire
()
return
ipos
return
ipos
src/ZODB/tests/PackableStorage.py
View file @
4128d7ae
...
@@ -29,10 +29,11 @@ import time
...
@@ -29,10 +29,11 @@ import time
from
ZODB
import
DB
from
ZODB
import
DB
from
persistent
import
Persistent
from
persistent
import
Persistent
from
persistent.mapping
import
PersistentMapping
from
ZODB.referencesf
import
referencesf
from
ZODB.referencesf
import
referencesf
from
ZODB.tests.MinPO
import
MinPO
from
ZODB.tests.MinPO
import
MinPO
from
ZODB.tests.StorageTestBase
import
snooze
from
ZODB.tests.StorageTestBase
import
snooze
from
ZODB.POSException
import
ConflictError
from
ZODB.POSException
import
ConflictError
,
StorageError
from
ZODB.tests.MTStorage
import
TestThread
from
ZODB.tests.MTStorage
import
TestThread
...
@@ -126,11 +127,10 @@ class PackableStorageBase:
...
@@ -126,11 +127,10 @@ class PackableStorageBase:
try
:
try
:
self
.
_storage
.
load
(
ZERO
,
''
)
self
.
_storage
.
load
(
ZERO
,
''
)
except
KeyError
:
except
KeyError
:
from
persistent
import
mapping
from
ZODB.Transaction
import
Transaction
from
ZODB.Transaction
import
Transaction
file
=
StringIO
()
file
=
StringIO
()
p
=
cPickle
.
Pickler
(
file
,
1
)
p
=
cPickle
.
Pickler
(
file
,
1
)
p
.
dump
((
mapping
.
PersistentMapping
,
None
))
p
.
dump
((
PersistentMapping
,
None
))
p
.
dump
({
'_container'
:
{}})
p
.
dump
({
'_container'
:
{}})
t
=
Transaction
()
t
=
Transaction
()
t
.
description
=
'initial database creation'
t
.
description
=
'initial database creation'
...
@@ -438,6 +438,50 @@ class PackableUndoStorage(PackableStorageBase):
...
@@ -438,6 +438,50 @@ class PackableUndoStorage(PackableStorageBase):
eq
(
root
[
'obj'
].
value
,
7
)
eq
(
root
[
'obj'
].
value
,
7
)
def
checkRedundantPack
(
self
):
# It is an error to perform a pack with a packtime earlier
# than a previous packtime. The storage can't do a full
# traversal as of the packtime, because the previous pack may
# have removed revisions necessary for a full traversal.
# It should be simple to test that a storage error is raised,
# but this test case goes to the trouble of constructing a
# scenario that would lose data if the earlier packtime was
# honored.
self
.
_initroot
()
db
=
DB
(
self
.
_storage
)
conn
=
db
.
open
()
root
=
conn
.
root
()
root
[
"d"
]
=
d
=
PersistentMapping
()
get_transaction
().
commit
()
snooze
()
obj
=
d
[
"obj"
]
=
C
()
obj
.
value
=
1
get_transaction
().
commit
()
snooze
()
packt1
=
time
.
time
()
lost_oid
=
obj
.
_p_oid
obj
=
d
[
"anotherobj"
]
=
C
()
obj
.
value
=
2
get_transaction
().
commit
()
snooze
()
packt2
=
time
.
time
()
db
.
pack
(
packt2
)
# BDBStorage allows the second pack, but doesn't lose data.
try
:
db
.
pack
(
packt1
)
except
StorageError
:
pass
# This object would be removed by the second pack, even though
# it is reachable.
self
.
_storage
.
load
(
lost_oid
,
""
)
def
checkPackUndoLog
(
self
):
def
checkPackUndoLog
(
self
):
self
.
_initroot
()
self
.
_initroot
()
# Create a `persistent' object
# Create a `persistent' object
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment