Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZODB
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
ZODB
Commits
eef59eb3
Commit
eef59eb3
authored
Jan 05, 2008
by
Jim Fulton
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Removed lots of version support. More version support, especially in
ZEO, still needs to be removed.
parent
7b044c7d
Changes
29
Show whitespace changes
Inline
Side-by-side
Showing
29 changed files
with
137 additions
and
1847 deletions
+137
-1847
src/ZEO/tests/Cache.py
src/ZEO/tests/Cache.py
+0
-48
src/ZEO/tests/InvalidationTests.py
src/ZEO/tests/InvalidationTests.py
+0
-68
src/ZEO/tests/testAuth.py
src/ZEO/tests/testAuth.py
+3
-3
src/ZEO/tests/testConversionSupport.py
src/ZEO/tests/testConversionSupport.py
+2
-3
src/ZEO/tests/testZEO.py
src/ZEO/tests/testZEO.py
+2
-22
src/ZEO/tests/test_cache.py
src/ZEO/tests/test_cache.py
+1
-1
src/ZODB/BaseStorage.py
src/ZODB/BaseStorage.py
+1
-1
src/ZODB/FileStorage/FileStorage.py
src/ZODB/FileStorage/FileStorage.py
+54
-334
src/ZODB/FileStorage/format.py
src/ZODB/FileStorage/format.py
+12
-84
src/ZODB/FileStorage/fsdump.py
src/ZODB/FileStorage/fsdump.py
+5
-17
src/ZODB/FileStorage/fsoids.py
src/ZODB/FileStorage/fsoids.py
+2
-3
src/ZODB/FileStorage/fspack.py
src/ZODB/FileStorage/fspack.py
+31
-151
src/ZODB/fsrecover.py
src/ZODB/fsrecover.py
+2
-2
src/ZODB/fstools.py
src/ZODB/fstools.py
+9
-21
src/ZODB/tests/BasicStorage.py
src/ZODB/tests/BasicStorage.py
+2
-36
src/ZODB/tests/HistoryStorage.py
src/ZODB/tests/HistoryStorage.py
+2
-170
src/ZODB/tests/IteratorStorage.py
src/ZODB/tests/IteratorStorage.py
+1
-30
src/ZODB/tests/MTStorage.py
src/ZODB/tests/MTStorage.py
+0
-11
src/ZODB/tests/PersistentStorage.py
src/ZODB/tests/PersistentStorage.py
+0
-6
src/ZODB/tests/ReadOnlyStorage.py
src/ZODB/tests/ReadOnlyStorage.py
+0
-7
src/ZODB/tests/RecoveryStorage.py
src/ZODB/tests/RecoveryStorage.py
+1
-99
src/ZODB/tests/StorageTestBase.py
src/ZODB/tests/StorageTestBase.py
+4
-25
src/ZODB/tests/Synchronization.py
src/ZODB/tests/Synchronization.py
+1
-31
src/ZODB/tests/TransactionalUndoVersionStorage.py
src/ZODB/tests/TransactionalUndoVersionStorage.py
+0
-195
src/ZODB/tests/VersionStorage.py
src/ZODB/tests/VersionStorage.py
+0
-406
src/ZODB/tests/testConnection.py
src/ZODB/tests/testConnection.py
+1
-5
src/ZODB/tests/testDemoStorage.py
src/ZODB/tests/testDemoStorage.py
+1
-21
src/ZODB/tests/testFileStorage.py
src/ZODB/tests/testFileStorage.py
+0
-42
src/ZODB/tests/testZODB.py
src/ZODB/tests/testZODB.py
+0
-5
No files found.
src/ZEO/tests/Cache.py
View file @
eef59eb3
...
...
@@ -52,51 +52,3 @@ class TransUndoStorageWithCache:
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
obj
=
zodb_unpickle
(
data
)
assert
obj
==
MinPO
(
24
)
class
StorageWithCache
:
def
checkAbortVersionInvalidation
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
1
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
2
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
3
),
version
=
"foo"
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
4
),
version
=
"foo"
)
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
abortVersion
(
"foo"
,
t
)
self
.
_storage
.
load
(
oid
,
"foo"
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
"foo"
)
obj
=
zodb_unpickle
(
data
)
assert
obj
==
MinPO
(
2
),
obj
def
checkCommitEmptyVersionInvalidation
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
1
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
2
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
3
),
version
=
"foo"
)
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
commitVersion
(
"foo"
,
""
,
t
)
self
.
_storage
.
load
(
oid
,
""
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
""
)
obj
=
zodb_unpickle
(
data
)
assert
obj
==
MinPO
(
3
),
obj
def
checkCommitVersionInvalidation
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
1
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
2
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
3
),
version
=
"foo"
)
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
commitVersion
(
"foo"
,
"bar"
,
t
)
self
.
_storage
.
load
(
oid
,
""
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
"bar"
)
obj
=
zodb_unpickle
(
data
)
assert
obj
==
MinPO
(
3
),
obj
src/ZEO/tests/InvalidationTests.py
View file @
eef59eb3
...
...
@@ -234,74 +234,6 @@ class LargeUpdatesThread(FailableThread):
self
.
added_keys
=
keys_added
.
keys
()
cn
.
close
()
class
VersionStressThread
(
FailableThread
):
def
__init__
(
self
,
db
,
stop
,
threadnum
,
commitdict
,
startnum
,
step
=
2
,
sleep
=
None
):
TestThread
.
__init__
(
self
)
self
.
db
=
db
self
.
stop
=
stop
self
.
threadnum
=
threadnum
self
.
startnum
=
startnum
self
.
step
=
step
self
.
sleep
=
sleep
self
.
added_keys
=
[]
self
.
commitdict
=
commitdict
def
_testrun
(
self
):
commit
=
0
key
=
self
.
startnum
while
not
self
.
stop
.
isSet
():
version
=
"%s:%s"
%
(
self
.
threadnum
,
key
)
commit
=
not
commit
if
self
.
oneupdate
(
version
,
key
,
commit
):
self
.
added_keys
.
append
(
key
)
self
.
commitdict
[
self
]
=
1
key
+=
self
.
step
def
oneupdate
(
self
,
version
,
key
,
commit
=
1
):
# The mess of sleeps below were added to reduce the number
# of VersionLockErrors, based on empirical observation.
# It looks like the threads don't switch enough without
# the sleeps.
cn
=
self
.
db
.
open
(
version
)
while
not
self
.
stop
.
isSet
():
try
:
tree
=
cn
.
root
()[
"tree"
]
break
except
(
ConflictError
,
KeyError
):
transaction
.
abort
()
while
not
self
.
stop
.
isSet
():
try
:
tree
[
key
]
=
self
.
threadnum
transaction
.
commit
()
if
self
.
sleep
:
time
.
sleep
(
self
.
sleep
)
break
except
(
VersionLockError
,
ReadConflictError
,
ConflictError
),
msg
:
transaction
.
abort
()
if
self
.
sleep
:
time
.
sleep
(
self
.
sleep
)
try
:
while
not
self
.
stop
.
isSet
():
try
:
if
commit
:
self
.
db
.
commitVersion
(
version
)
transaction
.
get
().
note
(
"commit version %s"
%
version
)
else
:
self
.
db
.
abortVersion
(
version
)
transaction
.
get
().
note
(
"abort version %s"
%
version
)
transaction
.
commit
()
if
self
.
sleep
:
time
.
sleep
(
self
.
sleep
)
return
commit
except
ConflictError
,
msg
:
transaction
.
abort
()
finally
:
cn
.
close
()
return
0
class
InvalidationTests
:
level
=
2
...
...
src/ZEO/tests/testAuth.py
View file @
eef59eb3
...
...
@@ -82,7 +82,7 @@ class AuthTest(CommonSetupTearDown):
self
.
_storage
.
_connection
.
poll
()
self
.
assert_
(
self
.
_storage
.
is_connected
())
# Make a call to make sure the mechanism is working
self
.
_storage
.
versions
()
self
.
_storage
.
undoInfo
()
def
testNOK
(
self
):
self
.
_storage
=
self
.
openClientStorage
(
wait
=
0
,
username
=
"foo"
,
...
...
@@ -101,11 +101,11 @@ class AuthTest(CommonSetupTearDown):
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self
.
wait
()
self
.
_storage
.
versions
()
self
.
_storage
.
undoInfo
()
# Manually clear the state of the hmac connection
self
.
_storage
.
_connection
.
_SizedMessageAsyncConnection__hmac_send
=
None
# Once the client stops using the hmac, it should be disconnected.
self
.
assertRaises
(
ClientDisconnected
,
self
.
_storage
.
versions
)
self
.
assertRaises
(
ClientDisconnected
,
self
.
_storage
.
undoInfo
)
class
PlainTextAuth
(
AuthTest
):
...
...
src/ZEO/tests/testConversionSupport.py
View file @
eef59eb3
...
...
@@ -17,10 +17,9 @@ from zope.testing import doctest
class
FakeStorageBase
:
def
__getattr__
(
self
,
name
):
if
name
in
(
'versionEmpty'
,
'versions'
,
'getTid'
,
'history'
,
'load'
,
'loadSerial'
,
'modifiedInVersion'
,
if
name
in
(
'getTid'
,
'history'
,
'load'
,
'loadSerial'
,
'lastTransaction'
,
'getSize'
,
'getName'
,
'supportsUndo'
,
'
supportsVersions'
,
'
tpc_transaction'
):
'tpc_transaction'
):
return
lambda
*
a
,
**
k
:
None
raise
AttributeError
(
name
)
...
...
src/ZEO/tests/testZEO.py
View file @
eef59eb3
...
...
@@ -38,8 +38,8 @@ import persistent
import
transaction
# ZODB test mixin classes
from
ZODB.tests
import
StorageTestBase
,
BasicStorage
,
VersionStorage
,
\
TransactionalUndoStorage
,
TransactionalUndoVersionStorage
,
\
from
ZODB.tests
import
StorageTestBase
,
BasicStorage
,
\
TransactionalUndoStorage
,
\
PackableStorage
,
Synchronization
,
ConflictResolution
,
RevisionStorage
,
\
MTStorage
,
ReadOnlyStorage
...
...
@@ -223,7 +223,6 @@ class GenericTests(
class
FullGenericTests
(
GenericTests
,
Cache
.
StorageWithCache
,
Cache
.
TransUndoStorageWithCache
,
CommitLockTests
.
CommitLockUndoTests
,
ConflictResolution
.
ConflictResolvingStorage
,
...
...
@@ -231,8 +230,6 @@ class FullGenericTests(
PackableStorage
.
PackableUndoStorage
,
RevisionStorage
.
RevisionStorage
,
TransactionalUndoStorage
.
TransactionalUndoStorage
,
TransactionalUndoVersionStorage
.
TransactionalUndoVersionStorage
,
VersionStorage
.
VersionStorage
,
):
"""Extend GenericTests with tests that MappingStorage can't pass."""
...
...
@@ -256,8 +253,6 @@ class MappingStorageTests(GenericTests):
class
DemoStorageTests
(
GenericTests
,
Cache
.
StorageWithCache
,
VersionStorage
.
VersionStorage
,
):
def
getConfig
(
self
):
...
...
@@ -269,21 +264,6 @@ class DemoStorageTests(
</demostorage>
"""
%
tempfile
.
mktemp
()
def
checkLoadBeforeVersion
(
self
):
# Doesn't implement loadBefore, except as a kind of place holder.
pass
# the next three pack tests depend on undo
def
checkPackVersionReachable
(
self
):
pass
def
checkPackVersions
(
self
):
pass
def
checkPackVersionsInPast
(
self
):
pass
class
HeartbeatTests
(
ZEO
.
tests
.
ConnectionTests
.
CommonSetupTearDown
):
"""Make sure a heartbeat is being sent and that it does no harm
...
...
src/ZEO/tests/test_cache.py
View file @
eef59eb3
...
...
@@ -11,7 +11,7 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Basic unit tests for a
multi-version
client cache."""
"""Basic unit tests for a client cache."""
import
os
import
random
...
...
src/ZODB/BaseStorage.py
View file @
eef59eb3
...
...
@@ -338,7 +338,7 @@ def copy(source, dest, verbose=0):
if
verbose
:
print
oid_repr
(
oid
),
r
.
version
,
len
(
r
.
data
)
if
restoring
:
dest
.
restore
(
oid
,
r
.
tid
,
r
.
data
,
r
.
version
,
dest
.
restore
(
oid
,
r
.
tid
,
r
.
data
,
''
,
r
.
data_txn
,
transaction
)
else
:
pre
=
preget
(
oid
,
None
)
...
...
src/ZODB/FileStorage/FileStorage.py
View file @
eef59eb3
...
...
@@ -31,7 +31,6 @@ fsync = getattr(os, "fsync", None)
from
ZODB
import
BaseStorage
,
ConflictResolution
,
POSException
from
ZODB.POSException
import
UndoError
,
POSKeyError
,
MultipleUndoErrors
from
ZODB.POSException
import
VersionLockError
from
persistent.TimeStamp
import
TimeStamp
from
ZODB.lock_file
import
LockFile
from
ZODB.utils
import
p64
,
u64
,
cp
,
z64
...
...
@@ -120,10 +119,8 @@ class FileStorage(BaseStorage.BaseStorage,
BaseStorage
.
BaseStorage
.
__init__
(
self
,
file_name
)
(
index
,
vindex
,
tindex
,
tvindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
)
=
self
.
_newIndexes
()
self
.
_initIndex
(
index
,
vindex
,
tindex
,
tvindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
)
(
index
,
tindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
)
=
self
.
_newIndexes
()
self
.
_initIndex
(
index
,
tindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
)
# Now open the file
...
...
@@ -155,18 +152,17 @@ class FileStorage(BaseStorage.BaseStorage,
r
=
self
.
_restore_index
()
if
r
is
not
None
:
self
.
_used_index
=
1
# Marker for testing
index
,
vindex
,
start
,
ltid
=
r
index
,
start
,
ltid
=
r
self
.
_initIndex
(
index
,
vindex
,
tindex
,
tvindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
)
self
.
_initIndex
(
index
,
tindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
)
self
.
_pos
,
self
.
_oid
,
tid
=
read_index
(
self
.
_file
,
file_name
,
index
,
vindex
,
tindex
,
stop
,
self
.
_file
,
file_name
,
index
,
tindex
,
stop
,
ltid
=
ltid
,
start
=
start
,
read_only
=
read_only
,
)
else
:
self
.
_used_index
=
0
# Marker for testing
self
.
_pos
,
self
.
_oid
,
tid
=
read_index
(
self
.
_file
,
file_name
,
index
,
vindex
,
tindex
,
stop
,
self
.
_file
,
file_name
,
index
,
tindex
,
stop
,
read_only
=
read_only
,
)
self
.
_save_index
()
...
...
@@ -195,20 +191,14 @@ class FileStorage(BaseStorage.BaseStorage,
# tid cache statistics.
self
.
_oid2tid_nlookups
=
self
.
_oid2tid_nhits
=
0
def
_initIndex
(
self
,
index
,
vindex
,
tindex
,
tvindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
):
def
_initIndex
(
self
,
index
,
tindex
,
oid2tid
,
toid2tid
,
toid2tid_delete
):
self
.
_index
=
index
self
.
_vindex
=
vindex
self
.
_tindex
=
tindex
self
.
_tvindex
=
tvindex
self
.
_index_get
=
index
.
get
self
.
_vindex_get
=
vindex
.
get
# .store() needs to compare the passed-in serial to the
# current tid in the database. _oid2tid caches the oid ->
# current tid mapping for non-version data (if the current
# record for oid is version data, the oid is not a key in
# _oid2tid). The point is that otherwise seeking into the
# current tid mapping. The point is that otherwise seeking into the
# storage is needed to extract the current tid, and that's
# an expensive operation. For example, if a transaction
# stores 4000 objects, and each random seek + read takes 7ms
...
...
@@ -220,8 +210,7 @@ class FileStorage(BaseStorage.BaseStorage,
# oid->tid map to transactionally add to _oid2tid.
self
.
_toid2tid
=
toid2tid
# Set of oids to transactionally delete from _oid2tid (e.g.,
# oids reverted by undo, or for which the most recent record
# becomes version data).
# oids reverted by undo).
self
.
_toid2tid_delete
=
toid2tid_delete
def
__len__
(
self
):
...
...
@@ -229,7 +218,7 @@ class FileStorage(BaseStorage.BaseStorage,
def
_newIndexes
(
self
):
# hook to use something other than builtin dict
return
fsIndex
(),
{},
{},
{},
{}
,
{},
{}
return
fsIndex
(),
{},
{},
{},
{}
_saved
=
0
def
_save_index
(
self
):
...
...
@@ -244,11 +233,7 @@ class FileStorage(BaseStorage.BaseStorage,
f
=
open
(
tmp_name
,
'wb'
)
p
=
Pickler
(
f
,
1
)
# Note: starting with ZODB 3.2.6, the 'oid' value stored is ignored
# by the code that reads the index. We still write it, so that
# .index files can still be read by older ZODBs.
info
=
{
'index'
:
self
.
_index
,
'pos'
:
self
.
_pos
,
'oid'
:
self
.
_oid
,
'vindex'
:
self
.
_vindex
}
info
=
{
'index'
:
self
.
_index
,
'pos'
:
self
.
_pos
}
p
.
dump
(
info
)
f
.
flush
()
...
...
@@ -338,10 +323,7 @@ class FileStorage(BaseStorage.BaseStorage,
def
_restore_index
(
self
):
"""Load database index to support quick startup."""
# Returns (index, vindex, pos, tid), or None in case of
# error.
# Starting with ZODB 3.2.6, the 'oid' value stored in the index
# is ignored.
# Returns (index, pos, tid), or None in case of error.
# The index returned is always an instance of fsIndex. If the
# index cached in the file is a Python dict, it's converted to
# fsIndex here, and, if we're not in read-only mode, the .index
...
...
@@ -365,8 +347,7 @@ class FileStorage(BaseStorage.BaseStorage,
return
None
index
=
info
.
get
(
'index'
)
pos
=
info
.
get
(
'pos'
)
vindex
=
info
.
get
(
'vindex'
)
if
index
is
None
or
pos
is
None
or
vindex
is
None
:
if
index
is
None
or
pos
is
None
:
return
None
pos
=
long
(
pos
)
...
...
@@ -393,7 +374,7 @@ class FileStorage(BaseStorage.BaseStorage,
if
not
tid
:
return
None
return
index
,
vindex
,
pos
,
tid
return
index
,
pos
,
tid
def
close
(
self
):
self
.
_file
.
close
()
...
...
@@ -428,83 +409,6 @@ class FileStorage(BaseStorage.BaseStorage,
return
result
def
abortVersion
(
self
,
src
,
transaction
):
return
self
.
commitVersion
(
src
,
''
,
transaction
,
abort
=
True
)
def
commitVersion
(
self
,
src
,
dest
,
transaction
,
abort
=
False
):
# We are going to commit by simply storing back pointers.
if
self
.
_is_read_only
:
raise
POSException
.
ReadOnlyError
()
if
not
(
src
and
isinstance
(
src
,
StringType
)
and
isinstance
(
dest
,
StringType
)):
raise
POSException
.
VersionCommitError
(
'Invalid source version'
)
if
src
==
dest
:
raise
POSException
.
VersionCommitError
(
"Can't commit to same version: %s"
%
repr
(
src
))
if
dest
and
abort
:
raise
POSException
.
VersionCommitError
(
"Internal error, can't abort to a version"
)
if
transaction
is
not
self
.
_transaction
:
raise
POSException
.
StorageTransactionError
(
self
,
transaction
)
self
.
_lock_acquire
()
try
:
return
self
.
_commitVersion
(
src
,
dest
,
transaction
,
abort
)
finally
:
self
.
_lock_release
()
def
_commitVersion
(
self
,
src
,
dest
,
transaction
,
abort
=
False
):
# call after checking arguments and acquiring lock
srcpos
=
self
.
_vindex_get
(
src
,
0
)
spos
=
p64
(
srcpos
)
# middle holds bytes 16:34 of a data record:
# pos of transaction, len of version name, data length
# commit version never writes data, so data length is always 0
middle
=
pack
(
">8sH8s"
,
p64
(
self
.
_pos
),
len
(
dest
),
z64
)
if
dest
:
sd
=
p64
(
self
.
_vindex_get
(
dest
,
0
))
heredelta
=
66
+
len
(
dest
)
else
:
sd
=
''
heredelta
=
50
here
=
self
.
_pos
+
(
self
.
_tfile
.
tell
()
+
self
.
_thl
)
oids
=
[]
current_oids
=
{}
while
srcpos
:
h
=
self
.
_read_data_header
(
srcpos
)
if
self
.
_index
.
get
(
h
.
oid
)
==
srcpos
:
# This is a current record!
self
.
_tindex
[
h
.
oid
]
=
here
oids
.
append
(
h
.
oid
)
self
.
_tfile
.
write
(
h
.
oid
+
self
.
_tid
+
spos
+
middle
)
if
dest
:
self
.
_tvindex
[
dest
]
=
here
self
.
_tfile
.
write
(
p64
(
h
.
pnv
)
+
sd
+
dest
)
sd
=
p64
(
here
)
self
.
_tfile
.
write
(
abort
and
p64
(
h
.
pnv
)
or
spos
)
# data backpointer to src data
here
+=
heredelta
current_oids
[
h
.
oid
]
=
1
else
:
# Hm. This is a non-current record. Is there a
# current record for this oid?
if
not
current_oids
.
has_key
(
h
.
oid
):
break
srcpos
=
h
.
vprev
spos
=
p64
(
srcpos
)
self
.
_toid2tid_delete
.
update
(
current_oids
)
return
self
.
_tid
,
oids
def
getSize
(
self
):
return
self
.
_pos
...
...
@@ -516,15 +420,14 @@ class FileStorage(BaseStorage.BaseStorage,
except
TypeError
:
raise
TypeError
(
"invalid oid %r"
%
(
oid
,))
def
load
(
self
,
oid
,
version
):
def
load
(
self
,
oid
,
version
=
''
):
"""Return pickle data and serial number."""
assert
not
version
self
.
_lock_acquire
()
try
:
pos
=
self
.
_lookup_pos
(
oid
)
h
=
self
.
_read_data_header
(
pos
,
oid
)
if
h
.
version
and
h
.
version
!=
version
:
data
=
self
.
_loadBack_impl
(
oid
,
h
.
pnv
)[
0
]
return
data
,
h
.
tid
if
h
.
plen
:
data
=
self
.
_file
.
read
(
h
.
plen
)
return
data
,
h
.
tid
...
...
@@ -537,8 +440,6 @@ class FileStorage(BaseStorage.BaseStorage,
self
.
_lock_release
()
def
loadSerial
(
self
,
oid
,
serial
):
# loadSerial must always return non-version data, because it
# is used by conflict resolution.
self
.
_lock_acquire
()
try
:
pos
=
self
.
_lookup_pos
(
oid
)
...
...
@@ -549,8 +450,6 @@ class FileStorage(BaseStorage.BaseStorage,
pos
=
h
.
prev
if
not
pos
:
raise
POSKeyError
(
oid
)
if
h
.
version
:
return
self
.
_loadBack_impl
(
oid
,
h
.
pnv
)[
0
]
if
h
.
plen
:
return
self
.
_file
.
read
(
h
.
plen
)
else
:
...
...
@@ -565,18 +464,6 @@ class FileStorage(BaseStorage.BaseStorage,
end_tid
=
None
while
True
:
h
=
self
.
_read_data_header
(
pos
,
oid
)
if
h
.
version
:
# Just follow the pnv pointer to the previous
# non-version data.
if
not
h
.
pnv
:
# Object was created in version. There is no
# before data to find.
return
None
pos
=
h
.
pnv
# The end_tid for the non-version data is not affected
# by versioned data records.
continue
if
h
.
tid
<
tid
:
break
...
...
@@ -594,20 +481,12 @@ class FileStorage(BaseStorage.BaseStorage,
finally
:
self
.
_lock_release
()
def
modifiedInVersion
(
self
,
oid
):
self
.
_lock_acquire
()
try
:
pos
=
self
.
_lookup_pos
(
oid
)
h
=
self
.
_read_data_header
(
pos
,
oid
)
return
h
.
version
finally
:
self
.
_lock_release
()
def
store
(
self
,
oid
,
oldserial
,
data
,
version
,
transaction
):
if
self
.
_is_read_only
:
raise
POSException
.
ReadOnlyError
()
if
transaction
is
not
self
.
_transaction
:
raise
POSException
.
StorageTransactionError
(
self
,
transaction
)
assert
not
version
self
.
_lock_acquire
()
try
:
...
...
@@ -620,10 +499,6 @@ class FileStorage(BaseStorage.BaseStorage,
cached_tid
=
self
.
_get_cached_tid
(
oid
)
if
cached_tid
is
None
:
h
=
self
.
_read_data_header
(
old
,
oid
)
if
h
.
version
:
if
h
.
version
!=
version
:
raise
VersionLockError
(
oid
,
h
.
version
)
pnv
=
h
.
pnv
cached_tid
=
h
.
tid
if
oldserial
!=
cached_tid
:
...
...
@@ -638,19 +513,8 @@ class FileStorage(BaseStorage.BaseStorage,
pos
=
self
.
_pos
here
=
pos
+
self
.
_tfile
.
tell
()
+
self
.
_thl
self
.
_tindex
[
oid
]
=
here
new
=
DataHeader
(
oid
,
self
.
_tid
,
old
,
pos
,
len
(
version
),
len
(
data
))
new
=
DataHeader
(
oid
,
self
.
_tid
,
old
,
pos
,
0
,
len
(
data
))
if
version
:
# Link to last record for this version:
pv
=
(
self
.
_tvindex
.
get
(
version
,
0
)
or
self
.
_vindex
.
get
(
version
,
0
))
if
pnv
is
None
:
pnv
=
old
new
.
setVersion
(
version
,
pnv
,
pv
)
self
.
_tvindex
[
version
]
=
here
self
.
_toid2tid_delete
[
oid
]
=
1
else
:
self
.
_toid2tid
[
oid
]
=
self
.
_tid
self
.
_tfile
.
write
(
new
.
asString
())
...
...
@@ -737,6 +601,8 @@ class FileStorage(BaseStorage.BaseStorage,
raise
POSException
.
ReadOnlyError
()
if
transaction
is
not
self
.
_transaction
:
raise
POSException
.
StorageTransactionError
(
self
,
transaction
)
if
version
:
raise
TypeError
(
"Versions are no-longer supported"
)
self
.
_lock_acquire
()
try
:
...
...
@@ -761,16 +627,7 @@ class FileStorage(BaseStorage.BaseStorage,
dlen
=
len
(
data
)
# Write the recovery data record
new
=
DataHeader
(
oid
,
serial
,
old
,
self
.
_pos
,
len
(
version
),
dlen
)
if
version
:
pnv
=
self
.
_restore_pnv
(
oid
,
old
,
version
,
prev_pos
)
or
old
vprev
=
self
.
_tvindex
.
get
(
version
,
0
)
if
not
vprev
:
vprev
=
self
.
_vindex
.
get
(
version
,
0
)
new
.
setVersion
(
version
,
pnv
,
vprev
)
self
.
_tvindex
[
version
]
=
here
self
.
_toid2tid_delete
[
oid
]
=
1
else
:
new
=
DataHeader
(
oid
,
serial
,
old
,
self
.
_pos
,
0
,
dlen
)
self
.
_toid2tid
[
oid
]
=
serial
self
.
_tfile
.
write
(
new
.
asString
())
...
...
@@ -788,38 +645,11 @@ class FileStorage(BaseStorage.BaseStorage,
finally
:
self
.
_lock_release
()
def
_restore_pnv
(
self
,
oid
,
prev
,
version
,
bp
):
# Find a valid pnv (previous non-version) pointer for this version.
# If there is no previous record, there can't be a pnv.
if
not
prev
:
return
None
# Load the record pointed to be prev
h
=
self
.
_read_data_header
(
prev
,
oid
)
if
h
.
version
:
return
h
.
pnv
if
h
.
back
:
# TODO: Not sure the following is always true:
# The previous record is not for this version, yet we
# have a backpointer to it. The current record must
# be an undo of an abort or commit, so the backpointer
# must be to a version record with a pnv.
h2
=
self
.
_read_data_header
(
h
.
back
,
oid
)
if
h2
.
version
:
return
h2
.
pnv
return
None
def
supportsUndo
(
self
):
return
1
def
supportsVersions
(
self
):
return
1
def
_clear_temp
(
self
):
self
.
_tindex
.
clear
()
self
.
_tvindex
.
clear
()
self
.
_toid2tid
.
clear
()
self
.
_toid2tid_delete
.
clear
()
if
self
.
_tfile
is
not
None
:
...
...
@@ -891,7 +721,6 @@ class FileStorage(BaseStorage.BaseStorage,
self
.
_pos
=
nextpos
self
.
_index
.
update
(
self
.
_tindex
)
self
.
_vindex
.
update
(
self
.
_tvindex
)
self
.
_oid2tid
.
update
(
self
.
_toid2tid
)
for
oid
in
self
.
_toid2tid_delete
.
keys
():
try
:
...
...
@@ -916,8 +745,8 @@ class FileStorage(BaseStorage.BaseStorage,
self
.
_nextpos
=
0
def
_undoDataInfo
(
self
,
oid
,
pos
,
tpos
):
"""Return the tid, data pointer,
data, and version for the oid
record at pos
"""
"""Return the tid, data pointer,
and data for the oid record at pos
"""
if
tpos
:
pos
=
tpos
-
self
.
_pos
-
self
.
_thl
tpos
=
self
.
_tfile
.
tell
()
...
...
@@ -938,7 +767,7 @@ class FileStorage(BaseStorage.BaseStorage,
if
tpos
:
self
.
_tfile
.
seek
(
tpos
)
# Restore temp file to end
return
h
.
tid
,
pos
,
data
,
h
.
version
return
h
.
tid
,
pos
,
data
def
getTid
(
self
,
oid
):
self
.
_lock_acquire
()
...
...
@@ -957,24 +786,17 @@ class FileStorage(BaseStorage.BaseStorage,
finally
:
self
.
_lock_release
()
def
_getVersion
(
self
,
oid
,
pos
):
h
=
self
.
_read_data_header
(
pos
,
oid
)
if
h
.
version
:
return
h
.
version
,
h
.
pnv
else
:
return
""
,
None
def
_transactionalUndoRecord
(
self
,
oid
,
pos
,
tid
,
pre
,
version
):
def
_transactionalUndoRecord
(
self
,
oid
,
pos
,
tid
,
pre
):
"""Get the undo information for a data record
'pos' points to the data header for 'oid' in the transaction
being undone. 'tid' refers to the transaction being undone.
'pre' is the 'prev' field of the same data header.
Return a
5-tuple consisting of a pickle, data pointer,
version, packed non-version data pointer, and current
po
sition. If the pickle is true, then the data pointer must
be 0, but the pickle can be empty *and* the
pointer 0.
Return a
3-tuple consisting of a pickle, data pointer, and
current position. If the pickle is true, then the data
po
inter must be 0, but the pickle can be empty *and* the
pointer 0.
"""
copy
=
1
# Can we just copy a data pointer
...
...
@@ -987,10 +809,7 @@ class FileStorage(BaseStorage.BaseStorage,
if
tipos
!=
pos
:
# Eek, a later transaction modified the data, but,
# maybe it is pointing at the same data we are.
ctid
,
cdataptr
,
cdata
,
cver
=
self
.
_undoDataInfo
(
oid
,
ipos
,
tpos
)
# Versions of undone record and current record *must* match!
if
cver
!=
version
:
raise
UndoError
(
'Current and undone versions differ'
,
oid
)
ctid
,
cdataptr
,
cdata
=
self
.
_undoDataInfo
(
oid
,
ipos
,
tpos
)
if
cdataptr
!=
pos
:
# We aren't sure if we are talking about the same data
...
...
@@ -1018,12 +837,11 @@ class FileStorage(BaseStorage.BaseStorage,
if
not
pre
:
# There is no previous revision, because the object creation
# is being undone.
return
""
,
0
,
""
,
""
,
ipos
return
""
,
0
,
ipos
version
,
snv
=
self
.
_getVersion
(
oid
,
pre
)
if
copy
:
# we can just copy our previous-record pointer forward
return
""
,
pre
,
version
,
snv
,
ipos
return
""
,
pre
,
ipos
try
:
bdata
=
self
.
_loadBack_impl
(
oid
,
pre
)[
0
]
...
...
@@ -1033,7 +851,7 @@ class FileStorage(BaseStorage.BaseStorage,
data
=
self
.
tryToResolveConflict
(
oid
,
ctid
,
tid
,
bdata
,
cdata
)
if
data
:
return
data
,
0
,
version
,
snv
,
ipos
return
data
,
0
,
ipos
raise
UndoError
(
"Some data were modified by a later transaction"
,
oid
)
...
...
@@ -1148,18 +966,13 @@ class FileStorage(BaseStorage.BaseStorage,
assert
base
+
self
.
_tfile
.
tell
()
==
here
,
(
here
,
base
,
self
.
_tfile
.
tell
())
try
:
p
,
prev
,
v
,
snv
,
ipos
=
self
.
_transactionalUndoRecord
(
h
.
oid
,
pos
,
h
.
tid
,
h
.
prev
,
h
.
version
)
p
,
prev
,
ipos
=
self
.
_transactionalUndoRecord
(
h
.
oid
,
pos
,
h
.
tid
,
h
.
prev
)
except
UndoError
,
v
:
# Don't fail right away. We may be redeemed later!
failures
[
h
.
oid
]
=
v
else
:
new
=
DataHeader
(
h
.
oid
,
self
.
_tid
,
ipos
,
otloc
,
len
(
v
),
len
(
p
))
if
v
:
vprev
=
self
.
_tvindex
.
get
(
v
,
0
)
or
self
.
_vindex
.
get
(
v
,
0
)
new
.
setVersion
(
v
,
snv
,
vprev
)
self
.
_tvindex
[
v
]
=
here
new
=
DataHeader
(
h
.
oid
,
self
.
_tid
,
ipos
,
otloc
,
0
,
len
(
p
))
# TODO: This seek shouldn't be necessary, but some other
# bit of code is messing with the file pointer.
...
...
@@ -1182,78 +995,17 @@ class FileStorage(BaseStorage.BaseStorage,
return
tindex
def
versionEmpty
(
self
,
version
):
if
not
version
:
# The interface is silent on this case. I think that this should
# be an error, but Barry thinks this should return 1 if we have
# any non-version data. This would be excruciatingly painful to
# test, so I must be right. ;)
raise
POSException
.
VersionError
(
'The version must be an non-empty string'
)
self
.
_lock_acquire
()
try
:
index
=
self
.
_index
file
=
self
.
_file
seek
=
file
.
seek
read
=
file
.
read
srcpos
=
self
.
_vindex_get
(
version
,
0
)
t
=
tstatus
=
None
while
srcpos
:
seek
(
srcpos
)
oid
=
read
(
8
)
if
index
[
oid
]
==
srcpos
:
return
0
h
=
read
(
50
)
# serial, prev(oid), tloc, vlen, plen, pnv, pv
tloc
=
h
[
16
:
24
]
if
t
!=
tloc
:
# We haven't checked this transaction before,
# get its status.
t
=
tloc
seek
(
u64
(
t
)
+
16
)
tstatus
=
read
(
1
)
if
tstatus
!=
'u'
:
return
1
spos
=
h
[
-
8
:]
srcpos
=
u64
(
spos
)
return
1
finally
:
self
.
_lock_release
()
def
versions
(
self
,
max
=
None
):
r
=
[]
a
=
r
.
append
keys
=
self
.
_vindex
.
keys
()
if
max
is
not
None
:
keys
=
keys
[:
max
]
for
version
in
keys
:
if
self
.
versionEmpty
(
version
):
continue
a
(
version
)
if
max
and
len
(
r
)
>=
max
:
return
r
return
r
def
history
(
self
,
oid
,
version
=
None
,
size
=
1
,
filter
=
None
):
assert
not
version
self
.
_lock_acquire
()
try
:
r
=
[]
pos
=
self
.
_lookup_pos
(
oid
)
wantver
=
version
while
1
:
if
len
(
r
)
>=
size
:
return
r
h
=
self
.
_read_data_header
(
pos
)
if
h
.
version
:
if
wantver
is
not
None
and
h
.
version
!=
wantver
:
if
h
.
prev
:
pos
=
h
.
prev
continue
else
:
return
r
else
:
version
=
""
wantver
=
None
th
=
self
.
_read_txn_header
(
h
.
tloc
)
if
th
.
ext
:
d
=
loads
(
th
.
ext
)
...
...
@@ -1264,7 +1016,6 @@ class FileStorage(BaseStorage.BaseStorage,
"user_name"
:
th
.
user
,
"description"
:
th
.
descr
,
"tid"
:
h
.
tid
,
"version"
:
h
.
version
,
"size"
:
h
.
plen
,
})
...
...
@@ -1341,7 +1092,7 @@ class FileStorage(BaseStorage.BaseStorage,
# OK, we're beyond the point of no return
os
.
rename
(
self
.
_file_name
+
'.pack'
,
self
.
_file_name
)
self
.
_file
=
open
(
self
.
_file_name
,
'r+b'
)
self
.
_initIndex
(
p
.
index
,
p
.
vindex
,
p
.
tindex
,
p
.
tv
index
,
self
.
_initIndex
(
p
.
index
,
p
.
t
index
,
p
.
oid2tid
,
p
.
toid2tid
,
p
.
toid2tid_delete
)
self
.
_pos
=
opos
...
...
@@ -1375,7 +1126,7 @@ class FileStorage(BaseStorage.BaseStorage,
pos
=
pos
-
8
-
u64
(
read
(
8
))
seek
(
0
)
return
[(
trans
.
tid
,
[(
r
.
oid
,
r
.
version
)
for
r
in
trans
])
return
[(
trans
.
tid
,
[(
r
.
oid
,
''
)
for
r
in
trans
])
for
trans
in
FileIterator
(
self
.
_file
,
pos
=
pos
)]
finally
:
self
.
_lock_release
()
...
...
@@ -1412,15 +1163,13 @@ class FileStorage(BaseStorage.BaseStorage,
except
ValueError
:
# "empty tree" error
next_oid
=
None
# ignore versions
# XXX if the object was created in a version, this will fail.
data
,
tid
=
self
.
load
(
oid
,
""
)
return
oid
,
tid
,
data
,
next_oid
def
shift_transactions_forward
(
index
,
vindex
,
tindex
,
file
,
pos
,
opos
):
def
shift_transactions_forward
(
index
,
tindex
,
file
,
pos
,
opos
):
"""Copy transactions forward in the data file
This might be done as part of a recovery effort
...
...
@@ -1432,7 +1181,6 @@ def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
write
=
file
.
write
index_get
=
index
.
get
vindex_get
=
vindex
.
get
# Initialize,
pv
=
z64
...
...
@@ -1482,18 +1230,10 @@ def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
seek
(
pos
)
h
=
read
(
DATA_HDR_LEN
)
oid
,
serial
,
sprev
,
stloc
,
vlen
,
splen
=
unpack
(
DATA_HDR
,
h
)
assert
not
vlen
plen
=
u64
(
splen
)
dlen
=
DATA_HDR_LEN
+
(
plen
or
8
)
if
vlen
:
dlen
=
dlen
+
(
16
+
vlen
)
pnv
=
u64
(
read
(
8
))
# skip position of previous version record
seek
(
8
,
1
)
version
=
read
(
vlen
)
pv
=
p64
(
vindex_get
(
version
,
0
))
if
status
!=
'u'
:
vindex
[
version
]
=
opos
tindex
[
oid
]
=
opos
if
plen
:
p
=
read
(
plen
)
...
...
@@ -1511,17 +1251,7 @@ def shift_transactions_forward(index, vindex, tindex, file, pos, opos):
seek
(
opos
)
sprev
=
p64
(
index_get
(
oid
,
0
))
write
(
pack
(
DATA_HDR
,
oid
,
serial
,
sprev
,
p64
(
otpos
),
vlen
,
splen
))
if
vlen
:
if
not
pnv
:
write
(
z64
)
else
:
if
pnv
>=
p2
:
pnv
=
pnv
-
offset
elif
pnv
>=
p1
:
pnv
=
index_get
(
oid
,
0
)
write
(
p64
(
pnv
))
write
(
pv
)
write
(
version
)
oid
,
serial
,
sprev
,
p64
(
otpos
),
0
,
splen
))
write
(
p
)
...
...
@@ -1557,11 +1287,9 @@ def search_back(file, pos):
def
recover
(
file_name
):
file
=
open
(
file_name
,
'r+b'
)
index
=
{}
vindex
=
{}
tindex
=
{}
pos
,
oid
,
tid
=
read_index
(
file
,
file_name
,
index
,
vindex
,
tindex
,
recover
=
1
)
pos
,
oid
,
tid
=
read_index
(
file
,
file_name
,
index
,
tindex
,
recover
=
1
)
if
oid
is
not
None
:
print
"Nothing to recover"
return
...
...
@@ -1569,9 +1297,7 @@ def recover(file_name):
opos
=
pos
pos
,
sz
=
search_back
(
file
,
pos
)
if
pos
<
sz
:
npos
=
shift_transactions_forward
(
index
,
vindex
,
tindex
,
file
,
pos
,
opos
,
)
npos
=
shift_transactions_forward
(
index
,
tindex
,
file
,
pos
,
opos
)
file
.
truncate
(
npos
)
...
...
@@ -1580,7 +1306,7 @@ def recover(file_name):
def
read_index
(
file
,
name
,
index
,
vindex
,
tindex
,
stop
=
'
\
377
'
*
8
,
def
read_index
(
file
,
name
,
index
,
tindex
,
stop
=
'
\
377
'
*
8
,
ltid
=
z64
,
start
=
4L
,
maxoid
=
z64
,
recover
=
0
,
read_only
=
0
):
"""Scan the file storage and update the index.
...
...
@@ -1591,7 +1317,6 @@ def read_index(file, name, index, vindex, tindex, stop='\377'*8,
file -- a file object (the Data.fs)
name -- the name of the file (presumably file.name)
index -- fsIndex, oid -> data record file offset
vindex -- dictionary, oid -> data record offset for version data
tindex -- dictionary, oid -> data record offset
tindex is cleared before return
...
...
@@ -1719,9 +1444,6 @@ def read_index(file, name, index, vindex, tindex, stop='\377'*8,
dlen
=
h
.
recordlen
()
tindex
[
h
.
oid
]
=
pos
if
h
.
version
:
vindex
[
h
.
version
]
=
pos
if
pos
+
dlen
>
tend
or
h
.
tloc
!=
tpos
:
if
recover
:
return
tpos
,
None
,
None
...
...
@@ -2006,8 +1728,7 @@ class RecordIterator(Iterator, BaseStorage.TransactionRecord,
else
:
if
h
.
back
==
0
:
# If the backpointer is 0, then this transaction
# undoes the object creation. It either aborts
# the version that created the object or undid the
# undoes the object creation. It undid the
# transaction that created it. Return None
# instead of a pickle to indicate this.
data
=
None
...
...
@@ -2017,17 +1738,16 @@ class RecordIterator(Iterator, BaseStorage.TransactionRecord,
# Should it go to the original data like BDBFullStorage?
prev_txn
=
self
.
getTxnFromData
(
h
.
oid
,
h
.
back
)
r
=
Record
(
h
.
oid
,
h
.
tid
,
h
.
version
,
data
,
prev_txn
,
pos
)
r
=
Record
(
h
.
oid
,
h
.
tid
,
data
,
prev_txn
,
pos
)
return
r
raise
IndexError
(
index
)
class
Record
(
BaseStorage
.
DataRecord
):
"""An abstract database record."""
def
__init__
(
self
,
oid
,
tid
,
version
,
data
,
prev
,
pos
):
def
__init__
(
self
,
oid
,
tid
,
data
,
prev
,
pos
):
self
.
oid
=
oid
self
.
tid
=
tid
self
.
version
=
version
self
.
data
=
data
self
.
data_txn
=
prev
self
.
pos
=
pos
...
...
src/ZODB/FileStorage/format.py
View file @
eef59eb3
...
...
@@ -64,19 +64,10 @@
#
# - 8-byte beginning of transaction record file position.
#
# - 2-byte
version length
# - 2-byte
s with zero values. (Was version length.)
#
# - 8-byte data length
#
# ? 8-byte position of non-version data record
# (if version length > 0)
#
# ? 8-byte position of previous record in this version
# (if version length > 0)
#
# ? version string
# (if version length > 0)
#
# ? data
# (data length > 0)
#
...
...
@@ -87,41 +78,10 @@
# Also, the object ids time stamps are big-endian, so comparisons
# are meaningful.
#
# Version handling
#
# There isn't a separate store for versions. Each record has a
# version field, indicating what version it is in. The records in a
# version form a linked list. Each record that has a non-empty
# version string has a pointer to the previous record in the version.
# Version back pointers are retained *even* when versions are
# committed or aborted or when transactions are undone.
#
# There is a notion of "current" version records, which are the
# records in a version that are the current records for their
# respective objects. When a version is comitted, the current records
# are committed to the destination version. When a version is
# aborted, the current records are aborted.
#
# When committing or aborting, we search backward through the linked
# list until we find a record for an object that does not have a
# current record in the version. If we find a record for which the
# non-version pointer is the same as the previous pointer, then we
# forget that the corresponding object had a current record in the
# version. This strategy allows us to avoid searching backward through
# previously committed or aborted version records.
#
# Of course, we ignore records in undone transactions when committing
# or aborting.
#
# Backpointers
#
# When we commit or abort a version, we don't copy (or delete)
# and data. Instead, we write records with back pointers.
#
# A version record *never* has a back pointer to a non-version
# record, because we never abort to a version. A non-version record
# may have a back pointer to a version record or to a non-version
# record.
# When we undo a record, we don't copy (or delete)
# data. Instead, we write records with back pointers.
import
struct
import
logging
...
...
@@ -156,7 +116,6 @@ DATA_HDR = ">8s8sQQHQ"
# constants to support various header sizes
TRANS_HDR_LEN
=
23
DATA_HDR_LEN
=
42
DATA_VERSION_HDR_LEN
=
58
assert
struct
.
calcsize
(
TRANS_HDR
)
==
TRANS_HDR_LEN
assert
struct
.
calcsize
(
DATA_HDR
)
==
DATA_HDR_LEN
...
...
@@ -180,9 +139,6 @@ class FileStorageFormatter(object):
If ois is not None, raise CorruptedDataError if oid passed
does not match oid in file.
If there is version data, reads the version part of the header.
If there is no pickle data, reads the back pointer.
"""
self
.
_file
.
seek
(
pos
)
s
=
self
.
_file
.
read
(
DATA_HDR_LEN
)
...
...
@@ -191,17 +147,10 @@ class FileStorageFormatter(object):
h
=
DataHeaderFromString
(
s
)
if
oid
is
not
None
and
oid
!=
h
.
oid
:
raise
CorruptedDataError
(
oid
,
s
,
pos
)
if
h
.
vlen
:
s
=
self
.
_file
.
read
(
16
+
h
.
vlen
)
h
.
parseVersion
(
s
)
if
not
h
.
plen
:
h
.
back
=
u64
(
self
.
_file
.
read
(
8
))
return
h
def
_write_version_header
(
self
,
file
,
pnv
,
vprev
,
version
):
s
=
struct
.
pack
(
">8s8s"
,
pnv
,
vprev
)
file
.
write
(
s
+
version
)
def
_read_txn_header
(
self
,
pos
,
tid
=
None
):
self
.
_file
.
seek
(
pos
)
s
=
self
.
_file
.
read
(
TRANS_HDR_LEN
)
...
...
@@ -284,47 +233,26 @@ def DataHeaderFromString(s):
class
DataHeader
(
object
):
"""Header for a data record."""
__slots__
=
(
"oid"
,
"tid"
,
"prev"
,
"tloc"
,
"vlen"
,
"plen"
,
"back"
,
# These three attributes are only defined when vlen > 0
"pnv"
,
"vprev"
,
"version"
)
__slots__
=
(
"oid"
,
"tid"
,
"prev"
,
"tloc"
,
"plen"
,
"back"
)
def
__init__
(
self
,
oid
,
tid
,
prev
,
tloc
,
vlen
,
plen
):
self
.
back
=
0
# default
self
.
version
=
""
# default
if
vlen
:
raise
ValueError
(
"Non-zero version length. Versions aren't supported."
)
self
.
oid
=
oid
self
.
tid
=
tid
self
.
prev
=
prev
self
.
tloc
=
tloc
self
.
vlen
=
vlen
self
.
plen
=
plen
self
.
back
=
0
# default
def
asString
(
self
):
s
=
struct
.
pack
(
DATA_HDR
,
self
.
oid
,
self
.
tid
,
self
.
prev
,
self
.
tloc
,
self
.
vlen
,
self
.
plen
)
if
self
.
version
:
v
=
struct
.
pack
(
">QQ"
,
self
.
pnv
,
self
.
vprev
)
return
s
+
v
+
self
.
version
else
:
return
s
def
setVersion
(
self
,
version
,
pnv
,
vprev
):
self
.
version
=
version
self
.
vlen
=
len
(
version
)
self
.
pnv
=
pnv
self
.
vprev
=
vprev
def
parseVersion
(
self
,
buf
):
pnv
,
vprev
=
struct
.
unpack
(
">QQ"
,
buf
[:
16
])
self
.
pnv
=
pnv
self
.
vprev
=
vprev
self
.
version
=
buf
[
16
:]
return
struct
.
pack
(
DATA_HDR
,
self
.
oid
,
self
.
tid
,
self
.
prev
,
self
.
tloc
,
0
,
self
.
plen
)
def
recordlen
(
self
):
rlen
=
DATA_HDR_LEN
+
(
self
.
plen
or
8
)
if
self
.
version
:
rlen
+=
16
+
self
.
vlen
return
rlen
return
DATA_HDR_LEN
+
(
self
.
plen
or
8
)
def
TxnHeaderFromString
(
s
):
return
TxnHeader
(
*
struct
.
unpack
(
TRANS_HDR
,
s
))
...
...
src/ZODB/FileStorage/fsdump.py
View file @
eef59eb3
...
...
@@ -14,8 +14,8 @@
import
struct
from
ZODB.FileStorage
import
FileIterator
from
ZODB.FileStorage.format
import
TRANS_HDR
,
TRANS_HDR_LEN
,
DATA_HDR
,
DATA_HDR_LEN
from
ZODB.FileStorage.format
import
DATA_HDR_LEN
from
ZODB.FileStorage.format
import
TRANS_HDR
,
TRANS_HDR_LEN
from
ZODB.FileStorage.format
import
DATA_HDR
,
DATA_HDR
_LEN
from
ZODB.TimeStamp
import
TimeStamp
from
ZODB.utils
import
u64
,
get_pickle_metadata
from
ZODB.tests.StorageTestBase
import
zodb_unpickle
...
...
@@ -41,11 +41,6 @@ def fsdump(path, file=None, with_offset=1):
size
=
" size=%d"
%
len
(
rec
.
data
)
fullclass
=
"%s.%s"
%
(
modname
,
classname
)
if
rec
.
version
:
version
=
" version=%r"
%
rec
.
version
else
:
version
=
""
if
rec
.
data_txn
:
# It would be nice to print the transaction number
# (i) but it would be expensive to keep track of.
...
...
@@ -53,8 +48,8 @@ def fsdump(path, file=None, with_offset=1):
else
:
bp
=
""
print
>>
file
,
(
" data #%05d oid=%016x%s
%s
class=%s%s"
%
(
j
,
u64
(
rec
.
oid
),
version
,
size
,
fullclass
,
bp
))
print
>>
file
,
(
" data #%05d oid=%016x%s class=%s%s"
%
(
j
,
u64
(
rec
.
oid
),
size
,
fullclass
,
bp
))
iter
.
close
()
def
fmt
(
p64
):
...
...
@@ -117,14 +112,7 @@ class Dumper:
print
>>
self
.
dest
,
"revid: %s"
%
fmt
(
revid
)
print
>>
self
.
dest
,
"previous record offset: %d"
%
prev
print
>>
self
.
dest
,
"transaction offset: %d"
%
tloc
if
vlen
:
pnv
=
self
.
file
.
read
(
8
)
sprevdata
=
self
.
file
.
read
(
8
)
version
=
self
.
file
.
read
(
vlen
)
print
>>
self
.
dest
,
"version: %r"
%
version
print
>>
self
.
dest
,
"non-version data offset: %d"
%
u64
(
pnv
)
print
>>
self
.
dest
,
(
"previous version data offset: %d"
%
u64
(
sprevdata
))
assert
not
vlen
print
>>
self
.
dest
,
"len(data): %d"
%
dlen
self
.
file
.
read
(
dlen
)
if
not
dlen
:
...
...
src/ZODB/FileStorage/fsoids.py
View file @
eef59eb3
...
...
@@ -148,7 +148,7 @@ class Tracer(object):
txn
.
_tpos
)
def
_save_references
(
self
,
drec
):
# drec has members oid, tid,
version,
data, data_txn
# drec has members oid, tid, data, data_txn
tid
,
oid
,
pick
,
pos
=
drec
.
tid
,
drec
.
oid
,
drec
.
data
,
drec
.
pos
if
pick
:
if
oid
in
self
.
oids
:
...
...
@@ -159,13 +159,12 @@ class Tracer(object):
self
.
_records_map
[
oid
]
=
drec
self
.
_records
.
append
(
drec
)
elif
oid
in
self
.
oids
:
# Or maybe it's a version abort.
self
.
_msg
(
oid
,
tid
,
"creation undo at"
,
pos
)
# Process next data record. If a message is produced, self._produced_msg
# will be set True.
def
_check_drec
(
self
,
drec
):
# drec has members oid, tid,
version,
data, data_txn
# drec has members oid, tid, data, data_txn
tid
,
oid
,
pick
,
pos
=
drec
.
tid
,
drec
.
oid
,
drec
.
data
,
drec
.
pos
ref2name
=
self
.
_ref2name
ref2name_get
=
ref2name
.
get
...
...
src/ZODB/FileStorage/fspack.py
View file @
eef59eb3
...
...
@@ -33,27 +33,13 @@ from ZODB.fsIndex import fsIndex
from
ZODB.FileStorage.format
import
FileStorageFormatter
,
CorruptedDataError
from
ZODB.FileStorage.format
import
DataHeader
,
TRANS_HDR_LEN
class
DataCopier
(
FileStorageFormatter
):
"""Mixin class for copying transactions into a storage.
class
PackCopier
(
FileStorageFormatter
):
The restore() and pack() methods share a need to copy data records
and update pointers to data in earlier transaction records. This
class provides the shared logic.
The mixin extends the FileStorageFormatter with a copy() method.
It also requires that the concrete class provides the following
attributes:
_file -- file with earlier destination data
_tfile -- destination file for copied data
_pos -- file pos of destination transaction
_tindex -- maps oid to data record file pos
_tvindex -- maps version name to data record file pos
_tindex and _tvindex are updated by copy().
The copy() method does not do any locking.
"""
def
__init__
(
self
,
f
,
index
,
tindex
):
self
.
_file
=
f
self
.
_index
=
index
self
.
_tindex
=
tindex
self
.
_pos
=
None
def
_txn_find
(
self
,
tid
,
stop_at_pack
):
# _pos always points just past the last transaction
...
...
@@ -107,43 +93,7 @@ class DataCopier(FileStorageFormatter):
pos
+=
h
.
recordlen
()
return
0
def
_restore_pnv
(
self
,
oid
,
prev
,
version
,
bp
):
# Find a valid pnv (previous non-version) pointer for this version.
# If there is no previous record, there can't be a pnv.
if
not
prev
:
return
None
pnv
=
None
h
=
self
.
_read_data_header
(
prev
,
oid
)
# If the previous record is for a version, it must have
# a valid pnv.
if
h
.
version
:
return
h
.
pnv
elif
bp
:
# Unclear: Not sure the following is always true:
# The previous record is not for this version, yet we
# have a backpointer to it. The current record must
# be an undo of an abort or commit, so the backpointer
# must be to a version record with a pnv.
h2
=
self
.
_read_data_header
(
bp
,
oid
)
if
h2
.
version
:
return
h2
.
pnv
else
:
warn
(
"restore could not find previous non-version data "
"at %d or %d"
,
prev
,
bp
)
return
None
def
_resolve_backpointer
(
self
,
prev_txn
,
oid
,
data
):
prev_pos
=
0
if
prev_txn
is
not
None
:
prev_txn_pos
=
self
.
_txn_find
(
prev_txn
,
0
)
if
prev_txn_pos
:
prev_pos
=
self
.
_data_find
(
prev_txn_pos
,
oid
,
data
)
return
prev_pos
def
copy
(
self
,
oid
,
serial
,
data
,
version
,
prev_txn
,
txnpos
,
datapos
):
def
copy
(
self
,
oid
,
serial
,
data
,
prev_txn
,
txnpos
,
datapos
):
prev_pos
=
self
.
_resolve_backpointer
(
prev_txn
,
oid
,
data
)
old
=
self
.
_index
.
get
(
oid
,
0
)
# Calculate the pos the record will have in the storage.
...
...
@@ -158,31 +108,34 @@ class DataCopier(FileStorageFormatter):
else
:
dlen
=
len
(
data
)
# Write the recovery data record
h
=
DataHeader
(
oid
,
serial
,
old
,
txnpos
,
len
(
version
),
dlen
)
if
version
:
h
.
version
=
version
pnv
=
self
.
_restore_pnv
(
oid
,
old
,
version
,
prev_pos
)
if
pnv
is
not
None
:
h
.
pnv
=
pnv
else
:
h
.
pnv
=
old
# Link to the last record for this version
h
.
vprev
=
self
.
_tvindex
.
get
(
version
,
0
)
if
not
h
.
vprev
:
h
.
vprev
=
self
.
_vindex
.
get
(
version
,
0
)
self
.
_tvindex
[
version
]
=
here
h
=
DataHeader
(
oid
,
serial
,
old
,
txnpos
,
0
,
dlen
)
self
.
_
t
file
.
write
(
h
.
asString
())
self
.
_file
.
write
(
h
.
asString
())
# Write the data or a backpointer
if
data
is
None
:
if
prev_pos
:
self
.
_
t
file
.
write
(
p64
(
prev_pos
))
self
.
_file
.
write
(
p64
(
prev_pos
))
else
:
# Write a zero backpointer, which indicates an
# un-creation transaction.
self
.
_
t
file
.
write
(
z64
)
self
.
_file
.
write
(
z64
)
else
:
self
.
_tfile
.
write
(
data
)
self
.
_file
.
write
(
data
)
def
setTxnPos
(
self
,
pos
):
self
.
_pos
=
pos
def
_resolve_backpointer
(
self
,
prev_txn
,
oid
,
data
):
pos
=
self
.
_file
.
tell
()
try
:
prev_pos
=
0
if
prev_txn
is
not
None
:
prev_txn_pos
=
self
.
_txn_find
(
prev_txn
,
0
)
if
prev_txn_pos
:
prev_pos
=
self
.
_data_find
(
prev_txn_pos
,
oid
,
data
)
return
prev_pos
finally
:
self
.
_file
.
seek
(
pos
)
class
GC
(
FileStorageFormatter
):
...
...
@@ -194,7 +147,6 @@ class GC(FileStorageFormatter):
# packpos: position of first txn header after pack time
self
.
packpos
=
None
self
.
oid2curpos
=
fsIndex
()
# maps oid to current data record position
self
.
oid2verpos
=
fsIndex
()
# maps oid to current version data
# The set of reachable revisions of each object.
#
...
...
@@ -228,7 +180,6 @@ class GC(FileStorageFormatter):
self
.
findReachableFromFuture
()
# These mappings are no longer needed and may consume a lot
# of space.
del
self
.
oid2verpos
del
self
.
oid2curpos
def
buildPackIndex
(
self
):
...
...
@@ -254,9 +205,6 @@ class GC(FileStorageFormatter):
while
pos
<
end
:
dh
=
self
.
_read_data_header
(
pos
)
self
.
checkData
(
th
,
tpos
,
dh
,
pos
)
if
dh
.
version
:
self
.
oid2verpos
[
dh
.
oid
]
=
pos
else
:
self
.
oid2curpos
[
dh
.
oid
]
=
pos
pos
+=
dh
.
recordlen
()
...
...
@@ -302,11 +250,6 @@ class GC(FileStorageFormatter):
L
.
append
(
pos
)
todo
.
extend
(
self
.
findrefs
(
pos
))
pos
=
self
.
oid2verpos
.
get
(
oid
)
if
pos
is
not
None
:
L
.
append
(
pos
)
todo
.
extend
(
self
.
findrefs
(
pos
))
if
not
L
:
continue
...
...
@@ -344,15 +287,6 @@ class GC(FileStorageFormatter):
else
:
self
.
reachable
[
dh
.
oid
]
=
dh
.
back
if
dh
.
version
and
dh
.
pnv
:
if
self
.
reachable
.
has_key
(
dh
.
oid
):
L
=
self
.
reach_ex
.
setdefault
(
dh
.
oid
,
[])
if
dh
.
pnv
not
in
L
:
L
.
append
(
dh
.
pnv
)
extra_roots
.
append
(
dh
.
pnv
)
else
:
self
.
reachable
[
dh
.
oid
]
=
dh
.
back
pos
+=
dh
.
recordlen
()
tlen
=
self
.
_read_num
(
pos
)
...
...
@@ -377,43 +311,6 @@ class GC(FileStorageFormatter):
else
:
return
[]
class
PackCopier
(
DataCopier
):
# PackCopier has to cope with _file and _tfile being the
# same file. The copy() implementation is written assuming
# that they are different, so that using one object doesn't
# mess up the file pointer for the other object.
# PackCopier overrides _resolve_backpointer() and _restore_pnv()
# to guarantee that they keep the file pointer for _tfile in
# the right place.
def
__init__
(
self
,
f
,
index
,
vindex
,
tindex
,
tvindex
):
self
.
_file
=
f
self
.
_tfile
=
f
self
.
_index
=
index
self
.
_vindex
=
vindex
self
.
_tindex
=
tindex
self
.
_tvindex
=
tvindex
self
.
_pos
=
None
def
setTxnPos
(
self
,
pos
):
self
.
_pos
=
pos
def
_resolve_backpointer
(
self
,
prev_txn
,
oid
,
data
):
pos
=
self
.
_tfile
.
tell
()
try
:
return
DataCopier
.
_resolve_backpointer
(
self
,
prev_txn
,
oid
,
data
)
finally
:
self
.
_tfile
.
seek
(
pos
)
def
_restore_pnv
(
self
,
oid
,
prev
,
version
,
bp
):
pos
=
self
.
_tfile
.
tell
()
try
:
return
DataCopier
.
_restore_pnv
(
self
,
oid
,
prev
,
version
,
bp
)
finally
:
self
.
_tfile
.
seek
(
pos
)
class
FileStoragePacker
(
FileStorageFormatter
):
# path is the storage file path.
...
...
@@ -447,23 +344,15 @@ class FileStoragePacker(FileStorageFormatter):
# The packer will use several indexes.
# index: oid -> pos
# vindex: version -> pos
# tindex: oid -> pos, for current txn
# tvindex: version -> pos, for current txn
# oid2tid: not used by the packer
self
.
index
=
fsIndex
()
self
.
vindex
=
{}
self
.
tindex
=
{}
self
.
tvindex
=
{}
self
.
oid2tid
=
{}
self
.
toid2tid
=
{}
self
.
toid2tid_delete
=
{}
# Index for non-version data. This is a temporary structure
# to reduce I/O during packing
self
.
nvindex
=
fsIndex
()
def
pack
(
self
):
# Pack copies all data reachable at the pack time or later.
#
...
...
@@ -486,8 +375,7 @@ class FileStoragePacker(FileStorageFormatter):
self
.
_file
.
seek
(
0
)
self
.
_tfile
.
write
(
self
.
_file
.
read
(
self
.
_metadata_size
))
self
.
_copier
=
PackCopier
(
self
.
_tfile
,
self
.
index
,
self
.
vindex
,
self
.
tindex
,
self
.
tvindex
)
self
.
_copier
=
PackCopier
(
self
.
_tfile
,
self
.
index
,
self
.
tindex
)
ipos
,
opos
=
self
.
copyToPacktime
()
assert
ipos
==
self
.
gc
.
packpos
...
...
@@ -623,13 +511,7 @@ class FileStoragePacker(FileStorageFormatter):
h
.
plen
=
len
(
data
)
h
.
tloc
=
new_tpos
pos
=
self
.
_tfile
.
tell
()
if
h
.
version
:
h
.
pnv
=
self
.
index
.
get
(
h
.
oid
,
0
)
h
.
vprev
=
self
.
vindex
.
get
(
h
.
version
,
0
)
self
.
vindex
[
h
.
version
]
=
pos
self
.
index
[
h
.
oid
]
=
pos
if
h
.
version
:
self
.
vindex
[
h
.
version
]
=
pos
self
.
_tfile
.
write
(
h
.
asString
())
self
.
_tfile
.
write
(
data
)
if
not
data
:
...
...
@@ -681,8 +563,8 @@ class FileStoragePacker(FileStorageFormatter):
if
h
.
back
:
prev_txn
=
self
.
getTxnFromData
(
h
.
oid
,
h
.
back
)
self
.
_copier
.
copy
(
h
.
oid
,
h
.
tid
,
data
,
h
.
versio
n
,
p
rev_txn
,
p
os
,
self
.
_tfile
.
tell
())
self
.
_copier
.
copy
(
h
.
oid
,
h
.
tid
,
data
,
prev_tx
n
,
pos
,
self
.
_tfile
.
tell
())
tlen
=
self
.
_tfile
.
tell
()
-
pos
assert
tlen
==
th
.
tlen
...
...
@@ -691,8 +573,6 @@ class FileStoragePacker(FileStorageFormatter):
self
.
index
.
update
(
self
.
tindex
)
self
.
tindex
.
clear
()
self
.
vindex
.
update
(
self
.
tvindex
)
self
.
tvindex
.
clear
()
if
self
.
_lock_counter
%
20
==
0
:
self
.
_commit_lock_acquire
()
return
ipos
src/ZODB/fsrecover.py
View file @
eef59eb3
...
...
@@ -338,8 +338,8 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
else
:
l
=
len
(
r
.
data
)
print
"%7d %s %s"
%
(
u64
(
r
.
oid
),
l
,
r
.
version
)
ofs
.
restore
(
r
.
oid
,
r
.
tid
,
r
.
data
,
r
.
version
,
r
.
data_txn
,
print
"%7d %s %s"
%
(
u64
(
r
.
oid
),
l
)
ofs
.
restore
(
r
.
oid
,
r
.
tid
,
r
.
data
,
''
,
r
.
data_txn
,
txn
)
nrec
+=
1
except
(
KeyboardInterrupt
,
SystemExit
):
...
...
src/ZODB/fstools.py
View file @
eef59eb3
...
...
@@ -22,7 +22,7 @@ import cPickle
import
struct
from
ZODB.FileStorage.format
import
TRANS_HDR
,
DATA_HDR
,
TRANS_HDR_LEN
from
ZODB.FileStorage.format
import
DATA_HDR_LEN
,
DATA_VERSION_HDR_LEN
from
ZODB.FileStorage.format
import
DATA_HDR_LEN
from
ZODB.utils
import
u64
from
persistent.TimeStamp
import
TimeStamp
...
...
@@ -103,12 +103,9 @@ class DataHeader:
serial 8-16 object serial numver
prev_rec_pos 16-24 position of previous data record for object
txn_pos 24-32 position of txn header
version_len 32-34 length of version
version_len 32-34 length of version
(always 0)
data_len 34-42 length of data
nonversion_pos 42-50* position of nonversion data record
prev_version_pos 50-58* pos of previous version data record
* these attributes are only present if version_len != 0.
"""
def
__init__
(
self
,
file
,
pos
):
...
...
@@ -118,27 +115,18 @@ class DataHeader:
def
_read_header
(
self
):
self
.
_file
.
seek
(
self
.
_pos
)
self
.
_hdr
=
self
.
_file
.
read
(
DATA_
VERSION_
HDR_LEN
)
self
.
_hdr
=
self
.
_file
.
read
(
DATA_HDR_LEN
)
# always read the longer header, just in case
(
self
.
oid
,
self
.
serial
,
prev_rec_pos
,
txn_pos
,
self
.
version_len
,
data_len
)
=
struct
.
unpack
(
DATA_HDR
,
self
.
_hdr
[:
DATA_HDR_LEN
])
(
self
.
oid
,
self
.
serial
,
prev_rec_pos
,
txn_pos
,
vlen
,
data_len
)
=
struct
.
unpack
(
DATA_HDR
,
self
.
_hdr
[:
DATA_HDR_LEN
])
assert
not
vlen
self
.
prev_rec_pos
=
u64
(
prev_rec_pos
)
self
.
txn_pos
=
u64
(
txn_pos
)
self
.
data_len
=
u64
(
data_len
)
if
self
.
version_len
:
s
=
self
.
_hdr
[
DATA_HDR_LEN
:]
self
.
nonversion_pos
=
u64
(
s
[:
8
])
self
.
prev_version_pos
=
u64
(
s
[
8
:])
else
:
self
.
nonversion_pos
=
None
self
.
prev_version_pos
=
None
def
next_offset
(
self
):
"""Return offset of next record."""
off
=
self
.
_pos
+
self
.
data_len
if
self
.
version_len
:
off
+=
self
.
version_len
+
DATA_VERSION_HDR_LEN
else
:
off
+=
DATA_HDR_LEN
if
self
.
data_len
==
0
:
off
+=
8
# backpointer
...
...
src/ZODB/tests/BasicStorage.py
View file @
eef59eb3
...
...
@@ -44,40 +44,12 @@ class BasicStorage:
self
.
_storage
.
store
,
0
,
0
,
0
,
0
,
transaction
.
Transaction
())
if
self
.
__supportsVersions
():
try
:
self
.
_storage
.
abortVersion
(
'dummy'
,
transaction
.
Transaction
())
except
(
POSException
.
StorageTransactionError
,
POSException
.
VersionCommitError
):
pass
# test passed ;)
else
:
assert
0
,
"Should have failed, invalid transaction."
try
:
self
.
_storage
.
commitVersion
(
'dummy'
,
'dummer'
,
transaction
.
Transaction
())
except
(
POSException
.
StorageTransactionError
,
POSException
.
VersionCommitError
):
pass
# test passed ;)
else
:
assert
0
,
"Should have failed, invalid transaction."
self
.
assertRaises
(
POSException
.
StorageTransactionError
,
self
.
_storage
.
store
,
0
,
1
,
2
,
3
,
transaction
.
Transaction
())
self
.
_storage
.
tpc_abort
(
t
)
def
__supportsVersions
(
self
):
storage
=
self
.
_storage
try
:
supportsVersions
=
storage
.
supportsVersions
except
AttributeError
:
return
False
else
:
return
supportsVersions
()
def
checkSerialIsNoneForInitialRevision
(
self
):
eq
=
self
.
assertEqual
oid
=
self
.
_storage
.
new_oid
()
...
...
@@ -95,13 +67,13 @@ class BasicStorage:
eq
(
value
,
MinPO
(
11
))
eq
(
revid
,
newrevid
)
def
check
NonVersion
Store
(
self
):
def
checkStore
(
self
):
revid
=
ZERO
newrevid
=
self
.
_dostore
(
revid
=
None
)
# Finish the transaction.
self
.
assertNotEqual
(
newrevid
,
revid
)
def
check
NonVersion
StoreAndLoad
(
self
):
def
checkStoreAndLoad
(
self
):
eq
=
self
.
assertEqual
oid
=
self
.
_storage
.
new_oid
()
self
.
_dostore
(
oid
=
oid
,
data
=
MinPO
(
7
))
...
...
@@ -115,12 +87,6 @@ class BasicStorage:
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
21
))
def
checkNonVersionModifiedInVersion
(
self
):
if
self
.
__supportsVersions
():
oid
=
self
.
_storage
.
new_oid
()
self
.
_dostore
(
oid
=
oid
)
self
.
assertEqual
(
self
.
_storage
.
modifiedInVersion
(
oid
),
''
)
def
checkConflicts
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
...
...
src/ZODB/tests/HistoryStorage.py
View file @
eef59eb3
...
...
@@ -23,7 +23,7 @@ from transaction import Transaction
class
HistoryStorage
:
def
checkSimpleHistory
(
self
):
eq
=
self
.
assertEqual
# Store a couple of
non-version
revisions of the object
# Store a couple of revisions of the object
oid
=
self
.
_storage
.
new_oid
()
self
.
assertRaises
(
KeyError
,
self
.
_storage
.
history
,
oid
)
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
...
...
@@ -34,197 +34,29 @@ class HistoryStorage:
eq
(
len
(
h
),
1
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
# Try to get 2 historical revisions
h
=
self
.
_storage
.
history
(
oid
,
size
=
2
)
eq
(
len
(
h
),
2
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
1
]
eq
(
d
[
'tid'
],
revid2
)
eq
(
d
[
'version'
],
''
)
# Try to get all 3 historical revisions
h
=
self
.
_storage
.
history
(
oid
,
size
=
3
)
eq
(
len
(
h
),
3
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
1
]
eq
(
d
[
'tid'
],
revid2
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
2
]
eq
(
d
[
'tid'
],
revid1
)
eq
(
d
[
'version'
],
''
)
# There should be no more than 3 revisions
h
=
self
.
_storage
.
history
(
oid
,
size
=
4
)
eq
(
len
(
h
),
3
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
1
]
eq
(
d
[
'tid'
],
revid2
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
2
]
eq
(
d
[
'tid'
],
revid1
)
eq
(
d
[
'version'
],
''
)
def
checkVersionHistory
(
self
):
if
not
self
.
_storage
.
supportsVersions
():
return
eq
=
self
.
assertEqual
# Store a couple of non-version revisions
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
revid2
=
self
.
_dostore
(
oid
,
revid
=
revid1
,
data
=
MinPO
(
12
))
revid3
=
self
.
_dostore
(
oid
,
revid
=
revid2
,
data
=
MinPO
(
13
))
# Now store some new revisions in a version
version
=
'test-version'
revid4
=
self
.
_dostore
(
oid
,
revid
=
revid3
,
data
=
MinPO
(
14
),
version
=
version
)
revid5
=
self
.
_dostore
(
oid
,
revid
=
revid4
,
data
=
MinPO
(
15
),
version
=
version
)
revid6
=
self
.
_dostore
(
oid
,
revid
=
revid5
,
data
=
MinPO
(
16
),
version
=
version
)
# Now, try to get the six historical revisions (first three are in
# 'test-version', followed by the non-version revisions).
h
=
self
.
_storage
.
history
(
oid
,
version
,
100
)
eq
(
len
(
h
),
6
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid6
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
1
]
eq
(
d
[
'tid'
],
revid5
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
2
]
eq
(
d
[
'tid'
],
revid4
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
3
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
4
]
eq
(
d
[
'tid'
],
revid2
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
5
]
eq
(
d
[
'tid'
],
revid1
)
eq
(
d
[
'version'
],
''
)
def
checkHistoryAfterVersionCommit
(
self
):
if
not
self
.
_storage
.
supportsVersions
():
return
eq
=
self
.
assertEqual
# Store a couple of non-version revisions
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
revid2
=
self
.
_dostore
(
oid
,
revid
=
revid1
,
data
=
MinPO
(
12
))
revid3
=
self
.
_dostore
(
oid
,
revid
=
revid2
,
data
=
MinPO
(
13
))
# Now store some new revisions in a version
version
=
'test-version'
revid4
=
self
.
_dostore
(
oid
,
revid
=
revid3
,
data
=
MinPO
(
14
),
version
=
version
)
revid5
=
self
.
_dostore
(
oid
,
revid
=
revid4
,
data
=
MinPO
(
15
),
version
=
version
)
revid6
=
self
.
_dostore
(
oid
,
revid
=
revid5
,
data
=
MinPO
(
16
),
version
=
version
)
# Now commit the version
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
commitVersion
(
version
,
''
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
# After consultation with Jim, we agreed that the semantics of
# revision id's after a version commit is that the committed object
# gets a new serial number (a.k.a. revision id). Note that
# FileStorage is broken here; the serial number in the post-commit
# non-version revision will be the same as the serial number of the
# previous in-version revision.
#
# BAW: Using load() is the only way to get the serial number of the
# current revision of the object. But at least this works for both
# broken and working storages.
ign
,
revid7
=
self
.
_storage
.
load
(
oid
,
''
)
# Now, try to get the six historical revisions (first three are in
# 'test-version', followed by the non-version revisions).
h
=
self
.
_storage
.
history
(
oid
,
version
,
100
)
eq
(
len
(
h
),
7
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid7
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
1
]
eq
(
d
[
'tid'
],
revid6
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
2
]
eq
(
d
[
'tid'
],
revid5
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
3
]
eq
(
d
[
'tid'
],
revid4
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
4
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
5
]
eq
(
d
[
'tid'
],
revid2
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
6
]
eq
(
d
[
'tid'
],
revid1
)
eq
(
d
[
'version'
],
''
)
def
checkHistoryAfterVersionAbort
(
self
):
if
not
self
.
_storage
.
supportsVersions
():
return
eq
=
self
.
assertEqual
# Store a couple of non-version revisions
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
revid2
=
self
.
_dostore
(
oid
,
revid
=
revid1
,
data
=
MinPO
(
12
))
revid3
=
self
.
_dostore
(
oid
,
revid
=
revid2
,
data
=
MinPO
(
13
))
# Now store some new revisions in a version
version
=
'test-version'
revid4
=
self
.
_dostore
(
oid
,
revid
=
revid3
,
data
=
MinPO
(
14
),
version
=
version
)
revid5
=
self
.
_dostore
(
oid
,
revid
=
revid4
,
data
=
MinPO
(
15
),
version
=
version
)
revid6
=
self
.
_dostore
(
oid
,
revid
=
revid5
,
data
=
MinPO
(
16
),
version
=
version
)
# Now commit the version
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
abortVersion
(
version
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
# After consultation with Jim, we agreed that the semantics of
# revision id's after a version commit is that the committed object
# gets a new serial number (a.k.a. revision id). Note that
# FileStorage is broken here; the serial number in the post-commit
# non-version revision will be the same as the serial number of the
# previous in-version revision.
#
# BAW: Using load() is the only way to get the serial number of the
# current revision of the object. But at least this works for both
# broken and working storages.
ign
,
revid7
=
self
.
_storage
.
load
(
oid
,
''
)
# Now, try to get the six historical revisions (first three are in
# 'test-version', followed by the non-version revisions).
h
=
self
.
_storage
.
history
(
oid
,
version
,
100
)
eq
(
len
(
h
),
7
)
d
=
h
[
0
]
eq
(
d
[
'tid'
],
revid7
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
1
]
eq
(
d
[
'tid'
],
revid6
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
2
]
eq
(
d
[
'tid'
],
revid5
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
3
]
eq
(
d
[
'tid'
],
revid4
)
eq
(
d
[
'version'
],
version
)
d
=
h
[
4
]
eq
(
d
[
'tid'
],
revid3
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
5
]
eq
(
d
[
'tid'
],
revid2
)
eq
(
d
[
'version'
],
''
)
d
=
h
[
6
]
eq
(
d
[
'tid'
],
revid1
)
eq
(
d
[
'version'
],
''
)
src/ZODB/tests/IteratorStorage.py
View file @
eef59eb3
...
...
@@ -34,7 +34,6 @@ class IteratorCompare:
for
rec
in
reciter
:
eq
(
rec
.
oid
,
oid
)
eq
(
rec
.
tid
,
revid
)
eq
(
rec
.
version
,
''
)
eq
(
zodb_unpickle
(
rec
.
data
),
MinPO
(
val
))
val
=
val
+
1
eq
(
val
,
val0
+
len
(
revids
))
...
...
@@ -59,34 +58,7 @@ class IteratorStorage(IteratorCompare):
txniter
.
close
()
self
.
assertRaises
(
IOError
,
txniter
.
__getitem__
,
0
)
def
checkVersionIterator
(
self
):
if
not
self
.
_storage
.
supportsVersions
():
return
self
.
_dostore
()
self
.
_dostore
(
version
=
'abort'
)
self
.
_dostore
()
self
.
_dostore
(
version
=
'abort'
)
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
abortVersion
(
'abort'
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
self
.
_dostore
(
version
=
'commit'
)
self
.
_dostore
()
self
.
_dostore
(
version
=
'commit'
)
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
commitVersion
(
'commit'
,
''
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
txniter
=
self
.
_storage
.
iterator
()
for
trans
in
txniter
:
for
data
in
trans
:
pass
def
checkUndoZombieNonVersion
(
self
):
def
checkUndoZombie
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
94
))
# Get the undo information
...
...
@@ -215,7 +187,6 @@ class IteratorDeepCompare:
for
rec1
,
rec2
in
zip
(
txn1
,
txn2
):
eq
(
rec1
.
oid
,
rec2
.
oid
)
eq
(
rec1
.
tid
,
rec2
.
tid
)
eq
(
rec1
.
version
,
rec2
.
version
)
eq
(
rec1
.
data
,
rec2
.
data
)
# Make sure there are no more records left in rec1 and rec2,
# meaning they were the same length.
...
...
src/ZODB/tests/MTStorage.py
View file @
eef59eb3
...
...
@@ -165,13 +165,6 @@ class ExtStorageClientThread(StorageClientThread):
names
=
[
"do_load"
]
storage
=
self
.
storage
try
:
supportsVersions
=
storage
.
supportsVersions
except
AttributeError
:
pass
else
:
if
supportsVersions
():
names
.
append
(
"do_modifiedInVersion"
)
try
:
supportsUndo
=
storage
.
supportsUndo
...
...
@@ -203,10 +196,6 @@ class ExtStorageClientThread(StorageClientThread):
oid
=
self
.
pick_oid
()
self
.
storage
.
loadSerial
(
oid
,
self
.
oids
[
oid
])
def
do_modifiedInVersion
(
self
):
oid
=
self
.
pick_oid
()
self
.
storage
.
modifiedInVersion
(
oid
)
def
do_undoLog
(
self
):
self
.
storage
.
undoLog
(
0
,
-
20
)
...
...
src/ZODB/tests/PersistentStorage.py
View file @
eef59eb3
...
...
@@ -28,8 +28,6 @@ class PersistentStorage:
self
.
_dostore
()
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
)
if
self
.
_storage
.
supportsVersions
():
self
.
_dostore
(
oid
,
revid
,
data
=
8
,
version
=
'b'
)
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
1
)
revid
=
self
.
_dostore
(
oid
,
revid
,
data
=
2
)
...
...
@@ -40,10 +38,6 @@ class PersistentStorage:
for
oid
in
oids
:
p
,
s
=
self
.
_storage
.
load
(
oid
,
''
)
objects
.
append
((
oid
,
''
,
p
,
s
))
ver
=
self
.
_storage
.
modifiedInVersion
(
oid
)
if
ver
:
p
,
s
=
self
.
_storage
.
load
(
oid
,
ver
)
objects
.
append
((
oid
,
ver
,
p
,
s
))
self
.
_storage
.
close
()
self
.
open
()
...
...
src/ZODB/tests/ReadOnlyStorage.py
View file @
eef59eb3
...
...
@@ -36,7 +36,6 @@ class ReadOnlyStorage:
for
oid
in
self
.
oids
.
keys
():
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
self
.
assertEqual
(
revid
,
self
.
oids
[
oid
])
self
.
assert_
(
not
self
.
_storage
.
modifiedInVersion
(
oid
))
# Storages without revisions may not have loadSerial().
try
:
_data
=
self
.
_storage
.
loadSerial
(
oid
,
revid
)
...
...
@@ -50,12 +49,6 @@ class ReadOnlyStorage:
t
=
transaction
.
Transaction
()
self
.
assertRaises
(
ReadOnlyError
,
self
.
_storage
.
tpc_begin
,
t
)
if
self
.
_storage
.
supportsVersions
():
self
.
assertRaises
(
ReadOnlyError
,
self
.
_storage
.
abortVersion
,
''
,
t
)
self
.
assertRaises
(
ReadOnlyError
,
self
.
_storage
.
commitVersion
,
''
,
''
,
t
)
self
.
assertRaises
(
ReadOnlyError
,
self
.
_storage
.
store
,
'
\
000
'
*
8
,
None
,
''
,
''
,
t
)
...
...
src/ZODB/tests/RecoveryStorage.py
View file @
eef59eb3
...
...
@@ -32,104 +32,6 @@ class RecoveryStorage(IteratorDeepCompare):
self
.
_dst
.
copyTransactionsFrom
(
self
.
_storage
)
self
.
compare
(
self
.
_storage
,
self
.
_dst
)
def
checkRecoveryAcrossVersions
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
21
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
22
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
23
,
version
=
'one'
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
34
,
version
=
'one'
)
# Now commit the version
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
commitVersion
(
'one'
,
''
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
self
.
_dst
.
copyTransactionsFrom
(
self
.
_storage
)
self
.
compare
(
self
.
_storage
,
self
.
_dst
)
def
checkRecoverAbortVersion
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
21
,
version
=
"one"
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
23
,
version
=
'one'
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
34
,
version
=
'one'
)
# Now abort the version and the creation
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
tid
,
oids
=
self
.
_storage
.
abortVersion
(
'one'
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
self
.
assertEqual
(
oids
,
[
oid
])
self
.
_dst
.
copyTransactionsFrom
(
self
.
_storage
)
self
.
compare
(
self
.
_storage
,
self
.
_dst
)
# Also make sure the the last transaction has a data record
# with None for its data attribute, because we've undone the
# object.
for
s
in
self
.
_storage
,
self
.
_dst
:
iter
=
s
.
iterator
()
for
trans
in
iter
:
pass
# iterate until we get the last one
data
=
trans
[
0
]
self
.
assertRaises
(
IndexError
,
lambda
i
,
t
=
trans
:
t
[
i
],
1
)
self
.
assertEqual
(
data
.
oid
,
oid
)
self
.
assertEqual
(
data
.
data
,
None
)
def
checkRecoverUndoInVersion
(
self
):
oid
=
self
.
_storage
.
new_oid
()
version
=
"aVersion"
revid_a
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
91
))
revid_b
=
self
.
_dostore
(
oid
,
revid
=
revid_a
,
version
=
version
,
data
=
MinPO
(
92
))
revid_c
=
self
.
_dostore
(
oid
,
revid
=
revid_b
,
version
=
version
,
data
=
MinPO
(
93
))
self
.
_undo
(
self
.
_storage
.
undoInfo
()[
0
][
'id'
],
[
oid
])
self
.
_commitVersion
(
version
,
''
)
self
.
_undo
(
self
.
_storage
.
undoInfo
()[
0
][
'id'
],
[
oid
])
# now copy the records to a new storage
self
.
_dst
.
copyTransactionsFrom
(
self
.
_storage
)
self
.
compare
(
self
.
_storage
,
self
.
_dst
)
# The last two transactions were applied directly rather than
# copied. So we can't use compare() to verify that they new
# transactions are applied correctly. (The new transactions
# will have different timestamps for each storage.)
self
.
_abortVersion
(
version
)
self
.
assert_
(
self
.
_storage
.
versionEmpty
(
version
))
self
.
_undo
(
self
.
_storage
.
undoInfo
()[
0
][
'id'
],
[
oid
])
self
.
assert_
(
not
self
.
_storage
.
versionEmpty
(
version
))
# check the data is what we expect it to be
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
92
))
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
91
))
# and swap the storages
tmp
=
self
.
_storage
self
.
_storage
=
self
.
_dst
self
.
_abortVersion
(
version
)
self
.
assert_
(
self
.
_storage
.
versionEmpty
(
version
))
self
.
_undo
(
self
.
_storage
.
undoInfo
()[
0
][
'id'
],
[
oid
])
self
.
assert_
(
not
self
.
_storage
.
versionEmpty
(
version
))
# check the data is what we expect it to be
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
92
))
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
91
))
# swap them back
self
.
_storage
=
tmp
# Now remove _dst and copy all the transactions a second time.
# This time we will be able to confirm via compare().
self
.
_dst
.
close
()
self
.
_dst
.
cleanup
()
self
.
_dst
=
self
.
new_dest
()
self
.
_dst
.
copyTransactionsFrom
(
self
.
_storage
)
self
.
compare
(
self
.
_storage
,
self
.
_dst
)
def
checkRestoreAcrossPack
(
self
):
db
=
DB
(
self
.
_storage
)
c
=
db
.
open
()
...
...
@@ -150,7 +52,7 @@ class RecoveryStorage(IteratorDeepCompare):
final
=
list
(
it
)[
-
1
]
self
.
_dst
.
tpc_begin
(
final
,
final
.
tid
,
final
.
status
)
for
r
in
final
:
self
.
_dst
.
restore
(
r
.
oid
,
r
.
tid
,
r
.
data
,
r
.
version
,
r
.
data_txn
,
self
.
_dst
.
restore
(
r
.
oid
,
r
.
tid
,
r
.
data
,
''
,
r
.
data_txn
,
final
)
it
.
close
()
self
.
_dst
.
tpc_vote
(
final
)
...
...
src/ZODB/tests/StorageTestBase.py
View file @
eef59eb3
...
...
@@ -160,14 +160,13 @@ class StorageTestBase(unittest.TestCase):
def
tearDown
(
self
):
self
.
_close
()
def
_dostore
(
self
,
oid
=
None
,
revid
=
None
,
data
=
None
,
version
=
None
,
def
_dostore
(
self
,
oid
=
None
,
revid
=
None
,
data
=
None
,
already_pickled
=
0
,
user
=
None
,
description
=
None
):
"""Do a complete storage transaction. The defaults are:
- oid=None, ask the storage for a new oid
- revid=None, use a revid of ZERO
- data=None, pickle up some arbitrary data (the integer 7)
- version=None, use the empty string version
Returns the object's new revision id.
"""
...
...
@@ -181,8 +180,6 @@ class StorageTestBase(unittest.TestCase):
data
=
MinPO
(
data
)
if
not
already_pickled
:
data
=
zodb_pickle
(
data
)
if
version
is
None
:
version
=
''
# Begin the transaction
t
=
transaction
.
Transaction
()
if
user
is
not
None
:
...
...
@@ -192,7 +189,7 @@ class StorageTestBase(unittest.TestCase):
try
:
self
.
_storage
.
tpc_begin
(
t
)
# Store an object
r1
=
self
.
_storage
.
store
(
oid
,
revid
,
data
,
version
,
t
)
r1
=
self
.
_storage
.
store
(
oid
,
revid
,
data
,
''
,
t
)
# Finish the transaction
r2
=
self
.
_storage
.
tpc_vote
(
t
)
revid
=
handle_serials
(
oid
,
r1
,
r2
)
...
...
@@ -202,9 +199,9 @@ class StorageTestBase(unittest.TestCase):
raise
return
revid
def
_dostoreNP
(
self
,
oid
=
None
,
revid
=
None
,
data
=
None
,
version
=
None
,
def
_dostoreNP
(
self
,
oid
=
None
,
revid
=
None
,
data
=
None
,
user
=
None
,
description
=
None
):
return
self
.
_dostore
(
oid
,
revid
,
data
,
version
,
1
,
user
,
description
)
return
self
.
_dostore
(
oid
,
revid
,
data
,
1
,
user
,
description
)
# The following methods depend on optional storage features.
...
...
@@ -222,21 +219,3 @@ class StorageTestBase(unittest.TestCase):
for
oid
in
expected_oids
:
self
.
assert_
(
oid
in
oids
)
return
self
.
_storage
.
lastTransaction
()
def
_commitVersion
(
self
,
src
,
dst
):
t
=
transaction
.
Transaction
()
t
.
note
(
"commit %r to %r"
%
(
src
,
dst
))
self
.
_storage
.
tpc_begin
(
t
)
tid
,
oids
=
self
.
_storage
.
commitVersion
(
src
,
dst
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
return
oids
def
_abortVersion
(
self
,
ver
):
t
=
transaction
.
Transaction
()
t
.
note
(
"abort %r"
%
ver
)
self
.
_storage
.
tpc_begin
(
t
)
tid
,
oids
=
self
.
_storage
.
abortVersion
(
ver
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
return
oids
src/ZODB/tests/Synchronization.py
View file @
eef59eb3
...
...
@@ -45,7 +45,7 @@ undo(): how's that handled?
Methods that have nothing to do with committing/non-committing:
load(), loadSerial(), getName(), getSize(), __len__(), history(),
undoLog(),
modifiedInVersion(), versionEmpty(), versions(),
pack().
undoLog(), pack().
Specific questions:
...
...
@@ -65,7 +65,6 @@ tested? Is it a general restriction?
from
transaction
import
Transaction
from
ZODB.POSException
import
StorageTransactionError
VERSION
=
"testversion"
OID
=
"
\
000
"
*
8
SERIALNO
=
"
\
000
"
*
8
TID
=
"
\
000
"
*
8
...
...
@@ -84,35 +83,6 @@ class SynchronizedStorage:
self
.
assertRaises
(
StorageTransactionError
,
callable
,
*
args
)
self
.
_storage
.
tpc_abort
(
t
)
def
__supportsVersions
(
self
):
storage
=
self
.
_storage
try
:
supportsVersions
=
storage
.
supportsVersions
except
AttributeError
:
return
False
return
supportsVersions
()
def
checkAbortVersionNotCommitting
(
self
):
if
self
.
__supportsVersions
():
self
.
verifyNotCommitting
(
self
.
_storage
.
abortVersion
,
VERSION
,
Transaction
())
def
checkAbortVersionWrongTrans
(
self
):
if
self
.
__supportsVersions
():
self
.
verifyWrongTrans
(
self
.
_storage
.
abortVersion
,
VERSION
,
Transaction
())
def
checkCommitVersionNotCommitting
(
self
):
if
self
.
__supportsVersions
():
self
.
verifyNotCommitting
(
self
.
_storage
.
commitVersion
,
VERSION
,
""
,
Transaction
())
def
checkCommitVersionWrongTrans
(
self
):
if
self
.
__supportsVersions
():
self
.
verifyWrongTrans
(
self
.
_storage
.
commitVersion
,
VERSION
,
""
,
Transaction
())
def
checkStoreNotCommitting
(
self
):
self
.
verifyNotCommitting
(
self
.
_storage
.
store
,
OID
,
SERIALNO
,
""
,
""
,
Transaction
())
...
...
src/ZODB/tests/TransactionalUndoVersionStorage.py
deleted
100644 → 0
View file @
7b044c7d
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# Check interactions between undo() and versions. Any storage that
# supports both undo() and versions must pass these tests.
import
time
import
transaction
from
ZODB.serialize
import
referencesf
from
ZODB.tests.MinPO
import
MinPO
from
ZODB.tests.StorageTestBase
import
zodb_unpickle
from
ZODB.tests.VersionStorage
import
loadEx
class
TransactionalUndoVersionStorage
:
# `hook` is a callable used by the ZRS tests.
def
checkUndoInVersion
(
self
,
hook
=
None
):
eq
=
self
.
assertEqual
unless
=
self
.
failUnless
def
check_objects
(
nonversiondata
,
versiondata
):
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
versiondata
))
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
nonversiondata
))
oid
=
self
.
_storage
.
new_oid
()
version
=
'one'
revid_a
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
91
))
revid_b
=
self
.
_dostore
(
oid
,
revid
=
revid_a
,
data
=
MinPO
(
92
),
version
=
version
)
revid_c
=
self
.
_dostore
(
oid
,
revid
=
revid_b
,
data
=
MinPO
(
93
),
version
=
version
)
info
=
self
.
_storage
.
undoInfo
()
self
.
_undo
(
info
[
0
][
'id'
],
[
oid
])
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
# load() always returns the tid of the most recent reversion in 3.4,
# so this old test of revid can't work anymore.
##eq(revid, revid_a)
# But the data should be correct for the non-version revision.
eq
(
zodb_unpickle
(
data
),
MinPO
(
91
))
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
unless
(
revid
>
revid_b
and
revid
>
revid_c
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
92
))
# Now commit the version...
oids
=
self
.
_commitVersion
(
version
,
""
)
eq
(
len
(
oids
),
1
)
eq
(
oids
[
0
],
oid
)
check_objects
(
92
,
92
)
# ...and undo the commit
info
=
self
.
_storage
.
undoInfo
()
self
.
_undo
(
info
[
0
][
'id'
],
[
oid
])
check_objects
(
91
,
92
)
if
hook
:
# ZRS passes a hook that arranges to start a secondary at this
# point in the test.
hook
()
# Now abort the version.
oids
=
self
.
_abortVersion
(
version
)
assert
len
(
oids
)
==
1
assert
oids
[
0
]
==
oid
check_objects
(
91
,
91
)
# Now undo the abort.
info
=
self
.
_storage
.
undoInfo
()
self
.
_undo
(
info
[
0
][
'id'
],
[
oid
])
# And the object should be back in versions 'one' and ''.
check_objects
(
91
,
92
)
def
checkUndoCommitVersion
(
self
):
def
load_value
(
oid
,
version
=
''
):
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
return
zodb_unpickle
(
data
).
value
# create a bunch of packable transactions
oid
=
self
.
_storage
.
new_oid
()
revid
=
'
\
000
'
*
8
for
i
in
range
(
4
):
revid
=
self
.
_dostore
(
oid
,
revid
,
description
=
'packable%d'
%
i
)
pt
=
time
.
time
()
time
.
sleep
(
1
)
oid1
=
self
.
_storage
.
new_oid
()
version
=
'version'
revid1
=
self
.
_dostore
(
oid1
,
data
=
MinPO
(
0
),
description
=
'create1'
)
revid2
=
self
.
_dostore
(
oid1
,
data
=
MinPO
(
1
),
revid
=
revid1
,
version
=
version
,
description
=
'version1'
)
self
.
_dostore
(
oid1
,
data
=
MinPO
(
2
),
revid
=
revid2
,
version
=
version
,
description
=
'version2'
)
self
.
_dostore
(
description
=
'create2'
)
t
=
transaction
.
Transaction
()
t
.
description
=
'commit version'
self
.
_storage
.
tpc_begin
(
t
)
self
.
_storage
.
commitVersion
(
version
,
''
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
info
=
self
.
_storage
.
undoInfo
()
t_id
=
info
[
0
][
'id'
]
self
.
assertEqual
(
load_value
(
oid1
),
2
)
self
.
assertEqual
(
load_value
(
oid1
,
version
),
2
)
self
.
_storage
.
pack
(
pt
,
referencesf
)
self
.
_undo
(
t_id
,
note
=
"undo commit version"
)
self
.
assertEqual
(
load_value
(
oid1
),
0
)
self
.
assertEqual
(
load_value
(
oid1
,
version
),
2
)
data
,
tid
=
self
.
_storage
.
load
(
oid1
,
""
)
# After undoing the version commit, the non-version data
# once again becomes the non-version data from 'create1'.
self
.
assertEqual
(
tid
,
self
.
_storage
.
lastTransaction
())
# The current version data comes from an undo record, which
# means that it gets data via the backpointer but tid from the
# current txn.
data
,
tid
,
ver
=
loadEx
(
self
.
_storage
,
oid1
,
version
)
self
.
assertEqual
(
ver
,
version
)
self
.
assertEqual
(
tid
,
self
.
_storage
.
lastTransaction
())
def
checkUndoAbortVersion
(
self
):
def
load_value
(
oid
,
version
=
''
):
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
return
zodb_unpickle
(
data
).
value
# create a bunch of packable transactions
oid
=
self
.
_storage
.
new_oid
()
revid
=
'
\
000
'
*
8
for
i
in
range
(
3
):
revid
=
self
.
_dostore
(
oid
,
revid
,
description
=
'packable%d'
%
i
)
pt
=
time
.
time
()
time
.
sleep
(
1
)
oid1
=
self
.
_storage
.
new_oid
()
version
=
'version'
revid1
=
self
.
_dostore
(
oid1
,
data
=
MinPO
(
0
),
description
=
'create1'
)
revid2
=
self
.
_dostore
(
oid1
,
data
=
MinPO
(
1
),
revid
=
revid1
,
version
=
version
,
description
=
'version1'
)
self
.
_dostore
(
oid1
,
data
=
MinPO
(
2
),
revid
=
revid2
,
version
=
version
,
description
=
'version2'
)
self
.
_dostore
(
description
=
'create2'
)
self
.
_abortVersion
(
version
)
info
=
self
.
_storage
.
undoInfo
()
t_id
=
info
[
0
][
'id'
]
self
.
assertEqual
(
load_value
(
oid1
),
0
)
# after abort, we should see non-version data
self
.
assertEqual
(
load_value
(
oid1
,
version
),
0
)
self
.
_undo
(
t_id
,
note
=
"undo abort version"
)
self
.
assertEqual
(
load_value
(
oid1
),
0
)
# t undo will re-create the version
self
.
assertEqual
(
load_value
(
oid1
,
version
),
2
)
info
=
self
.
_storage
.
undoInfo
()
t_id
=
info
[
0
][
'id'
]
self
.
_storage
.
pack
(
pt
,
referencesf
)
self
.
_undo
(
t_id
,
note
=
"undo undo"
)
# undo of undo will put as back where we started
self
.
assertEqual
(
load_value
(
oid1
),
0
)
# after abort, we should see non-version data
self
.
assertEqual
(
load_value
(
oid1
,
version
),
0
)
src/ZODB/tests/VersionStorage.py
deleted
100644 → 0
View file @
7b044c7d
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Run the version related tests for a storage.
Any storage that supports versions should be able to pass all these tests.
"""
import
time
,
warnings
import
transaction
from
transaction
import
Transaction
from
ZODB
import
POSException
from
ZODB.serialize
import
referencesf
from
ZODB.tests.MinPO
import
MinPO
from
ZODB.tests.StorageTestBase
import
zodb_unpickle
,
snooze
from
ZODB
import
DB
def
loadEx
(
storage
,
oid
,
version
):
v
=
storage
.
modifiedInVersion
(
oid
)
if
v
==
version
:
data
,
serial
=
storage
.
load
(
oid
,
version
)
return
data
,
serial
,
version
else
:
data
,
serial
=
storage
.
load
(
oid
,
''
)
return
data
,
serial
,
''
warnings
.
filterwarnings
(
'ignore'
,
message
=
'Versions are deprecated'
,
module
=
__name__
)
class
VersionStorage
:
def
checkCommitVersionSerialno
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
12
))
revid2
=
self
.
_dostore
(
oid
,
revid
=
revid1
,
data
=
MinPO
(
13
),
version
=
"version"
)
oids
=
self
.
_commitVersion
(
"version"
,
""
)
self
.
assertEqual
([
oid
],
oids
)
data
,
revid3
=
self
.
_storage
.
load
(
oid
,
""
)
# use repr() to avoid getting binary data in a traceback on error
self
.
assertNotEqual
(
`revid1`
,
`revid3`
)
self
.
assertNotEqual
(
`revid2`
,
`revid3`
)
def
checkAbortVersionSerialno
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
12
))
revid2
=
self
.
_dostore
(
oid
,
revid
=
revid1
,
data
=
MinPO
(
13
),
version
=
"version"
)
data
,
tid
,
ver
=
loadEx
(
self
.
_storage
,
oid
,
"version"
)
self
.
assertEqual
(
revid2
,
tid
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
13
))
oids
=
self
.
_abortVersion
(
"version"
)
self
.
assertEqual
([
oid
],
oids
)
data
,
revid3
=
self
.
_storage
.
load
(
oid
,
""
)
# use repr() to avoid getting binary data in a traceback on error
self
.
assertNotEqual
(
revid1
,
revid3
)
self
.
assertNotEqual
(
revid2
,
revid3
)
data
,
tid
=
self
.
_storage
.
load
(
oid
,
""
)
self
.
assertEqual
(
revid3
,
tid
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
12
))
self
.
assertEqual
(
tid
,
self
.
_storage
.
lastTransaction
())
def
checkVersionedStoreAndLoad
(
self
):
eq
=
self
.
assertEqual
# Store a couple of non-version revisions of the object
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
revid1
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
12
))
# And now store some new revisions in a version
version
=
'test-version'
revid
=
self
.
_dostore
(
oid
,
revid
=
revid1
,
data
=
MinPO
(
13
),
version
=
version
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
14
),
version
=
version
)
revid2
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
15
),
version
=
version
)
# Now read back the object in both the non-version and version and
# make sure the values jive.
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
12
))
data
,
vrevid
=
self
.
_storage
.
load
(
oid
,
version
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
15
))
if
hasattr
(
self
.
_storage
,
'getTid'
):
s
=
self
.
_storage
.
getTid
(
oid
)
eq
(
s
,
max
(
revid
,
vrevid
))
data
,
tid
,
ver
=
loadEx
(
self
.
_storage
,
oid
,
version
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
15
))
eq
(
tid
,
revid2
)
data
,
tid
,
ver
=
loadEx
(
self
.
_storage
,
oid
,
"other version"
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
12
))
eq
(
tid
,
revid2
)
# loadSerial returns non-version data
try
:
data
=
self
.
_storage
.
loadSerial
(
oid
,
revid
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
12
))
data
=
self
.
_storage
.
loadSerial
(
oid
,
revid2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
12
))
except
POSException
.
Unsupported
:
pass
def
checkVersionedLoadErrors
(
self
):
oid
=
self
.
_storage
.
new_oid
()
version
=
'test-version'
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
12
),
version
=
version
)
# Try to load a bogus oid
self
.
assertRaises
(
KeyError
,
self
.
_storage
.
load
,
self
.
_storage
.
new_oid
(),
''
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
'bogus'
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
11
))
def
checkVersionLock
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
version
=
'test-version'
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
12
),
version
=
version
)
self
.
assertRaises
(
POSException
.
VersionLockError
,
self
.
_dostore
,
oid
,
revid
=
revid
,
data
=
MinPO
(
14
),
version
=
'another-version'
)
def
checkVersionEmpty
(
self
):
# Before we store anything, these versions ought to be empty
version
=
'test-version'
self
.
failUnless
(
self
.
_storage
.
versionEmpty
(
version
))
# Now store some objects
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
11
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
12
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
13
),
version
=
version
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
14
),
version
=
version
)
# The blank version should not be empty
# Neither should 'test-version'
self
.
failUnless
(
not
self
.
_storage
.
versionEmpty
(
version
))
# But this non-existant version should be empty
self
.
failUnless
(
self
.
_storage
.
versionEmpty
(
'bogus'
))
def
checkVersions
(
self
):
unless
=
self
.
failUnless
# Store some objects in the non-version
oid1
=
self
.
_storage
.
new_oid
()
oid2
=
self
.
_storage
.
new_oid
()
oid3
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid1
,
data
=
MinPO
(
11
))
revid2
=
self
.
_dostore
(
oid2
,
data
=
MinPO
(
12
))
revid3
=
self
.
_dostore
(
oid3
,
data
=
MinPO
(
13
))
# Now create some new versions
revid1
=
self
.
_dostore
(
oid1
,
revid
=
revid1
,
data
=
MinPO
(
14
),
version
=
'one'
)
revid2
=
self
.
_dostore
(
oid2
,
revid
=
revid2
,
data
=
MinPO
(
15
),
version
=
'two'
)
revid3
=
self
.
_dostore
(
oid3
,
revid
=
revid3
,
data
=
MinPO
(
16
),
version
=
'three'
)
# Ask for the versions
versions
=
self
.
_storage
.
versions
()
unless
(
'one'
in
versions
)
unless
(
'two'
in
versions
)
unless
(
'three'
in
versions
)
# Now flex the `max' argument
versions
=
self
.
_storage
.
versions
(
1
)
self
.
assertEqual
(
len
(
versions
),
1
)
unless
(
'one'
in
versions
or
'two'
in
versions
or
'three'
in
versions
)
def
_setup_version
(
self
,
version
=
'test-version'
):
# Store some revisions in the non-version
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
MinPO
(
49
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
50
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
51
))
# Now do some stores in a version
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
52
),
version
=
version
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
53
),
version
=
version
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
54
),
version
=
version
)
return
oid
,
version
def
checkAbortVersion
(
self
):
eq
=
self
.
assertEqual
oid
,
version
=
self
.
_setup_version
()
# Not sure I can write a test for getTid() in the
# presence of aborted versions, because FileStorage and
# Berkeley storage give a different answer. I think Berkeley
# is right and FS is wrong.
oids
=
self
.
_abortVersion
(
version
)
eq
(
len
(
oids
),
1
)
eq
(
oids
[
0
],
oid
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
def
checkAbortVersionNonCurrent
(
self
):
# Make sure the non-current serial number is correctly
# after a version is aborted.
oid
,
version
=
self
.
_setup_version
()
self
.
_abortVersion
(
version
)
data
,
tid
,
ver
=
loadEx
(
self
.
_storage
,
oid
,
""
)
# write a new revision of oid so that the aborted-version txn
# is not current
self
.
_dostore
(
oid
,
revid
=
tid
,
data
=
MinPO
(
17
))
ltid
=
self
.
_storage
.
lastTransaction
()
ncdata
,
ncstart
,
end
=
self
.
_storage
.
loadBefore
(
oid
,
ltid
)
self
.
assertEqual
(
data
,
ncdata
)
self
.
assertEqual
(
tid
,
ncstart
)
def
checkAbortVersionErrors
(
self
):
eq
=
self
.
assertEqual
oid
,
version
=
self
.
_setup_version
()
# Now abort a bogus version
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
# And try to abort the empty version
self
.
assertRaises
(
POSException
.
VersionError
,
self
.
_storage
.
abortVersion
,
''
,
t
)
# But now we really try to abort the version
tid
,
oids
=
self
.
_storage
.
abortVersion
(
version
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
eq
(
len
(
oids
),
1
)
eq
(
oids
[
0
],
oid
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
def
checkCommitVersionErrors
(
self
):
eq
=
self
.
assertEqual
oid1
,
version1
=
self
.
_setup_version
(
'one'
)
data
,
revid1
=
self
.
_storage
.
load
(
oid1
,
version1
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
try
:
self
.
assertRaises
(
POSException
.
VersionCommitError
,
self
.
_storage
.
commitVersion
,
'one'
,
'one'
,
t
)
finally
:
self
.
_storage
.
tpc_abort
(
t
)
def
checkNewSerialOnCommitVersionToVersion
(
self
):
oid
,
version
=
self
.
_setup_version
()
data
,
vtid
=
self
.
_storage
.
load
(
oid
,
version
)
data
,
ntid
=
self
.
_storage
.
load
(
oid
,
''
)
version2
=
'test version 2'
self
.
_commitVersion
(
version
,
version2
)
data
,
tid
=
self
.
_storage
.
load
(
oid
,
version2
)
self
.
failUnless
(
tid
!=
vtid
and
tid
!=
ntid
,
"New tid, %r, should be different from the old "
"version, %r, and non-version, %r, tids."
%
(
tid
,
vtid
,
ntid
))
def
checkModifyAfterAbortVersion
(
self
):
eq
=
self
.
assertEqual
oid
,
version
=
self
.
_setup_version
()
self
.
_abortVersion
(
version
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
# And modify it a few times
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
52
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
53
))
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
54
))
data
,
newrevid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
newrevid
,
revid
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
def
checkCommitToNonVersion
(
self
):
eq
=
self
.
assertEqual
oid
,
version
=
self
.
_setup_version
()
data
,
revid
=
self
.
_storage
.
load
(
oid
,
version
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
self
.
_commitVersion
(
version
,
''
)
data
,
revid
=
self
.
_storage
.
load
(
oid
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
def
checkCommitToOtherVersion
(
self
):
eq
=
self
.
assertEqual
oid1
,
version1
=
self
.
_setup_version
(
'one'
)
data
,
revid1
=
self
.
_storage
.
load
(
oid1
,
version1
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
oid2
,
version2
=
self
.
_setup_version
(
'two'
)
data
,
revid2
=
self
.
_storage
.
load
(
oid2
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
# make sure we see the non-version data when appropriate
data
,
revid2
=
self
.
_storage
.
load
(
oid1
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
data
,
revid2
=
self
.
_storage
.
load
(
oid2
,
version1
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
data
,
revid2
=
self
.
_storage
.
load
(
oid1
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
# Okay, now let's commit object1 to version2
oids
=
self
.
_commitVersion
(
version1
,
version2
)
eq
(
len
(
oids
),
1
)
eq
(
oids
[
0
],
oid1
)
data
,
revid
=
self
.
_storage
.
load
(
oid1
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
data
,
revid
=
self
.
_storage
.
load
(
oid2
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
# an object can only exist in one version, so a load from
# version1 should now give the non-version data
data
,
revid2
=
self
.
_storage
.
load
(
oid1
,
version1
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
# as should a version that has never been used
data
,
revid2
=
self
.
_storage
.
load
(
oid1
,
'bela lugosi'
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
def
checkAbortOneVersionCommitTheOther
(
self
):
eq
=
self
.
assertEqual
oid1
,
version1
=
self
.
_setup_version
(
'one'
)
data
,
revid1
=
self
.
_storage
.
load
(
oid1
,
version1
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
oid2
,
version2
=
self
.
_setup_version
(
'two'
)
data
,
revid2
=
self
.
_storage
.
load
(
oid2
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
# Let's make sure we can't get object1 in version2
data
,
revid2
=
self
.
_storage
.
load
(
oid1
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
oids
=
self
.
_abortVersion
(
version1
)
eq
(
len
(
oids
),
1
)
eq
(
oids
[
0
],
oid1
)
data
,
revid
=
self
.
_storage
.
load
(
oid1
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
data
,
revid
=
self
.
_storage
.
load
(
oid1
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
data
,
revid
=
self
.
_storage
.
load
(
oid1
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
data
,
revid
=
self
.
_storage
.
load
(
oid2
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
data
,
revid
=
self
.
_storage
.
load
(
oid2
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
# Okay, now let's commit version2 back to the trunk
oids
=
self
.
_commitVersion
(
version2
,
''
)
eq
(
len
(
oids
),
1
)
eq
(
oids
[
0
],
oid2
)
data
,
revid
=
self
.
_storage
.
load
(
oid1
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
51
))
# But the trunk should be up to date now
data
,
revid
=
self
.
_storage
.
load
(
oid2
,
''
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
data
,
revid
=
self
.
_storage
.
load
(
oid2
,
version2
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
54
))
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
MinPO
(
54
),
version
=
'one'
)
self
.
assertRaises
(
KeyError
,
self
.
_storage
.
load
,
oid
,
''
)
self
.
assertRaises
(
KeyError
,
self
.
_storage
.
load
,
oid
,
'two'
)
def
checkCreateObjectInVersionWithAbort
(
self
):
oid
=
self
.
_storage
.
new_oid
()
revid
=
self
.
_dostore
(
oid
,
data
=
21
,
version
=
"one"
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
23
,
version
=
'one'
)
revid
=
self
.
_dostore
(
oid
,
revid
=
revid
,
data
=
34
,
version
=
'one'
)
# Now abort the version and the creation
t
=
Transaction
()
self
.
_storage
.
tpc_begin
(
t
)
tid
,
oids
=
self
.
_storage
.
abortVersion
(
'one'
,
t
)
self
.
_storage
.
tpc_vote
(
t
)
self
.
_storage
.
tpc_finish
(
t
)
self
.
assertEqual
(
oids
,
[
oid
])
def
checkLoadBeforeVersion
(
self
):
eq
=
self
.
assertEqual
oid
=
self
.
_storage
.
new_oid
()
revid1
=
self
.
_dostore
(
oid
,
data
=
1
)
revid2
=
self
.
_dostore
(
oid
,
data
=
2
,
revid
=
revid1
,
version
=
"kobe"
)
revid3
=
self
.
_dostore
(
oid
,
data
=
3
,
revid
=
revid2
,
version
=
"kobe"
)
data
,
start_tid
,
end_tid
=
self
.
_storage
.
loadBefore
(
oid
,
revid3
)
eq
(
zodb_unpickle
(
data
),
MinPO
(
1
))
eq
(
start_tid
,
revid1
)
eq
(
end_tid
,
None
)
src/ZODB/tests/testConnection.py
View file @
eef59eb3
...
...
@@ -145,9 +145,6 @@ class UserMethodTests(unittest.TestCase):
# add isn't tested here, because there are a bunch of traditional
# unit tests for it.
# The version tests would require a storage that supports versions
# which is a bit more work.
def
test_root
(
self
):
r"""doctest of root() method
...
...
@@ -547,7 +544,6 @@ class StubStorage:
Only one concurrent transaction is supported.
Voting is not supported.
Versions are not supported.
Inspect self._stored and self._finished to see how the storage has been
used during a unit test. Whenever an object is stored in the store()
...
...
@@ -607,7 +603,7 @@ class StubStorage:
self
.
_transdata
.
clear
()
self
.
_transstored
=
[]
def
load
(
self
,
oid
,
version
):
def
load
(
self
,
oid
,
version
=
''
):
if
version
!=
''
:
raise
TypeError
(
'StubStorage does not support versions.'
)
return
self
.
_data
[
oid
]
...
...
src/ZODB/tests/testDemoStorage.py
View file @
eef59eb3
...
...
@@ -17,12 +17,11 @@ import transaction
from
ZODB.DB
import
DB
import
ZODB.utils
import
ZODB.DemoStorage
from
ZODB.tests
import
StorageTestBase
,
BasicStorage
,
VersionStorage
from
ZODB.tests
import
StorageTestBase
,
BasicStorage
from
ZODB.tests
import
Synchronization
class
DemoStorageTests
(
StorageTestBase
.
StorageTestBase
,
BasicStorage
.
BasicStorage
,
VersionStorage
.
VersionStorage
,
Synchronization
.
SynchronizedStorage
,
):
...
...
@@ -38,25 +37,6 @@ class DemoStorageTests(StorageTestBase.StorageTestBase,
# have this limit, so we inhibit this test here.
pass
def
checkAbortVersionNonCurrent
(
self
):
# TODO: Need to implement a real loadBefore for DemoStorage?
pass
def
checkLoadBeforeVersion
(
self
):
# TODO: Need to implement a real loadBefore for DemoStorage?
pass
# the next three pack tests depend on undo
def
checkPackVersionReachable
(
self
):
pass
def
checkPackVersions
(
self
):
pass
def
checkPackVersionsInPast
(
self
):
pass
def
checkLoadDelegation
(
self
):
# Minimal test of loadEX w/o version -- ironically
db
=
DB
(
self
.
_storage
)
# creates object 0. :)
...
...
src/ZODB/tests/testFileStorage.py
View file @
eef59eb3
...
...
@@ -19,7 +19,6 @@ from ZODB import POSException
from
ZODB
import
DB
from
ZODB.tests
import
StorageTestBase
,
BasicStorage
,
TransactionalUndoStorage
from
ZODB.tests
import
VersionStorage
,
TransactionalUndoVersionStorage
from
ZODB.tests
import
PackableStorage
,
Synchronization
,
ConflictResolution
from
ZODB.tests
import
HistoryStorage
,
IteratorStorage
,
Corruption
from
ZODB.tests
import
RevisionStorage
,
PersistentStorage
,
MTStorage
...
...
@@ -44,8 +43,6 @@ class FileStorageTests(
BasicStorage
.
BasicStorage
,
TransactionalUndoStorage
.
TransactionalUndoStorage
,
RevisionStorage
.
RevisionStorage
,
VersionStorage
.
VersionStorage
,
TransactionalUndoVersionStorage
.
TransactionalUndoVersionStorage
,
PackableStorage
.
PackableStorage
,
PackableStorage
.
PackableUndoStorage
,
Synchronization
.
SynchronizedStorage
,
...
...
@@ -182,45 +179,6 @@ class FileStorageTests(
self
.
open
()
self
.
assertEqual
(
self
.
_storage
.
_saved
,
1
)
def
check_index_oid_ignored
(
self
):
# Prior to ZODB 3.2.6, the 'oid' value stored in the .index file
# was believed. But there were cases where adding larger oids
# didn't update the FileStorage ._oid attribute -- the restore()
# method in particular didn't update it, and that's about the only
# method copyTransactionsFrom() uses. A database copy created that
# way then stored an 'oid' of z64 in the .index file. This created
# torturous problems, as when that file was opened, "new" oids got
# generated starting over from 0 again.
# Now the cached 'oid' value is ignored: verify that this is so.
import
cPickle
as
pickle
from
ZODB.utils
import
z64
# Create some data.
db
=
DB
(
self
.
_storage
)
conn
=
db
.
open
()
conn
.
root
()[
'xyz'
]
=
1
transaction
.
commit
()
true_max_oid
=
self
.
_storage
.
_oid
# Save away the index, and poke in a bad 'oid' value by hand.
db
.
close
()
f
=
open
(
'FileStorageTests.fs.index'
,
'r+b'
)
p
=
pickle
.
Unpickler
(
f
)
data
=
p
.
load
()
saved_oid
=
data
[
'oid'
]
self
.
assertEqual
(
true_max_oid
,
saved_oid
)
data
[
'oid'
]
=
z64
f
.
seek
(
0
)
f
.
truncate
()
p
=
pickle
.
Pickler
(
f
,
1
)
p
.
dump
(
data
)
f
.
close
()
# Verify that we get the correct oid again when we reopen, despite
# that we stored nonsense in the .index file's 'oid'.
self
.
open
()
self
.
assertEqual
(
self
.
_storage
.
_oid
,
true_max_oid
)
# This would make the unit tests too slow
# check_save_after_load_that_worked_hard(self)
...
...
src/ZODB/tests/testZODB.py
View file @
eef59eb3
...
...
@@ -25,11 +25,6 @@ from persistent import Persistent
from
persistent.mapping
import
PersistentMapping
import
transaction
# deprecated39 remove when versions go away
warnings
.
filterwarnings
(
"ignore"
,
"Versions are deprecated"
,
DeprecationWarning
,
__name__
)
class
P
(
Persistent
):
pass
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment