Commit 0c8c1b52 authored by Jim Fulton's avatar Jim Fulton

Refactored most of the blob-storage tests to be usable with different

blob-storage implementations.
parent 59425057
...@@ -52,6 +52,7 @@ entry_points = """ ...@@ -52,6 +52,7 @@ entry_points = """
zeopasswd = ZEO.zeopasswd:main zeopasswd = ZEO.zeopasswd:main
mkzeoinst = ZEO.mkzeoinst:main mkzeoinst = ZEO.mkzeoinst:main
zeoctl = ZEO.zeoctl:main zeoctl = ZEO.zeoctl:main
remove-old-zeo-cached-blobs = ZEO.ClientStorage:check_blob_size_script
""" """
scripts = [] scripts = []
......
##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
Connection support for Blobs tests Connection support for Blobs tests
================================== ==================================
...@@ -30,13 +16,8 @@ We also need a database with a blob supporting storage. (We're going to use ...@@ -30,13 +16,8 @@ We also need a database with a blob supporting storage. (We're going to use
FileStorage rather than MappingStorage here because we will want ``loadBefore`` FileStorage rather than MappingStorage here because we will want ``loadBefore``
for one of our examples.) for one of our examples.)
>>> import ZODB.FileStorage >>> blob_storage = create_storage()
>>> from ZODB.blob import BlobStorage
>>> from ZODB.DB import DB >>> from ZODB.DB import DB
>>> base_storage = ZODB.FileStorage.FileStorage(
... 'BlobTests.fs', create=True)
>>> blob_dir = 'blobs'
>>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage) >>> database = DB(blob_storage)
Putting a Blob into a Connection works like every other object: Putting a Blob into a Connection works like every other object:
......
...@@ -17,25 +17,13 @@ Import/export support for blob data ...@@ -17,25 +17,13 @@ Import/export support for blob data
Set up: Set up:
>>> from ZODB.FileStorage import FileStorage >>> import ZODB.blob, transaction
>>> from ZODB.blob import Blob, BlobStorage
>>> from ZODB.DB import DB
>>> from persistent.mapping import PersistentMapping >>> from persistent.mapping import PersistentMapping
>>> import shutil
>>> import transaction
>>> storagefile1 = 'Data.fs.1'
>>> blob_dir1 = 'blobs1'
>>> storagefile2 = 'Data.fs.2'
>>> blob_dir2 = 'blobs2'
We need an database with an undoing blob supporting storage: We need an database with an undoing blob supporting storage:
>>> base_storage1 = FileStorage(storagefile1) >>> database1 = ZODB.DB(create_storage('1'))
>>> blob_storage1 = BlobStorage(blob_dir1, base_storage1) >>> database2 = ZODB.DB(create_storage('2'))
>>> base_storage2 = FileStorage(storagefile2)
>>> blob_storage2 = BlobStorage(blob_dir2, base_storage2)
>>> database1 = DB(blob_storage1)
>>> database2 = DB(blob_storage2)
Create our root object for database1: Create our root object for database1:
...@@ -46,12 +34,11 @@ Put a couple blob objects in our database1 and on the filesystem: ...@@ -46,12 +34,11 @@ Put a couple blob objects in our database1 and on the filesystem:
>>> import time, os >>> import time, os
>>> nothing = transaction.begin() >>> nothing = transaction.begin()
>>> tid = blob_storage1._tid
>>> data1 = 'x'*100000 >>> data1 = 'x'*100000
>>> blob1 = Blob() >>> blob1 = ZODB.blob.Blob()
>>> blob1.open('w').write(data1) >>> blob1.open('w').write(data1)
>>> data2 = 'y'*100000 >>> data2 = 'y'*100000
>>> blob2 = Blob() >>> blob2 = ZODB.blob.Blob()
>>> blob2.open('w').write(data2) >>> blob2.open('w').write(data2)
>>> d = PersistentMapping({'blob1':blob1, 'blob2':blob2}) >>> d = PersistentMapping({'blob1':blob1, 'blob2':blob2})
>>> root1['blobdata'] = d >>> root1['blobdata'] = d
...@@ -85,17 +72,7 @@ Make sure our data exists: ...@@ -85,17 +72,7 @@ Make sure our data exists:
True True
>>> transaction.get().abort() >>> transaction.get().abort()
Clean up our blob directory: .. cleanup
>>> base_storage1.close() >>> database1.close()
>>> base_storage2.close() >>> database2.close()
>>> import ZODB.blob
>>> ZODB.blob.remove_committed_dir(blob_dir1)
>>> ZODB.blob.remove_committed_dir(blob_dir2)
>>> os.unlink(exportfile)
>>> os.unlink(storagefile1)
>>> os.unlink(storagefile1+".index")
>>> os.unlink(storagefile1+".tmp")
>>> os.unlink(storagefile2)
>>> os.unlink(storagefile2+".index")
>>> os.unlink(storagefile2+".tmp")
...@@ -17,20 +17,16 @@ Transaction support for Blobs ...@@ -17,20 +17,16 @@ Transaction support for Blobs
We need a database with a blob supporting storage:: We need a database with a blob supporting storage::
>>> from ZODB.MappingStorage import MappingStorage >>> import ZODB.blob, transaction
>>> from ZODB.blob import Blob, BlobStorage
>>> from ZODB.DB import DB
>>> import transaction
>>> base_storage = MappingStorage("test")
>>> blob_dir = 'blobs' >>> blob_dir = 'blobs'
>>> blob_storage = BlobStorage(blob_dir, base_storage) >>> blob_storage = create_storage(blob_dir=blob_dir)
>>> database = DB(blob_storage) >>> database = ZODB.DB(blob_storage)
>>> connection1 = database.open() >>> connection1 = database.open()
>>> root1 = connection1.root() >>> root1 = connection1.root()
Putting a Blob into a Connection works like any other Persistent object:: Putting a Blob into a Connection works like any other Persistent object::
>>> blob1 = Blob() >>> blob1 = ZODB.blob.Blob()
>>> blob1.open('w').write('this is blob 1') >>> blob1.open('w').write('this is blob 1')
>>> root1['blob1'] = blob1 >>> root1['blob1'] = blob1
>>> 'blob1' in root1 >>> 'blob1' in root1
...@@ -130,7 +126,7 @@ when we start):: ...@@ -130,7 +126,7 @@ when we start)::
We can open more than one blob object during the course of a single We can open more than one blob object during the course of a single
transaction:: transaction::
>>> blob2 = Blob() >>> blob2 = ZODB.blob.Blob()
>>> blob2.open('w').write('this is blob 3') >>> blob2.open('w').write('this is blob 3')
>>> root2['blob2'] = blob2 >>> root2['blob2'] = blob2
>>> transaction.commit() >>> transaction.commit()
...@@ -189,16 +185,6 @@ connections:: ...@@ -189,16 +185,6 @@ connections::
>>> root4['blob1'].open('r').read() >>> root4['blob1'].open('r').read()
'this is blob 1woot!this is from connection 3' 'this is blob 1woot!this is from connection 3'
BlobStorages implementation of getSize() does not include the blob data and
only returns what the underlying storages do. (We need to ensure the last
number to be an int, otherwise it will be a long on 32-bit platforms and an
int on 64-bit)::
>>> underlying_size = base_storage.getSize()
>>> blob_size = blob_storage.getSize()
>>> int(blob_size - underlying_size)
0
You can't commit a transaction while blob files are open: You can't commit a transaction while blob files are open:
>>> f = root3['blob1'].open('w') >>> f = root3['blob1'].open('w')
...@@ -227,7 +213,7 @@ We do support optimistic savepoints: ...@@ -227,7 +213,7 @@ We do support optimistic savepoints:
>>> connection5 = database.open() >>> connection5 = database.open()
>>> root5 = connection5.root() >>> root5 = connection5.root()
>>> blob = Blob() >>> blob = ZODB.blob.Blob()
>>> blob_fh = blob.open("w") >>> blob_fh = blob.open("w")
>>> blob_fh.write("I'm a happy blob.") >>> blob_fh.write("I'm a happy blob.")
>>> blob_fh.close() >>> blob_fh.close()
...@@ -297,7 +283,7 @@ file that can be opened. ...@@ -297,7 +283,7 @@ file that can be opened.
>>> connection6 = database.open() >>> connection6 = database.open()
>>> root6 = connection6.root() >>> root6 = connection6.root()
>>> blob = Blob() >>> blob = ZODB.blob.Blob()
>>> blob_fh = blob.open("w") >>> blob_fh = blob.open("w")
>>> blob_fh.write("I'm a happy blob.") >>> blob_fh.write("I'm a happy blob.")
>>> blob_fh.close() >>> blob_fh.close()
...@@ -330,7 +316,7 @@ and doesn't prevent us from opening the blob for writing: ...@@ -330,7 +316,7 @@ and doesn't prevent us from opening the blob for writing:
An exception is raised if we call committed on a blob that has An exception is raised if we call committed on a blob that has
uncommitted changes: uncommitted changes:
>>> blob = Blob() >>> blob = ZODB.blob.Blob()
>>> blob.committed() >>> blob.committed()
Traceback (most recent call last): Traceback (most recent call last):
... ...
...@@ -375,55 +361,55 @@ You can't open a committed blob file for writing: ...@@ -375,55 +361,55 @@ You can't open a committed blob file for writing:
... ...
IOError: ... IOError: ...
tpc_abort with dirty data tpc_abort
------------------------- ---------
When `tpc_abort` is called during the first commit phase we need to be able to
clean up dirty files:
>>> class DummyBaseStorage(object):
... def tpc_abort(self):
... pass
>>> base_storage = DummyBaseStorage()
>>> blob_dir2 = 'blobs2'
>>> blob_storage2 = BlobStorage(blob_dir2, base_storage)
>>> committed_blob_dir = blob_storage2.fshelper.getPathForOID(0)
>>> os.makedirs(committed_blob_dir)
>>> committed_blob_file = blob_storage2.fshelper.getBlobFilename(0, 0)
>>> open(os.path.join(committed_blob_file), 'w').write('foo')
>>> os.path.exists(committed_blob_file)
True
Now, telling the storage that Blob 0 and Blob 1 (both with serial 0) are dirty If a transaction is aborted in the middle of 2-phase commit, any data
will: remove the committed file for Blob 0 and ignore the fact that Blob 1 is stored are discarded.
set to dirty but doesn't actually have an existing file:
>>> blob_storage2.dirty_oids = [(0, 0), (1, 0)] >>> olddata, oldserial = blob_storage.load(blob._p_oid, '')
>>> blob_storage2.tpc_abort() >>> t = transaction.get()
>>> os.path.exists(committed_blob_file) >>> blob_storage.tpc_begin(t)
False >>> open('blobfile', 'w').write('This data should go away')
>>> s1 = blob_storage.storeBlob(blob._p_oid, oldserial, olddata, 'blobfile',
... '', t)
>>> new_oid = blob_storage.new_oid()
>>> open('blobfile2', 'w').write('This data should go away too')
>>> s2 = blob_storage.storeBlob(new_oid, '\0'*8, olddata, 'blobfile2',
... '', t)
>>> blob_storage.tpc_abort(t)
Now, the serial for the existing blob should be the same:
Note: This is a counter measure against regression of bug #126007. >>> blob_storage.load(blob._p_oid, '') == (olddata, oldserial)
True
`getSize` iterates over the existing blob files in the blob directory and adds And we shouldn't be able to read the data that we saved:
up their size. The blob directory sometimes contains temporary files that the
getSize function needs to ignore:
>>> garbage_file = os.path.join(blob_dir, 'garbage') >>> blob_storage.loadBlob(blob._p_oid, s1)
>>> open(garbage_file, 'w').write('garbage') Traceback (most recent call last):
>>> int(blob_storage.getSize()) ...
2673 POSKeyError: 'No blob file'
Note: This is a counter measer against regression of bug #12991. Of course the old data should be unaffected:
Teardown >>> open(blob_storage.loadBlob(blob._p_oid, oldserial)).read()
-------- "I'm a happy blob."
Similarly, the new object wasn't added to the storage:
>>> blob_storage.load(new_oid, '')
Traceback (most recent call last):
...
POSKeyError: 0x06
>>> blob_storage.loadBlob(blob._p_oid, s2)
Traceback (most recent call last):
...
POSKeyError: 'No blob file'
We don't need the storage directory and databases anymore:: .. clean up
>>> tm1.abort() >>> tm1.abort()
>>> tm2.abort() >>> tm2.abort()
>>> database.close() >>> database.close()
>>> rmtree(blob_dir)
>>> rmtree(blob_dir2)
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment