Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZODB
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
ZODB
Commits
1213f8a6
Commit
1213f8a6
authored
Jun 20, 2008
by
Christian Theune
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Implemented new bushy layout for blob directories including backwards
compatibility for the old `lawn` layout.
parent
de31996b
Changes
8
Show whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
351 additions
and
73 deletions
+351
-73
src/CHANGES.txt
src/CHANGES.txt
+5
-0
src/ZEO/ClientStorage.py
src/ZEO/ClientStorage.py
+3
-16
src/ZEO/tests/testZEO.py
src/ZEO/tests/testZEO.py
+8
-11
src/ZODB/blob.py
src/ZODB/blob.py
+165
-39
src/ZODB/tests/blob_layout.txt
src/ZODB/tests/blob_layout.txt
+164
-0
src/ZODB/tests/blob_tempdir.txt
src/ZODB/tests/blob_tempdir.txt
+1
-1
src/ZODB/tests/blob_transaction.txt
src/ZODB/tests/blob_transaction.txt
+3
-3
src/ZODB/tests/testblob.py
src/ZODB/tests/testblob.py
+2
-3
No files found.
src/CHANGES.txt
View file @
1213f8a6
...
...
@@ -8,6 +8,11 @@ Change History
New Features
------------
- Changed layout strategy for the blob directory to a bushy approach (8 levels
deep, at most ~256 entries per directory level, one directory for each
blob). Old directories are automatically detected and will be handled with
the old strategy.
- Versions are no-longer supported.
- ZEO cache files can be larger than 4G. Note that older ZEO cache
...
...
src/ZEO/ClientStorage.py
View file @
1213f8a6
...
...
@@ -855,9 +855,7 @@ class ClientStorage(object):
def
_storeBlob_shared
(
self
,
oid
,
serial
,
data
,
filename
,
txn
):
# First, move the blob into the blob directory
dir
=
self
.
fshelper
.
getPathForOID
(
oid
)
if
not
os
.
path
.
exists
(
dir
):
os
.
mkdir
(
dir
)
self
.
fshelper
.
getPathForOID
(
oid
,
create
=
True
)
fd
,
target
=
self
.
fshelper
.
blob_mkstemp
(
oid
,
serial
)
os
.
close
(
fd
)
...
...
@@ -924,14 +922,7 @@ class ClientStorage(object):
raise
POSException
.
POSKeyError
(
"No blob file"
,
oid
,
serial
)
# First, we'll create the directory for this oid, if it doesn't exist.
targetpath
=
self
.
fshelper
.
getPathForOID
(
oid
)
if
not
os
.
path
.
exists
(
targetpath
):
try
:
os
.
makedirs
(
targetpath
,
0700
)
except
OSError
:
# We might have lost a race. If so, the directory
# must exist now
assert
os
.
path
.
exists
(
targetpath
)
targetpath
=
self
.
fshelper
.
getPathForOID
(
oid
,
create
=
True
)
# OK, it's not here and we (or someone) needs to get it. We
# want to avoid getting it multiple times. We want to avoid
...
...
@@ -1118,19 +1109,15 @@ class ClientStorage(object):
assert
s
==
tid
,
(
s
,
tid
)
self
.
_cache
.
store
(
oid
,
s
,
None
,
data
)
if
self
.
fshelper
is
not
None
:
blobs
=
self
.
_tbuf
.
blobs
while
blobs
:
oid
,
blobfilename
=
blobs
.
pop
()
targetpath
=
self
.
fshelper
.
getPathForOID
(
oid
)
if
not
os
.
path
.
exists
(
targetpath
):
os
.
makedirs
(
targetpath
,
0700
)
targetpath
=
self
.
fshelper
.
getPathForOID
(
oid
,
create
=
True
)
rename_or_copy_blob
(
blobfilename
,
self
.
fshelper
.
getBlobFilename
(
oid
,
tid
),
)
self
.
_tbuf
.
clear
()
def
undo
(
self
,
trans_id
,
txn
):
...
...
src/ZEO/tests/testZEO.py
View file @
1213f8a6
...
...
@@ -515,8 +515,7 @@ class CommonBlobTests:
self
.
_storage
.
tpc_abort
(
t
)
raise
self
.
assert_
(
not
os
.
path
.
exists
(
tfname
))
filename
=
os
.
path
.
join
(
self
.
blobdir
,
oid_repr
(
oid
),
tid_repr
(
revid
)
+
BLOB_SUFFIX
)
filename
=
self
.
_storage
.
fshelper
.
getBlobFilename
(
oid
,
revid
)
self
.
assert_
(
os
.
path
.
exists
(
filename
))
self
.
assertEqual
(
somedata
,
open
(
filename
).
read
())
...
...
@@ -631,16 +630,14 @@ class BlobAdaptedFileStorageTests(GenericTests, CommonBlobTests):
d2
=
somedata
.
read
(
8096
)
self
.
assertEqual
(
d1
,
d2
)
# The file should have been copied to the server:
filename
=
os
.
path
.
join
(
self
.
blobdir
,
oid_repr
(
oid
),
tid_repr
(
revid
)
+
BLOB_SUFFIX
)
# The file should be in the cache ...
filename
=
self
.
_storage
.
fshelper
.
getBlobFilename
(
oid
,
revid
)
check_data
(
filename
)
#
It should also be in the cache:
filename
=
os
.
path
.
join
(
self
.
blob_cache_dir
,
oid_repr
(
oid
),
tid_repr
(
revid
)
+
BLOB_SUFFIX
)
check_data
(
filename
)
#
... and on the server
server_filename
=
filename
.
replace
(
self
.
blob_cache_dir
,
self
.
blobdir
)
self
.
assert_
(
server_filename
.
startswith
(
self
.
blobdir
)
)
check_data
(
server_
filename
)
# If we remove it from the cache and call loadBlob, it should
# come back. We can do this in many threads. We'll instrument
...
...
src/ZODB/blob.py
View file @
1213f8a6
...
...
@@ -15,8 +15,10 @@
"""
import
base64
import
binascii
import
logging
import
os
import
re
import
shutil
import
stat
import
sys
...
...
@@ -43,6 +45,9 @@ logger = logging.getLogger('ZODB.blob')
BLOB_SUFFIX
=
".blob"
SAVEPOINT_SUFFIX
=
".spb"
LAYOUT_MARKER
=
'.layout'
LAYOUTS
=
{}
valid_modes
=
'r'
,
'w'
,
'r+'
,
'a'
# Threading issues:
...
...
@@ -292,21 +297,42 @@ class FilesystemHelper:
# with blobs and storages needn't indirect through this if they
# want to perform blob storage differently.
def
__init__
(
self
,
base_dir
):
self
.
base_dir
=
base_dir
def
__init__
(
self
,
base_dir
,
layout_name
=
'automatic'
):
self
.
base_dir
=
os
.
path
.
normpath
(
base_dir
)
+
'/'
self
.
temp_dir
=
os
.
path
.
join
(
base_dir
,
'tmp'
)
if
layout_name
==
'automatic'
:
layout_name
=
auto_layout_select
(
base_dir
)
if
layout_name
==
'lawn'
:
log
(
'The `lawn` blob directory layout is deprecated due to '
'scalability issues on some file systems, please consider '
'migrating to the `bushy` layout.'
,
level
=
logging
.
WARN
)
self
.
layout_name
=
layout_name
self
.
layout
=
LAYOUTS
[
layout_name
]
def
create
(
self
):
if
not
os
.
path
.
exists
(
self
.
base_dir
):
os
.
makedirs
(
self
.
base_dir
,
0700
)
log
(
"Blob cache directory '%s' does not exist. "
"Created new directory."
%
self
.
base_dir
,
level
=
logging
.
INFO
)
log
(
"Blob directory '%s' does not exist. "
"Created new directory."
%
self
.
base_dir
)
if
not
os
.
path
.
exists
(
self
.
temp_dir
):
os
.
makedirs
(
self
.
temp_dir
,
0700
)
log
(
"Blob temporary directory '%s' does not exist. "
"Created new directory."
%
self
.
temp_dir
,
level
=
logging
.
INFO
)
"Created new directory."
%
self
.
temp_dir
)
if
not
os
.
path
.
exists
(
os
.
path
.
join
(
self
.
base_dir
,
LAYOUT_MARKER
)):
layout_marker
=
open
(
os
.
path
.
join
(
self
.
base_dir
,
LAYOUT_MARKER
),
'wb'
)
layout_marker
.
write
(
self
.
layout_name
)
else
:
layout_marker
=
open
(
os
.
path
.
join
(
self
.
base_dir
,
LAYOUT_MARKER
),
'rb'
)
layout
=
layout_marker
.
read
().
strip
()
if
layout
!=
self
.
layout_name
:
raise
ValueError
(
"Directory layout `%s` selected for blob directory %s, but "
"marker found for layout `%s`"
%
(
self
.
layout_name
,
self
.
base_dir
,
layout
))
def
isSecure
(
self
,
path
):
"""Ensure that (POSIX) path mode bits are 0700."""
...
...
@@ -317,12 +343,40 @@ class FilesystemHelper:
log
(
'Blob dir %s has insecure mode setting'
%
self
.
base_dir
,
level
=
logging
.
WARNING
)
def
getPathForOID
(
self
,
oid
):
def
getPathForOID
(
self
,
oid
,
create
=
False
):
"""Given an OID, return the path on the filesystem where
the blob data relating to that OID is stored.
If the create flag is given, the path is also created if it didn't
exist already.
"""
return
os
.
path
.
join
(
self
.
base_dir
,
utils
.
oid_repr
(
oid
))
# OIDs are numbers and sometimes passed around as integers. For our
# computations we rely on the 64-bit packed string representation.
if
isinstance
(
oid
,
int
):
oid
=
utils
.
p64
(
oid
)
path
=
self
.
layout
.
oid_to_path
(
oid
)
path
=
os
.
path
.
join
(
self
.
base_dir
,
path
)
if
create
and
not
os
.
path
.
exists
(
path
):
try
:
os
.
makedirs
(
path
,
0700
)
except
OSError
:
# We might have lost a race. If so, the directory
# must exist now
assert
os
.
path
.
exists
(
targetpath
)
return
path
def
getOIDForPath
(
self
,
path
):
"""Given a path, return an OID, if the path is a valid path for an
OID. The inverse function to `getPathForOID`.
Raises ValueError if the path is not valid for an OID.
"""
path
=
path
[
len
(
self
.
base_dir
):]
return
self
.
layout
.
path_to_oid
(
path
)
def
getBlobFilename
(
self
,
oid
,
tid
):
"""Given an oid and a tid, return the full filename of the
...
...
@@ -330,6 +384,10 @@ class FilesystemHelper:
"""
oid_path
=
self
.
getPathForOID
(
oid
)
# TIDs are numbers and sometimes passed around as integers. For our
# computations we rely on the 64-bit packed string representation
if
isinstance
(
tid
,
int
):
tid
=
utils
.
p64
(
tid
)
filename
=
"%s%s"
%
(
utils
.
tid_repr
(
tid
),
BLOB_SUFFIX
)
return
os
.
path
.
join
(
oid_path
,
filename
)
...
...
@@ -359,10 +417,9 @@ class FilesystemHelper:
if
not
filename
.
endswith
(
BLOB_SUFFIX
):
return
None
,
None
path
,
filename
=
os
.
path
.
split
(
filename
)
oid
=
os
.
path
.
split
(
path
)[
1
]
oid
=
self
.
getOIDForPath
(
path
)
serial
=
filename
[:
-
len
(
BLOB_SUFFIX
)]
oid
=
utils
.
repr_to_oid
(
oid
)
serial
=
utils
.
repr_to_oid
(
serial
)
return
oid
,
serial
...
...
@@ -372,24 +429,104 @@ class FilesystemHelper:
"""
oids
=
[]
base_dir
=
self
.
base_dir
for
oidpath
in
os
.
listdir
(
base_dir
):
for
filename
in
os
.
listdir
(
os
.
path
.
join
(
base_dir
,
oidpath
)):
blob_path
=
os
.
path
.
join
(
base_dir
,
oidpath
,
filename
)
for
oid
,
oidpath
in
self
.
listOIDs
():
for
filename
in
os
.
listdir
(
oidpath
):
blob_path
=
os
.
path
.
join
(
oidpath
,
filename
)
oid
,
serial
=
self
.
splitBlobFilename
(
blob_path
)
if
search_serial
==
serial
:
oids
.
append
(
oid
)
return
oids
def
listOIDs
(
self
):
"""
Lists all OIDs and their paths.
"""
Iterates over all paths under the base directory that contain blob
files.
"""
for
candidate
in
os
.
listdir
(
self
.
base_dir
):
if
candidate
==
'tmp'
:
for
path
,
dirs
,
files
in
os
.
walk
(
self
.
base_dir
):
try
:
oid
=
self
.
getOIDForPath
(
path
)
except
ValueError
:
continue
oid
=
utils
.
repr_to_oid
(
candidate
)
yield
oid
,
self
.
getPathForOID
(
oid
)
yield
oid
,
path
def
auto_layout_select
(
path
):
# A heuristic to look at a path and determine which directory layout to
# use. Basically we try to figure out if the directory is either already
# used and contains an explicit marker, is unused or used without a
# marker.
layout_marker
=
os
.
path
.
join
(
path
,
LAYOUT_MARKER
)
if
not
os
.
path
.
exists
(
path
):
log
(
'Blob directory %s does not exist. '
'Selected `bushy` layout. '
%
path
)
layout
=
'bushy'
elif
len
(
os
.
listdir
(
path
))
==
0
:
log
(
'Blob directory %s is unused and has no layout marker set.'
'Selected `bushy` layout. '
%
path
)
layout
=
'bushy'
elif
LAYOUT_MARKER
not
in
os
.
listdir
(
path
):
log
(
'Blob directory %s is used but has no layout marker set.'
'Selected `lawn` layout. '
%
path
)
layout
=
'lawn'
else
:
layout
=
open
(
layout_marker
,
'rb'
).
read
()
layout
=
layout
.
strip
()
log
(
'Blob directory %s has layout marker set.'
'Selected `%s` layout. '
%
(
path
,
layout
))
return
layout
class
BushyLayout
(
object
):
"""A bushy directory layout for blob directories.
Creates an 8-level directory structure (one level per byte) in
little-endian order from the OID of an object.
"""
blob_path_pattern
=
r'^'
+
(
r'0x[0-9]{1,2}/*'
*
8
)
+
r'$'
blob_path_pattern
=
re
.
compile
(
blob_path_pattern
)
def
oid_to_path
(
self
,
oid
):
directories
=
[]
# Create the bushy directory structure with the least significant byte
# first
for
byte
in
reversed
(
str
(
oid
)):
directories
.
append
(
'0x%s'
%
binascii
.
hexlify
(
byte
))
return
'/'
.
join
(
directories
)
def
path_to_oid
(
self
,
path
):
if
self
.
blob_path_pattern
.
match
(
path
)
is
None
:
raise
ValueError
(
"Not a valid OID path: %s"
%
path
)
# The path always has a leading slash that we need to ignore.
path
=
path
.
split
(
'/'
)
# The path contains the OID in little endian form but the OID itself
# is big endian.
path
.
reverse
()
# Each path segment stores a byte in hex representation. Turn it into
# an int and then get the character for our byte string.
oid
=
''
.
join
(
binascii
.
unhexlify
(
byte
[
2
:])
for
byte
in
path
)
return
oid
LAYOUTS
[
'bushy'
]
=
BushyLayout
()
class
LawnLayout
(
object
):
"""A shallow directory layout for blob directories.
Creates a single level of directories (one for each oid).
"""
def
oid_to_path
(
self
,
oid
):
return
utils
.
oid_repr
(
oid
)
def
path_to_oid
(
self
,
path
):
try
:
return
utils
.
repr_to_oid
(
path
)
except
TypeError
:
raise
ValueError
(
'Not a valid OID path: %s'
%
path
)
LAYOUTS
[
'lawn'
]
=
LawnLayout
()
class
BlobStorage
(
SpecificationDecoratorBase
):
...
...
@@ -401,13 +538,13 @@ class BlobStorage(SpecificationDecoratorBase):
# us to have instance attributes explicitly on the proxy.
__slots__
=
(
'fshelper'
,
'dirty_oids'
,
'_BlobStorage__supportsUndo'
)
def
__new__
(
self
,
base_directory
,
storage
):
def
__new__
(
self
,
base_directory
,
storage
,
layout
=
'automatic'
):
return
SpecificationDecoratorBase
.
__new__
(
self
,
storage
)
def
__init__
(
self
,
base_directory
,
storage
):
def
__init__
(
self
,
base_directory
,
storage
,
layout
=
'automatic'
):
# XXX Log warning if storage is ClientStorage
SpecificationDecoratorBase
.
__init__
(
self
,
storage
)
self
.
fshelper
=
FilesystemHelper
(
base_directory
)
self
.
fshelper
=
FilesystemHelper
(
base_directory
,
layout
)
self
.
fshelper
.
create
()
self
.
fshelper
.
checkSecure
()
self
.
dirty_oids
=
[]
...
...
@@ -438,10 +575,7 @@ class BlobStorage(SpecificationDecoratorBase):
self
.
_lock_acquire
()
try
:
targetpath
=
self
.
fshelper
.
getPathForOID
(
oid
)
if
not
os
.
path
.
exists
(
targetpath
):
os
.
makedirs
(
targetpath
,
0700
)
self
.
fshelper
.
getPathForOID
(
oid
,
create
=
True
)
targetname
=
self
.
fshelper
.
getBlobFilename
(
oid
,
serial
)
rename_or_copy_blob
(
blobfilename
,
targetname
)
...
...
@@ -487,14 +621,12 @@ class BlobStorage(SpecificationDecoratorBase):
# if they are still needed by attempting to load the revision
# of that object from the database. This is maybe the slowest
# possible way to do this, but it's safe.
base_dir
=
self
.
fshelper
.
base_dir
for
oid
,
oid_path
in
self
.
fshelper
.
listOIDs
():
files
=
os
.
listdir
(
oid_path
)
for
filename
in
files
:
filepath
=
os
.
path
.
join
(
oid_path
,
filename
)
whatever
,
serial
=
self
.
fshelper
.
splitBlobFilename
(
filepath
)
try
:
fn
=
self
.
fshelper
.
getBlobFilename
(
oid
,
serial
)
self
.
loadSerial
(
oid
,
serial
)
except
POSKeyError
:
remove_committed
(
filepath
)
...
...
@@ -504,7 +636,6 @@ class BlobStorage(SpecificationDecoratorBase):
@
non_overridable
def
_packNonUndoing
(
self
,
packtime
,
referencesf
):
base_dir
=
self
.
fshelper
.
base_dir
for
oid
,
oid_path
in
self
.
fshelper
.
listOIDs
():
exists
=
True
try
:
...
...
@@ -552,15 +683,11 @@ class BlobStorage(SpecificationDecoratorBase):
"""Return the size of the database in bytes."""
orig_size
=
getProxiedObject
(
self
).
getSize
()
blob_size
=
0
base_dir
=
self
.
fshelper
.
base_dir
for
oid
in
os
.
listdir
(
base_dir
):
sub_dir
=
os
.
path
.
join
(
base_dir
,
oid
)
if
not
os
.
path
.
isdir
(
sub_dir
):
continue
for
serial
in
os
.
listdir
(
sub_dir
):
for
oid
,
path
in
self
.
fshelper
.
listOIDs
():
for
serial
in
os
.
listdir
(
path
):
if
not
serial
.
endswith
(
BLOB_SUFFIX
):
continue
file_path
=
os
.
path
.
join
(
base_dir
,
oid
,
serial
)
file_path
=
os
.
path
.
join
(
path
,
serial
)
blob_size
+=
os
.
stat
(
file_path
).
st_size
return
orig_size
+
blob_size
...
...
@@ -583,7 +710,6 @@ class BlobStorage(SpecificationDecoratorBase):
# we get all the blob oids on the filesystem related to the
# transaction we want to undo.
for
oid
in
self
.
fshelper
.
getOIDsForSerial
(
serial_id
):
# we want to find the serial id of the previous revision
# of this blob object.
load_result
=
self
.
loadBefore
(
oid
,
serial_id
)
...
...
src/ZODB/tests/blob_layout.txt
0 → 100644
View file @
1213f8a6
======================
Blob directory layouts
======================
The internal structure of the blob directories is governed by so called
`layouts`. The current default layout is called `bushy`.
The original blob implementation used a layout that we now call `lawn` and
which is still available for backwards compatibility.
Layouts implement two methods: one for computing a relative path for an
OID and one for turning a relative path back into an OID.
Our terminology is roughly the same as used in `DirectoryStorage`.
The `bushy` layout
==================
The bushy layout splits the OID into the 8 byte parts, reverses them and
creates one directory level for each part, named by the hexlified
representation of the byte value. This results in 8 levels of directories, the
leaf directories being used for the revisions of the blobs and at most 256
entries per directory level:
>>> from ZODB.blob import BushyLayout
>>> bushy = BushyLayout()
>>> bushy.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x00')
'0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x00'
>>> bushy.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x01')
'0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00'
>>> bushy.path_to_oid('0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00')
'\x00\x00\x00\x00\x00\x00\x00\x01'
Paths that do not represent an OID will cause a ValueError:
>>> bushy.path_to_oid('tmp')
Traceback (most recent call last):
ValueError: Not a valid OID path: tmp
The `lawn` layout
=================
The lawn layout creates on directory for each blob named by the blob's hex
representation of its OID. This has some limitations on various file systems
like performance penalties or the inability to store more than a given number
of blobs at the same time (e.g. 32k on ext3).
>>> from ZODB.blob import LawnLayout
>>> lawn = LawnLayout()
>>> lawn.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x00')
'0x00'
>>> lawn.oid_to_path('\x00\x00\x00\x00\x00\x00\x00\x01')
'0x01'
>>> lawn.path_to_oid('0x01')
'\x00\x00\x00\x00\x00\x00\x00\x01'
Paths that do not represent an OID will cause a ValueError:
>>> lawn.path_to_oid('tmp')
Traceback (most recent call last):
ValueError: Not a valid OID path: tmp
Auto-detecting the layout of a directory
========================================
To allow easier migration, we provide an auto-detection feature that analyses a
blob directory and decides for a strategy to use. In general it prefers to
choose the `bushy` layout, except if it determines that the directory has
already been used to create a lawn structure.
>>> from ZODB.blob import auto_layout_select
1. Non-existing directories will trigger a bushy layout:
>>> import tempfile
>>> import shutil
>>> d = tempfile.mkdtemp()
>>> shutil.rmtree(d)
>>> auto_layout_select(d)
'bushy'
2. Empty directories will trigger a bushy layout too:
>>> d = tempfile.mkdtemp()
>>> auto_layout_select(d)
'bushy'
3. If the directory contains a marker for the strategy it will be used:
>>> from ZODB.blob import LAYOUT_MARKER
>>> import os.path
>>> open(os.path.join(d, LAYOUT_MARKER), 'wb').write('bushy')
>>> auto_layout_select(d)
'bushy'
>>> open(os.path.join(d, LAYOUT_MARKER), 'wb').write('lawn')
>>> auto_layout_select(d)
'lawn'
>>> shutil.rmtree(d)
4. If the directory does not contain a marker but other files, we assume that
it was created with an earlier version of the blob implementation and uses our
`lawn` layout:
>>> d = tempfile.mkdtemp()
>>> open(os.path.join(d, '0x0101'), 'wb').write('foo')
>>> auto_layout_select(d)
'lawn'
>>> shutil.rmtree(d)
Directory layout markers
========================
When the file system helper (FSH) is asked to create the directory structure,
it will leave a marker with the choosen layout if no marker exists yet:
>>> from ZODB.blob import FilesystemHelper
>>> d = tempfile.mkdtemp()
>>> blobs = os.path.join(d, 'blobs')
>>> fsh = FilesystemHelper(blobs)
>>> fsh.layout_name
'bushy'
>>> fsh.create()
>>> open(os.path.join(blobs, LAYOUT_MARKER), 'rb').read()
'bushy'
If the FSH finds a marker, then it verifies whether its content matches the
strategy that was chosen. It will raise an exception if we try to work with a
directory that has a different marker than the chosen strategy:
>>> fsh = FilesystemHelper(blobs, 'lawn')
>>> fsh.layout_name
'lawn'
>>> fsh.create() # doctest: +ELLIPSIS
Traceback (most recent call last):
ValueError: Directory layout `lawn` selected for blob directory /.../blobs/, but marker found for layout `bushy`
>>> shutil.rmtree(blobs)
This function interacts with the automatic detection in the way, that an
unmarked directory will be marked the first time when it is auto-guessed and
the marker will be used in the future:
>>> import ZODB.FileStorage
>>> from ZODB.blob import BlobStorage
>>> datafs = os.path.join(d, 'data.fs')
>>> base_storage = ZODB.FileStorage.FileStorage(datafs)
>>> os.mkdir(blobs)
>>> open(os.path.join(blobs, 'foo'), 'wb').write('foo')
>>> blob_storage = BlobStorage(blobs, base_storage)
>>> blob_storage.fshelper.layout_name
'lawn'
>>> open(os.path.join(blobs, LAYOUT_MARKER), 'rb').read()
'lawn'
>>> blob_storage = BlobStorage(blobs, base_storage, layout='bushy') # doctest: +ELLIPSIS
Traceback (most recent call last):
ValueError: Directory layout `bushy` selected for blob directory /.../blobs/, but marker found for layout `lawn`
>>> shutil.rmtree(d)
src/ZODB/tests/blob_tempdir.txt
View file @
1213f8a6
...
...
@@ -32,7 +32,7 @@ First, we need a datatabase with blob support::
>>> from ZODB.DB import DB
>>> from tempfile import mkdtemp
>>> import os.path
>>> base_storage = MappingStorage(
"test"
)
>>> base_storage = MappingStorage(
'test'
)
>>> blob_dir = mkdtemp()
>>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage)
...
...
src/ZODB/tests/blob_transaction.txt
View file @
1213f8a6
...
...
@@ -322,9 +322,9 @@ clean up dirty files:
>>> base_storage = DummyBaseStorage()
>>> blob_dir2 = mkdtemp()
>>> blob_storage2 = BlobStorage(blob_dir2, base_storage)
>>> committed_blob_dir =
os.path.join(blob_dir2, '0'
)
>>>
committed_blob_file = os.path.join(committed_blob_dir, '0.blob'
)
>>>
os.mkdir(committed_blob_dir
)
>>> committed_blob_dir =
blob_storage2.fshelper.getPathForOID(0
)
>>>
os.makedirs(committed_blob_dir
)
>>>
committed_blob_file = blob_storage2.fshelper.getBlobFilename(0, 0
)
>>> open(os.path.join(committed_blob_file), 'w').write('foo')
>>> os.path.exists(committed_blob_file)
True
...
...
src/ZODB/tests/testblob.py
View file @
1213f8a6
...
...
@@ -105,7 +105,6 @@ class BlobUndoTests(unittest.TestCase):
self
.
here
=
os
.
getcwd
()
os
.
chdir
(
self
.
test_dir
)
self
.
storagefile
=
'Data.fs'
os
.
mkdir
(
'blobs'
)
self
.
blob_dir
=
'blobs'
def
tearDown
(
self
):
...
...
@@ -483,7 +482,7 @@ def loadblob_tmpstore():
We can access the blob correctly:
>>> tmpstore.loadBlob(blob_oid, tid) # doctest: +ELLIPSIS
'.../0x01/0x...blob'
'.../0x01/0x
00/0x00/0x00/0x00/0x00/0x00/0x00/0x
...blob'
Clean up:
...
...
@@ -503,7 +502,7 @@ def test_suite():
suite
.
addTest
(
doctest
.
DocFileSuite
(
"blob_basic.txt"
,
"blob_connection.txt"
,
"blob_transaction.txt"
,
"blob_packing.txt"
,
"blob_importexport.txt"
,
"blob_consume.txt"
,
"blob_tempdir.txt"
,
"blob_tempdir.txt"
,
"blob_layout.txt"
,
setUp
=
ZODB
.
tests
.
util
.
setUp
,
tearDown
=
ZODB
.
tests
.
util
.
tearDown
,
))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment