Commit 4ac3ab3e authored by Tim Peters's avatar Tim Peters

Merge tim-simpler_connection branch.

There's no longer a hard limit on # of open connections per DB.

Introduced a sane scheme for raising deprecation warnings.
Sane ==

1. The machinery ensures that a "this will be removed in ZODB 3.6"
   blurb gets attached to all deprecation warnings.

and

2. It will dead easy to find these when it's time for 3.6.
parent 5485da8a
......@@ -2,6 +2,33 @@ What's new in ZODB3 3.4?
========================
Release date: DD-MMM-2004
DB
--
- There is no longer a hard limit on the number of connections that
``DB.open()`` will create. In other words, ``DB.open()`` never blocks
anymore waiting for an earlier connection to close, and ``DB.open()``
always returns a connection now (while it wasn't documented, it was
possible for ``DB.open()`` to return ``None`` before).
``pool_size`` continues to default to 7, but its meaning has changed:
if more than ``pool_size`` connections are obtained from ``DB.open()``
and not closed, a warning is logged; if more than twice ``pool_size``, a
critical problem is logged. ``pool_size`` should be set to the maximum
number of connections from the ``DB`` instance you expect to have open
simultaneously.
In addition, if a connection obtained from ``DB.open()`` becomes
unreachable without having been explicitly closed, when Python's garbage
collection reclaims that connection it no longer counts against the
``pool_size`` thresholds for logging messages.
The following optional arguments to ``DB.open()`` are deprecated:
``transaction``, ``waitflag``, ``force`` and ``temporary``. If one
is specified, its value is ignored, and ``DeprecationWarning`` is
raised. In ZODB 3.6, these optional arguments will be removed.
Tools
-----
......
......@@ -182,6 +182,7 @@ def copy_other_files(cmd, outputbase):
"ZConfig/tests/library/widget",
"ZEO",
"ZODB",
"ZODB/tests",
"zdaemon",
"zdaemon/tests",
]:
......
......@@ -34,6 +34,8 @@ from ZODB.TmpStore import TmpStore
from ZODB.utils import u64, oid_repr, z64, positive_id
from ZODB.serialize import ObjectWriter, ConnectionObjectReader, myhasattr
from ZODB.interfaces import IConnection
from ZODB.utils import DEPRECATED_ARGUMENT, deprecated36
from zope.interface import implements
global_reset_counter = 0
......@@ -262,9 +264,8 @@ class Connection(ExportImport, object):
method. You can pass a transaction manager (TM) to DB.open()
to control which TM the Connection uses.
"""
warnings.warn("getTransaction() is deprecated. "
"Use the txn_mgr argument to DB.open() instead.",
DeprecationWarning)
deprecated36("getTransaction() is deprecated. "
"Use the txn_mgr argument to DB.open() instead.")
return self._txn_mgr.get()
def setLocalTransaction(self):
......@@ -276,9 +277,8 @@ class Connection(ExportImport, object):
can pass a transaction manager (TM) to DB.open() to control
which TM the Connection uses.
"""
warnings.warn("setLocalTransaction() is deprecated. "
"Use the txn_mgr argument to DB.open() instead.",
DeprecationWarning)
deprecated36("setLocalTransaction() is deprecated. "
"Use the txn_mgr argument to DB.open() instead.")
if self._txn_mgr is transaction.manager:
if self._synch:
self._txn_mgr.unregisterSynch(self)
......@@ -486,14 +486,14 @@ class Connection(ExportImport, object):
def cacheFullSweep(self, dt=None):
# XXX needs doc string
warnings.warn("cacheFullSweep is deprecated. "
"Use cacheMinimize instead.", DeprecationWarning)
deprecated36("cacheFullSweep is deprecated. "
"Use cacheMinimize instead.")
if dt is None:
self._cache.full_sweep()
else:
self._cache.full_sweep(dt)
def cacheMinimize(self, dt=None):
def cacheMinimize(self, dt=DEPRECATED_ARGUMENT):
"""Deactivate all unmodified objects in the cache.
Call _p_deactivate() on each cached object, attempting to turn
......@@ -503,9 +503,8 @@ class Connection(ExportImport, object):
:Parameters:
- `dt`: ignored. It is provided only for backwards compatibility.
"""
if dt is not None:
warnings.warn("The dt argument to cacheMinimize is ignored.",
DeprecationWarning)
if dt is not DEPRECATED_ARGUMENT:
deprecated36("cacheMinimize() dt= is ignored.")
self._cache.minimize()
def cacheGC(self):
......@@ -781,8 +780,8 @@ class Connection(ExportImport, object):
# an oid is being registered. I can't think of any way to
# achieve that without assignment to _p_jar. If there is
# a way, this will be a very confusing warning.
warnings.warn("Assigning to _p_jar is deprecated",
DeprecationWarning)
deprecated36("Assigning to _p_jar is deprecated, and will be "
"changed to raise an exception.")
elif obj._p_oid in self._added:
# It was registered before it was added to _added.
return
......
This diff is collapsed.
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
Here we exercise the connection management done by the DB class.
>>> from ZODB import DB
>>> from ZODB.MappingStorage import MappingStorage as Storage
Capturing log messages from DB is important for some of the examples:
>>> from zope.testing.loggingsupport import InstalledHandler
>>> handler = InstalledHandler('ZODB.DB')
Create a storage, and wrap it in a DB wrapper:
>>> st = Storage()
>>> db = DB(st)
By default, we can open 7 connections without any log messages:
>>> conns = [db.open() for dummy in range(7)]
>>> handler.records
[]
Open one more, and we get a warning:
>>> conns.append(db.open())
>>> len(handler.records)
1
>>> msg = handler.records[0]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB WARNING DB.open() has 8 open connections with a pool_size of 7
Open 6 more, and we get 6 more warnings:
>>> conns.extend([db.open() for dummy in range(6)])
>>> len(conns)
14
>>> len(handler.records)
7
>>> msg = handler.records[-1]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB WARNING DB.open() has 14 open connections with a pool_size of 7
Add another, so that it's more than twice the default, and the level
rises to critical:
>>> conns.append(db.open())
>>> len(conns)
15
>>> len(handler.records)
8
>>> msg = handler.records[-1]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB CRITICAL DB.open() has 15 open connections with a pool_size of 7
While it's boring, it's important to verify that the same relationships
hold if the default pool size is overridden.
>>> handler.clear()
>>> st.close()
>>> st = Storage()
>>> PS = 2 # smaller pool size
>>> db = DB(st, pool_size=PS)
>>> conns = [db.open() for dummy in range(PS)]
>>> handler.records
[]
A warning for opening one more:
>>> conns.append(db.open())
>>> len(handler.records)
1
>>> msg = handler.records[0]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB WARNING DB.open() has 3 open connections with a pool_size of 2
More warnings through 4 connections:
>>> conns.extend([db.open() for dummy in range(PS-1)])
>>> len(conns)
4
>>> len(handler.records)
2
>>> msg = handler.records[-1]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB WARNING DB.open() has 4 open connections with a pool_size of 2
And critical for going beyond that:
>>> conns.append(db.open())
>>> len(conns)
5
>>> len(handler.records)
3
>>> msg = handler.records[-1]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB CRITICAL DB.open() has 5 open connections with a pool_size of 2
We can change the pool size on the fly:
>>> handler.clear()
>>> db.setPoolSize(6)
>>> conns.append(db.open())
>>> handler.records # no log msg -- the pool is bigger now
[]
>>> conns.append(db.open()) # but one more and there's a warning again
>>> len(handler.records)
1
>>> msg = handler.records[0]
>>> print msg.name, msg.levelname, msg.getMessage()
ZODB.DB WARNING DB.open() has 7 open connections with a pool_size of 6
Enough of that.
>>> handler.clear()
>>> st.close()
More interesting is the stack-like nature of connection reuse. So long as
we keep opening new connections, and keep them alive, all connections
returned are distinct:
>>> st = Storage()
>>> db = DB(st)
>>> c1 = db.open()
>>> c2 = db.open()
>>> c3 = db.open()
>>> c1 is c2 or c1 is c3 or c2 is c3
False
Let's put some markers on the connections, so we can identify these
specific objects later:
>>> c1.MARKER = 'c1'
>>> c2.MARKER = 'c2'
>>> c3.MARKER = 'c3'
Now explicitly close c1 and c2:
>>> c1.close()
>>> c2.close()
Reaching into the internals, we can see that db's connection pool now has
two connections available for reuse, and knows about three connections in
all:
>>> pool = db._pools['']
>>> len(pool.available)
2
>>> len(pool.all)
3
Since we closed c2 last, it's at the top of the available stack, so will
be reused by the next open():
>>> c1 = db.open()
>>> c1.MARKER
'c2'
>>> len(pool.available), len(pool.all)
(1, 3)
>>> c3.close() # now the stack has c3 on top, then c1
>>> c2 = db.open()
>>> c2.MARKER
'c3'
>>> len(pool.available), len(pool.all)
(1, 3)
>>> c3 = db.open()
>>> c3.MARKER
'c1'
>>> len(pool.available), len(pool.all)
(0, 3)
What about the 3 in pool.all? We've seen that closing connections doesn't
reduce pool.all, and it would be bad if DB kept connections alive forever.
In fact pool.all is a "weak set" of connections -- it holds weak references
to connections. That alone doesn't keep connection objects alive. The
weak set allows DB's statistics methods to return info about connections
that are still alive.
>>> len(db.cacheDetailSize()) # one result for each connection's cache
3
If a connection object is abandoned (it becomes unreachable), then it
will vanish from pool.all automatically. However, connections are
involved in cycles, so exactly when a connection vanishes from pool.all
isn't predictable. It can be forced by running gc.collect():
>>> import gc
>>> dummy = gc.collect()
>>> len(pool.all)
3
>>> c3 = None
>>> dummy = gc.collect() # removes c3 from pool.all
>>> len(pool.all)
2
Note that c3 is really gone; in particular it didn't get added back to
the stack of available connections by magic:
>>> len(pool.available)
0
Nothing in that last block should have logged any msgs:
>>> handler.records
[]
If "too many" connections are open, then closing one may kick an older
closed one out of the available connection stack.
>>> st.close()
>>> st = Storage()
>>> db = DB(st, pool_size=3)
>>> conns = [db.open() for dummy in range(6)]
>>> len(handler.records) # 3 warnings for the "excess" connections
3
>>> pool = db._pools['']
>>> len(pool.available), len(pool.all)
(0, 6)
Let's mark them:
>>> for i, c in enumerate(conns):
... c.MARKER = i
Closing connections adds them to the stack:
>>> for i in range(3):
... conns[i].close()
>>> len(pool.available), len(pool.all)
(3, 6)
>>> del conns[:3] # leave the ones with MARKERs 3, 4 and 5
Closing another one will purge the one with MARKER 0 from the stack
(since it was the first added to the stack):
>>> [c.MARKER for c in pool.available]
[0, 1, 2]
>>> conns[0].close() # MARKER 3
>>> len(pool.available), len(pool.all)
(3, 5)
>>> [c.MARKER for c in pool.available]
[1, 2, 3]
Similarly for the other two:
>>> conns[1].close(); conns[2].close()
>>> len(pool.available), len(pool.all)
(3, 3)
>>> [c.MARKER for c in pool.available]
[3, 4, 5]
Reducing the pool size may also purge the oldest closed connections:
>>> db.setPoolSize(2) # gets rid of MARKER 3
>>> len(pool.available), len(pool.all)
(2, 2)
>>> [c.MARKER for c in pool.available]
[4, 5]
Since MARKER 5 is still the last one added to the stack, it will be the
first popped:
>>> c1 = db.open(); c2 = db.open()
>>> c1.MARKER, c2.MARKER
(5, 4)
>>> len(pool.available), len(pool.all)
(0, 2)
Clean up.
>>> st.close()
>>> handler.uninstall()
......@@ -414,8 +414,9 @@ class UserMethodTests(unittest.TestCase):
>>> len(hook.warnings)
1
>>> message, category, filename, lineno = hook.warnings[0]
>>> message
'The dt argument to cacheMinimize is ignored.'
>>> print message
This will be removed in ZODB 3.6:
cacheMinimize() dt= is ignored.
>>> category.__name__
'DeprecationWarning'
>>> hook.clear()
......@@ -434,8 +435,9 @@ class UserMethodTests(unittest.TestCase):
>>> len(hook.warnings)
2
>>> message, category, filename, lineno = hook.warnings[0]
>>> message
'cacheFullSweep is deprecated. Use cacheMinimize instead.'
>>> print message
This will be removed in ZODB 3.6:
cacheFullSweep is deprecated. Use cacheMinimize instead.
>>> category.__name__
'DeprecationWarning'
>>> message, category, filename, lineno = hook.warnings[1]
......
......@@ -23,6 +23,10 @@ import ZODB.FileStorage
from ZODB.tests.MinPO import MinPO
# Return total number of connections across all pools in a db._pools.
def nconn(pools):
return sum([len(pool.all) for pool in pools.values()])
class DBTests(unittest.TestCase):
def setUp(self):
......@@ -75,22 +79,22 @@ class DBTests(unittest.TestCase):
c12.close() # return to pool
self.assert_(c1 is c12) # should be same
pools, pooll = self.db._pools
pools = self.db._pools
self.assertEqual(len(pools), 3)
self.assertEqual(len(pooll), 3)
self.assertEqual(nconn(pools), 3)
self.db.removeVersionPool('v1')
self.assertEqual(len(pools), 2)
self.assertEqual(len(pooll), 2)
self.assertEqual(nconn(pools), 2)
c12 = self.db.open('v1')
c12.close() # return to pool
self.assert_(c1 is not c12) # should be different
self.assertEqual(len(pools), 3)
self.assertEqual(len(pooll), 3)
self.assertEqual(nconn(pools), 3)
def _test_for_leak(self):
self.dowork()
......@@ -112,27 +116,27 @@ class DBTests(unittest.TestCase):
c12 = self.db.open('v1')
self.assert_(c1 is c12) # should be same
pools, pooll = self.db._pools
pools = self.db._pools
self.assertEqual(len(pools), 3)
self.assertEqual(len(pooll), 3)
self.assertEqual(nconn(pools), 3)
self.db.removeVersionPool('v1')
self.assertEqual(len(pools), 2)
self.assertEqual(len(pooll), 2)
self.assertEqual(nconn(pools), 2)
c12.close() # should leave pools alone
self.assertEqual(len(pools), 2)
self.assertEqual(len(pooll), 2)
self.assertEqual(nconn(pools), 2)
c12 = self.db.open('v1')
c12.close() # return to pool
self.assert_(c1 is not c12) # should be different
self.assertEqual(len(pools), 3)
self.assertEqual(len(pooll), 3)
self.assertEqual(nconn(pools), 3)
def test_suite():
......
......@@ -243,9 +243,13 @@ class ZODBTests(unittest.TestCase):
self.assertEqual(r1['item'], 2)
self.assertEqual(r2['item'], 2)
for msg, obj, filename, lineno in hook.warnings:
self.assert_(
msg.startswith("setLocalTransaction() is deprecated.") or
msg.startswith("getTransaction() is deprecated."))
self.assert_(msg in [
"This will be removed in ZODB 3.6:\n"
"setLocalTransaction() is deprecated. "
"Use the txn_mgr argument to DB.open() instead.",
"This will be removed in ZODB 3.6:\n"
"getTransaction() is deprecated. "
"Use the txn_mgr argument to DB.open() instead."])
finally:
conn1.close()
conn2.close()
......
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from zope.testing.doctestunit import DocFileSuite
def test_suite():
return DocFileSuite("dbopen.txt")
......@@ -18,6 +18,8 @@ from struct import pack, unpack
from binascii import hexlify
import cPickle
import cStringIO
import weakref
import warnings
from persistent.TimeStamp import TimeStamp
......@@ -34,8 +36,27 @@ __all__ = ['z64',
'positive_id',
'get_refs',
'readable_tid_repr',
'WeakSet',
'DEPRECATED_ARGUMENT',
'deprecated36',
]
# A unique marker to give as the default value for a deprecated argument.
# The method should then do a
#
# if that_arg is not DEPRECATED_ARGUMENT:
# complain
#
# dance.
DEPRECATED_ARGUMENT = object()
# Raise DeprecationWarning, noting that the deprecated thing will go
# away in ZODB 3.6. Point to the caller of our caller (i.e., at the
# code using the deprecated thing).
def deprecated36(msg):
warnings.warn("This will be removed in ZODB 3.6:\n%s" % msg,
DeprecationWarning, stacklevel=3)
z64 = '\0'*8
# TODO The purpose of t32 is unclear. Code that uses it is usually
......@@ -164,3 +185,46 @@ def get_refs(pickle):
u.noload() # class info
u.noload() # instance state info
return refs
# A simple implementation of weak sets, supplying just enough of Python's
# sets.Set interface for our needs.
class WeakSet(object):
"""A set of objects that doesn't keep its elements alive.
The objects in the set must be weakly referencable.
The objects need not be hashable, and need not support comparison.
Two objects are considered to be the same iff their id()s are equal.
When the only references to an object are weak references (including
those from WeakSets), the object can be garbage-collected, and
will vanish from any WeakSets it may be a member of at that time.
"""
def __init__(self):
# Map id(obj) to obj. By using ids as keys, we avoid requiring
# that the elements be hashable or comparable.
self.data = weakref.WeakValueDictionary()
def __len__(self):
return len(self.data)
def __contains__(self, obj):
return id(obj) in self.data
# Same as a Set, add obj to the collection.
def add(self, obj):
self.data[id(obj)] = obj
# Same as a Set, remove obj from the collection, and raise
# KeyError if obj not in the collection.
def remove(self, obj):
del self.data[id(obj)]
# Return a list of all the objects in the collection.
# Because a weak dict is used internally, iteration
# is dicey (the underlying dict may change size during
# iteration, due to gc or activity from other threads).
# as_list() attempts to be safe.
def as_list(self):
return self.data.values()
......@@ -11,44 +11,16 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import doctest
import os
import sys
import unittest
import persistent.tests
from persistent import Persistent
from zope.testing.doctestunit import DocFileSuite
class P(Persistent):
def __init__(self):
self.x = 0
def inc(self):
self.x += 1
try:
DocFileSuite = doctest.DocFileSuite # >= Python 2.4.0a2
except AttributeError:
# <= Python 2.4.0a1
def DocFileSuite(path, globs=None):
# It's not entirely obvious how to connection this single string
# with unittest. For now, re-use the _utest() function that comes
# standard with doctest in Python 2.3. One problem is that the
# error indicator doesn't point to the line of the doctest file
# that failed.
path = os.path.join(persistent.tests.__path__[0], path)
source = open(path).read()
if globs is None:
globs = sys._getframe(1).f_globals
t = doctest.Tester(globs=globs)
def runit():
doctest._utest(t, path, source, path, 0)
f = unittest.FunctionTestCase(runit,
description="doctest from %s" % path)
suite = unittest.TestSuite()
suite.addTest(f)
return suite
def test_suite():
return DocFileSuite("persistent.txt", globs={"P": P})
......@@ -18,7 +18,6 @@ are associated with the right transaction.
"""
import thread
import weakref
from transaction._transaction import Transaction
......@@ -28,48 +27,16 @@ from transaction._transaction import Transaction
# practice not to explicitly close Connection objects, and keeping
# a Connection alive keeps a potentially huge number of other objects
# alive (e.g., the cache, and everything reachable from it too).
# Therefore we use "weak sets" internally.
#
# Therefore we use "weak sets" internally. The implementation here
# implements just enough of Python's sets.Set interface for our needs.
class WeakSet(object):
"""A set of objects that doesn't keep its elements alive.
The objects in the set must be weakly referencable.
The objects need not be hashable, and need not support comparison.
Two objects are considered to be the same iff their id()s are equal.
When the only references to an object are weak references (including
those from WeakSets), the object can be garbage-collected, and
will vanish from any WeakSets it may be a member of at that time.
"""
def __init__(self):
# Map id(obj) to obj. By using ids as keys, we avoid requiring
# that the elements be hashable or comparable.
self.data = weakref.WeakValueDictionary()
# Same as a Set, add obj to the collection.
def add(self, obj):
self.data[id(obj)] = obj
# Same as a Set, remove obj from the collection, and raise
# KeyError if obj not in the collection.
def remove(self, obj):
del self.data[id(obj)]
# Return a list of all the objects in the collection.
# Because a weak dict is used internally, iteration
# is dicey (the underlying dict may change size during
# iteration, due to gc or activity from other threads).
# as_list() attempts to be safe.
def as_list(self):
return self.data.values()
# Obscure: because of the __init__.py maze, we can't import WeakSet
# at top level here.
class TransactionManager(object):
def __init__(self):
from ZODB.utils import WeakSet
self._txn = None
self._synchs = WeakSet()
......@@ -135,6 +102,8 @@ class ThreadTransactionManager(object):
del self._txns[tid]
def registerSynch(self, synch):
from ZODB.utils import WeakSet
tid = thread.get_ident()
ws = self._synchs.get(tid)
if ws is None:
......
......@@ -261,9 +261,10 @@ class Transaction(object):
self._resources.append(adapter)
def begin(self):
warnings.warn("Transaction.begin() should no longer be used; use "
"the begin() method of a transaction manager.",
DeprecationWarning, stacklevel=2)
from ZODB.utils import deprecated36
deprecated36("Transaction.begin() should no longer be used; use "
"the begin() method of a transaction manager.")
if (self._resources or
self._sub or
self._nonsub or
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment