Commit bffc9db9 authored by Jim Fulton's avatar Jim Fulton

Rewrote zeopack:

The zeopack script has gotten a numner of improvements:

- Simplified command-line interface. (The old interface is still
    supported, except that support for ZEO version 1 servers has been
    dropped.)

- Multiple storages can be packed in sequence.

- This simplifies pack scheduling on servers serving multiple
      databases.

- All storages are packed to the same time.

- You can now specify a time of day to pack to.

- The script will now time out if it can't connect to s storage in
    60 seconds.

- It now has a test. :)
parent c8477e6a
...@@ -8,6 +8,30 @@ ...@@ -8,6 +8,30 @@
New Features New Features
------------ ------------
3.9.0a13 (2009-??-??)
=====================
New Features
------------
- The zeopack script has gotten a numner of improvements:
- Simplified command-line interface. (The old interface is still
supported, except that support for ZEO version 1 servers has been
dropped.)
- Multiple storages can be packed in sequence.
- This simplifies pack scheduling on servers serving multiple
databases.
- All storages are packed to the same time.
- You can now specify a time of day to pack to.
- The script will now time out if it can't connect to s storage in
60 seconds.
- The connection now estimates the object size based on its pickle size - The connection now estimates the object size based on its pickle size
and informs the cache about size changes. and informs the cache about size changes.
...@@ -19,15 +43,6 @@ New Features ...@@ -19,15 +43,6 @@ New Features
There are corresponding methods to read and set the new configuration There are corresponding methods to read and set the new configuration
parameters. parameters.
XXX There are known issues with this implementation that need to be
sorted out before it is "released".
- Moved a blob related logging message to debug level, for the case of
everything working as intended.
- Changed the url in the package metadata to point to PyPi. Also removed
references to download.zope.org.
3.9.0a12 (2009-02-26) 3.9.0a12 (2009-02-26)
===================== =====================
......
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""XXX short summary goes here.
$Id$
"""
import unittest
from zope.testing import doctest
def test_suite():
return unittest.TestSuite((
doctest.DocFileSuite('zeopack.test'),
))
#!/usr/bin/env python2.3 #!/usr/bin/env python2.3
"""Connect to a ZEO server and ask it to pack. import logging
import optparse
Usage: zeopack.py [options] import socket
import sys
Options: import time
import traceback
-p port -- port to connect to import ZEO.ClientStorage
-h host -- host to connect to (default is current host)
-U path -- Unix-domain socket to connect to
-S name -- storage name (default is '1') usage = """Usage: %prog [options] [servers]
-d days -- pack objects more than days old Pack one or more storages hosted by ZEO servers.
-1 -- Connect to a ZEO 1 server The positional arguments specify 0 or more tcp servers to pack, where
each is of the form:
-W -- wait for server to come up. Normally the script tries to host:port[:name]
connect for 10 seconds, then exits with an error. The -W
option is only supported with ZEO 1.
You must specify either -p and -h or -U.
""" """
import getopt
import socket
import sys
import time
from ZEO.ClientStorage import ClientStorage
WAIT = 10 # wait no more than 10 seconds for client to connect WAIT = 10 # wait no more than 10 seconds for client to connect
def connect(storage): def _main(args=None, prog=None):
# The connect-on-startup logic that ZEO provides isn't too useful if args is None:
# for this script. We'd like to client to attempt to startup, but args = sys.argv[1:]
# fail if it can't get through to the server after a reasonable
# amount of time. There's no external support for this, so we'll parser = optparse.OptionParser(usage, prog=prog)
# expose the ZEO 1.0 internals. (consenting adults only)
t0 = time.time() parser.add_option(
while t0 + WAIT > time.time(): "-d", "--days", dest="days", type='int', default=0,
storage._call.connect() help=("Pack objects that are older than this number of days")
if storage._connected: )
return
raise RuntimeError("Unable to connect to ZEO server") parser.add_option(
"-t", "--time", dest="time",
def pack1(addr, storage, days, wait): help=("Time of day to pack to of the form: HH[:MM[:SS]]. "
cs = ClientStorage(addr, storage=storage, "Defaults to current time.")
wait_for_server_on_startup=wait) )
if wait:
# _startup() is an artifact of the way ZEO 1.0 works. The parser.add_option(
# ClientStorage doesn't get fully initialized until registerDB() "-u", "--unix", dest="unix_sockets", action="append",
# is called. The only thing we care about, though, is that help=("A unix-domain-socket server to connect to, of the form: "
# registerDB() calls _startup(). "path[:name]")
cs._startup() )
else:
connect(cs) parser.remove_option('-h')
cs.invalidator = None parser.add_option(
cs.pack(wait=1, days=days) "-h", dest="host",
cs.close() help=("Deprecated: "
"Used with the -p and -S options, specified the host to "
def pack2(addr, storage, days): "connect to.")
cs = ClientStorage(addr, storage=storage, wait=1, read_only=1) )
cs.pack(wait=1, days=days)
cs.close() parser.add_option(
"-p", type="int", dest="port",
def usage(exit=1): help=("Deprecated: "
print __doc__ "Used with the -h and -S options, specifies "
print " ".join(sys.argv) "the port to connect to.")
sys.exit(exit) )
def main(): parser.add_option(
host = None "-S", dest="name", default='1',
port = None help=("Deprecated: Used with the -h and -p, options, or with the "
unix = None "-U option specified the storage name to use. Defaults to 1.")
storage = '1' )
days = 0
wait = 0 parser.add_option(
zeoversion = 2 "-U", dest="unix",
help=("Deprecated: Used with the -S option, "
"Unix-domain socket to connect to.")
)
if not args:
parser.print_help()
return
def error(message):
sys.stderr.write("Error:\n%s\n" % message)
sys.exit(1)
options, args = parser.parse_args(args)
packt = time.time()
if options.time:
time_ = map(int, options.time.split(':'))
if len(time_) == 1:
time_ += (0, 0)
elif len(time_) == 2:
time_ += (0,)
elif len(time_) > 3:
error("Invalid time value: %r" % options.time)
packt = time.localtime(packt)
packt = time.mktime(packt[:3]+tuple(time_)+packt[6:])
packt -= options.days * 86400
servers = []
if options.host:
if not options.port:
error("If host (-h) is specified then a port (-p) must be "
"specified as well.")
servers.append(((options.host, options.port), options.name))
elif options.port:
error("If port (-p) is specified then a host (-h) must be "
"specified as well.")
if options.unix:
servers.append((options.unix, options.name))
for server in args:
data = server.split(':')
if len(data) in (2, 3):
host = data[0]
try:
port = int(data[1])
except ValueError:
error("Invalid port in server specification: %r" % server)
addr = host, port
if len(data) == 2:
name = '1'
else:
name = data[2]
else:
error("Invalid server specification: %r" % server)
servers.append((addr, name))
for server in options.unix_sockets or ():
data = server.split(':')
if len(data) == 1:
addr = data[0]
name = '1'
elif len(data) == 2:
addr = data[0]
name = data[1]
else:
error("Invalid server specification: %r" % server)
servers.append((addr, name))
if not servers:
error("No servers specified.")
for addr, name in servers:
try:
cs = ZEO.ClientStorage.ClientStorage(
addr, storage=name, wait=False, read_only=1)
for i in range(60):
if cs.is_connected():
break
time.sleep(1)
else:
sys.stderr.write("Couldn't connect to: %r\n"
% ((addr, name), ))
cs.close()
continue
cs.pack(packt, wait=True)
cs.close()
except:
traceback.print_exception(*(sys.exc_info()+(99, sys.stderr)))
error("Error packing storage %s in %r" % (name, addr))
def main(*args):
root_logger = logging.getLogger()
old_level = root_logger.getEffectiveLevel()
logging.getLogger().setLevel(logging.WARNING)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(
"%(name)s %(levelname)s %(message)s"))
logging.getLogger().addHandler(handler)
try: try:
opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:d:W1') _main(*args)
for o, a in opts: finally:
if o == '-p': logging.getLogger().setLevel(old_level)
port = int(a) logging.getLogger().removeHandler(handler)
elif o == '-h':
host = a
elif o == '-U':
unix = a
elif o == '-S':
storage = a
elif o == '-d':
days = int(a)
elif o == '-W':
wait = 1
elif o == '-1':
zeoversion = 1
except Exception, err:
print err
usage()
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
if port is None:
usage()
addr = host, port
if zeoversion == 1:
pack1(addr, storage, days, wait)
else:
pack2(addr, storage, days)
if __name__ == "__main__": if __name__ == "__main__":
try: main()
main()
except Exception, err:
print err
sys.exit(1)
zeopack
=======
The zeopack script can be used to pack one or more storages. It uses
ClientStorage to do this. To test it's behavior, we'll replace the
normal ClientStorage with a fake one that echos information we'll want
for our test:
>>> class ClientStorage:
... connect_wait = 0
... def __init__(self, *args, **kw):
... if args[0] == 'bad':
... import logging
... logging.getLogger('test.ClientStorage').error(
... "I hate this address, %r", args[0])
... raise ValueError("Bad address")
... print "ClientStorage(%s %s)" % (
... repr(args)[1:-1],
... ', '.join("%s=%r" % i for i in sorted(kw.items())),
... )
... def pack(self, *args, **kw):
... print "pack(%s %s)" % (
... repr(args)[1:-1],
... ', '.join("%s=%r" % i for i in sorted(kw.items())),
... )
... def is_connected(self):
... self.connect_wait -= 1
... print 'is_connected', self.connect_wait < 0
... return self.connect_wait < 0
... def close(self):
... print "close()"
>>> import ZEO
>>> ClientStorage_orig = ZEO.ClientStorage.ClientStorage
>>> ZEO.ClientStorage.ClientStorage = ClientStorage
Now, we're ready to try the script:
>>> from ZEO.scripts.zeopack import main
If we call it with no arguments, we get help:
>>> main([], 'zeopack')
Usage: zeopack [options] [servers]
<BLANKLINE>
Pack one or more storages hosted by ZEO servers.
<BLANKLINE>
The positional arguments specify 0 or more tcp servers to pack, where
each is of the form:
<BLANKLINE>
host:port[:name]
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
Options:
-d DAYS, --days=DAYS Pack objects that are older than this number of days
-t TIME, --time=TIME Time of day to pack to of the form: HH[:MM[:SS]].
Defaults to current time.
-u UNIX_SOCKETS, --unix=UNIX_SOCKETS
A unix-domain-socket server to connect to, of the
form: path[:name]
-h HOST Deprecated: Used with the -p and -S options, specified
the host to connect to.
-p PORT Deprecated: Used with the -h and -S options, specifies
the port to connect to.
-S NAME Deprecated: Used with the -h and -p, options, or with
the -U option specified the storage name to use.
Defaults to 1.
-U UNIX Deprecated: Used with the -S option, Unix-domain
socket to connect to.
Since packing involved time, we'd better have our way with it:
>>> import time
>>> time_orig = time.time
>>> time.time = lambda : 1237906517.0
>>> import os
>>> oldtz = os.environ.get('TZ')
>>> os.environ['TZ'] = 'PST+07'
>>> sleep_orig = time.sleep
>>> def sleep(t):
... print 'sleep(%r)' % t
>>> time.sleep = sleep
Normally, we pass one or more TCP server specifications:
>>> main(["host1:8100", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(1237906517.0, wait=True)
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(1237906517.0, wait=True)
close()
We can also pass unix-domain-sockey servers using the -u option:
>>> main(["-ufoo", "-ubar:spam", "host1:8100", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(1237906517.0, wait=True)
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(1237906517.0, wait=True)
close()
ClientStorage('foo', read_only=1, storage='1', wait=False)
is_connected True
pack(1237906517.0, wait=True)
close()
ClientStorage('bar', read_only=1, storage='spam', wait=False)
is_connected True
pack(1237906517.0, wait=True)
close()
The -d option causes a pack time the given number of days earlier to
be used:
>>> main(["-ufoo", "-ubar:spam", "-d3", "host1:8100", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected True
pack(1237647317.0, wait=True)
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(1237647317.0, wait=True)
close()
ClientStorage('foo', read_only=1, storage='1', wait=False)
is_connected True
pack(1237647317.0, wait=True)
close()
ClientStorage('bar', read_only=1, storage='spam', wait=False)
is_connected True
pack(1237647317.0, wait=True)
close()
The -t option allows us to control the time of day:
>>> main(["-ufoo", "-d3", "-t1:30", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(1237624200.0, wait=True)
close()
ClientStorage('foo', read_only=1, storage='1', wait=False)
is_connected True
pack(1237624200.0, wait=True)
close()
Connection timeout
------------------
The zeopack script tells ClientStorage not to wait for connections
before returning from the constructor, but will time out after 60
seconds of waiting for a connect.
>>> ClientStorage.connect_wait = 3
>>> main(["-d3", "-t1:30", "host1:8100:2"])
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected False
sleep(1)
is_connected False
sleep(1)
is_connected False
sleep(1)
is_connected True
pack(1237624200.0, wait=True)
close()
>>> def call_main(args):
... import sys
... old_stderr = sys.stderr
... sys.stderr = sys.stdout
... try:
... try:
... main(args)
... except SystemExit, v:
... print "Exited", v
... finally:
... sys.stderr = old_stderr
>>> ClientStorage.connect_wait = 999
>>> call_main(["-d3", "-t1:30", "host1:8100", "host1:8100:2"])
... # doctest: +ELLIPSIS
ClientStorage(('host1', 8100), read_only=1, storage='1', wait=False)
is_connected False
sleep(1)
...
is_connected False
sleep(1)
Couldn't connect to: (('host1', 8100), '1')
close()
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected False
sleep(1)
...
is_connected False
sleep(1)
Couldn't connect to: (('host1', 8100), '2')
close()
>>> ClientStorage.connect_wait = 0
Legacy support
--------------
>>> main(["-d3", "-h", "host1", "-p", "8100", "-S", "2"])
ClientStorage(('host1', 8100), read_only=1, storage='2', wait=False)
is_connected True
pack(1237647317.0, wait=True)
close()
>>> main(["-d3", "-U", "foo/bar", "-S", "2"])
ClientStorage('foo/bar', read_only=1, storage='2', wait=False)
is_connected True
pack(1237647317.0, wait=True)
close()
Error handling
--------------
>>> call_main(["-d3"])
Error:
No servers specified.
Exited 1
>>> call_main(["-d3", "a"])
Error:
Invalid server specification: 'a'
Exited 1
>>> call_main(["-d3", "a:b:c:d"])
Error:
Invalid server specification: 'a:b:c:d'
Exited 1
>>> call_main(["-d3", "a:b:2"])
Error:
Invalid port in server specification: 'a:b:2'
Exited 1
>>> call_main(["-d3", "-u", "a:b:2"])
Error:
Invalid server specification: 'a:b:2'
Exited 1
>>> call_main(["-d3", "-u", "bad"]) # doctest: +ELLIPSIS
test.ClientStorage ERROR I hate this address, 'bad'
Traceback (most recent call last):
...
ValueError: Bad address
Error:
Error packing storage 1 in 'bad'
Exited 1
Note that in the previous example, the first line was output through logging.
.. tear down
>>> ZEO.ClientStorage.ClientStorage = ClientStorage_orig
>>> time.time = time_orig
>>> time.sleep = sleep_orig
>>> if oldtz is None:
... del os.environ['TZ']
... else:
... os.environ['TZ'] = oldtz
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment