Commit dbf223db authored by Julien Muchembled's avatar Julien Muchembled

publish_early: rework API

Commit ba6c3331 does not handle non-string
values properly. While fixing this bug, I also found that '-init' was bad:
sections were uninstalled on next runs, which is a problem with recipes
whose install() return paths.
parent 66e05850
...@@ -32,30 +32,26 @@ from .librecipe import GenericBaseRecipe ...@@ -32,30 +32,26 @@ from .librecipe import GenericBaseRecipe
class Cluster(object): class Cluster(object):
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.buildout = buildout masters = options.setdefault('masters', '')
self.options = options
def publish_early(self, publish_dict):
masters = publish_dict.setdefault('masters', '')
result_dict = { result_dict = {
'connection-admin': [], 'connection-admin': [],
'connection-master': [], 'connection-master': [],
} }
node_list = [] node_list = []
for node in sorted(self.options['nodes'].split()): for node in sorted(options['nodes'].split()):
node = self.buildout[node] node = buildout[node]
node_list.append(node) node_list.append(node)
for k, v in result_dict.iteritems(): for k, v in result_dict.iteritems():
x = node[k] x = node[k]
if x: if x:
v.append(x) v.append(x)
publish_dict['admins'] = ' '.join(result_dict.pop('connection-admin')) options['admins'] = ' '.join(result_dict.pop('connection-admin'))
x = ' '.join(result_dict.pop('connection-master')) x = ' '.join(result_dict.pop('connection-master'))
if masters != x: if masters != x:
publish_dict['masters'] = x options['masters'] = x
for node in node_list: for node in node_list:
node['config-masters'] = x node['config-masters'] = x
node.recipe.__init__(self.buildout, node.name, node) node.recipe.__init__(buildout, node.name, node)
install = update = lambda self: None install = update = lambda self: None
......
...@@ -28,20 +28,17 @@ ...@@ -28,20 +28,17 @@
from collections import defaultdict from collections import defaultdict
from .librecipe import unwrap, wrap, GenericSlapRecipe from .librecipe import unwrap, wrap, GenericSlapRecipe
def patchOptions(options, override): def volatileOptions(options, volatile):
def get(option, *args, **kw): def keys():
try: keys = options_keys()
return override[option] for key in volatile:
except KeyError: try:
return options_get(option, *args, **kw) keys.remove(key)
try: except ValueError:
options_get = options._get pass
except AttributeError: return keys
options_get = options.get options_keys = options.keys
options.get = get options.keys = keys
else:
options._get = get
class Recipe(GenericSlapRecipe): class Recipe(GenericSlapRecipe):
""" """
...@@ -57,8 +54,6 @@ class Recipe(GenericSlapRecipe): ...@@ -57,8 +54,6 @@ class Recipe(GenericSlapRecipe):
-init = -init =
foo gen-foo:x foo gen-foo:x
bar gen-bar:y bar gen-bar:y
-update =
baz update-baz:z
bar = z bar = z
[gen-foo] [gen-foo]
...@@ -69,72 +64,74 @@ class Recipe(GenericSlapRecipe): ...@@ -69,72 +64,74 @@ class Recipe(GenericSlapRecipe):
-extends = publish-early -extends = publish-early
... ...
${publish-early:foo} is initialized with the value of the published Just before the recipe of [gen-foo] is instantiated, 'x' is overridden with
parameter 'foo', or ${gen-foo:x} if it hasn't been published yet the published value 'foo' if it exists. If its __init__ modifies 'x', the new
(and in this case, it is published immediately as a way to save the value). value is published. To prevent [gen-foo] from being accessed too early, 'x'
is then removed and the value can only be accessed with ${publish-early:foo}.
Generated values don't end up in the buildout installed file, which is good
if they're secret. Note however that buildout won't detect if values change
and it may only call update().
${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored): ${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored):
a line like 'bar = z' is usually rendered conditionally with Jinja2. a line like 'bar = z' is usually rendered conditionally with Jinja2.
The '-update' option has the same syntax than '-init'. The recipes of the
specified sections must implement 'publish_early(publish_dict)':
- it is always called, just before early publishing
- publish_dict is a dict with already published values
- 'publish_early' can change published values by modifying publish_dict.
In the above example:
- publish_dict is {'z': ...}
- during the execution of 'publish_early', other sections can access the
value with ${update-baz:z}
- once [publish-early] is initialized, the value should be accessed with
${publish-early:bar} ([update-baz] does not have it if it's accessed
before [publish-early])
""" """
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
GenericSlapRecipe.__init__(self, buildout, name, options) GenericSlapRecipe.__init__(self, buildout, name, options)
init = defaultdict(dict) init = defaultdict(dict)
update = defaultdict(dict) for line in options['-init'].splitlines():
for d, k in (init, '-init'), (update, '-update'): if line:
for line in options.get(k, '').splitlines(): k, v = line.split()
if line: if k not in options:
k, v = line.split() section, v = v.split(':')
if k not in options: init[section][k] = v
section, v = v.split(':') if init:
d[section][k] = v
if init or update:
self.slap.initializeConnection(self.server_url, self.key_file, self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file) self.cert_file)
computer_partition = self.slap.registerComputerPartition( computer_partition = self.slap.registerComputerPartition(
self.computer_id, self.computer_partition_id) self.computer_id, self.computer_partition_id)
published_dict = unwrap(computer_partition.getConnectionParameterDict()) published_dict = unwrap(computer_partition.getConnectionParameterDict())
Options = buildout.Options
if 'Options' in buildout.__dict__:
def revertOptions():
buildout.Options = Options
else:
def revertOptions():
try:
del buildout.Options
except AttributeError:
pass
def newOptions(buildout, section, data):
assert section == init_section, (section, init_section)
revertOptions()
self = buildout.Options(buildout, section, data)
self.update(override)
return self
publish = False publish = False
publish_dict = {} publish_dict = {}
for section, init in init.iteritems(): try:
for k, v in init.iteritems(): for init_section, init in init.iteritems():
try: override = {}
publish_dict[k] = published_dict[k] for k, v in init.iteritems():
except KeyError: try:
publish_dict[k] = buildout[section][v] override[v] = published_dict[k]
except KeyError:
pass
buildout.Options = newOptions
init_section = buildout[init_section]
assert buildout.Options is Options
new = {}
for k, v in init.iteritems():
try:
publish_dict[k] = new[v] = init_section.pop(v)
except KeyError:
pass
if new != override:
publish = True publish = True
finally:
for section, update in update.iteritems(): revertOptions()
override = {}
for k, v in update.iteritems():
try:
override[v] = published_dict[k]
except KeyError:
pass
section = buildout[section]
patchOptions(section, override)
old = override.copy()
section.recipe.publish_early(override)
if override != old:
publish = True
for k, v in update.iteritems():
try:
publish_dict[k] = override[v]
except KeyError:
pass
if publish: if publish:
computer_partition.setConnectionDict(wrap(publish_dict)) computer_partition.setConnectionDict(wrap(publish_dict))
...@@ -143,6 +140,7 @@ class Recipe(GenericSlapRecipe): ...@@ -143,6 +140,7 @@ class Recipe(GenericSlapRecipe):
if k != 'recipe' and not k.startswith('-')] if k != 'recipe' and not k.startswith('-')]
publish += publish_dict publish += publish_dict
publish_dict['-publish'] = ' '.join(publish) publish_dict['-publish'] = ' '.join(publish)
patchOptions(options, publish_dict) volatileOptions(options, list(publish_dict))
options.update(publish_dict)
install = update = lambda self: None install = update = lambda self: None
...@@ -36,8 +36,7 @@ import errno ...@@ -36,8 +36,7 @@ import errno
import os import os
import random import random
import string import string
from .librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import GenericBaseRecipe
class Integer(object): class Integer(object):
""" """
...@@ -54,7 +53,9 @@ class Integer(object): ...@@ -54,7 +53,9 @@ class Integer(object):
Resulting integer. Resulting integer.
""" """
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
options['value'] = random.randint(int(options['minimum']), int(options['maximum'])) if 'value' not in options:
options['value'] = random.randint(int(options['minimum']),
int(options['maximum']))
def install(self): def install(self):
pass pass
...@@ -65,10 +66,9 @@ class Time(object): ...@@ -65,10 +66,9 @@ class Time(object):
"""Generate a random time from a 24h time clock""" """Generate a random time from a 24h time clock"""
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.name = name if 'time' not in options:
self.buildout = buildout options['time'] = "%u:%02u" % (
self.options = options random.randint(0, 23), random.randint(0, 59))
self.options['time'] = "%d:%d" % (random.randint(0, 23), random.randint(0, 59))
def install(self): def install(self):
pass pass
...@@ -76,26 +76,33 @@ class Time(object): ...@@ -76,26 +76,33 @@ class Time(object):
update = install update = install
class Mac(GenericBaseRecipe): class Mac(object):
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
if os.path.exists(options['storage-path']): self.storage_path = options['storage-path']
open_file = open(options['storage-path'], 'r') mac = options.get('mac-address')
options['mac-address'] = open_file.read() if not mac:
open_file.close() try:
with open(self.storage_path) as f:
if options.get('mac-address', '') == '': mac = f.read()
# First octet has to represent a locally administered address except IOError as e:
octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)] if e.errno != errno.ENOENT:
options['mac-address'] = ':'.join(['%02x' % x for x in octet_list]) raise
return GenericBaseRecipe.__init__(self, buildout, name, options) if not mac:
# First octet has to represent a locally administered address
octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)]
mac = ':'.join(['%02x' % x for x in octet_list])
self.update = self.install
options['mac-address'] = mac
self.mac = mac
def install(self): def install(self):
open_file = open(self.options['storage-path'], 'w') with open(self.storage_path, 'w') as f:
open_file.write(self.options['mac-address']) f.write(self.mac)
open_file.close() return self.storage_path
return [self.options['storage-path']]
def update(self):
pass
def generatePassword(length): def generatePassword(length):
return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length)) return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
...@@ -130,30 +137,20 @@ class Password(object): ...@@ -130,30 +137,20 @@ class Password(object):
except KeyError: except KeyError:
self.storage_path = options['storage-path'] = os.path.join( self.storage_path = options['storage-path'] = os.path.join(
buildout['buildout']['parts-directory'], name) buildout['buildout']['parts-directory'], name)
passwd = None passwd = options.get('passwd')
if self.storage_path:
try:
with open(self.storage_path) as f:
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd: if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8'))) if self.storage_path:
self.update = self.install try:
with open(self.storage_path) as f:
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8')))
self.update = self.install
options['passwd'] = passwd
self.passwd = passwd self.passwd = passwd
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
def get(option, *args, **kw):
return passwd if option == 'passwd' else options_get(option, *args, **kw)
try:
options_get = options._get
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
generatePassword = staticmethod(generatePassword) generatePassword = staticmethod(generatePassword)
...@@ -179,4 +176,4 @@ class Password(object): ...@@ -179,4 +176,4 @@ class Password(object):
return self.storage_path return self.storage_path
def update(self): def update(self):
return () pass
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment