Commit 02bb91c3 authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

parents 24d1d7cf 4f15f8a1
Changes Changes
======= =======
1.0.119 (2019-08-14)
--------------------
* publish_early: rework API
1.0.118 (2019-08-13) 1.0.118 (2019-08-13)
-------------------- --------------------
......
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '1.0.118' version = '1.0.119'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \ long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n" open("CHANGES.rst").read() + "\n"
......
...@@ -32,30 +32,26 @@ from .librecipe import GenericBaseRecipe ...@@ -32,30 +32,26 @@ from .librecipe import GenericBaseRecipe
class Cluster(object): class Cluster(object):
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.buildout = buildout masters = options.setdefault('masters', '')
self.options = options
def publish_early(self, publish_dict):
masters = publish_dict.setdefault('masters', '')
result_dict = { result_dict = {
'connection-admin': [], 'connection-admin': [],
'connection-master': [], 'connection-master': [],
} }
node_list = [] node_list = []
for node in sorted(self.options['nodes'].split()): for node in sorted(options['nodes'].split()):
node = self.buildout[node] node = buildout[node]
node_list.append(node) node_list.append(node)
for k, v in result_dict.iteritems(): for k, v in result_dict.iteritems():
x = node[k] x = node[k]
if x: if x:
v.append(x) v.append(x)
publish_dict['admins'] = ' '.join(result_dict.pop('connection-admin')) options['admins'] = ' '.join(result_dict.pop('connection-admin'))
x = ' '.join(result_dict.pop('connection-master')) x = ' '.join(result_dict.pop('connection-master'))
if masters != x: if masters != x:
publish_dict['masters'] = x options['masters'] = x
for node in node_list: for node in node_list:
node['config-masters'] = x node['config-masters'] = x
node.recipe.__init__(self.buildout, node.name, node) node.recipe.__init__(buildout, node.name, node)
install = update = lambda self: None install = update = lambda self: None
......
...@@ -28,20 +28,14 @@ ...@@ -28,20 +28,14 @@
from collections import defaultdict from collections import defaultdict
from .librecipe import unwrap, wrap, GenericSlapRecipe from .librecipe import unwrap, wrap, GenericSlapRecipe
def patchOptions(options, override): def volatileOptions(options, volatile):
def get(option, *args, **kw): def copy():
try: copy = options_copy()
return override[option] for key in volatile:
except KeyError: copy.pop(key, None)
return options_get(option, *args, **kw) return copy
try: options_copy = options.copy
options_get = options._get options.copy = copy
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
class Recipe(GenericSlapRecipe): class Recipe(GenericSlapRecipe):
""" """
...@@ -57,8 +51,6 @@ class Recipe(GenericSlapRecipe): ...@@ -57,8 +51,6 @@ class Recipe(GenericSlapRecipe):
-init = -init =
foo gen-foo:x foo gen-foo:x
bar gen-bar:y bar gen-bar:y
-update =
baz update-baz:z
bar = z bar = z
[gen-foo] [gen-foo]
...@@ -69,72 +61,74 @@ class Recipe(GenericSlapRecipe): ...@@ -69,72 +61,74 @@ class Recipe(GenericSlapRecipe):
-extends = publish-early -extends = publish-early
... ...
${publish-early:foo} is initialized with the value of the published Just before the recipe of [gen-foo] is instantiated, 'x' is overridden with
parameter 'foo', or ${gen-foo:x} if it hasn't been published yet the published value 'foo' if it exists. If its __init__ modifies 'x', the new
(and in this case, it is published immediately as a way to save the value). value is published. To prevent [gen-foo] from being accessed too early, 'x'
is then removed and the value can only be accessed with ${publish-early:foo}.
Generated values don't end up in the buildout installed file, which is good
if they're secret. Note however that buildout won't detect if values change
and it may only call update().
${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored): ${publish-early:bar} is forced to 'z' (${gen-bar:y} ignored):
a line like 'bar = z' is usually rendered conditionally with Jinja2. a line like 'bar = z' is usually rendered conditionally with Jinja2.
The '-update' option has the same syntax than '-init'. The recipes of the
specified sections must implement 'publish_early(publish_dict)':
- it is always called, just before early publishing
- publish_dict is a dict with already published values
- 'publish_early' can change published values by modifying publish_dict.
In the above example:
- publish_dict is {'z': ...}
- during the execution of 'publish_early', other sections can access the
value with ${update-baz:z}
- once [publish-early] is initialized, the value should be accessed with
${publish-early:bar} ([update-baz] does not have it if it's accessed
before [publish-early])
""" """
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
GenericSlapRecipe.__init__(self, buildout, name, options) GenericSlapRecipe.__init__(self, buildout, name, options)
init = defaultdict(dict) init = defaultdict(dict)
update = defaultdict(dict) for line in options['-init'].splitlines():
for d, k in (init, '-init'), (update, '-update'): if line:
for line in options.get(k, '').splitlines(): k, v = line.split()
if line: if k not in options:
k, v = line.split() section, v = v.split(':')
if k not in options: init[section][k] = v
section, v = v.split(':') if init:
d[section][k] = v
if init or update:
self.slap.initializeConnection(self.server_url, self.key_file, self.slap.initializeConnection(self.server_url, self.key_file,
self.cert_file) self.cert_file)
computer_partition = self.slap.registerComputerPartition( computer_partition = self.slap.registerComputerPartition(
self.computer_id, self.computer_partition_id) self.computer_id, self.computer_partition_id)
published_dict = unwrap(computer_partition.getConnectionParameterDict()) published_dict = unwrap(computer_partition.getConnectionParameterDict())
Options = buildout.Options
if 'Options' in buildout.__dict__:
def revertOptions():
buildout.Options = Options
else:
def revertOptions():
try:
del buildout.Options
except AttributeError:
pass
def newOptions(buildout, section, data):
assert section == init_section, (section, init_section)
revertOptions()
self = buildout.Options(buildout, section, data)
self.update(override)
return self
publish = False publish = False
publish_dict = {} publish_dict = {}
for section, init in init.iteritems(): try:
for k, v in init.iteritems(): for init_section, init in init.iteritems():
try: override = {}
publish_dict[k] = published_dict[k] for k, v in init.iteritems():
except KeyError: try:
publish_dict[k] = buildout[section][v] override[v] = published_dict[k]
except KeyError:
pass
buildout.Options = newOptions
init_section = buildout[init_section]
assert buildout.Options is Options
new = {}
for k, v in init.iteritems():
try:
publish_dict[k] = new[v] = init_section.pop(v)
except KeyError:
pass
if new != override:
publish = True publish = True
finally:
for section, update in update.iteritems(): revertOptions()
override = {}
for k, v in update.iteritems():
try:
override[v] = published_dict[k]
except KeyError:
pass
section = buildout[section]
patchOptions(section, override)
old = override.copy()
section.recipe.publish_early(override)
if override != old:
publish = True
for k, v in update.iteritems():
try:
publish_dict[k] = override[v]
except KeyError:
pass
if publish: if publish:
computer_partition.setConnectionDict(wrap(publish_dict)) computer_partition.setConnectionDict(wrap(publish_dict))
...@@ -143,6 +137,7 @@ class Recipe(GenericSlapRecipe): ...@@ -143,6 +137,7 @@ class Recipe(GenericSlapRecipe):
if k != 'recipe' and not k.startswith('-')] if k != 'recipe' and not k.startswith('-')]
publish += publish_dict publish += publish_dict
publish_dict['-publish'] = ' '.join(publish) publish_dict['-publish'] = ' '.join(publish)
patchOptions(options, publish_dict) volatileOptions(options, list(publish_dict))
options.update(publish_dict)
install = update = lambda self: None install = update = lambda self: None
...@@ -36,8 +36,8 @@ import errno ...@@ -36,8 +36,8 @@ import errno
import os import os
import random import random
import string import string
from .librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import GenericBaseRecipe from .publish_early import volatileOptions
class Integer(object): class Integer(object):
""" """
...@@ -54,7 +54,9 @@ class Integer(object): ...@@ -54,7 +54,9 @@ class Integer(object):
Resulting integer. Resulting integer.
""" """
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
options['value'] = random.randint(int(options['minimum']), int(options['maximum'])) if 'value' not in options:
options['value'] = random.randint(int(options['minimum']),
int(options['maximum']))
def install(self): def install(self):
pass pass
...@@ -65,10 +67,9 @@ class Time(object): ...@@ -65,10 +67,9 @@ class Time(object):
"""Generate a random time from a 24h time clock""" """Generate a random time from a 24h time clock"""
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.name = name if 'time' not in options:
self.buildout = buildout options['time'] = "%u:%02u" % (
self.options = options random.randint(0, 23), random.randint(0, 59))
self.options['time'] = "%d:%d" % (random.randint(0, 23), random.randint(0, 59))
def install(self): def install(self):
pass pass
...@@ -76,26 +77,33 @@ class Time(object): ...@@ -76,26 +77,33 @@ class Time(object):
update = install update = install
class Mac(GenericBaseRecipe): class Mac(object):
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
if os.path.exists(options['storage-path']): self.storage_path = options['storage-path']
open_file = open(options['storage-path'], 'r') mac = options.get('mac-address')
options['mac-address'] = open_file.read() if not mac:
open_file.close() try:
with open(self.storage_path) as f:
if options.get('mac-address', '') == '': mac = f.read()
# First octet has to represent a locally administered address except IOError as e:
octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)] if e.errno != errno.ENOENT:
options['mac-address'] = ':'.join(['%02x' % x for x in octet_list]) raise
return GenericBaseRecipe.__init__(self, buildout, name, options) if not mac:
# First octet has to represent a locally administered address
octet_list = [254] + [random.randint(0x00, 0xff) for x in range(5)]
mac = ':'.join(['%02x' % x for x in octet_list])
self.update = self.install
options['mac-address'] = mac
self.mac = mac
def install(self): def install(self):
open_file = open(self.options['storage-path'], 'w') with open(self.storage_path, 'w') as f:
open_file.write(self.options['mac-address']) f.write(self.mac)
open_file.close() return self.storage_path
return [self.options['storage-path']]
def update(self):
pass
def generatePassword(length): def generatePassword(length):
return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length)) return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
...@@ -130,30 +138,24 @@ class Password(object): ...@@ -130,30 +138,24 @@ class Password(object):
except KeyError: except KeyError:
self.storage_path = options['storage-path'] = os.path.join( self.storage_path = options['storage-path'] = os.path.join(
buildout['buildout']['parts-directory'], name) buildout['buildout']['parts-directory'], name)
passwd = None passwd = options.get('passwd')
if self.storage_path:
try:
with open(self.storage_path) as f:
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd: if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8'))) if self.storage_path:
self.update = self.install try:
self.passwd = passwd with open(self.storage_path) as f:
passwd = f.read().strip('\n')
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '8')))
self.update = self.install
options['passwd'] = passwd
# Password must not go into .installed file, for 2 reasons: # Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall. # security of course but also to prevent buildout to always reinstall.
def get(option, *args, **kw): # publish_early already does it, but this recipe may also be used alone.
return passwd if option == 'passwd' else options_get(option, *args, **kw) volatileOptions(options, ('passwd',))
self.passwd = passwd
try:
options_get = options._get
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
generatePassword = staticmethod(generatePassword) generatePassword = staticmethod(generatePassword)
...@@ -179,4 +181,4 @@ class Password(object): ...@@ -179,4 +181,4 @@ class Password(object):
return self.storage_path return self.storage_path
def update(self): def update(self):
return () pass
...@@ -8,7 +8,6 @@ extends = ...@@ -8,7 +8,6 @@ extends =
../../software/erp5/software.cfg ../../software/erp5/software.cfg
parts += parts +=
wendelin wendelin
erp5-bin
scipy scipy
msgpack-python msgpack-python
msgpack-numpy msgpack-numpy
...@@ -61,14 +60,10 @@ initialization = ...@@ -61,14 +60,10 @@ initialization =
${testrunner:initialization} ${testrunner:initialization}
[erp5_repository_list] [erp5_repository_list]
repository_id_list += repository_id_list += wendelin
wendelin
erp5-bin
[local-bt5-repository] [local-bt5-repository]
list += list += ${wendelin:location}/bt5
${wendelin:location}/bt5
${erp5-bin:location}/bt5
# Jupyter is by default enabled in Wendelin # Jupyter is by default enabled in Wendelin
[erp5-defaults] [erp5-defaults]
...@@ -80,12 +75,6 @@ git-executable = ${git:location}/bin/git ...@@ -80,12 +75,6 @@ git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/wendelin.git repository = https://lab.nexedi.com/nexedi/wendelin.git
branch = master branch = master
[erp5-bin]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/erp5-bin.git
branch = master
[versions] [versions]
msgpack = 0.6.1 msgpack = 0.6.1
msgpack-numpy = 0.4.4.3 msgpack-numpy = 0.4.4.3
......
...@@ -136,7 +136,7 @@ pyparsing = 2.2.0 ...@@ -136,7 +136,7 @@ pyparsing = 2.2.0
pytz = 2016.10 pytz = 2016.10
requests = 2.13.0 requests = 2.13.0
six = 1.11.0 six = 1.11.0
slapos.cookbook = 1.0.118 slapos.cookbook = 1.0.119
slapos.core = 1.4.26 slapos.core = 1.4.26
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.extension.shared = 1.0 slapos.extension.shared = 1.0
...@@ -195,7 +195,7 @@ enum34 = 1.1.6 ...@@ -195,7 +195,7 @@ enum34 = 1.1.6
# Required by: # Required by:
# slapos.toolbox==0.94 # slapos.toolbox==0.94
erp5.util = 0.4.59 erp5.util = 0.4.59.1
# Required by: # Required by:
# slapos.toolbox==0.94 # slapos.toolbox==0.94
...@@ -218,7 +218,7 @@ pyrsistent = 0.14.5 ...@@ -218,7 +218,7 @@ pyrsistent = 0.14.5
ipaddress = 1.0.18 ipaddress = 1.0.18
# Required by: # Required by:
# slapos.cookbook==1.0.118 # slapos.cookbook==1.0.119
jsonschema = 3.0.0a3 jsonschema = 3.0.0a3
# Required by: # Required by:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment