Commit c81bfd99 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents 8d19951b 02a0acd5
......@@ -162,7 +162,8 @@ install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), location)
${:post-install}
post-install =
[geckodriver]
# Current geckodriver installed as ${buildout:bin-directory}/geckodriver
......
......@@ -17,7 +17,7 @@ recipe = hexagonit.recipe.download
filename = en-default.pyrnn.gz
md5sum = cedd140c7d7650e910f0550ad0f04727
download-only = true
url = http://www.tmbdev.net/en-default.pyrnn.gz
url = https://raw.githubusercontent.com/zuphilip/ocropy-models/master/en-default.pyrnn.gz
[ocropy-env]
OCROPY_MODEL_PATH = ${ocropy-eng-traineddata:location}/${ocropy-eng-traineddata:filename}
......
......@@ -17,8 +17,8 @@ parts =
[openssl]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.openssl.org/source/openssl-1.1.1d.tar.gz
md5sum = 3be209000dbc7e1b95bcdf47980a3baa
url = https://www.openssl.org/source/openssl-1.1.1g.tar.gz
md5sum = 76766e98997660138cdaf13a187bd234
location = @@LOCATION@@
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with DESTDIR). Used by slapos.package.git/obs
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.143'
version = '1.0.145'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......
......@@ -30,11 +30,14 @@ class Recipe(GenericBaseRecipe):
def install(self):
if not self.options['lockfile'].endswith('.lock'):
raise ValueError('lockfile parameter must end with .lock as equeue process will add .lock suffix')
args = [
self.options['equeue-binary'],
'--database', self.options['database'],
'--logfile', self.options['log'],
'--lockfile', self.options['lockfile']
'--lockfile', self.options['lockfile'][:-5]
]
if 'takeover-triggered-file-path' in self.options:
......
......@@ -46,16 +46,18 @@ class Recipe(GenericBaseRecipe):
class Callback(GenericBaseRecipe):
def install(self):
options = self.options
notification_id = options['on-notification-id']
# Note: this function is also used in pbs recipe
def createCallback(self, notification_id, callback_list):
# XXX: hashing the name here and in
# slapos.toolbox/slapos/pubsub/__init__.py is completely messed up and
# prevent any debug.
callback_id = sha512(str2bytes(notification_id)).hexdigest()
return self.createFile(os.path.join(self.options['callbacks-directory'], callback_id), callback_list)
return self.createFile(os.path.join(options['directory'], callback_id),
options['callbacks'])
def install(self):
options = self.options
notification_id = options['on-notification-id']
return self.createCallback(notification_id, options['callback-list'])
class Notify(GenericBaseRecipe):
......
......@@ -57,7 +57,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
def _options(self, options):
options['rdiff-backup-data-folder'] = ""
if 'slave-instance-list' in options:
for slave in json.loads(options['slave-instance-list']):
for slave in options['slave-instance-list']:
if slave['type'] == 'pull':
options['rdiff-backup-data-folder'] = str(os.path.join(options['directory'], slave['name'], 'rdiff-backup-data'))
......@@ -299,8 +299,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
if self.optionIsTrue('client', True):
self.logger.info("Client mode")
slaves = json.loads(self.options['slave-instance-list'])
slaves = self.options['slave-instance-list']
known_hosts = KnownHostsFile(self.options['known-hosts'])
with known_hosts:
for slave in slaves:
......
......@@ -204,7 +204,7 @@ class Recipe(GenericBaseRecipe):
password = self.options['password']
# encrypt the password to avoid storing in the logs
enc_password = 'md5' + hashlib.md5(password+user).hexdigest()
enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest()
self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
......@@ -228,7 +228,7 @@ class Recipe(GenericBaseRecipe):
'postgres',
], stdin=subprocess.PIPE)
p.communicate(cmd+'\n')
p.communicate((cmd + '\n').encode())
except subprocess.CalledProcessError:
raise UserError('Could not create database %s' % pgdata)
......
......@@ -82,7 +82,7 @@ class PBSTest(unittest.TestCase):
'run-directory': run_directory,
'cron-entries': cron_directory,
'known-hosts': 'TEST_KNOWN_HOSTS',
'slave-instance-list': '''[
'slave-instance-list': [
{
"url": "http://url.to.pull/",
"type": "pull",
......@@ -101,7 +101,6 @@ class PBSTest(unittest.TestCase):
"frequency": "TEST_FREQUENCY"
}
]
'''
})
recipe._install()
......
......@@ -26,7 +26,7 @@ md5sum = 9e76028df7e93d3e32982884d5dc0913
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 97cfcd64357162271d641e1dece98443
md5sum = 0fda1ba57fb0c4f1b51a6a94a2bd91d3
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -42,7 +42,7 @@ md5sum = b666d7c4a5c1fd8020713aa53b44a386
[template-custom-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 97cfcd64357162271d641e1dece98443
md5sum = 0fda1ba57fb0c4f1b51a6a94a2bd91d3
[template-not-found-html]
filename = templates/notfound.html
......
......@@ -231,22 +231,6 @@ extra-context =
{{ '\n' }}
{% set check_error_log_section_title = 'check-%s-error-log-last-hour' % slave_instance.get('slave_reference') -%}
{% do part_list.append(check_error_log_section_title) -%}
[{{ check_error_log_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/check-error-on-apache-log -l {{ slave_instance.get('error_log') }} -d 3600
filename = {{ check_error_log_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% set check_error_log_section_title = 'check-%s-error-log-last-day' % slave_instance.get('slave_reference') -%}
{% do part_list.append(check_error_log_section_title) -%}
[{{ check_error_log_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/check-error-on-apache-log -l {{ slave_instance.get('error_log') }} -d 86400
filename = {{ check_error_log_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% set monitor_ipv6_test = slave_instance.get('monitor-ipv6-test', '') %}
{% if monitor_ipv6_test %}
{% set monitor_ipv6_section_title = 'check-%s-ipv6-packet-list-test' % slave_instance.get('slave_reference') %}
......
......@@ -237,9 +237,22 @@ def getQUIC(url, ip, port):
def getPluginParameterDict(software_path, filepath):
"""Load the slapos monitor plugin and returns the configuration used by this plugin.
This allow to check that monitoring plugin are using a proper config.
"""
# This is implemented by creating a wrapper script that loads the plugin wrapper
# script and returns its `extra_config_dict`. This might have to be adjusted if
# internals of slapos promise plugins change.
bin_file = os.path.join(software_path, 'bin', 'test-plugin-promise')
monitor_python_with_eggs = os.path.join(software_path, 'bin', 'monitor-pythonwitheggs')
if not os.path.exists(monitor_python_with_eggs):
raise ValueError("Monitoring stack's python does not exist at %s" % monitor_python_with_eggs)
with open(bin_file, 'w') as f:
f.write("""#!%s/bin/pythonwitheggs
f.write("""#!%s
import os
import importlib
import sys
......@@ -251,9 +264,9 @@ filename = os.path.basename(filepath)
module = importlib.import_module(os.path.splitext(filename)[0])
print json.dumps(module.extra_config_dict)
""" % software_path)
""" % monitor_python_with_eggs)
os.chmod(bin_file, 0755)
os.chmod(bin_file, 0o755)
result = subprocess_output([bin_file, filepath]).strip()
try:
return json.loads(result)
......@@ -836,7 +849,7 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
parameter_dict = cls.slave_connection_parameter_dict_dict[
cls.check_slave_id
]
wait_time = 60
wait_time = 600
begin = time.time()
try_num = 0
cls.logger.debug('waitForCaddy for %is' % (wait_time,))
......
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = 42e7d1825f5582fbe2c55977e5abc574
md5sum = ab08e75c26d06d95674042d4a2934716
[template-nginx-service]
filename = template-nginx-service.sh.in
......
......@@ -53,19 +53,20 @@ recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${xserver:location}/bin/Xvfb
:$${:display}
$${:display}
-screen 0 1024x768x24
-fbdir $${directory:framebuffer}
environment =
XORG_LOCK_DIR=$${:lock-dir}
display = 0
display = :$${:display-number}
display-number = 0
lock-dir = $${directory:run}
[xvfb-promise]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:promise}/$${:_buildout_section_name_}
command-line = bash -c "[ -S $${xvfb-instance:lock-dir}/.X11-unix/X$${xvfb-instance:display} ]"
command-line = bash -c "[ -S $${xvfb-instance:lock-dir}/.X11-unix/X$${xvfb-instance:display-number} ]"
#################################
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 07873511a15df74f0be761947819ac9b
md5sum = 1b5d5a72d0d0e3156770ce8ecc5d19ce
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......
......@@ -49,7 +49,7 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${buildout:bin-directory}/surykatka
surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
......
......@@ -61,27 +61,9 @@ url = ${:_profile_base_location_}/script/${:filename}
filename = collect_csv_dump.py
output = ${:destination}/${:filename}
[extra-eggs]
<= monitor-eggs
interpreter = pythonwitheggs
eggs +=
${lxml-python:egg}
${pycurl:egg}
${python-cryptography:egg}
slapos.toolbox
slapos.core
surykatka
scripts =
networkbench
onetimedownload
monitor.bootstrap
monitor.collect
monitor.runpromise
monitor.genstatus
monitor.configwrite
check-computer-memory
surykatka
[surykatka]
recipe = zc.recipe.egg
script-name = ${:_buildout_section_name_}
[monitor-eggs]
recipe = zc.recipe.egg
......
......@@ -37,6 +37,7 @@ import plantuml
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import ImageComparisonTestCase
setUpModule, PlantUMLTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -44,7 +45,7 @@ setUpModule, PlantUMLTestCase = makeModuleSetUpAndTestCaseClass(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestSimpleDiagram(PlantUMLTestCase):
class TestSimpleDiagram(PlantUMLTestCase, ImageComparisonTestCase):
def setUp(self):
self.url = self.computer_partition.getConnectionParameterDict()["url"]
self.plantuml = plantuml.PlantUML(
......@@ -52,24 +53,6 @@ class TestSimpleDiagram(PlantUMLTestCase):
http_opts={"disable_ssl_certificate_validation": True}
)
def assertImagesSimilar(self, i1, i2, tolerance=5):
"""Assert images difference between images is less than `tolerance` %.
taken from https://rosettacode.org/wiki/Percentage_difference_between_images
"""
pairs = zip(i1.getdata(), i2.getdata())
if len(i1.getbands()) == 1:
# for gray-scale jpegs
dif = sum(abs(p1-p2) for p1,p2 in pairs)
else:
dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
ncomponents = i1.size[0] * i1.size[1] * 3
self.assertLessEqual((dif / 255.0 * 100) / ncomponents, tolerance)
def assertImagesSame(self, i1, i2):
"""Assert images are exactly same."""
self.assertImagesSimilar(i1, i2, 0)
def test_sequence_diagram(self):
png = self.plantuml.processes(textwrap.dedent("""\
@startuml
......
......@@ -11,6 +11,7 @@ extends =
../../component/firefox/buildout.cfg
../../component/ffmpeg/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/fonts/buildout.cfg
../../stack/slapos.cfg
./buildout.hash.cfg
......@@ -22,6 +23,27 @@ parts =
firefox-wrapper
geckodriver
# XXX firefox 68 does not seem to honor <dir> from fontconfig's
# fonts.conf and only loads from system locations and from the
# fonts folder located in firefox part, so we copy (actually, symlink)
# some fonts there, otherwise firefox cannot find its standard fonts.
[symlink-extra-fonts-to-firefox-fonts-dir]
extra-fonts =
${android-fonts:location}
${dejavu-fonts:location}
${ipa-fonts:location}
${ipaex-fonts:location}
${liberation-fonts:location}
${ocrb-fonts:location}
install =
import os
for extra_font_dir in '''${:extra-fonts}'''.splitlines():
dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
os.symlink(extra_font_dir, dst)
[firefox]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[macro-template]
recipe = slapos.recipe.template
......
......@@ -44,6 +44,33 @@ output = ${buildout:directory}/template.cfg
<= macro-template
output = ${buildout:directory}/template-selenium.cfg
# XXX firefox 62 and firefox 68 does not seem to honor <dir> from
# fonts.conf and only loads from system locations and from the fonts
# folder located in firefox part, so we copy (actually, symlink) some
# fonts there, otherwise firefox cannot find its standard fonts.
[symlink-extra-fonts-to-firefox-fonts-dir]
extra-fonts =
${android-fonts:location}
${dejavu-fonts:location}
${ipa-fonts:location}
${ipaex-fonts:location}
${liberation-fonts:location}
${ocrb-fonts:location}
install =
import os
for extra_font_dir in '''${:extra-fonts}'''.splitlines():
dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
os.symlink(extra_font_dir, dst)
[firefox-60]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[firefox-68]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 4.4
......@@ -48,7 +48,7 @@ from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort
from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase
setUpModule, SeleniumServerTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
......@@ -80,11 +80,13 @@ class WebServerMixin(object):
<html>
<title>Test page</title>
<body>
<style> p { font-family: Arial; } </style>
<form action="/" method="POST" enctype="multipart/form-data">
<input name="q" type="text"></input>
<input name="f" type="file" ></input>
<input type="submit" value="I'm feeling lucky"></input>
</form>
<p>the quick brown fox jumps over the lazy dog</p>
</body>
</html>''')
......@@ -168,8 +170,18 @@ class BrowserCompatibilityMixin(WebServerMixin):
def test_screenshot(self):
self.driver.get(self.server_url)
screenshot = Image.open(BytesIO(self.driver.get_screenshot_as_png()))
# just check it's not a white screen
self.assertGreater(len(screenshot.getcolors(maxcolors=512)), 2)
reference_filename = os.path.join(
os.path.dirname(__file__), "data",
self.id() + ".png")
# save the screenshot somewhere in a path that will be in snapshot folder.
# XXX we could use a better folder name ...
screenshot.save(
os.path.join(self.slap.instance_directory, 'etc',
self.id() + ".png"))
reference = Image.open(reference_filename)
self.assertImagesSame(screenshot, reference)
def test_window_and_screen_size(self):
size = json.loads(
......@@ -388,7 +400,11 @@ class TestSSHServer(SeleniumServerTestCase):
self.assertIn("Welcome to SlapOS Selenium Server.", received)
class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
class TestFirefox52(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='52.9.0esr')
user_agent = 'Gecko/20100101 Firefox/52.0'
# resizing window is not supported on firefox 52 geckodriver
......@@ -396,16 +412,28 @@ class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
BrowserCompatibilityMixin.test_resize_window)
class TestFirefox60(BrowserCompatibilityMixin, SeleniumServerTestCase):
class TestFirefox60(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='60.0.2esr')
user_agent = 'Gecko/20100101 Firefox/60.0'
class TestFirefox68(BrowserCompatibilityMixin, SeleniumServerTestCase):
class TestFirefox68(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
user_agent = 'Gecko/20100101 Firefox/68.0'
class TestChrome69(BrowserCompatibilityMixin, SeleniumServerTestCase):
class TestChrome69(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
user_agent = 'Chrome/69.0.3497.0'
......@@ -656,7 +656,7 @@ oauth2client = 4.0.0
objgraph = 3.1.0
ply = 3.10
polib = 1.0.8
pprofile = 2.0.1
pprofile = 2.0.4
pyasn1-modules = 0.0.8
pycountry = 17.1.8
pycrypto = 2.6.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment