Update Release Candidate

component/firefox/buildout.cfg

@@ -162,7 +162,8 @@ install =
   url, md5sum = options[guessPlatform()].split()
   extract_dir = self.extract(self.download(url, md5sum))
   self.copyTree(guessworkdir(extract_dir), location)
-
+  ${:post-install}
+post-install =
 
 [geckodriver]
 # Current geckodriver installed as ${buildout:bin-directory}/geckodriver

component/ocropy/buildout.cfg

@@ -17,7 +17,7 @@ recipe = hexagonit.recipe.download
 filename = en-default.pyrnn.gz
 md5sum = cedd140c7d7650e910f0550ad0f04727
 download-only = true
-url = http://www.tmbdev.net/en-default.pyrnn.gz
+url = https://raw.githubusercontent.com/zuphilip/ocropy-models/master/en-default.pyrnn.gz
 
 [ocropy-env]
 OCROPY_MODEL_PATH = ${ocropy-eng-traineddata:location}/${ocropy-eng-traineddata:filename}

component/openssl/buildout.cfg

@@ -17,8 +17,8 @@ parts =
 [openssl]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.openssl.org/source/openssl-1.1.1d.tar.gz
-md5sum = 3be209000dbc7e1b95bcdf47980a3baa
+url = https://www.openssl.org/source/openssl-1.1.1g.tar.gz
+md5sum = 76766e98997660138cdaf13a187bd234
 location = @@LOCATION@@
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with DESTDIR). Used by slapos.package.git/obs

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.143'
+version = '1.0.145'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/equeue.py

@@ -30,11 +30,14 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
 
+    if not self.options['lockfile'].endswith('.lock'):
+      raise ValueError('lockfile parameter must end with .lock as equeue process will add .lock suffix')
+
     args = [
       self.options['equeue-binary'],
       '--database', self.options['database'],
       '--logfile', self.options['log'],
-      '--lockfile', self.options['lockfile']
+      '--lockfile', self.options['lockfile'][:-5]
     ]
 
     if 'takeover-triggered-file-path' in self.options:

slapos/recipe/notifier.py

@@ -46,16 +46,18 @@ class Recipe(GenericBaseRecipe):
 
 class Callback(GenericBaseRecipe):
 
-  def install(self):
-    options = self.options
-    notification_id = options['on-notification-id']
+  # Note: this function is also used in pbs recipe
+  def createCallback(self, notification_id, callback_list):
     # XXX: hashing the name here and in
     # slapos.toolbox/slapos/pubsub/__init__.py is completely messed up and
     # prevent any debug.
     callback_id = sha512(str2bytes(notification_id)).hexdigest()
+    return self.createFile(os.path.join(self.options['callbacks-directory'], callback_id), callback_list)
 
-    return self.createFile(os.path.join(options['directory'], callback_id),
-                           options['callbacks'])
+  def install(self):
+    options = self.options
+    notification_id = options['on-notification-id']
+    return self.createCallback(notification_id, options['callback-list'])
 
 class Notify(GenericBaseRecipe):
 

slapos/recipe/pbs.py

@@ -57,7 +57,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
   def _options(self, options):
     options['rdiff-backup-data-folder'] = ""
     if 'slave-instance-list' in options:
-      for slave in json.loads(options['slave-instance-list']):
+      for slave in options['slave-instance-list']:
         if slave['type'] == 'pull':
           options['rdiff-backup-data-folder'] = str(os.path.join(options['directory'], slave['name'], 'rdiff-backup-data'))
 
@@ -299,8 +299,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     if self.optionIsTrue('client', True):
       self.logger.info("Client mode")
-
-      slaves = json.loads(self.options['slave-instance-list'])
+      slaves = self.options['slave-instance-list']
       known_hosts = KnownHostsFile(self.options['known-hosts'])
       with known_hosts:
         for slave in slaves:

slapos/recipe/postgres.py

@@ -204,7 +204,7 @@ class Recipe(GenericBaseRecipe):
         password = self.options['password']
 
         # encrypt the password to avoid storing in the logs
-        enc_password = 'md5' + hashlib.md5(password+user).hexdigest()
+        enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest()
 
         self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
 
@@ -228,7 +228,7 @@ class Recipe(GenericBaseRecipe):
                                   'postgres',
                                   ], stdin=subprocess.PIPE)
 
-            p.communicate(cmd+'\n')
+            p.communicate((cmd + '\n').encode())
         except subprocess.CalledProcessError:
             raise UserError('Could not create database %s' % pgdata)
 

slapos/test/recipe/test_pbs.py

@@ -82,7 +82,7 @@ class PBSTest(unittest.TestCase):
             'run-directory': run_directory,
             'cron-entries': cron_directory,
             'known-hosts': 'TEST_KNOWN_HOSTS',
-            'slave-instance-list': '''[
+            'slave-instance-list': [
                 {
                  "url": "http://url.to.pull/",
                  "type": "pull",
@@ -101,7 +101,6 @@ class PBSTest(unittest.TestCase):
                  "frequency": "TEST_FREQUENCY"
                         }
                 ]
-                '''
             })
 
         recipe._install()

software/apache-frontend/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 9e76028df7e93d3e32982884d5dc0913
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 97cfcd64357162271d641e1dece98443
+md5sum = 0fda1ba57fb0c4f1b51a6a94a2bd91d3
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -42,7 +42,7 @@ md5sum = b666d7c4a5c1fd8020713aa53b44a386
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 97cfcd64357162271d641e1dece98443
+md5sum = 0fda1ba57fb0c4f1b51a6a94a2bd91d3
 
 [template-not-found-html]
 filename = templates/notfound.html

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -231,22 +231,6 @@ extra-context =
 {{ '\n' }}
 
 
-{%   set check_error_log_section_title = 'check-%s-error-log-last-hour' % slave_instance.get('slave_reference') -%}
-{%   do part_list.append(check_error_log_section_title) -%}
-[{{ check_error_log_section_title }}]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/check-error-on-apache-log -l {{ slave_instance.get('error_log') }} -d 3600 
-filename = {{ check_error_log_section_title }}
-wrapper-path = {{ promise_directory }}/${:filename}
-
-{%   set check_error_log_section_title = 'check-%s-error-log-last-day' % slave_instance.get('slave_reference') -%}
-{%   do part_list.append(check_error_log_section_title) -%}
-[{{ check_error_log_section_title }}]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/check-error-on-apache-log -l {{ slave_instance.get('error_log') }} -d 86400
-filename = {{ check_error_log_section_title }}
-wrapper-path = {{ promise_directory }}/${:filename}
-
 {%   set monitor_ipv6_test = slave_instance.get('monitor-ipv6-test', '') %}
 {%   if monitor_ipv6_test %}
 {%     set monitor_ipv6_section_title = 'check-%s-ipv6-packet-list-test' % slave_instance.get('slave_reference') %}

software/caddy-frontend/test/test.py

@@ -237,9 +237,22 @@ def getQUIC(url, ip, port):
 
 
 def getPluginParameterDict(software_path, filepath):
+  """Load the slapos monitor plugin and returns the configuration used by this plugin.
+
+  This allow to check that monitoring plugin are using a proper config.
+  """
+  # This is implemented by creating a wrapper script that loads the plugin wrapper
+  # script and returns its `extra_config_dict`. This might have to be adjusted if
+  # internals of slapos promise plugins change.
+
   bin_file = os.path.join(software_path, 'bin', 'test-plugin-promise')
+
+  monitor_python_with_eggs = os.path.join(software_path, 'bin', 'monitor-pythonwitheggs')
+  if not os.path.exists(monitor_python_with_eggs):
+    raise ValueError("Monitoring stack's python does not exist at %s" % monitor_python_with_eggs)
+
   with open(bin_file, 'w') as f:
-    f.write("""#!%s/bin/pythonwitheggs
+    f.write("""#!%s
 import os
 import importlib
 import sys
@@ -251,9 +264,9 @@ filename = os.path.basename(filepath)
 module = importlib.import_module(os.path.splitext(filename)[0])
 
 print json.dumps(module.extra_config_dict)
-    """ % software_path)
+    """ % monitor_python_with_eggs)
 
-  os.chmod(bin_file, 0755)
+  os.chmod(bin_file, 0o755)
   result = subprocess_output([bin_file, filepath]).strip()
   try:
     return json.loads(result)
@@ -836,7 +849,7 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
     parameter_dict = cls.slave_connection_parameter_dict_dict[
       cls.check_slave_id
     ]
-    wait_time = 60
+    wait_time = 600
     begin = time.time()
     try_num = 0
     cls.logger.debug('waitForCaddy for %is' % (wait_time,))

software/jstestnode/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 42e7d1825f5582fbe2c55977e5abc574
+md5sum = ab08e75c26d06d95674042d4a2934716
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in

software/jstestnode/instance.cfg.in

@@ -53,19 +53,20 @@ recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
   ${xserver:location}/bin/Xvfb
-    :$${:display}
+    $${:display}
     -screen 0 1024x768x24
     -fbdir $${directory:framebuffer}
 environment =
   XORG_LOCK_DIR=$${:lock-dir}
 
-display = 0
+display = :$${:display-number}
+display-number = 0
 lock-dir = $${directory:run}
 
 [xvfb-promise]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:promise}/$${:_buildout_section_name_}
-command-line = bash -c "[ -S $${xvfb-instance:lock-dir}/.X11-unix/X$${xvfb-instance:display} ]"
+command-line = bash -c "[ -S $${xvfb-instance:lock-dir}/.X11-unix/X$${xvfb-instance:display-number} ]"
 
 
 #################################

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 07873511a15df74f0be761947819ac9b
+md5sum = 1b5d5a72d0d0e3156770ce8ecc5d19ce
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2

software/monitor/instance.cfg

@@ -39,7 +39,7 @@ context = import json_module json
           section slap_configuration slap-configuration
           raw software_type edgetest
           key instance_base_monitor instance-base-monitor:rendered
-          key slave_instance_list slap-configuration:slave-instance-list 
+          key slave_instance_list slap-configuration:slave-instance-list
           raw buildout_bin ${buildout:bin-directory}
           raw template_json_edgetest_test ${json-test-template:target}
 mode = 0644
@@ -49,7 +49,7 @@ recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgebot:target}
 rendered = $${buildout:directory}/template-monitor-edgebot.cfg
 extensions = jinja2.ext.do
-surykatka-binary = ${buildout:bin-directory}/surykatka
+surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
 template-surykatka-ini = ${template-surykatka-ini:target}
 
 context = import json_module json

software/monitor/software.cfg

@@ -10,7 +10,7 @@ extends =
 
 python = python3
 
-parts = 
+parts =
   slapos-cookbook
   network-bench-cfg
   json-test-template
@@ -20,13 +20,13 @@ parts =
   monitor-collect-csv-dump
 
 [template]
-recipe = slapos.recipe.template 
+recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [template-monitor]
-recipe = slapos.recipe.build:download 
+recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 mode = 0644
 
@@ -61,31 +61,13 @@ url = ${:_profile_base_location_}/script/${:filename}
 filename = collect_csv_dump.py
 output = ${:destination}/${:filename}
 
-[extra-eggs]
-<= monitor-eggs
-interpreter = pythonwitheggs
-eggs +=
-  ${lxml-python:egg}
-  ${pycurl:egg}
-  ${python-cryptography:egg}
-  slapos.toolbox
-  slapos.core
-  surykatka
-
-scripts =
-  networkbench
-  onetimedownload
-  monitor.bootstrap
-  monitor.collect
-  monitor.runpromise
-  monitor.genstatus
-  monitor.configwrite
-  check-computer-memory
-  surykatka
+[surykatka]
+recipe = zc.recipe.egg
+script-name = ${:_buildout_section_name_}
 
 [monitor-eggs]
 recipe = zc.recipe.egg
-eggs += 
+eggs +=
   slapos.cookbook
   slapos.recipe.cmmi
   hexagonit.recipe.download

software/plantuml/test/test.py

@@ -37,6 +37,7 @@ import plantuml
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.utils import ImageComparisonTestCase
 
 
 setUpModule, PlantUMLTestCase = makeModuleSetUpAndTestCaseClass(
@@ -44,7 +45,7 @@ setUpModule, PlantUMLTestCase = makeModuleSetUpAndTestCaseClass(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
-class TestSimpleDiagram(PlantUMLTestCase):
+class TestSimpleDiagram(PlantUMLTestCase, ImageComparisonTestCase):
   def setUp(self):
     self.url = self.computer_partition.getConnectionParameterDict()["url"]
     self.plantuml = plantuml.PlantUML(
@@ -52,24 +53,6 @@ class TestSimpleDiagram(PlantUMLTestCase):
       http_opts={"disable_ssl_certificate_validation": True}
     )
 
-  def assertImagesSimilar(self, i1, i2, tolerance=5):
-    """Assert images difference between images is less than `tolerance` %.
-   taken from https://rosettacode.org/wiki/Percentage_difference_between_images
-    """
-    pairs = zip(i1.getdata(), i2.getdata())
-    if len(i1.getbands()) == 1:
-      # for gray-scale jpegs
-      dif = sum(abs(p1-p2) for p1,p2 in pairs)
-    else:
-      dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
-
-    ncomponents = i1.size[0] * i1.size[1] * 3
-    self.assertLessEqual((dif / 255.0 * 100) / ncomponents, tolerance)
-
-  def assertImagesSame(self, i1, i2):
-    """Assert images are exactly same."""
-    self.assertImagesSimilar(i1, i2, 0)
-
   def test_sequence_diagram(self):
     png = self.plantuml.processes(textwrap.dedent("""\
     @startuml

software/seleniumrunner/software.cfg

@@ -11,6 +11,7 @@ extends =
    ../../component/firefox/buildout.cfg
    ../../component/ffmpeg/buildout.cfg
    ../../component/coreutils/buildout.cfg
+   ../../component/fonts/buildout.cfg
    ../../stack/slapos.cfg
    ./buildout.hash.cfg
 
@@ -22,6 +23,27 @@ parts =
    firefox-wrapper
    geckodriver
 
+# XXX firefox 68 does not seem to honor <dir> from fontconfig's
+# fonts.conf and only loads from system locations and from the
+# fonts folder located in firefox part, so we copy (actually, symlink)
+# some fonts there, otherwise firefox cannot find its standard fonts.
+[symlink-extra-fonts-to-firefox-fonts-dir]
+extra-fonts =
+  ${android-fonts:location}
+  ${dejavu-fonts:location}
+  ${ipa-fonts:location}
+  ${ipaex-fonts:location}
+  ${liberation-fonts:location}
+  ${ocrb-fonts:location}
+install =
+  import os
+  for extra_font_dir in '''${:extra-fonts}'''.splitlines():
+    dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
+    os.symlink(extra_font_dir, dst)
+
+[firefox]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}
 
 [macro-template]
 recipe = slapos.recipe.template

software/seleniumserver/software.cfg

@@ -44,6 +44,33 @@ output = ${buildout:directory}/template.cfg
 <= macro-template
 output = ${buildout:directory}/template-selenium.cfg
 
+
+# XXX firefox 62 and firefox 68 does not seem to honor <dir> from
+# fonts.conf and only loads from system locations and from the fonts
+# folder located in firefox part, so we copy (actually, symlink) some
+# fonts there, otherwise firefox cannot find its standard fonts.
+[symlink-extra-fonts-to-firefox-fonts-dir]
+extra-fonts =
+  ${android-fonts:location}
+  ${dejavu-fonts:location}
+  ${ipa-fonts:location}
+  ${ipaex-fonts:location}
+  ${liberation-fonts:location}
+  ${ocrb-fonts:location}
+install =
+  import os
+  for extra_font_dir in '''${:extra-fonts}'''.splitlines():
+    dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
+    os.symlink(extra_font_dir, dst)
+
+[firefox-60]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}
+
+[firefox-68]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}
+
 [versions]
 plone.recipe.command = 1.1
 slapos.recipe.template = 4.4

software/seleniumserver/test/test.py

@@ -48,7 +48,7 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.testing.utils import findFreeTCPPort
+from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase
 
 setUpModule, SeleniumServerTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
@@ -80,11 +80,13 @@ class WebServerMixin(object):
           <html>
             <title>Test page</title>
             <body>
+              <style> p { font-family: Arial; } </style>
               <form action="/" method="POST" enctype="multipart/form-data">
                 <input name="q" type="text"></input>
                 <input name="f" type="file" ></input>
                 <input type="submit" value="I'm feeling lucky"></input>
               </form>
+              <p>the quick brown fox jumps over the lazy dog</p>
             </body>
           </html>''')
 
@@ -168,8 +170,18 @@ class BrowserCompatibilityMixin(WebServerMixin):
   def test_screenshot(self):
     self.driver.get(self.server_url)
     screenshot = Image.open(BytesIO(self.driver.get_screenshot_as_png()))
-    # just check it's not a white screen
-    self.assertGreater(len(screenshot.getcolors(maxcolors=512)), 2)
+    reference_filename = os.path.join(
+        os.path.dirname(__file__), "data",
+        self.id() + ".png")
+
+    # save the screenshot somewhere in a path that will be in snapshot folder.
+    # XXX we could use a better folder name ...
+    screenshot.save(
+        os.path.join(self.slap.instance_directory, 'etc',
+                     self.id() + ".png"))
+
+    reference = Image.open(reference_filename)
+    self.assertImagesSame(screenshot, reference)
 
   def test_window_and_screen_size(self):
     size = json.loads(
@@ -388,7 +400,11 @@ class TestSSHServer(SeleniumServerTestCase):
       self.assertIn("Welcome to SlapOS Selenium Server.", received)
 
 
-class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestFirefox52(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='52.9.0esr')
   user_agent = 'Gecko/20100101 Firefox/52.0'
   # resizing window is not supported on firefox 52 geckodriver
@@ -396,16 +412,28 @@ class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
       BrowserCompatibilityMixin.test_resize_window)
 
 
-class TestFirefox60(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestFirefox60(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='60.0.2esr')
   user_agent = 'Gecko/20100101 Firefox/60.0'
 
 
-class TestFirefox68(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestFirefox68(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
   user_agent = 'Gecko/20100101 Firefox/68.0'
 
 
-class TestChrome69(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestChrome69(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
   user_agent = 'Chrome/69.0.3497.0'

stack/erp5/buildout.cfg

@@ -656,7 +656,7 @@ oauth2client = 4.0.0
 objgraph = 3.1.0
 ply = 3.10
 polib = 1.0.8
-pprofile = 2.0.1
+pprofile = 2.0.4
 pyasn1-modules = 0.0.8
 pycountry = 17.1.8
 pycrypto = 2.6.1