Cloudooo cluster fixes

 - make cloudooo services use same name as in haproxy, this was off by one (service cloudooo-0 was cloudooo_0 in haproxy)
 - test cluster functionality (to catch bug from cloudooo!29 )
 - update cloudooo with that fix
 - make sure we set an existant locale ( by using `C.UTF-8` ) and exercice a bit conversions involving characters encoding

See merge request !975

component/cloudooo/buildout.cfg

@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/cloudooo.git
 branch = master
 git-executable = ${git:location}/bin/git
-revision = be6c35c0156e028f31da3ccb205afc6f95728d97
+revision = 0b5ff71a2ede76499e81659aed392057ae910917
 
 [cloudooo]
 recipe = zc.recipe.egg

software/cloudooo/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
 
 [template-cloudooo-instance]
 filename = instance-cloudooo.cfg.in
-md5sum = 9e1a66cf18d7c30c14afeb66c20afb46
+md5sum = 440f2b82b119cbfa6f8c7d27652c3170

software/cloudooo/instance-cloudooo.cfg.in

@@ -162,6 +162,7 @@ environment =
   LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-conf:rendered}
   PATH = ${binary-link:target-directory}
+  LANG = C.UTF-8
 mimetype_entry_addition =
 {% for entry in mimetype_entry_addition.splitlines() -%}
 {{ "  " ~ entry.strip() }}
@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
 ooo-paster = {{ bin_directory }}/cloudooo_paster
 ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
 
-{% for index in range(backend_count) -%}
+{% for index in range(1, backend_count + 1) -%}
 {%   set name = 'cloudooo-' ~ index -%}
 [{{ cloudooo(name) }}]
 < = cloudooo-base
@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory}
 run-unit-test = ${buildout:bin-directory}/runUnitTest
 run-test-suite = ${buildout:bin-directory}/runTestSuite
 
-ooo-paster = ${cloudooo-0:ooo-paster}
-configuration-file = ${cloudooo-0:configuration-file}
+ooo-paster = ${cloudooo-1:ooo-paster}
+configuration-file = ${cloudooo-1:configuration-file}
 
 run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
 run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite

software/cloudooo/test/setup.py

@@ -44,6 +44,7 @@ setup(name=name,
         'slapos.core',
         'slapos.cookbook',
         'slapos.libnetworkcache',
+        'requests',
         'six',
         'PyPDF2',
       ],

software/cloudooo/test/test.py

@@ -26,23 +26,43 @@
 #
 ##############################################################################
 
+import codecs
+import csv
+import multiprocessing
 import os
 import json
 import six.moves.xmlrpc_client as xmlrpclib
+import six.moves.urllib.parse as urllib_parse
 import ssl
 import base64
 import io
 
+import requests
 import PyPDF2
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
-setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
+setUpModule, _CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
-# Cloudooo needs a lot of time before being available.
-CloudOooTestCase.instance_max_retry = 30
+
+class CloudOooTestCase(_CloudOooTestCase):
+  # Cloudooo needs a lot of time before being available.
+  instance_max_retry = 30
+
+  def setUp(self):
+    self.url = json.loads(
+        self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
+    # XXX ignore certificate errors
+    ssl_context = ssl.create_default_context()
+    ssl_context.check_hostname = False
+    ssl_context.verify_mode = ssl.CERT_NONE
+    self.server = xmlrpclib.ServerProxy(
+        self.url,
+        context=ssl_context,
+        allow_none=True,
+    )
 
 
 def normalizeFontName(font_name):
@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin:
     """Convert the HTML source to pdf bytes.
     """
 
-  def setUp(self):
-    self.url = json.loads(
-        self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
-    # XXX ignore certificate errors
-    ssl_context = ssl.create_default_context()
-    ssl_context.check_hostname = False
-    ssl_context.verify_mode = ssl.CERT_NONE
-    self.server = xmlrpclib.ServerProxy(
-        self.url,
-        context=ssl_context,
-        allow_none=True,
-    )
-
   def test(self):
     actual_font_mapping_mapping = {}
     for font in self.expected_font_mapping:
@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
             'html',
             'pdf',
         ).encode())
+
+
+class TestLibreOfficeTextConversion(CloudOooTestCase):
+  __partition_reference__ = 'txt'
+
+  def test_html_to_text(self):
+    self.assertEqual(
+        base64.decodestring(
+            self.server.convertFile(
+                base64.encodestring(
+                    u'<html>héhé</html>'.encode('utf-8')).decode(),
+                'html',
+                'txt',
+            ).encode()),
+        codecs.BOM_UTF8 + b'h\xc3\xa9h\xc3\xa9\n',
+    )
+
+
+class TestLibreOfficeCluster(CloudOooTestCase):
+  __partition_reference__ = 'lc'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'backend-count': 4}
+
+  def test_multiple_conversions(self):
+    # make this function global so that it can be picked and used by multiprocessing
+    global _convert_html_to_text
+
+    def _convert_html_to_text(src_html):
+      return base64.decodestring(
+          self.server.convertFile(
+              base64.encodestring(src_html.encode()).decode(),
+              'html',
+              'txt',
+          ).encode())
+
+    pool = multiprocessing.Pool(5)
+    # TODO py3: use with pool
+    converted = pool.map(_convert_html_to_text,
+                         ['<html><body>hello</body></html>'] * 100)
+    pool.terminate()
+    pool.join()
+
+    self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
+
+    # haproxy stats are exposed
+    res = requests.get(
+        urllib_parse.urljoin(self.url, '/haproxy;csv'),
+        verify=False,
+        stream=True,
+    )
+    reader = csv.DictReader(res.raw)
+    line_list = list(reader)
+    # requests have been balanced
+    total_hrsp_2xx = {
+        line['svname']: int(line['hrsp_2xx'])
+        for line in line_list
+    }
+    self.assertEqual(total_hrsp_2xx['FRONTEND'], 100)
+    self.assertEqual(total_hrsp_2xx['BACKEND'], 100)
+    for backend in 'cloudooo_1', 'cloudooo_2', 'cloudooo_3', 'cloudooo_4':
+      # ideally there should be 25% of requests on each backend, because we use
+      # round robin scheduling, but it can happen that some backend take longer
+      # to start, so we are tolerant here and just check that each backend
+      # process at least 15% of requests. 
+      self.assertGreater(total_hrsp_2xx[backend], 15)
+    # no errors
+    total_eresp = {
+        line['svname']: int(line['eresp'] or 0)
+        for line in line_list
+    }
+    self.assertEqual(
+        total_eresp, {
+            'FRONTEND': 0,
+            'cloudooo_1': 0,
+            'cloudooo_2': 0,
+            'cloudooo_3': 0,
+            'cloudooo_4': 0,
+            'BACKEND': 0,
+        })