Fix fonts for firefox >= 60 in selenium

As we can see in the testcases added, since we started to use Firefox 60, fonts were no longer selected and some garbage characters were displayed instead of the text.

Before these changes, it looked like this:

![screenshot_test.TestFirefox60.test_screenshot_before](/uploads/7ffe2eb7678b9f62ae13a56f4c8b57cf/screenshot_test.TestFirefox60.test_screenshot_before.png)

now it looks like this:

![screenshot_test.TestFirefox60.test_screenshot_after](/uploads/6d178ebdac388e534206149e97de08a1/screenshot_test.TestFirefox60.test_screenshot_after.png)

which is of course better, but still wrong, because the test case uses Arial and this font is not Arial. This part will be fixed later.

It seems firefox no longer accepts fonts from slapos directories, before the fix, firefox prints this on console:
    
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.942: failed to create cairo scaled font, expect ugly output. the offending font is 'IPAGothic 9.9990234375'
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.942: font_face status is: file not found
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.942: scaled_font status is: file not found
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.942: shaping failure, expect ugly output. shape-engine='PangoFcShapeEngine', font='IPAGothic 9.9990234375', text='?'
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.943: failed to create cairo scaled font, expect ugly output. the offending font is 'IPAGothic 9.9990234375'
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.943: font_face status is: file not found
    (./parts/firefox-60/firefox:7277): Pango-WARNING **: 08:06:33.943: scaled_font status is: file not found

but the file exists... Maybe this is a bug in firefox ( https://bugzilla.mozilla.org/show_bug.cgi?id=1336049#c2 looks similar, even though for us in firefox 52 it was fine and the problem appeared between 52 and 60).

A simple workaround is to copy all fonts in firefox fonts' directory.

See merge request !736

component/firefox/buildout.cfg

@@ -162,7 +162,8 @@ install =
   url, md5sum = options[guessPlatform()].split()
   extract_dir = self.extract(self.download(url, md5sum))
   self.copyTree(guessworkdir(extract_dir), location)
-
+  ${:post-install}
+post-install =
 
 [geckodriver]
 # Current geckodriver installed as ${buildout:bin-directory}/geckodriver

software/plantuml/test/test.py

@@ -37,6 +37,7 @@ import plantuml
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.utils import ImageComparisonTestCase
 
 
 setUpModule, PlantUMLTestCase = makeModuleSetUpAndTestCaseClass(
@@ -44,7 +45,7 @@ setUpModule, PlantUMLTestCase = makeModuleSetUpAndTestCaseClass(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
-class TestSimpleDiagram(PlantUMLTestCase):
+class TestSimpleDiagram(PlantUMLTestCase, ImageComparisonTestCase):
   def setUp(self):
     self.url = self.computer_partition.getConnectionParameterDict()["url"]
     self.plantuml = plantuml.PlantUML(
@@ -52,24 +53,6 @@ class TestSimpleDiagram(PlantUMLTestCase):
       http_opts={"disable_ssl_certificate_validation": True}
     )
 
-  def assertImagesSimilar(self, i1, i2, tolerance=5):
-    """Assert images difference between images is less than `tolerance` %.
-   taken from https://rosettacode.org/wiki/Percentage_difference_between_images
-    """
-    pairs = zip(i1.getdata(), i2.getdata())
-    if len(i1.getbands()) == 1:
-      # for gray-scale jpegs
-      dif = sum(abs(p1-p2) for p1,p2 in pairs)
-    else:
-      dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
-
-    ncomponents = i1.size[0] * i1.size[1] * 3
-    self.assertLessEqual((dif / 255.0 * 100) / ncomponents, tolerance)
-
-  def assertImagesSame(self, i1, i2):
-    """Assert images are exactly same."""
-    self.assertImagesSimilar(i1, i2, 0)
-
   def test_sequence_diagram(self):
     png = self.plantuml.processes(textwrap.dedent("""\
     @startuml

software/seleniumrunner/software.cfg

@@ -11,6 +11,7 @@ extends =
    ../../component/firefox/buildout.cfg
    ../../component/ffmpeg/buildout.cfg
    ../../component/coreutils/buildout.cfg
+   ../../component/fonts/buildout.cfg
    ../../stack/slapos.cfg
    ./buildout.hash.cfg
 
@@ -22,6 +23,27 @@ parts =
    firefox-wrapper
    geckodriver
 
+# XXX firefox 68 does not seem to honor <dir> from fontconfig's
+# fonts.conf and only loads from system locations and from the
+# fonts folder located in firefox part, so we copy (actually, symlink)
+# some fonts there, otherwise firefox cannot find its standard fonts.
+[symlink-extra-fonts-to-firefox-fonts-dir]
+extra-fonts =
+  ${android-fonts:location}
+  ${dejavu-fonts:location}
+  ${ipa-fonts:location}
+  ${ipaex-fonts:location}
+  ${liberation-fonts:location}
+  ${ocrb-fonts:location}
+install =
+  import os
+  for extra_font_dir in '''${:extra-fonts}'''.splitlines():
+    dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
+    os.symlink(extra_font_dir, dst)
+
+[firefox]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}
 
 [macro-template]
 recipe = slapos.recipe.template

software/seleniumserver/software.cfg

@@ -44,6 +44,33 @@ output = ${buildout:directory}/template.cfg
 <= macro-template
 output = ${buildout:directory}/template-selenium.cfg
 
+
+# XXX firefox 62 and firefox 68 does not seem to honor <dir> from
+# fonts.conf and only loads from system locations and from the fonts
+# folder located in firefox part, so we copy (actually, symlink) some
+# fonts there, otherwise firefox cannot find its standard fonts.
+[symlink-extra-fonts-to-firefox-fonts-dir]
+extra-fonts =
+  ${android-fonts:location}
+  ${dejavu-fonts:location}
+  ${ipa-fonts:location}
+  ${ipaex-fonts:location}
+  ${liberation-fonts:location}
+  ${ocrb-fonts:location}
+install =
+  import os
+  for extra_font_dir in '''${:extra-fonts}'''.splitlines():
+    dst = os.path.join(location, 'fonts', os.path.basename(extra_font_dir))
+    os.symlink(extra_font_dir, dst)
+
+[firefox-60]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}
+
+[firefox-68]
+post-install =
+  ${symlink-extra-fonts-to-firefox-fonts-dir:install}
+
 [versions]
 plone.recipe.command = 1.1
 slapos.recipe.template = 4.4

software/seleniumserver/test/test.py

@@ -48,7 +48,7 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.testing.utils import findFreeTCPPort
+from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase
 
 setUpModule, SeleniumServerTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
@@ -80,11 +80,13 @@ class WebServerMixin(object):
           <html>
             <title>Test page</title>
             <body>
+              <style> p { font-family: Arial; } </style>
               <form action="/" method="POST" enctype="multipart/form-data">
                 <input name="q" type="text"></input>
                 <input name="f" type="file" ></input>
                 <input type="submit" value="I'm feeling lucky"></input>
               </form>
+              <p>the quick brown fox jumps over the lazy dog</p>
             </body>
           </html>''')
 
@@ -168,8 +170,18 @@ class BrowserCompatibilityMixin(WebServerMixin):
   def test_screenshot(self):
     self.driver.get(self.server_url)
     screenshot = Image.open(BytesIO(self.driver.get_screenshot_as_png()))
-    # just check it's not a white screen
-    self.assertGreater(len(screenshot.getcolors(maxcolors=512)), 2)
+    reference_filename = os.path.join(
+        os.path.dirname(__file__), "data",
+        self.id() + ".png")
+
+    # save the screenshot somewhere in a path that will be in snapshot folder.
+    # XXX we could use a better folder name ...
+    screenshot.save(
+        os.path.join(self.slap.instance_directory, 'etc',
+                     self.id() + ".png"))
+
+    reference = Image.open(reference_filename)
+    self.assertImagesSame(screenshot, reference)
 
   def test_window_and_screen_size(self):
     size = json.loads(
@@ -388,7 +400,11 @@ class TestSSHServer(SeleniumServerTestCase):
       self.assertIn("Welcome to SlapOS Selenium Server.", received)
 
 
-class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestFirefox52(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='52.9.0esr')
   user_agent = 'Gecko/20100101 Firefox/52.0'
   # resizing window is not supported on firefox 52 geckodriver
@@ -396,16 +412,28 @@ class TestFirefox52(BrowserCompatibilityMixin, SeleniumServerTestCase):
       BrowserCompatibilityMixin.test_resize_window)
 
 
-class TestFirefox60(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestFirefox60(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='60.0.2esr')
   user_agent = 'Gecko/20100101 Firefox/60.0'
 
 
-class TestFirefox68(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestFirefox68(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
   user_agent = 'Gecko/20100101 Firefox/68.0'
 
 
-class TestChrome69(BrowserCompatibilityMixin, SeleniumServerTestCase):
+class TestChrome69(
+    BrowserCompatibilityMixin,
+    SeleniumServerTestCase,
+    ImageComparisonTestCase,
+):
   desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
   user_agent = 'Chrome/69.0.3497.0'