Update to jsonschema to support Draft 7 and fix some invalid json

Update jsonschema eggs and some dependent eggs (including setuptools - but it looks OK) so that we can validate draft7 of json schema.

Because jsonschema now bundles some schemas for json schemas, we don't need to include it here.

/reviewed-on nexedi/slapos!425

slapos/test/metaschema.json

-{
-  "id": "http://json-schema.org/draft-04/schema#",
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Core schema meta-schema",
-  "definitions": {
-    "schemaArray": {
-      "type": "array",
-      "minItems": 1,
-      "items": {
-        "$ref": "#"
-      }
-    },
-    "positiveInteger": {
-      "type": "integer",
-      "minimum": 0
-    },
-    "positiveIntegerDefault0": {
-      "allOf": [
-        {
-          "$ref": "#/definitions/positiveInteger"
-        },
-        {
-          "default": 0
-        }
-      ]
-    },
-    "simpleTypes": {
-      "enum": [
-        "array",
-        "boolean",
-        "integer",
-        "null",
-        "number",
-        "object",
-        "string"
-      ]
-    },
-    "stringArray": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      },
-      "minItems": 1,
-      "uniqueItems": true
-    }
-  },
-  "type": "object",
-  "properties": {
-    "id": {
-      "type": "string",
-      "format": "uri"
-    },
-    "$schema": {
-      "type": "string",
-      "format": "uri"
-    },
-    "title": {
-      "type": "string"
-    },
-    "description": {
-      "type": "string"
-    },
-    "default": {},
-    "multipleOf": {
-      "type": "number",
-      "minimum": 0,
-      "exclusiveMinimum": true
-    },
-    "maximum": {
-      "type": "number"
-    },
-    "exclusiveMaximum": {
-      "type": "boolean",
-      "default": false
-    },
-    "minimum": {
-      "type": "number"
-    },
-    "exclusiveMinimum": {
-      "type": "boolean",
-      "default": false
-    },
-    "maxLength": {
-      "$ref": "#/definitions/positiveInteger"
-    },
-    "minLength": {
-      "$ref": "#/definitions/positiveIntegerDefault0"
-    },
-    "pattern": {
-      "type": "string",
-      "format": "regex"
-    },
-    "additionalItems": {
-      "anyOf": [
-        {
-          "type": "boolean"
-        },
-        {
-          "$ref": "#"
-        }
-      ],
-      "default": {}
-    },
-    "items": {
-      "anyOf": [
-        {
-          "$ref": "#"
-        },
-        {
-          "$ref": "#/definitions/schemaArray"
-        }
-      ],
-      "default": {}
-    },
-    "maxItems": {
-      "$ref": "#/definitions/positiveInteger"
-    },
-    "minItems": {
-      "$ref": "#/definitions/positiveIntegerDefault0"
-    },
-    "uniqueItems": {
-      "type": "boolean",
-      "default": false
-    },
-    "maxProperties": {
-      "$ref": "#/definitions/positiveInteger"
-    },
-    "minProperties": {
-      "$ref": "#/definitions/positiveIntegerDefault0"
-    },
-    "required": {
-      "$ref": "#/definitions/stringArray"
-    },
-    "additionalProperties": {
-      "anyOf": [
-        {
-          "type": "boolean"
-        },
-        {
-          "$ref": "#"
-        }
-      ],
-      "default": {}
-    },
-    "definitions": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#"
-      },
-      "default": {}
-    },
-    "properties": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#"
-      },
-      "default": {}
-    },
-    "patternProperties": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#"
-      },
-      "default": {}
-    },
-    "dependencies": {
-      "type": "object",
-      "additionalProperties": {
-        "anyOf": [
-          {
-            "$ref": "#"
-          },
-          {
-            "$ref": "#/definitions/stringArray"
-          }
-        ]
-      }
-    },
-    "enum": {
-      "type": "array",
-      "minItems": 1,
-      "uniqueItems": true
-    },
-    "type": {
-      "anyOf": [
-        {
-          "$ref": "#/definitions/simpleTypes"
-        },
-        {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/simpleTypes"
-          },
-          "minItems": 1,
-          "uniqueItems": true
-        }
-      ]
-    },
-    "allOf": {
-      "$ref": "#/definitions/schemaArray"
-    },
-    "anyOf": {
-      "$ref": "#/definitions/schemaArray"
-    },
-    "oneOf": {
-      "$ref": "#/definitions/schemaArray"
-    },
-    "not": {
-      "$ref": "#"
-    }
-  },
-  "dependencies": {
-    "exclusiveMaximum": [
-      "maximum"
-    ],
-    "exclusiveMinimum": [
-      "minimum"
-    ]
-  },
-  "default": {}
-}

slapos/test/test_json_schema.py

@@ -34,23 +34,36 @@ import slapos.test
 import jsonschema
 
 
-def getSchemaValidator(filename):
-  schema_json_file = "/".join(slapos.test.__file__.split("/")[:-1])
-  schema_json_file += "/%s" % filename
-  with open(schema_json_file, "r") as json_file:
-    json_dict = json.loads(json_file.read())
-    json_file.close()
-  return json_dict
-
-def createValidatorTest(path, json_dict):
-  # Test that json is valid
+def createInstanceParameterSchemaValidatorTest(path):
+  # Test that json is a valid json schema, supports several
+  # validator, depending on the `$schema` defined in the json.
+  validator_dict = {
+    "http://json-schema.org/draft-03/schema#": jsonschema.Draft3Validator,
+    "http://json-schema.org/draft-04/schema": jsonschema.Draft4Validator,
+    "http://json-schema.org/draft-04/schema#": jsonschema.Draft4Validator,
+    "http://json-schema.org/draft-06/schema#": jsonschema.Draft6Validator,
+    "http://json-schema.org/draft-07/schema#": jsonschema.Draft7Validator,
+  }
   def run(self, *args, **kwargs):
     with open(path, "r") as json_file:
-      self.assertEqual(jsonschema.validate(json.load(json_file), json_dict), None)
+      json_dict = json.load(json_file)
+      validator = validator_dict.get(
+        json_dict.get('$schema'),
+        jsonschema.Draft7Validator)
+      validator.check_schema(json_dict)
+  return run
 
+
+def createSoftwareCfgValidatorTest(path, software_cfg_schema):
+  # Test that software json follows the schema for softwares json,
+  # which is defined in schema.json in this directory
+  def run(self, *args, **kwargs):
+    with open(path, "r") as json_file:
+      jsonschema.validate(json.load(json_file), software_cfg_schema)
   return run
 
-def createFormatTest(path, json_dict):
+
+def createFormatTest(path):
   # Test that json match our formatting rules
   def run(self, *args, **kwargs):
     with open(path, "r") as json_file:
@@ -69,23 +82,26 @@ def createFormatTest(path, json_dict):
 
 
 def generateSoftwareCfgTest():
-  json_dict = getSchemaValidator("schema.json")
+  software_cfg_schema = json.load(
+    open(os.path.join(
+        os.path.dirname(slapos.test.__file__),
+        "schema.json"), 'r'))
   base_path = "/".join(slapos.test.__file__.split("/")[:-3])
   for path in glob.glob("%s/software/*/software.cfg.json" % base_path):
     test_name = "test_%s_software_cfg_json" % path.split("/")[-2]
-    setattr(TestJSONSchemaValidation, test_name, createValidatorTest(path, json_dict))
-    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path, json_dict))
+    setattr(TestJSONSchemaValidation, test_name, createSoftwareCfgValidatorTest(path, software_cfg_schema))
+    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path))
 
 
 def generateJSONSchemaTest():
-  json_dict = getSchemaValidator("metaschema.json")
   base_path = "/".join(slapos.test.__file__.split("/")[:-3])
   for path in glob.glob("%s/software/*/*schema.json" % base_path):
     software_type = path.split("/")[-2]
     filename = path.split("/")[-1].replace("-", "_").replace(".", "_")
     test_name = "test_schema_%s_%s" % (software_type, filename)
-    setattr(TestJSONSchemaValidation, test_name, createValidatorTest(path, json_dict))
-    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path, json_dict))
+    setattr(TestJSONSchemaValidation, test_name, createInstanceParameterSchemaValidatorTest(path))
+    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path))
+
 
 class TestJSONSchemaValidation(unittest.TestCase):
   pass

software/erp5/instance-erp5-input-schema.json

@@ -297,7 +297,6 @@
                 "description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. The partition reference is 'zodb'.",
                 "$ref": "./instance-zeo-schema.json"
               }
-
             }
           },
           {
@@ -363,27 +362,27 @@
         "$ref": "../caucase/instance-caucase-input-schema.json"
       },
       "type": "object"
-    }
-  },
-  "test-runner": {
-    "description": "Test runner parameters.",
-    "properties": {
-      "enabled": {
-        "description": "Generate helper scripts to run test suite.",
-        "default": true,
-        "type": "boolean"
-      },
-      "node-count": {
-        "description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
-        "default": 3,
-        "type": "integer"
-      },
-      "extra-database-count": {
-        "description": "Number of extra databases this instance tests will need.",
-        "default": 3,
-        "type": "integer"
-      }
     },
-    "type": "object"
+    "test-runner": {
+      "description": "Test runner parameters.",
+      "properties": {
+        "enabled": {
+          "description": "Generate helper scripts to run test suite.",
+          "default": true,
+          "type": "boolean"
+        },
+        "node-count": {
+          "description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
+          "default": 3,
+          "type": "integer"
+        },
+        "extra-database-count": {
+          "description": "Number of extra databases this instance tests will need.",
+          "default": 3,
+          "type": "integer"
+        }
+      },
+      "type": "object"
+    }
   }
 }

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 9f22db89a2679534aa8fd37dbca86782
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = bd9ff3543f0dfaf2702624e3ed74d334
+md5sum = b44268d46a41042a879f47babb66c922

software/jstestnode/instance-jstestnode-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate JSTestNode",
   "additionalProperties": false,
+  "required": [
+    "test-suite",
+    "test-runner"
+  ],
   "properties": {
     "test-suite": {
       "description": "The test suite to run",
@@ -18,68 +22,86 @@
       "default": "(the web server started by this instance)",
       "example": "https://softinst1234.host.vifib.net/"
     },
-    "oneOf": [
-      {
-        "title": "selenium server",
-        "description": "Configuration for running tests on selenium server",
-        "properties": {
-          "target": {
-            "description": "Target system",
-            "const": "selenium-server"
-          },
-          "server-url": {
-            "description": "URL of the selenium server",
-            "type": "string"
-          },
-          "verify-server-certificate": {
-            "description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
-            "type": "boolean",
-            "default": true
-          },
-          "server-ca-certificate": {
-            "description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
-            "type": "string",
-            "default": "root certificates from http://certifi.io/en/latest/"
-          },
-          "desired-capabilities": {
-            "description": "Desired browser capabilities",
-            "type": "object",
-            "properties": {
-              "browserName": {
-                "description": "Name of the browser being used, for example firefox, chrome",
-                "type": "string",
-                "required": true
-              },
-              "version": {
-                "description": "The browser version",
-                "type": "string"
-              }
+    "test-runner": {
+      "oneOf": [
+        {
+          "type": "object",
+          "title": "Selenium Server",
+          "description": "Configuration for Selenium server",
+          "additionalProperties": false,
+          "required": [
+            "desired-capabilities",
+            "server-url",
+            "target"
+          ],
+          "properties": {
+            "target": {
+              "description": "Target system",
+              "type": "string",
+              "const": "selenium-server"
+            },
+            "server-url": {
+              "description": "URL of the selenium server",
+              "type": "string",
+              "format": "uri"
+            },
+            "verify-server-certificate": {
+              "description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
+              "type": "boolean",
+              "default": true
+            },
+            "server-ca-certificate": {
+              "description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
+              "type": "string",
+              "default": "root certificates from http://certifi.io/en/latest/"
             },
-            "additionalProperties": true
+            "desired-capabilities": {
+              "description": "Desired browser capabilities",
+              "required": [
+                "browserName"
+              ],
+              "type": "object",
+              "properties": {
+                "browserName": {
+                  "description": "Name of the browser being used, for example firefox, chrome",
+                  "type": "string"
+                },
+                "version": {
+                  "description": "The browser version",
+                  "type": "string"
+                }
+              }
+            }
           }
-        }
-      },
-      {
-        "title": "firefox",
-        "description": "Configuration for running tests on local firefox process",
-        "properties": {
-          "target": {
-            "description": "Target system",
-            "const": "firefox",
-            "default": "firefox"
+        },
+        {
+          "type": "object",
+          "title": "Firefox",
+          "description": "Configuration for Firefox",
+          "additionalProperties": false,
+          "properties": {
+            "target": {
+              "description": "Target system",
+              "const": "firefox",
+              "type": "string",
+              "default": "firefox"
+            }
           }
-        }
-      },
-      {
-        "title": "node",
-        "description": "Configuration for running tests on local nodejs",
-        "properties": {
-          "target": {
-            "description": "Target system",
-            "const": "node"
+        },
+        {
+          "type": "object",
+          "title": "NodeJS",
+          "description": "Configuration for NodeJS",
+          "additionalProperties": false,
+          "properties": {
+            "target": {
+              "description": "Target system",
+              "const": "node",
+              "type": "string"
+            }
           }
         }
-      }
-    ]
+      ]
+    }
   }
 }

software/jstestnode/runTestSuite.in

@@ -25,8 +25,14 @@ BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
 ETC_DIRECTORY = '$${directory:etc}'
 
 def main():
+  parsed_parameters = json.load(
+      open('$${runTestSuite-config-file:rendered}', 'rb'))
+  test_runner = parsed_parameters.get('test-runner', {})
+
   parser = argparse.ArgumentParser(description='Run a test suite.')
-  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite', help='The test suite name',
+                      default=parsed_parameters.get('test-suite', ''),
+                      required=not parsed_parameters.has_key('test-suite'))
   parser.add_argument('--test_suite_title', help='The test suite title')
   parser.add_argument('--test_node_title', help='The test node title')
   parser.add_argument('--project_title', help='The project title')
@@ -40,9 +46,6 @@ def main():
 
   args = parser.parse_args()
 
-  parsed_parameters = json.load(
-      open('$${runTestSuite-config-file:rendered}', 'rb'))
-
   is_browser_running = False
   try:
     test_suite_title = args.test_suite_title or args.test_suite
@@ -62,7 +65,7 @@ def main():
     ##########################
     # Run all tests
     ##########################
-    target = parsed_parameters.get('target', 'firefox')
+    target = test_runner.get('target', 'firefox')
     if target == 'node':
       # Execute NodeJS tests
       result_string = check_output(['${nodejs-output:node}', '${jio-repository.git:location}/test/node.js'],
@@ -92,24 +95,24 @@ def main():
             firefox_binary='${firefox-wrapper:location}',
             executable_path='${geckodriver:location}')
       else:
-        assert target == 'selenium-server', "Unsupported target {}".format(parsed_parameters['target'])
+        assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
         # use a remote connection which verifies TLS certificate
         # workaround for https://github.com/SeleniumHQ/selenium/issues/6534
-        executor = RemoteConnection(parsed_parameters['server-url'], keep_alive=True)
+        executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
         cert_reqs = 'CERT_REQUIRED'
         ca_certs = certifi.where()
-        if not parsed_parameters.get('verify-server-certificate', True):
+        if not test_runner.get('verify-server-certificate', True):
           cert_reqs = 'CERT_NONE'
           ca_certs = None
-        if parsed_parameters.get('server-ca-certificate'):
+        if test_runner.get('server-ca-certificate'):
           ca_certs = os.path.join(ETC_DIRECTORY, "cacerts.pem")
           with open(ca_certs, 'w') as f:
-            f.write(parsed_parameters.get('server-ca-certificate'))
+            f.write(test_runner.get('server-ca-certificate'))
         executor._conn = urllib3.PoolManager(cert_reqs=cert_reqs, ca_certs=ca_certs)
 
         browser = webdriver.Remote(
             command_executor=executor,
-            desired_capabilities=parsed_parameters['desired-capabilities'],
+            desired_capabilities=test_runner['desired-capabilities'],
         )
 
       # adjust path for remote test url

software/kvm/instance-kvm-cluster-input-schema.json

@@ -368,11 +368,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": [
-                "host-name",
-                "ip-address",
-                "ipv6"
-              ],
+              "format": "internet-address",
               "default": "debian.nbd.vifib.net"
             },
             "nbd-port": {

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -64,11 +64,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": [
-                "host-name",
-                "ip-address",
-                "ipv6"
-              ],
+              "format": "internet-address",
               "default": "debian.nbd.vifib.net"
             },
             "nbd-port": {

software/kvm/instance-kvm-input-schema.json

@@ -172,11 +172,7 @@
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",
       "type": "string",
-      "format": [
-        "host-name",
-        "ip-address",
-        "ipv6"
-      ],
+      "format": "internet-address",
       "default": "debian.nbd.vifib.net"
     },
     "nbd-port": {
@@ -191,11 +187,7 @@
       "title": "Second NBD hostname",
       "description": "hostname (or IP) of the second NBD server (containing drivers for example).",
       "type": "string",
-      "format": [
-        "host-name",
-        "ip-address",
-        "ipv6"
-      ]
+      "format": "internet-address"
     },
     "nbd2-port": {
       "title": "Second NBD port",

stack/slapos.cfg

@@ -98,7 +98,7 @@ eggs =
   slapos.libnetworkcache
 
 [versions]
-setuptools = 33.1.1
+setuptools = 40.4.3
 # Use SlapOS patched zc.buildout
 zc.buildout = 2.5.2+slapos013
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
@@ -133,12 +133,12 @@ pyOpenSSL = 18.0.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
-six = 1.10.0
+six = 1.11.0
 slapos.cookbook = 1.0.75
 slapos.core = 1.4.11
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
-slapos.libnetworkcache = 0.15
+slapos.libnetworkcache = 0.16
 slapos.rebootstrap = 4.1
 slapos.recipe.build = 0.39
 slapos.recipe.cmmi = 0.8
@@ -193,16 +193,24 @@ erp5.util = 0.4.51
 feedparser = 5.2.1
 
 # Required by:
-# jsonschema==2.6.0
+# jsonschema==3.0.0a3
 functools32 = 3.2.3.post2
 
+# Required by:
+# jsonschema==3.0.0a3
+attrs = 18.2.0
+
+# Required by:
+# jsonschema==3.0.0a3
+pyrsistent = 0.14.5
+
 # Required by:
 # cryptography==1.8.1
 ipaddress = 1.0.18
 
 # Required by:
 # slapos.cookbook==1.0.62
-jsonschema = 2.6.0
+jsonschema = 3.0.0a3
 
 # Required by:
 # slapos.toolbox==0.81