slapproxy: skip instanciation if nothing has changed

slapos/grid/slapgrid.py

@@ -43,6 +43,7 @@ import logging
 import json
 import shutil
 import six
+import errno
 
 if sys.version_info < (2, 6):
   warnings.warn('Used python version (%s) is old and has problems with'
@@ -966,10 +967,7 @@ stderr_logfile_backups=1
         COMPUTER_PARTITION_TIMESTAMP_FILENAME
     )
     parameter_dict = computer_partition.getInstanceParameterDict()
-    if 'timestamp' in parameter_dict:
-      timestamp = parameter_dict['timestamp']
-    else:
-      timestamp = None
+    timestamp = parameter_dict.get('timestamp')
 
     error_output_file = os.path.join(
         instance_path,
@@ -1031,18 +1029,23 @@ stderr_logfile_backups=1
     # If not: it's not worth processing this partition (nothing has
     # changed).
     if (computer_partition_id not in self.computer_partition_filter_list and
-          not self.develop and os.path.exists(timestamp_path)):
-      with open(timestamp_path) as f:
-        old_timestamp = f.read()
-      last_runtime = int(os.path.getmtime(timestamp_path))
-      if timestamp:
-        try:
-          if periodicity == 0:
-            os.remove(timestamp_path)
-          elif int(timestamp) <= int(old_timestamp):
+          not self.develop and timestamp and periodicity):
+      try:
+        last_runtime = os.path.getmtime(timestamp_path)
+      except OSError as e:
+        if e.errno != errno.ENOENT:
+          raise
+      else:
+        with open(timestamp_path) as f:
+          try:
+            old_timestamp = float(f.read())
+          except ValueError:
+            self.logger.exception('')
+            old_timestamp = 0
+        if float(timestamp) <= old_timestamp:
             # Check periodicity, i.e if periodicity is one day, partition
             # should be processed at least every day.
-            if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0:
+            if time.time() <= last_runtime + periodicity or periodicity < 0:
               # check promises anomaly
               if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
                 self.logger.debug('Partition already up-to-date.')
@@ -1055,13 +1058,7 @@ stderr_logfile_backups=1
                 manager.instanceTearDown(local_partition)
 
               return
-            else:
-              # Periodicity forced processing this partition. Removing
-              # the timestamp file in case it fails.
-              os.remove(timestamp_path)
-        except ValueError:
-          os.remove(timestamp_path)
-          self.logger.exception('')
+        os.remove(timestamp_path)
 
     # Include Partition Logging
     log_folder_path = "%s/.slapgrid/log" % instance_path
@@ -1168,7 +1165,7 @@ stderr_logfile_backups=1
     # If partition has been successfully processed, write timestamp
     if timestamp:
       with open(timestamp_path, 'w') as f:
-        f.write(timestamp)
+        f.write(str(timestamp))
 
   def FilterComputerPartitionList(self, computer_partition_list):
     """

slapos/proxy/schema.sql

---version:13
+--version:14
 CREATE TABLE IF NOT EXISTS software%(version)s (
   url VARCHAR(255),
   computer_reference VARCHAR(255) DEFAULT '%(computer)s',
@@ -26,6 +26,7 @@ CREATE TABLE IF NOT EXISTS partition%(version)s (
   requested_by VARCHAR(255), -- only used for debugging,
                              -- slapproxy does not support proper scope
   requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
+  timestamp REAL,
   CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
 );
 

slapos/proxy/views.py

@@ -31,6 +31,7 @@
 from lxml import etree
 import random
 import string
+import time
 from datetime import datetime
 from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
@@ -91,7 +92,8 @@ def partitiondict2partition(partition):
   slap_partition._software_release_document = None
   slap_partition._requested_state = 'destroyed'
   slap_partition._need_modification = 0
-  slap_partition._instance_guid = '%s-%s' % (partition['computer_reference'], partition['reference'])
+  slap_partition._instance_guid = '%(computer_reference)s-%(reference)s' \
+    % partition
 
   root_partition = getRootPartition(partition['reference'])
 
@@ -118,6 +120,9 @@ def partitiondict2partition(partition):
           loads(partition['slave_instance_list'].encode('utf-8'))
     else:
       slap_partition._parameter_dict['slave_instance_list'] = []
+    timestamp = partition['timestamp']
+    if timestamp:
+      slap_partition._parameter_dict['timestamp'] = str(timestamp)
     slap_partition._connection_dict = xml2dict(partition['connection_xml'])
     slap_partition._software_release_document = SoftwareRelease(
       software_release=partition['software_release'],
@@ -267,12 +272,18 @@ def setComputerPartitionConnectionXml():
     query = 'UPDATE %s SET connection_xml=? WHERE reference=? AND computer_reference=?'
     argument_list = [connection_xml, computer_partition_id, computer_id]
     execute_db('partition', query, argument_list)
-    return 'done'
+    # Update timestamp of parent partition.
+    requested_by = execute_db('partition',
+      'SELECT requested_by FROM %s WHERE reference=? AND computer_reference=?',
+      (computer_partition_id, computer_id), one=True)['requested_by']
+    execute_db('partition',
+      "UPDATE %s SET timestamp=? WHERE reference=? AND computer_reference=?",
+      (time.time(), requested_by, computer_id))
   else:
     query = 'UPDATE %s SET connection_xml=? , hosted_by=? WHERE reference=?'
     argument_list = [connection_xml, computer_partition_id, slave_reference]
     execute_db('slave', query, argument_list)
-    return 'done'
+  return 'done'
 
 @app.route('/buildingSoftwareRelease', methods=['POST'])
 def buildingSoftwareRelease():
@@ -569,26 +580,23 @@ def getRootPartition(reference):
       reference, execute_db("partition", "select reference, requested_by from %s")))
     return None
 
-  parent_partition = execute_db('partition', p, [partition['requested_by']], one=True)
-  while (parent_partition is not None and
-         parent_partition['requested_by'] and
-         parent_partition['requested_by'] != reference):
+  while True:
+    requested_by = partition['requested_by']
+    if requested_by is None or requested_by == reference:
+      return partition
+    parent_partition = execute_db('partition', p, (requested_by,), one=True)
+    if parent_partition is None:
+      return partition
     partition = parent_partition
-    reference = parent_partition['requested_by']
-    parent_partition = execute_db('partition', p, [reference], one=True)
-
-  return partition
+    reference = requested_by
 
 def requestNotSlave(software_release, software_type, partition_reference, partition_id, partition_parameter_kw, filter_kw, requested_state):
   instance_xml = dict2xml(partition_parameter_kw)
   requested_computer_id = filter_kw['computer_guid']
 
-  args = []
-  a = args.append
-  q = 'SELECT * FROM %s WHERE partition_reference=?'
-  a(partition_reference)
-
-  partition = execute_db('partition', q, args, one=True)
+  partition = execute_db('partition',
+    'SELECT * FROM %s WHERE partition_reference=?',
+    (partition_reference,), one=True)
 
   args = []
   a = args.append
@@ -601,8 +609,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
     if partition is None:
       app.logger.warning('No more free computer partition')
       abort(404)
-    q += ' ,software_release=?'
-    a(software_release)
     if partition_reference:
       q += ' ,partition_reference=?'
       a(partition_reference)
@@ -612,10 +618,6 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
     if not software_type:
       software_type = 'RootSoftwareInstance'
   else:
-    # XXX Check if software_release should be updated
-    if partition['software_release'] != software_release:
-      q += ' ,software_release=?'
-      a(software_release)
     if partition['requested_by']:
       root_partition = getRootPartition(partition['requested_by'])
       if root_partition and root_partition['requested_state'] != "started":
@@ -623,49 +625,49 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
         # child can be stopped or destroyed while parent is started
         requested_state = root_partition['requested_state']
 
-  if requested_state:
-    q += ', requested_state=?'
-    a(requested_state)
-
-  #
-  # XXX change software_type when requested
-  #
-  if software_type:
-    q += ' ,software_type=?'
-    a(software_type)
+  timestamp = partition['timestamp']
+  changed = timestamp is None
+  for k, v in (('requested_state', requested_state or
+                                   partition['requested_state']),
+               ('software_release', software_release),
+               ('software_type', software_type),
+               ('xml', instance_xml)):
+    if partition[k] != v:
+      q += ', %s=?' % k
+      a(v)
+      changed = True
+
+  if changed:
+    timestamp = time.time()
+    q += ', timestamp=?'
+    a(timestamp)
 
-  # Else: only update partition parameters
-  if instance_xml:
-    q += ' ,xml=?'
-    a(instance_xml)
   q += ' WHERE reference=? AND computer_reference=?'
   a(partition['reference'])
   a(partition['computer_reference'])
 
   execute_db('partition', q, args)
-  args = []
   partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
       [partition['reference'], partition['computer_reference']], one=True)
   address_list = []
   for address in execute_db('partition_network', 'SELECT * FROM %s WHERE partition_reference=?', [partition['reference']]):
     address_list.append((address['reference'], address['address']))
 
-  if not requested_state:
-    requested_state = 'started'
   # XXX it should be ComputerPartition, not a SoftwareInstance
-  software_instance = SoftwareInstance(_connection_dict=xml2dict(partition['connection_xml']),
-                                       _parameter_dict=xml2dict(partition['xml']),
-                                       connection_xml=partition['connection_xml'],
-                                       slap_computer_id=partition['computer_reference'],
-                                       slap_computer_partition_id=partition['reference'],
-                                       slap_software_release_url=partition['software_release'],
-                                       slap_server_url='slap_server_url',
-                                       slap_software_type=partition['software_type'],
-                                       _instance_guid='%s-%s' % (partition['computer_reference'], partition['reference']),
-                                       _requested_state=requested_state,
-                                       ip_list=address_list)
-  return software_instance
-
+  parameter_dict = xml2dict(partition['xml'])
+  parameter_dict['timestamp'] = str(partition['timestamp'])
+  return SoftwareInstance(
+    _connection_dict=xml2dict(partition['connection_xml']),
+    _parameter_dict=parameter_dict,
+    connection_xml=partition['connection_xml'],
+    slap_computer_id=partition['computer_reference'],
+    slap_computer_partition_id=partition['reference'],
+    slap_software_release_url=partition['software_release'],
+    slap_server_url='slap_server_url',
+    slap_software_type=partition['software_type'],
+    _instance_guid='%(computer_reference)s-%(reference)s' % partition,
+    _requested_state=requested_state or 'started',
+    ip_list=address_list)
 
 def requestSlave(software_release, software_type, partition_reference, partition_id, partition_parameter_kw, filter_kw, requested_state):
   """
@@ -737,7 +739,6 @@ def requestSlave(software_release, software_type, partition_reference, partition
   a(partition['reference'])
   a(requested_computer_id)
   execute_db('partition', q, args)
-  args = []
   partition = execute_db('partition', 'SELECT * FROM %s WHERE reference=? and computer_reference=?',
       [partition['reference'], requested_computer_id], one=True)
 
@@ -758,16 +759,15 @@ def requestSlave(software_release, software_type, partition_reference, partition
     address_list.append((address['reference'], address['address']))
 
   # XXX it should be ComputerPartition, not a SoftwareInstance
-  software_instance = SoftwareInstance(_connection_dict=xml2dict(slave['connection_xml']),
-                                       _parameter_dict=xml2dict(instance_xml),
-                                       slap_computer_id=partition['computer_reference'],
-                                       slap_computer_partition_id=slave['hosted_by'],
-                                       slap_software_release_url=partition['software_release'],
-                                       slap_server_url='slap_server_url',
-                                       slap_software_type=partition['software_type'],
-                                       ip_list=address_list)
-
-  return software_instance
+  return SoftwareInstance(
+    _connection_dict=xml2dict(slave['connection_xml']),
+    _parameter_dict=xml2dict(instance_xml),
+    slap_computer_id=partition['computer_reference'],
+    slap_computer_partition_id=slave['hosted_by'],
+    slap_software_release_url=partition['software_release'],
+    slap_server_url='slap_server_url',
+    slap_software_type=partition['software_type'],
+    ip_list=address_list)
 
 @app.route('/softwareInstanceRename', methods=['POST'])
 def softwareInstanceRename():

slapos/slap/slap.py

@@ -627,16 +627,16 @@ class ComputerPartition(SlapRequester):
 
   def getInstanceParameter(self, key):
     parameter_dict = getattr(self, '_parameter_dict', None) or {}
-    if key in parameter_dict:
+    try:
       return parameter_dict[key]
-    else:
+    except KeyError:
       raise NotFoundError("%s not found" % key)
 
   def getConnectionParameter(self, key):
     connection_dict = self.getConnectionParameterDict()
-    if key in connection_dict:
+    try:
       return connection_dict[key]
-    else:
+    except KeyError:
       raise NotFoundError("%s not found" % key)
 
   def setUsage(self, usage_log):

slapos/tests/test_slapproxy/database_dump_version_13.sql

+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE software13 (
+  url VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  requested_state VARCHAR(255) DEFAULT 'available',
+  CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
+);
+INSERT INTO "software13" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer','available');
+CREATE TABLE computer13 (
+  reference VARCHAR(255) DEFAULT 'computer',
+  address VARCHAR(255),
+  netmask VARCHAR(255),
+  CONSTRAINT uniq PRIMARY KEY (reference)
+);
+INSERT INTO "computer13" VALUES('computer','127.0.0.1','255.255.255.255');
+CREATE TABLE partition13 (
+  reference VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  slap_state VARCHAR(255) DEFAULT 'free',
+  software_release VARCHAR(255),
+  xml TEXT,
+  connection_xml TEXT,
+  slave_instance_list TEXT,
+  software_type VARCHAR(255),
+  partition_reference VARCHAR(255), -- name of the instance
+  requested_by VARCHAR(255), -- only used for debugging,
+                             -- slapproxy does not support proper scope
+  requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
+  CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
+);
+INSERT INTO "partition13" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="json">{
+  "site-id": "erp5"
+  }
+}</parameter>
+</instance>
+',NULL,NULL,'production','slapos',NULL,'started');
+INSERT INTO "partition13" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
+</instance>
+',NULL,'mariadb','MariaDB DataBase','slappart0','started');
+INSERT INTO "partition13" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="cloudooo-json"></parameter>
+</instance>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
+</instance>
+',NULL,'cloudooo','Cloudooo','slappart0','started');
+INSERT INTO "partition13" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">memcached://127.0.0.1:11000/</parameter>
+</instance>
+',NULL,'memcached','Memcached','slappart0','started');
+INSERT INTO "partition13" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">memcached://127.0.0.1:13301/</parameter>
+</instance>
+',NULL,'kumofs','KumoFS','slappart0','started');
+INSERT INTO "partition13" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
+  <parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
+  <parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
+</instance>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">https://[fc00::1]:10001</parameter>
+</instance>
+',NULL,'tidstorage','TidStorage','slappart0','started');
+INSERT INTO "partition13" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+INSERT INTO "partition13" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+INSERT INTO "partition13" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+INSERT INTO "partition13" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+CREATE TABLE slave13 (
+  reference VARCHAR(255), -- unique slave reference
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  connection_xml TEXT,
+  hosted_by VARCHAR(255),
+  asked_by VARCHAR(255) -- only used for debugging,
+                        -- slapproxy does not support proper scope
+);
+CREATE TABLE partition_network13 (
+  partition_reference VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT 'computer',
+  reference VARCHAR(255),
+  address VARCHAR(255),
+  netmask VARCHAR(255)
+);
+INSERT INTO "partition_network13" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network13" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network13" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
+CREATE TABLE forwarded_partition_request13 (
+  partition_reference VARCHAR(255), -- a.k.a source_instance_id
+  master_url VARCHAR(255)
+);
+COMMIT;