slapproxy: Store local software urls relatively

Slapproxy now takes a mandatory 'root-path' argument.
When a local software url is a subpath of this root path, it is stored
as a relative path to the root path in the proxy database.

The urls are automatically expanded to their absolute path when read
from the datable, but they must be carefully adapted to be relative
at every place they are stored or compared to values in the database.

This makes it possible to seamlessly backup the slapproxy and restore
it on a different machine along with the local software releases.

slapos/cli/proxy_start.py

@@ -47,6 +47,8 @@ class ProxyStartCommand(ConfigCommand):
                         help='Port to use')
         ap.add_argument('--host',
                         help='Host to use')
+        ap.add_argument('--root-path', required=True,
+                        help='Root path for all local SR')
 
         return ap
 
@@ -67,4 +69,4 @@ class ProxyStartCommand(ConfigCommand):
 
         conf.setConfig()
 
-        do_proxy(conf=conf)
+        do_proxy(conf=conf, root_path=args.root_path)

slapos/proxy/__init__.py

@@ -99,7 +99,7 @@ def connectDB():
   conn = sqlite_connect(app.config['DATABASE_URI'])
   conn.close()
 
-def do_proxy(conf):
+def do_proxy(conf, root_path):
   for handler in conf.logger.handlers:
     app.logger.addHandler(handler)
 
@@ -107,6 +107,7 @@ def do_proxy(conf):
   app.logger.removeHandler(default_handler)
 
   setupFlaskConfiguration(conf)
+  app.config['root-path'] = root_path
   connectDB()
   app.run(host=conf.host, port=int(conf.port), threaded=True)
 

slapos/proxy/schema.sql

---version:14
+--version:15
 CREATE TABLE IF NOT EXISTS software%(version)s (
   url VARCHAR(255),
   computer_reference VARCHAR(255) DEFAULT '%(computer)s',

slapos/proxy/views.py

@@ -31,6 +31,7 @@
 import random
 import string
 import time
+import os
 from datetime import datetime
 from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
@@ -99,6 +100,31 @@ def partitiondict2partition(partition):
   return slap_partition
 
 
+def get_root_path():
+  return app.config['root-path']
+
+def store_url(url):
+  if not url or urlparse(url).scheme:
+    return url
+  rel = os.path.relpath(url, g.root_path)
+  if rel.startswith(os.pardir + os.sep):
+    return url
+  return rel
+
+def retrieve_url(url):
+  if not url or urlparse(url).scheme or os.path.isabs(url):
+    return url
+  return os.path.join(g.root_path, url)
+
+table_to_urlkey = {
+  'software': 'url',
+  'partition': 'software_release',
+}
+
+def retrieve_row(row, urlkey):
+  return { key : retrieve_url(value) if key == urlkey else value for key, value in six.iteritems(row) }
+
+
 def execute_db(table, query, args=(), one=False, db_version=None, db=None):
   if not db:
     db = g.db
@@ -113,6 +139,9 @@ def execute_db(table, query, args=(), one=False, db_version=None, db=None):
     raise
   rv = [dict((cur.description[idx][0], value)
     for idx, value in enumerate(row)) for row in cur.fetchall()]
+  urlkey = table_to_urlkey.get(table, None)
+  if urlkey:
+    rv = [retrieve_row(row, urlkey) for row in rv]
   return (rv[0] if rv else None) if one else rv
 
 
@@ -191,6 +220,8 @@ is_schema_already_executed = False
 @app.before_request
 def before_request():
   g.db = connect_db()
+  g.root_path = get_root_path()
+  app.logger.debug("Root path is %s", g.root_path)
   global is_schema_already_executed
   if not is_schema_already_executed:
     _upgradeDatabaseIfNeeded()
@@ -259,7 +290,7 @@ def destroyedSoftwareRelease():
   execute_db(
     'software',
     'DELETE FROM %s WHERE url = ? and computer_reference=? ',
-    [request.form['url'], request.form['computer_id']])
+    [store_url(request.form['url']), request.form['computer_id']])
   return 'OK'
 
 @app.route('/availableSoftwareRelease', methods=['POST'])
@@ -392,7 +423,7 @@ def supplySupply():
   execute_db(
     'software',
     'INSERT OR REPLACE INTO %s VALUES(?, ?, ?)',
-    [url, computer_id, state])
+    [store_url(url), computer_id, state])
 
   return 'Supplied %r to be %s' % (url, state)
 
@@ -735,7 +766,7 @@ def requestNotSlave(software_release, software_type, partition_reference, partit
                ('xml', instance_xml)):
     if partition[k] != v:
       q += ', %s=?' % k
-      a(v)
+      a(store_url(v) if k == 'software_release' else v)
       changed = True
 
   if changed:
@@ -788,7 +819,7 @@ def requestSlave(software_release, software_type, partition_reference, partition
   args = []
   a = args.append
   q = 'SELECT * FROM %s WHERE software_release=? and computer_reference=?'
-  a(software_release)
+  a(store_url(software_release))
   a(requested_computer_id)
   if software_type:
     q += ' AND software_type=?'