use memory-based kumofs instead of memcached to have no limitation for key length and data size.

slapos/recipe/generic_kumofs/__init__.py

@@ -35,6 +35,10 @@ class Recipe(GenericBaseRecipe):
     kumo_server_listen_port = int(self.options['server-listen-port'])
     kumo_gateway_port = int(self.options['gateway-port'])
     path_list = []
+    if 'data-path' in self.options:
+      kumo_server_storage = self.options.get('data-path')
+    else:
+      kumo_server_storage = os.path.join(self.options['data-directory'], "kumodb.tch")
     # XXX: kumo is not storing pid in file, unless it is not running as daemon
     #      but running daemons is incompatible with SlapOS, so there is currently
     #      no way to have Kumo's pid files to rotate logs and send signals to them
@@ -48,7 +52,7 @@ class Recipe(GenericBaseRecipe):
       kumo_server_binary=self.options['kumo-server-binary'],
       kumo_server_ip=ip,
       kumo_server_log=self.options['kumo-server-log'],
-      kumo_server_storage=os.path.join(self.options['data-directory'], "kumodb.tch"),
+      kumo_server_storage=kumo_server_storage,
       kumo_manager_port=kumo_manager_port,
       kumo_server_port=kumo_server_port,
       kumo_server_listen_port=kumo_server_listen_port,

stack/erp5/buildout.cfg

@@ -40,7 +40,6 @@ extends =
   ../../component/lxml-python/buildout.cfg
   ../../component/percona-toolkit/buildout.cfg
   ../../component/mariadb/buildout.cfg
-  ../../component/memcached/buildout.cfg
   ../../component/mysql-python/buildout.cfg
   ../../component/pil-python/buildout.cfg
   ../../component/pycrypto-python/buildout.cfg
@@ -74,7 +73,6 @@ parts =
   apache-antiloris
   file
   graphviz
-  memcached
   haproxy
   varnish-3.0
   w3m
@@ -291,10 +289,13 @@ extra-context =
 [template-memcached]
 < = template-jinja2-base
 filename = instance-memcached.cfg
-md5sum = 44c507e9eec1003beb99f9063b5e09be
+md5sum = 2de1801236eb78651ecfd50ada46dd3b
 extra-context =
     key dash_location dash:location
-    key memcached_location memcached:location
+    key dcron_location dcron:location
+    key gzip_location gzip:location
+    key kumo_location kumo:location
+    key logrotate_location logrotate:location
 
 [template-erp5-single]
 recipe = slapos.recipe.build:download

stack/erp5/instance-memcached.cfg.in

-#############################
-#
-# Instanciate memcached
-#
-# No slapos parameter needed
-#
-#############################
+# memcached-compatible volatile cache using kumofs
+# that has no limitation for key length and data size
+
 [buildout]
 parts =
-  memcached-instance
-  publish-memcached-connection-information
-  promise
+  publish-kumofs-connection-information
+  kumofs-instance
+  logrotate
+  logrotate-entry-kumofs
+  cron
+  cron-entry-logrotate
+  promise-kumofs-server
+  promise-kumofs-server-listen
+  promise-kumofs-gateway
+  promise-kumofs-manager
 
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
+[publish-kumofs-connection-information]
+recipe = slapos.cookbook:publishurl
+url = memcached://${kumofs-instance:ip}:${kumofs-instance:gateway-port}/
+
+[kumofs-instance]
+recipe = slapos.cookbook:generic.kumofs
+
+# Network options
+ip = ${slap-network-information:local-ipv4}
+manager-port = 13101
+server-port = 13201
+server-listen-port = 13202
+# previous memcached configuration
+gateway-port = 11000
+# previous memcached configuration
+storage-size = 64m
+
+# Paths: Running wrappers
+gateway-wrapper = ${basedirectory:services}/volatile_kumofs_gateway
+manager-wrapper = ${basedirectory:services}/volatile_kumofs_manager
+server-wrapper = ${basedirectory:services}/volatile_kumofs_server
+
+# Paths: Data
+data-path = *#capsiz=${:storage-size}
+
+# Paths: Logs
+kumo-gateway-log = ${basedirectory:log}/kumo-gateway.log
+kumo-manager-log = ${basedirectory:log}/kumo-manager.log
+kumo-server-log = ${basedirectory:log}/kumo-server.log
+
+# Binary information
+kumo-gateway-binary = {{ kumo_location }}/bin/kumo-gateway
+kumo-manager-binary = {{ kumo_location }}/bin/kumo-manager
+kumo-server-binary = {{ kumo_location }}/bin/kumo-server
+shell-path = {{ dash_location }}/bin/dash
+
+[logrotate-entry-kumofs]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = kumofs
+log = ${kumofs-instance:kumo-gateway-log} ${kumofs-instance:kumo-manager-log}
+  ${kumofs-instance:kumo-server-log}
+
+# rest of parts are candidates for some generic stuff
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
+gzip-binary = {{ gzip_location }}/bin/gzip
+gunzip-binary = {{ gzip_location }}/bin/gunzip
+# Directories
+wrapper = ${rootdirectory:bin}/logrotate
+conf = ${rootdirectory:etc}/logrotate.conf
+logrotate-entries = ${directory:logrotate-entries}
+backup = ${directory:logrotate-backup}
+state-file = ${rootdirectory:srv}/logrotate.status
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = ${rootdirectory:var}/log
+services = ${rootdirectory:etc}/run
+promise = ${rootdirectory:etc}/promise
+run = ${rootdirectory:var}/run
+backup = ${rootdirectory:srv}/backup
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
+cron-entries = ${rootdirectory:etc}/cron.d
+crontabs = ${rootdirectory:etc}/crontabs
+cronstamps = ${rootdirectory:etc}/cronstamps
+logrotate-backup = ${basedirectory:backup}/logrotate
+logrotate-entries = ${rootdirectory:etc}/logrotate.d
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+srv = ${buildout:directory}/srv
 bin = ${buildout:directory}/bin
-service = ${:etc}/run
-promise = ${:etc}/promise
 
-[memcached-instance]
-recipe = slapos.cookbook:generic.memcached
-wrapper_path = ${directory:service}/memcached
-binary_path = {{ memcached_location }}/bin/memcached
-shell-path = {{ dash_location }}/bin/dash
-ip = ${slap-network-information:local-ipv4}
-port = 11000
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = {{ dcron_location }}/sbin/crond
+cron-entries = ${directory:cron-entries}
+crontabs = ${directory:crontabs}
+cronstamps = ${directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${basedirectory:services}/crond
 
-[publish-memcached-connection-information]
-recipe = slapos.cookbook:publishurl
-url = memcached://${memcached-instance:ip}:${memcached-instance:port}/
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${rootdirectory:bin}/cron_simplelogger
+log = ${basedirectory:log}/cron.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
 
 # Deploy zope promises scripts
-[promise]
+[promise-template]
 recipe = slapos.cookbook:check_port_listening
-path = ${directory:promise}/memcache
-hostname = ${memcached-instance:ip}
-port = ${memcached-instance:port}
+hostname = ${kumofs-instance:ip}
+port = ${kumofs-instance:server-listen-port}
+
+[promise-kumofs-server]
+<= promise-template
+path = ${basedirectory:promise}/kumofs-server
+port = ${kumofs-instance:server-port}
+
+[promise-kumofs-server-listen]
+<= promise-template
+path = ${basedirectory:promise}/kumofs-server-listen
+port = ${kumofs-instance:server-listen-port}
+
+[promise-kumofs-gateway]
+<= promise-template
+path = ${basedirectory:promise}/kumofs-gateway
+port = ${kumofs-instance:gateway-port}
+
+[promise-kumofs-manager]
+<= promise-template
+path = ${basedirectory:promise}/kumofs-manager
+port = ${kumofs-instance:manager-port}