Added Cache Tool

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@11028 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ERP5/Tool/CacheTool.py

+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                     Ivan Tyagov <ivan@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+""" Cache Tool module for ERP5 """
+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type.Tool.BaseTool import BaseTool
+from Products.ERP5Type import Permissions
+from Globals import InitializeClass, DTMLFile, PersistentMapping
+from Products.ERP5 import _dtmldir
+from Products.ERP5Type.Cache import CachingMethod, CacheFactory
+from Products.ERP5Type.CachePlugins.RamCache import RamCache
+from Products.ERP5Type.CachePlugins.DistributedRamCache import DistributedRamCache
+from Products.ERP5Type.CachePlugins.SQLCache import SQLCache
+
+
+class CacheTool(BaseTool):
+  """ Caches tool wrapper for ERP5 """
+    
+  id = "portal_caches"
+  meta_type = "ERP5 Cache Tool"
+  portal_type = "Cache Tool"
+
+  security = ClassSecurityInfo()
+  manage_options = ({'label': 'Configure',
+                                'action': 'cache_tool_configure',
+                    },) + BaseTool.manage_options
+
+  security.declareProtected( Permissions.ManagePortal, 'cache_tool_configure')
+  cache_tool_configure = DTMLFile('cache_tool_configure', _dtmldir)
+  
+  def __init__(self):
+    BaseTool.__init__(self)
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'getCacheFactoryList')
+  def getCacheFactoryList(self):
+    """ Return available cache factories """
+    rd ={}
+    for cf in self.objectValues('ERP5 Cache Factory'):
+      cache_scope = cf.getId()
+      rd[cache_scope] = {}
+      rd[cache_scope]['cache_plugins'] = []
+      rd[cache_scope]['cache_params'] = {}
+      for cp in cf.getCachePluginList():
+        cp_meta_type = cp.meta_type
+        if cp_meta_type == 'ERP5 Ram Cache Plugin':
+          cache_obj = RamCache()
+        elif cp_meta_type == 'ERP5 Distributed Ram Cache Plugin':
+          cache_obj = DistributedRamCache({'server':cp.getServer()})
+        elif cp_meta_type == 'ERP5 SQL Cache Plugin':
+          ## use connection details from 'erp5_sql_transactionless_connection' ZMySLQDA object
+          connection_string = self.erp5_sql_transactionless_connection.connection_string
+          kw = self.parseDBConnectionString(connection_string)
+          kw['cache_table_name'] = cp.getCacheTableName()
+          cache_obj = SQLCache(kw)
+        ## set cache expire check interval
+        cache_obj.cache_expire_check_interval = cp.getCacheExpireCheckInterval() 
+        rd[cache_scope]['cache_plugins'].append(cache_obj)
+        rd[cache_scope]['cache_params']['cache_duration'] = cf.getCacheDuration() 
+    return rd
+
+  ##
+  ## DB structure
+  ##
+  security.declareProtected(Permissions.ModifyPortalContent, 'createDBCacheTable')
+  def createDBCacheTable(self, cache_table_name="cache", REQUEST=None):
+    """ create in MySQL DB cache table """
+    my_query = SQLCache.create_table_sql %cache_table_name
+    try:
+      self.erp5_sql_transactionless_connection.manage_test("DROP TABLE %s" %cache_table_name)
+    except:
+      pass
+    self.erp5_sql_transactionless_connection.manage_test(my_query)
+    if REQUEST:
+      self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache table successfully created.')
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'parseDBConnectionString')
+  def parseDBConnectionString(self, connection_string):
+    """ Parse given connection string. Code "borrowed" from ZMySLQDA.db """
+    kwargs = {}
+    items = connection_string.split()
+    if not items: 
+      return kwargs
+    lockreq, items = items[0], items[1:]
+    if lockreq[0] == "*":
+      db_host, items = items[0], items[1:]
+    else:
+      db_host = lockreq
+    if '@' in db_host:
+      db, host = split(db_host,'@',1)
+      kwargs['db'] = db
+      if ':' in host:
+        host, port = split(host,':',1)
+        kwargs['port'] = int(port)
+      kwargs['host'] = host
+    else:
+      kwargs['db'] = db_host
+    if kwargs['db'] and kwargs['db'][0] in ('+', '-'):
+      kwargs['db'] = kwargs['db'][1:]
+    if not kwargs['db']:
+      del kwargs['db']
+    if not items: 
+      return kwargs
+    kwargs['user'], items = items[0], items[1:]
+    if not items: 
+      return kwargs
+    kwargs['passwd'], items = items[0], items[1:]
+    if not items: 
+      return kwargs
+    kwargs['unix_socket'], items = items[0], items[1:]
+    return kwargs
+    
+  ##
+  ## RAM cache structure
+  ##
+  security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheRoot')
+  def getRamCacheRoot(self):
+    """ Return RAM based cache root """
+    return CachingMethod.factories
+
+  security.declareProtected(Permissions.ModifyPortalContent, 'updateCache')
+  def updateCache(self, REQUEST=None):
+    """ Clear and update cache structure """
+    #erp5_site_id = self.getPortalObject().getId()
+    for cf in CachingMethod.factories:
+      for cp in  CachingMethod.factories[cf].getCachePluginList():
+        del cp
+    CachingMethod.factories = {}
+    ## read configuration from ZODB
+    for key,item in self.getCacheFactoryList().items():
+      if len(item['cache_plugins'])!=0:
+        CachingMethod.factories[key] = CacheFactory(item['cache_plugins'], item['cache_params'])    
+    if REQUEST:
+      self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache updated.')
+    
+  security.declareProtected(Permissions.ModifyPortalContent, 'clearCache')
+  def clearCache(self, REQUEST=None):
+    """ Clear whole cache structure """
+    ram_cache_root = self.getRamCacheRoot()
+    for cf in ram_cache_root:
+      for cp in ram_cache_root[cf].getCachePluginList():
+        cp.clearCache()
+    if REQUEST:
+      self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache cleared.')
+
+  security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactory')
+  def clearCacheFactory(self, cache_factory_id, REQUEST=None):
+    """ Clear only cache factory. """
+    ram_cache_root = self.getRamCacheRoot()
+    if ram_cache_root.has_key(cache_factory_id):
+      ram_cache_root[cache_factory_id].clearCache()
+    if REQUEST:
+      self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory %s cleared.' %cache_factory_id)
+    
+  security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactoryScope')
+  def clearCacheFactoryScope(self, cache_factory_id, scope, REQUEST=None):
+    """ Clear only cache factory. """
+    ram_cache_root = self.getRamCacheRoot()
+    if ram_cache_root.has_key(cache_factory_id):
+      ram_cache_root[cache_factory_id].clearCacheForScope(scope)
+    if REQUEST:
+      self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory scope %s cleared.' %cache_factory_id)  
+  

product/ERP5/__init__.py

@@ -46,7 +46,7 @@ product_path = package_home( globals() )
 # Define object classes and tools
 from Tool import CategoryTool, SimulationTool, RuleTool, IdTool, TemplateTool,\
                  TestTool, DomainTool, AlarmTool, OrderTool, DeliveryTool,\
-                 TrashTool
+                 TrashTool, CacheTool
 import ERP5Site
 object_classes = ( ERP5Site.ERP5Site,
                  )
@@ -61,6 +61,7 @@ portal_tools = ( CategoryTool.CategoryTool,
                  OrderTool.OrderTool,
                  DeliveryTool.DeliveryTool,
                  TrashTool.TrashTool,
+		 CacheTool.CacheTool,
                 )
 content_classes = ()
 content_constructors = ()

product/ERP5/dtml/cache_tool_configure.dtml

+<dtml-var manage_page_header>
+<dtml-var manage_tabs>
+
+<b><br/>
+  <dtml-var expr="REQUEST.get('portal_status_message', '')">
+</b>
+
+<h3>Cache invalidation</h3>
+  <form action="clearCache" method="POST">
+    <input type="submit" value="Clear all cache factories"/>
+  </form>
+  <dtml-in expr="objectIds('ERP5 Cache Factory')">
+    <form action="clearCacheFactory" method="POST">
+      <input type="hidden" name="cache_factory_id" value="<dtml-var sequence-item>">
+      <input type="submit" value="Clear <dtml-var sequence-item>"/>
+    </form>
+  </dtml-in>
+  
+<h3>SQLCache configuration</h3>
+<p>Create SQL cache table(Note: you need to adjust later each SQLCache plugin to use this cache table name manually. Generally it is a good idea to use default sql cache table name)</p>
+<form action="createDBCacheTable" method="POST">
+  <input name="cache_table_name" value="cache">
+  <br/>
+  <input type="submit" value="Create (Recreate) sql cache table"/>
+</form>
+
+<dtml-var manage_page_footer>

product/ERP5/tests/testCache.py

+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                     Ivan Tyagov <ivan@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import random
+import unittest
+import time
+import base64, md5
+from ERP5Cache.CachePlugins.RamCache import RamCache
+from ERP5Cache.CachePlugins.DistributedRamCache import DistributedRamCache
+from ERP5Cache.CachePlugins.SQLCache import SQLCache
+from ERP5Cache.CachePlugins.BaseCache import CacheEntry
+
+
+class Foo:
+  my_field = (1,2,3,4,5)
+
+class TestRamCache(unittest.TestCase):
+    
+  def setUp(self):
+    self.cache_plugins = (RamCache(), 
+                          DistributedRamCache({'servers': '127.0.0.1:11211',
+                                                 'debugLevel': 7,}),
+                          SQLCache( {'server': '',
+                                     'user': '',
+                                     'passwd': '',
+                                     'db': 'test',
+                                     'cache_table_name': 'cache',
+                                      }),
+                        )
+
+  def testScope(self):
+    """ test scope functions """
+    ## create some sample scopes
+    iterations = 10
+    test_scopes = []
+    for i in range(0, iterations):
+        test_scopes.append("my_scope_%s" %i)
+    test_scopes.sort()
+    
+    ## remove DistributedRamCache since it's a flat storage
+    filtered_cache_plugins = filter(lambda x: not isinstance(x, DistributedRamCache), self.cache_plugins)
+    
+    for cache_plugin in filtered_cache_plugins:
+      print "TESTING (scope): ", cache_plugin
+
+      ## clear cache for this plugin
+      cache_plugin.clearCache()
+      
+      ## should exists no scopes in cache
+      self.assertEqual([], cache_plugin.getScopeList())
+      
+      ## set some sample values 
+      for scope in test_scopes:
+        cache_id = '%s_cache_id' %scope
+        cache_plugin.set(cache_id, scope, scope*10)
+        
+        ## we set ONLY one value per scope -> check if we get the same cache_id
+        self.assertEqual([cache_id], cache_plugin.getScopeKeyList(scope))
+        print "\t", cache_id, scope, "\t\tOK"
+      
+      ## get list of scopes which must be the same as test_scopes since we clear cache initially
+      scopes_from_cache = cache_plugin.getScopeList()
+      scopes_from_cache.sort()  
+      self.assertEqual(test_scopes, scopes_from_cache)
+      
+      ## remove scope one by one
+      count = 1 
+      for scope in test_scopes:
+        cache_plugin.clearCacheForScope(scope)
+        ## .. and check that  we should have 1 less cache scope 
+        scopes_from_cache = cache_plugin.getScopeList()
+        self.assertEqual(iterations - count, len(scopes_from_cache))
+        count = count + 1
+        
+      ## .. we shouldn't have any cache scopes 
+      scopes_from_cache = cache_plugin.getScopeList()
+      self.assertEqual([], scopes_from_cache)
+
+      
+  def testSetGet(self):
+    """ set value to cache and then get it back """
+    for cache_plugin in self.cache_plugins:
+      self.generaltestSetGet(cache_plugin, 100)
+    
+  def testExpire(self):
+    """ Check expired by setting a key, wit for its timeout and check if in cache"""
+    for cache_plugin in self.cache_plugins:
+      self.generalExpire(cache_plugin, 2)
+
+            
+  def generalExpire(self, cache_plugin, iterations):
+    print "TESTING (expire): ", cache_plugin
+    base_timeout = 1
+    values = self.prepareValues(iterations)
+    scope = "peter"
+    count = 0
+    for value in values:
+      count = count +1
+      cache_timeout = base_timeout + random.random()*2
+      cache_id = "mycache_id_to_expire_%s" %(count)
+      print "\t", cache_id, " ==> timeout (s) = ", cache_timeout, 
+
+      ## set to cache
+      cache_plugin.set(cache_id, scope, value, cache_timeout)
+        
+      ## sleep for timeout +1
+      time.sleep(cache_timeout + 1)
+        
+      ## should remove from cache expired cache entries 
+      cache_plugin.expireOldCacheEntries(forceCheck=True)
+        
+      ##  check it, we MUST NOT have this key any more in cache
+      self.assertEqual(False, cache_plugin.has_key(cache_id, scope))
+      print "\t\tOK"
+     
+  def generaltestSetGet(self, cache_plugin, iterations):
+    print "TESTING (set/get/has/del): ", cache_plugin
+    values = self.prepareValues(iterations)
+    cache_duration = 30
+    scope = "peter"
+    count = 0
+    for value in values:
+      count = count +1
+      cache_id = "mycache_id_to_set_get_has_del_%s" %(count)
+        
+      ## set to cache
+      cache_plugin.set(cache_id, scope, value, cache_duration)
+      print "\t", cache_id, 
+        
+      ## check has_key()
+      self.assertEqual(True, cache_plugin.has_key(cache_id, scope))
+        
+      ## check get()
+      cache_entry = cache_plugin.get(cache_id, scope)
+      if isinstance(value, Foo):
+        ## when memcached or sql cached we have a new object created for user
+        ## just compare one field from it
+        self.assertEqual(value.my_field, cache_entry.getValue().my_field)
+      else:
+        ## primitive types, direct comparision
+        self.assertEqual(value, cache_entry.getValue())
+        
+      ## is returned result proper cache entry?
+      self.assertEqual(True, isinstance(cache_entry, CacheEntry))
+        
+      ## is returned result proper type?
+      self.assertEqual(type(value), type(cache_entry.getValue()))
+        
+      ## check delete(), key should be removed from there
+      cache_plugin.delete(cache_id, scope)
+      self.assertEqual(False, cache_plugin.has_key(cache_id, scope))
+        
+      print "\t\tOK"
+    
+  def prepareValues(self, iterations):
+    """ generate a big list of values """
+    values = []
+    my_text = "".join(map(chr, range(50,200))) * 10 ## long string (150*x)
+    for i in range(0, iterations):
+      values.append(random.random()*i)
+      values.append(random.random()*i/1000)
+      values.append(my_text)
+      values.append(Foo())
+    return values
+
+if __name__ == '__main__':
+  unittest.main()