several bug fixes and improvement

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@10634 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ERP5Cache/CachePlugins/DistributedRamCache.py

@@ -48,18 +48,35 @@ class DistributedRamCache(BaseCache):
   def __init__(self, params):
     self._servers = params.get('server', '')
     self._debugLevel = params.get('debugLevel', 7)
-    self._cache = memcache.Client(self._servers.split('\n'), self._debugLevel)
     self._last_cache_conn_creation_time = time()
     BaseCache.__init__(self)
         
   def getCacheStorage(self):
     ## if we use one connection object this causes  "MemCached: while expecting 'STORED', got unexpected response 'END'"
     ## messages in log files and thus sometimes can block the thread. For the moment we create
-    ## a new conn object for every memcache access which in turns cmeans another socket. 
+    ## a new conn object for every memcache access which in turns means another socket. 
     ## See  addiionaly expireOldCacheEntries() comments for one or many connections.
-    self._cache = memcache.Client(self._servers.split('\n'), debug=self._debugLevel)    
-    return self._cache
-
+    try:
+      from Products.ERP5Type.Utils import get_request
+      request = get_request()
+    except ImportError:
+      request = None
+      
+    if request:
+      ## Zope/ERP5 environment
+      memcache_conn = request.get('_erp5_memcache_connection', None)
+      if not memcache_conn:
+        ## we have not memcache_conn for this request
+        memcache_conn = memcache.Client(self._servers.split('\n'), debug=self._debugLevel)
+        request.set('_erp5_memcache_connection', memcache_conn)
+        return memcache_conn      
+      else:
+        ## we have memcache_conn for this request
+        return memcache_conn
+    else:
+      ## run from unit tests
+      return  memcache.Client(self._servers.split('\n'), debug=self._debugLevel)
+       
   def checkAndFixCacheId(self, cache_id, scope):
     ## memcached doesn't support namespaces (cache scopes) so to "emmulate"
     ## such behaviour when constructing cache_id we add scope in front

product/ERP5Cache/CachePlugins/RamCache.py

+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                     Ivan Tyagov <ivan@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
 """
 Local RAM based cache plugin.
 """

product/ERP5Cache/CachePlugins/SQLCache.py

@@ -114,15 +114,37 @@ class SQLCache(BaseCache):
     
   def getCacheStorage(self):
     """ 
-    Return current DB connection or create a new one.
+    Return current DB connection or create a new one for his thread.
     See http://sourceforge.net/docman/display_doc.php?docid=32071&group_id=22307
     especially threadsafety part why we create every time a new MySQL db connection object.
     """
-    dbConn = MySQLdb.connect(host=self._db_server, \
+    try:
+      from Products.ERP5Type.Utils import get_request
+      request = get_request()
+    except ImportError:
+      request = None
+      
+    if request:
+      ## Zope/ERP5 environment
+      dbConn = request.get('_erp5_dbcache_connection', None)
+      if not dbConn:
+        ## we have not dbConn for this request
+        dbConn = MySQLdb.connect(host=self._db_server, \
+                                 user=self._db_user,\
+                                 passwd=self._db_passwd, \
+                                 db=self._db_name)
+        request.set('_erp5_dbcache_connection', dbConn)
+        return dbConn      
+      else:
+        ## we have already dbConn for this request
+        return dbConn
+    else:
+      ## run from unit tests
+      dbConn = MySQLdb.connect(host=self._db_server, \
                              user=self._db_user,\
                              passwd=self._db_passwd, \
                              db=self._db_name)
-    return dbConn
+      return dbConn
     
   def get(self, cache_id, scope, default=None):
     sql_query = self.get_key_sql %(self._db_cache_table_name, cache_id, scope)
@@ -162,7 +184,6 @@ class SQLCache(BaseCache):
     now = time.time()
     if forceCheck or (now > (self._last_cache_expire_check_at + self.cache_expire_check_interval)):
       ## time to check for expired cache items
-      #print "EXPIRE", self, self.cache_expire_check_interval
       self._last_cache_expire_check_at = now
       my_query = self.delete_expired_keys_sql %(self._db_cache_table_name, now)
       self.execSQLQuery(my_query)

product/ERP5Cache/CacheTool.py

@@ -38,13 +38,6 @@ from Products.ERP5Cache.CachePlugins.RamCache import RamCache
 from Products.ERP5Cache.CachePlugins.DistributedRamCache import DistributedRamCache
 from Products.ERP5Cache.CachePlugins.SQLCache import SQLCache
 
-##try:
-##  from Products.TimerService import getTimerService
-##except ImportError:
-##  def getTimerService(self):
-##    pass
-
-
 class CacheTool(BaseTool):
   """ Caches tool wrapper for ERP5 """
     
@@ -183,63 +176,13 @@ class CacheTool(BaseTool):
       ram_cache_root[cache_factory_id].clearCache()
     if REQUEST:
       self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory %s cleared.' %cache_factory_id)
-
+    
+  security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactoryScope')
+  def clearCacheFactoryScope(self, cache_factory_id, scope, REQUEST=None):
+    """ Clear only cache factory. """
+    ram_cache_root = self.getRamCacheRoot()
+    if ram_cache_root.has_key(cache_factory_id):
+      ram_cache_root[cache_factory_id].clearCacheForScope(scope)
+    if REQUEST:
+      self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory scope %s cleared.' %cache_factory_id)  
   
-  # Timer - checks for cache expiration triggered by Zope's TimerService
-##  def isSubscribed(self):
-##      """
-##      return True, if we are subscribed to TimerService.
-##      Otherwise return False.
-##      """
-##      service = getTimerService(self)
-##      if not service:
-##          LOG('AlarmTool', INFO, 'TimerService not available')
-##          return False
-##
-##      path = '/'.join(self.getPhysicalPath())
-##      if path in service.lisSubscriptions():
-##          return True
-##      return False
-##
-##  security.declareProtected(Permissions.ManageProperties, 'subscribe')
-##  def subscribe(self):
-##    """
-##      Subscribe to the global Timer Service.
-##    """
-##    service = getTimerService(self)
-##    if not service:
-##      LOG('AlarmTool', INFO, 'TimerService not available')
-##      return
-##    service.subscribe(self)
-##    return "Subscribed to Timer Service"
-##
-##  security.declareProtected(Permissions.ManageProperties, 'unsubscribe')
-##  def unsubscribe(self):
-##    """
-##      Unsubscribe from the global Timer Service.
-##    """
-##    service = getTimerService(self)
-##    if not service:
-##      LOG('AlarmTool', INFO, 'TimerService not available')
-##      return
-##    service.unsubscribe(self)
-##    return "Usubscribed from Timer Service"
-##
-##  def manage_beforeDelete(self, item, container):
-##    self.unsubscribe()
-##    BaseTool.inheritedAttribute('manage_beforeDelete')(self, item, container)
-##
-##  def manage_afterAdd(self, item, container):
-##    self.subscribe()
-##    BaseTool.inheritedAttribute('manage_afterAdd')(self, item, container)
-##
-##  security.declarePrivate('process_timer')
-##  def process_timer(self, interval, tick, prev="", next=""):
-##    """
-##      This method is called by TimerService in the interval given
-##      in zope.conf. The Default is every 5 seconds. This method will
-##      try to expire cache entries.
-##    """
-##    ram_cache_root = self.getRamCacheRoot()
-##    for cf_id, cf_obj in ram_cache_root.items():
-##      cf_obj.expire()

product/ERP5Cache/Document/DistributedRamCachePlugin.py

@@ -30,6 +30,7 @@ from AccessControl import ClassSecurityInfo
 from Products.CMFCore import CMFCorePermissions
 from Products.ERP5Type.XMLObject import XMLObject
 from Products.ERP5Type import PropertySheet
+from Products.ERP5.PropertySheet.SortIndex import SortIndex
 from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin
 from Products.ERP5Cache.PropertySheet.DistributedRamCachePlugin import DistributedRamCachePlugin
 
@@ -57,5 +58,6 @@ class DistributedRamCachePlugin(XMLObject):
                     , PropertySheet.SimpleItem
                     , PropertySheet.Folder
                     , BaseCachePlugin
+                    , SortIndex
                     , DistributedRamCachePlugin 
                     )

product/ERP5Cache/Document/SQLCachePlugin.py

@@ -31,6 +31,7 @@ from Products.CMFCore import CMFCorePermissions
 from Products.ERP5Type.Base import Base
 from Products.ERP5Type.XMLObject import XMLObject
 from Products.ERP5Type import PropertySheet
+from Products.ERP5.PropertySheet.SortIndex import SortIndex
 from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin
 from Products.ERP5Cache.PropertySheet.SQLCachePlugin import SQLCachePlugin
 
@@ -58,5 +59,6 @@ class SQLCachePlugin(XMLObject):
                     , PropertySheet.SimpleItem
                     , PropertySheet.Folder
                     , BaseCachePlugin
+                    , SortIndex
                     , SQLCachePlugin 
                     )

product/ERP5Cache/ERP5Cache.e3p

 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE Project SYSTEM "Project-3.7.dtd">
 <!-- Project file for project ERP5Cache -->
-<!-- Saved: 2006-10-03, 20:15:31 -->
+<!-- Saved: 2006-10-06, 19:59:47 -->
 <!-- Copyright (C) 2006 Ivan Tyagov, ivan.tyagov@brmtec.com -->
 <Project version="3.7">
   <ProgLanguage mixed="0">Python</ProgLanguage>
@@ -99,6 +99,10 @@
       <Dir>dtml</Dir>
       <Name>cache_tool_configure.dtml</Name>
     </Source>
+    <Source>
+      <Dir>tests</Dir>
+      <Name>testCacheTool.py</Name>
+    </Source>
   </Sources>
   <Forms>
   </Forms>

product/ERP5Cache/ERP5Cache.e3t

 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE Tasks SYSTEM "Tasks-3.7.dtd">
 <!-- Tasks file for project ERP5Cache -->
-<!-- Saved: 2006-10-05, 17:54:47 -->
+<!-- Saved: 2006-10-06, 20:07:54 -->
 <Tasks version="3.7">
   <Task priority="1" completed="0">
     <Description>TODO: move result file path generation to runBenchmark.</Description>
@@ -38,35 +38,13 @@
   </Task>
   <Task priority="1" completed="0">
     <Description>TODO: Based on above data we can have a different invalidation policy</Description>
-    <Created>2006-09-28, 13:08:57</Created>
+    <Created>2006-10-06, 20:00:38</Created>
     <Resource>
       <Filename>
         <Dir>CachePlugins</Dir>
         <Name>BaseCache.py</Name>
       </Filename>
-      <Linenumber>19</Linenumber>
-    </Resource>
-  </Task>
-  <Task priority="1" completed="0">
-    <Description>TODO: make check not always but each 100 or n calls</Description>
-    <Created>2006-09-28, 13:08:57</Created>
-    <Resource>
-      <Filename>
-        <Dir>CachePlugins</Dir>
-        <Name>BaseCache.py</Name>
-      </Filename>
-      <Linenumber>64</Linenumber>
-    </Resource>
-  </Task>
-  <Task priority="1" completed="0">
-    <Description>TODO: check how to avoid problems with memcache whe using one connection to</Description>
-    <Created>2006-10-05, 16:42:13</Created>
-    <Resource>
-      <Filename>
-        <Dir>CachePlugins</Dir>
-        <Name>DistributedRamCache.py</Name>
-      </Filename>
-      <Linenumber>25</Linenumber>
+      <Linenumber>47</Linenumber>
     </Resource>
   </Task>
 </Tasks>

product/ERP5Cache/tests/testCache.py

+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                     Ivan Tyagov <ivan@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
 import random
 import unittest
 import time
@@ -14,15 +42,15 @@ class Foo:
 class TestRamCache(unittest.TestCase):
     
   def setUp(self):
-    self.cache_plugins = (#RamCache(), 
+    self.cache_plugins = (RamCache(), 
                           DistributedRamCache({'servers': '127.0.0.1:11211',
                                                  'debugLevel': 7,}),
-                          #SQLCache( {'server': '',
-                          #           'user': '',
-                          #           'passwd': '',
-                          #           'db': 'test',
-                          #           'cache_table_name': 'cache',
-                          #            }),
+                          SQLCache( {'server': '',
+                                     'user': '',
+                                     'passwd': '',
+                                     'db': 'test',
+                                     'cache_table_name': 'cache',
+                                      }),
                         )
 
   def testScope(self):
@@ -77,13 +105,13 @@ class TestRamCache(unittest.TestCase):
   def testSetGet(self):
     """ set value to cache and then get it back """
     for cache_plugin in self.cache_plugins:
-      self.generaltestSetGet(cache_plugin, 1000)
+      self.generaltestSetGet(cache_plugin, 100)
     
-##  def testExpire(self):
-##    """ Check expired by setting a key, wit for its timeout and check if in cache"""
-##    for cache_plugin in self.cache_plugins:
-##      self.generalExpire(cache_plugin, 2)
-##    pass
+  def testExpire(self):
+    """ Check expired by setting a key, wit for its timeout and check if in cache"""
+    for cache_plugin in self.cache_plugins:
+      self.generalExpire(cache_plugin, 2)
+
             
   def generalExpire(self, cache_plugin, iterations):
     print "TESTING (expire): ", cache_plugin