From 5512c51539b886f1a65ef9a434a3c21eb9bcb41d Mon Sep 17 00:00:00 2001 From: Sebastien Robin <seb@nexedi.com> Date: Mon, 9 Oct 2006 21:39:47 +0000 Subject: [PATCH] several bug fixes and improvement git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@10634 20353a03-c40f-0410-a6d1-a30d3c3de9de --- .../CachePlugins/DistributedRamCache.py | 27 +++++-- product/ERP5Cache/CachePlugins/RamCache.py | 29 +++++++ product/ERP5Cache/CachePlugins/SQLCache.py | 29 ++++++- product/ERP5Cache/CacheTool.py | 75 +++---------------- .../Document/DistributedRamCachePlugin.py | 2 + product/ERP5Cache/Document/SQLCachePlugin.py | 2 + product/ERP5Cache/ERP5Cache.e3p | 6 +- product/ERP5Cache/ERP5Cache.e3t | 28 +------ product/ERP5Cache/tests/testCache.py | 54 +++++++++---- 9 files changed, 138 insertions(+), 114 deletions(-) diff --git a/product/ERP5Cache/CachePlugins/DistributedRamCache.py b/product/ERP5Cache/CachePlugins/DistributedRamCache.py index 6ba8374acb..0873f164ab 100644 --- a/product/ERP5Cache/CachePlugins/DistributedRamCache.py +++ b/product/ERP5Cache/CachePlugins/DistributedRamCache.py @@ -48,18 +48,35 @@ class DistributedRamCache(BaseCache): def __init__(self, params): self._servers = params.get('server', '') self._debugLevel = params.get('debugLevel', 7) - self._cache = memcache.Client(self._servers.split('\n'), self._debugLevel) self._last_cache_conn_creation_time = time() BaseCache.__init__(self) def getCacheStorage(self): ## if we use one connection object this causes "MemCached: while expecting 'STORED', got unexpected response 'END'" ## messages in log files and thus sometimes can block the thread. For the moment we create - ## a new conn object for every memcache access which in turns cmeans another socket. + ## a new conn object for every memcache access which in turns means another socket. ## See addiionaly expireOldCacheEntries() comments for one or many connections. - self._cache = memcache.Client(self._servers.split('\n'), debug=self._debugLevel) - return self._cache - + try: + from Products.ERP5Type.Utils import get_request + request = get_request() + except ImportError: + request = None + + if request: + ## Zope/ERP5 environment + memcache_conn = request.get('_erp5_memcache_connection', None) + if not memcache_conn: + ## we have not memcache_conn for this request + memcache_conn = memcache.Client(self._servers.split('\n'), debug=self._debugLevel) + request.set('_erp5_memcache_connection', memcache_conn) + return memcache_conn + else: + ## we have memcache_conn for this request + return memcache_conn + else: + ## run from unit tests + return memcache.Client(self._servers.split('\n'), debug=self._debugLevel) + def checkAndFixCacheId(self, cache_id, scope): ## memcached doesn't support namespaces (cache scopes) so to "emmulate" ## such behaviour when constructing cache_id we add scope in front diff --git a/product/ERP5Cache/CachePlugins/RamCache.py b/product/ERP5Cache/CachePlugins/RamCache.py index 66288f98a2..cf7704c6e2 100644 --- a/product/ERP5Cache/CachePlugins/RamCache.py +++ b/product/ERP5Cache/CachePlugins/RamCache.py @@ -1,3 +1,32 @@ +############################################################################## +# +# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved. +# Ivan Tyagov <ivan@nexedi.com> +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + + """ Local RAM based cache plugin. """ diff --git a/product/ERP5Cache/CachePlugins/SQLCache.py b/product/ERP5Cache/CachePlugins/SQLCache.py index bf070c203c..e1e6e5c329 100644 --- a/product/ERP5Cache/CachePlugins/SQLCache.py +++ b/product/ERP5Cache/CachePlugins/SQLCache.py @@ -114,15 +114,37 @@ class SQLCache(BaseCache): def getCacheStorage(self): """ - Return current DB connection or create a new one. + Return current DB connection or create a new one for his thread. See http://sourceforge.net/docman/display_doc.php?docid=32071&group_id=22307 especially threadsafety part why we create every time a new MySQL db connection object. """ - dbConn = MySQLdb.connect(host=self._db_server, \ + try: + from Products.ERP5Type.Utils import get_request + request = get_request() + except ImportError: + request = None + + if request: + ## Zope/ERP5 environment + dbConn = request.get('_erp5_dbcache_connection', None) + if not dbConn: + ## we have not dbConn for this request + dbConn = MySQLdb.connect(host=self._db_server, \ + user=self._db_user,\ + passwd=self._db_passwd, \ + db=self._db_name) + request.set('_erp5_dbcache_connection', dbConn) + return dbConn + else: + ## we have already dbConn for this request + return dbConn + else: + ## run from unit tests + dbConn = MySQLdb.connect(host=self._db_server, \ user=self._db_user,\ passwd=self._db_passwd, \ db=self._db_name) - return dbConn + return dbConn def get(self, cache_id, scope, default=None): sql_query = self.get_key_sql %(self._db_cache_table_name, cache_id, scope) @@ -162,7 +184,6 @@ class SQLCache(BaseCache): now = time.time() if forceCheck or (now > (self._last_cache_expire_check_at + self.cache_expire_check_interval)): ## time to check for expired cache items - #print "EXPIRE", self, self.cache_expire_check_interval self._last_cache_expire_check_at = now my_query = self.delete_expired_keys_sql %(self._db_cache_table_name, now) self.execSQLQuery(my_query) diff --git a/product/ERP5Cache/CacheTool.py b/product/ERP5Cache/CacheTool.py index 67d318d94d..d9d330a001 100644 --- a/product/ERP5Cache/CacheTool.py +++ b/product/ERP5Cache/CacheTool.py @@ -38,13 +38,6 @@ from Products.ERP5Cache.CachePlugins.RamCache import RamCache from Products.ERP5Cache.CachePlugins.DistributedRamCache import DistributedRamCache from Products.ERP5Cache.CachePlugins.SQLCache import SQLCache -##try: -## from Products.TimerService import getTimerService -##except ImportError: -## def getTimerService(self): -## pass - - class CacheTool(BaseTool): """ Caches tool wrapper for ERP5 """ @@ -183,63 +176,13 @@ class CacheTool(BaseTool): ram_cache_root[cache_factory_id].clearCache() if REQUEST: self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory %s cleared.' %cache_factory_id) - + + security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactoryScope') + def clearCacheFactoryScope(self, cache_factory_id, scope, REQUEST=None): + """ Clear only cache factory. """ + ram_cache_root = self.getRamCacheRoot() + if ram_cache_root.has_key(cache_factory_id): + ram_cache_root[cache_factory_id].clearCacheForScope(scope) + if REQUEST: + self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory scope %s cleared.' %cache_factory_id) - # Timer - checks for cache expiration triggered by Zope's TimerService -## def isSubscribed(self): -## """ -## return True, if we are subscribed to TimerService. -## Otherwise return False. -## """ -## service = getTimerService(self) -## if not service: -## LOG('AlarmTool', INFO, 'TimerService not available') -## return False -## -## path = '/'.join(self.getPhysicalPath()) -## if path in service.lisSubscriptions(): -## return True -## return False -## -## security.declareProtected(Permissions.ManageProperties, 'subscribe') -## def subscribe(self): -## """ -## Subscribe to the global Timer Service. -## """ -## service = getTimerService(self) -## if not service: -## LOG('AlarmTool', INFO, 'TimerService not available') -## return -## service.subscribe(self) -## return "Subscribed to Timer Service" -## -## security.declareProtected(Permissions.ManageProperties, 'unsubscribe') -## def unsubscribe(self): -## """ -## Unsubscribe from the global Timer Service. -## """ -## service = getTimerService(self) -## if not service: -## LOG('AlarmTool', INFO, 'TimerService not available') -## return -## service.unsubscribe(self) -## return "Usubscribed from Timer Service" -## -## def manage_beforeDelete(self, item, container): -## self.unsubscribe() -## BaseTool.inheritedAttribute('manage_beforeDelete')(self, item, container) -## -## def manage_afterAdd(self, item, container): -## self.subscribe() -## BaseTool.inheritedAttribute('manage_afterAdd')(self, item, container) -## -## security.declarePrivate('process_timer') -## def process_timer(self, interval, tick, prev="", next=""): -## """ -## This method is called by TimerService in the interval given -## in zope.conf. The Default is every 5 seconds. This method will -## try to expire cache entries. -## """ -## ram_cache_root = self.getRamCacheRoot() -## for cf_id, cf_obj in ram_cache_root.items(): -## cf_obj.expire() diff --git a/product/ERP5Cache/Document/DistributedRamCachePlugin.py b/product/ERP5Cache/Document/DistributedRamCachePlugin.py index 56059d7aee..ca923a112a 100644 --- a/product/ERP5Cache/Document/DistributedRamCachePlugin.py +++ b/product/ERP5Cache/Document/DistributedRamCachePlugin.py @@ -30,6 +30,7 @@ from AccessControl import ClassSecurityInfo from Products.CMFCore import CMFCorePermissions from Products.ERP5Type.XMLObject import XMLObject from Products.ERP5Type import PropertySheet +from Products.ERP5.PropertySheet.SortIndex import SortIndex from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin from Products.ERP5Cache.PropertySheet.DistributedRamCachePlugin import DistributedRamCachePlugin @@ -57,5 +58,6 @@ class DistributedRamCachePlugin(XMLObject): , PropertySheet.SimpleItem , PropertySheet.Folder , BaseCachePlugin + , SortIndex , DistributedRamCachePlugin ) diff --git a/product/ERP5Cache/Document/SQLCachePlugin.py b/product/ERP5Cache/Document/SQLCachePlugin.py index 7b28b157fe..5bcede1716 100644 --- a/product/ERP5Cache/Document/SQLCachePlugin.py +++ b/product/ERP5Cache/Document/SQLCachePlugin.py @@ -31,6 +31,7 @@ from Products.CMFCore import CMFCorePermissions from Products.ERP5Type.Base import Base from Products.ERP5Type.XMLObject import XMLObject from Products.ERP5Type import PropertySheet +from Products.ERP5.PropertySheet.SortIndex import SortIndex from Products.ERP5Cache.PropertySheet.BaseCachePlugin import BaseCachePlugin from Products.ERP5Cache.PropertySheet.SQLCachePlugin import SQLCachePlugin @@ -58,5 +59,6 @@ class SQLCachePlugin(XMLObject): , PropertySheet.SimpleItem , PropertySheet.Folder , BaseCachePlugin + , SortIndex , SQLCachePlugin ) diff --git a/product/ERP5Cache/ERP5Cache.e3p b/product/ERP5Cache/ERP5Cache.e3p index aefe1bf05f..c63e9bc779 100644 --- a/product/ERP5Cache/ERP5Cache.e3p +++ b/product/ERP5Cache/ERP5Cache.e3p @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE Project SYSTEM "Project-3.7.dtd"> <!-- Project file for project ERP5Cache --> -<!-- Saved: 2006-10-03, 20:15:31 --> +<!-- Saved: 2006-10-06, 19:59:47 --> <!-- Copyright (C) 2006 Ivan Tyagov, ivan.tyagov@brmtec.com --> <Project version="3.7"> <ProgLanguage mixed="0">Python</ProgLanguage> @@ -99,6 +99,10 @@ <Dir>dtml</Dir> <Name>cache_tool_configure.dtml</Name> </Source> + <Source> + <Dir>tests</Dir> + <Name>testCacheTool.py</Name> + </Source> </Sources> <Forms> </Forms> diff --git a/product/ERP5Cache/ERP5Cache.e3t b/product/ERP5Cache/ERP5Cache.e3t index b781e9a680..75cee1494d 100644 --- a/product/ERP5Cache/ERP5Cache.e3t +++ b/product/ERP5Cache/ERP5Cache.e3t @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE Tasks SYSTEM "Tasks-3.7.dtd"> <!-- Tasks file for project ERP5Cache --> -<!-- Saved: 2006-10-05, 17:54:47 --> +<!-- Saved: 2006-10-06, 20:07:54 --> <Tasks version="3.7"> <Task priority="1" completed="0"> <Description>TODO: move result file path generation to runBenchmark.</Description> @@ -38,35 +38,13 @@ </Task> <Task priority="1" completed="0"> <Description>TODO: Based on above data we can have a different invalidation policy</Description> - <Created>2006-09-28, 13:08:57</Created> + <Created>2006-10-06, 20:00:38</Created> <Resource> <Filename> <Dir>CachePlugins</Dir> <Name>BaseCache.py</Name> </Filename> - <Linenumber>19</Linenumber> - </Resource> - </Task> - <Task priority="1" completed="0"> - <Description>TODO: make check not always but each 100 or n calls</Description> - <Created>2006-09-28, 13:08:57</Created> - <Resource> - <Filename> - <Dir>CachePlugins</Dir> - <Name>BaseCache.py</Name> - </Filename> - <Linenumber>64</Linenumber> - </Resource> - </Task> - <Task priority="1" completed="0"> - <Description>TODO: check how to avoid problems with memcache whe using one connection to</Description> - <Created>2006-10-05, 16:42:13</Created> - <Resource> - <Filename> - <Dir>CachePlugins</Dir> - <Name>DistributedRamCache.py</Name> - </Filename> - <Linenumber>25</Linenumber> + <Linenumber>47</Linenumber> </Resource> </Task> </Tasks> diff --git a/product/ERP5Cache/tests/testCache.py b/product/ERP5Cache/tests/testCache.py index de25e3a694..d0b21c14ef 100644 --- a/product/ERP5Cache/tests/testCache.py +++ b/product/ERP5Cache/tests/testCache.py @@ -1,3 +1,31 @@ +############################################################################## +# +# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved. +# Ivan Tyagov <ivan@nexedi.com> +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + import random import unittest import time @@ -14,15 +42,15 @@ class Foo: class TestRamCache(unittest.TestCase): def setUp(self): - self.cache_plugins = (#RamCache(), + self.cache_plugins = (RamCache(), DistributedRamCache({'servers': '127.0.0.1:11211', 'debugLevel': 7,}), - #SQLCache( {'server': '', - # 'user': '', - # 'passwd': '', - # 'db': 'test', - # 'cache_table_name': 'cache', - # }), + SQLCache( {'server': '', + 'user': '', + 'passwd': '', + 'db': 'test', + 'cache_table_name': 'cache', + }), ) def testScope(self): @@ -77,13 +105,13 @@ class TestRamCache(unittest.TestCase): def testSetGet(self): """ set value to cache and then get it back """ for cache_plugin in self.cache_plugins: - self.generaltestSetGet(cache_plugin, 1000) + self.generaltestSetGet(cache_plugin, 100) -## def testExpire(self): -## """ Check expired by setting a key, wit for its timeout and check if in cache""" -## for cache_plugin in self.cache_plugins: -## self.generalExpire(cache_plugin, 2) -## pass + def testExpire(self): + """ Check expired by setting a key, wit for its timeout and check if in cache""" + for cache_plugin in self.cache_plugins: + self.generalExpire(cache_plugin, 2) + def generalExpire(self, cache_plugin, iterations): print "TESTING (expire): ", cache_plugin -- 2.30.9