Use Distributed Cache for Session

Until now, portal_sessions was only reliably usable when using a family with only one zope node, because the session data was using RAM cache.
When used by authenticated users it was more or less usable, because of haproxy
sticky cookie that we set for authenticated users, but for non authenticated users
this was basically unusable.
This was especially a problem for CaptchaField, for which users are generally not
authenticated.

This changes portal_sessions to use a distributed cache, which brings several
differences:
 - sessions are now shared between all zopes of a cluster.
 - storing ERP5 temp documents is still possible, but modifying a temp document in
   session does not automatically save the changes in session, for next session read
   to be using the modified document it's required to save the document explicitly.
 - session respects transaction semantics, changes are only persisted in session when transaction commits successfully.
 - `portal_caches.clearAllCaches` API no longer clear all sessions.

See merge request nexedi/erp5!1451

bt5/erp5_advanced_ecommerce/SkinTemplateItem/portal_skins/erp5_advanced_ecommerce/Resource_addToShoppingCart.py

@@ -88,6 +88,9 @@ if not line_found:
     method_id(category)
   order_line.setPrice(context.getPrice(supply_path_type=["Sale Supply Line", "Sale Supply Cell"], context=order_line))
 context.WebSection_updateShoppingCartTradeCondition(shopping_cart, None)
+
+context.getPortalObject().portal_sessions[session_id].update(shopping_cart=shopping_cart)
+
 if checkout:
   website = context.getWebSiteValue()
   if website is not None:

bt5/erp5_advanced_ecommerce/SkinTemplateItem/portal_skins/erp5_advanced_ecommerce/WebSection_editShoppingCart.py

@@ -7,7 +7,8 @@
   - use Base_edit and a real ERP5 Form in order to  benefit from
     field reusability and property validation
 """
-translateString = context.Base_translateString 
+portal = context.getPortalObject()
+translateString = portal.Base_translateString 
 if field_my_buy_quantity is None:
   field_my_buy_quantity = context.REQUEST.get("field_my_buy_quantity", None)
 
@@ -44,7 +45,7 @@ if field_my_shipping_method not in ['', None]:
   line = getattr(shopping_cart, 'shipping_method', None)
   if line is not None:
     shopping_cart.manage_delObjects(line.getId())
-  shipping = context.getPortalObject().restrictedTraverse(field_my_shipping_method)
+  shipping = portal.restrictedTraverse(field_my_shipping_method)
   # create new shipping method order line
   shopping_cart.newContent(
                  id='shipping_method', 
@@ -57,7 +58,9 @@ if field_my_comment is not None:
   shopping_cart.setComment(field_my_comment)
 
 context.WebSection_updateShoppingCartTradeCondition(shopping_cart, field_my_payment_mode, preserve=True)
-  
+
+portal.portal_sessions[container.REQUEST['session_id']].update(shopping_cart=shopping_cart)
+
 if redirect:
   # Hardcode redirection.
   return context.Base_redirect("WebSection_viewShoppingCart", \

bt5/erp5_advanced_ecommerce/SkinTemplateItem/portal_skins/erp5_advanced_ecommerce/WebSection_updateShoppingCartTradeCondition.py

 """
   Update Trade Condition with the appropriated Trade Condition.
 """
+portal = context.getPortalObject()
 if payment_mode is None and preserve:
   current_trade_condition = shopping_cart.getSpecialiseValue()
   if current_trade_condition is not None:
@@ -16,7 +17,7 @@ if context.REQUEST.get("loyalty_reward", "") == "enable" and context.getSiteLoya
 
 if payment_mode:
   reference = '%s_%s' % (reference, payment_mode.lower())
-sale_trade_condition = context.portal_catalog.getResultValue(
+sale_trade_condition = portal.portal_catalog.getResultValue(
   portal_type='Sale Trade Condition',
   reference='%' + reference + '%',
   validation_state='published',
@@ -27,3 +28,6 @@ if sale_trade_condition:
   shopping_cart.setSpecialiseValue(sale_trade_condition.getObject())
 else:
   shopping_cart.setSpecialise(context.WebSection_getDefaultTradeCondition())
+
+
+portal.portal_sessions[container.REQUEST['session_id']].update(shopping_cart=shopping_cart)

bt5/erp5_advanced_ecommerce/SkinTemplateItem/portal_skins/erp5_advanced_ecommerce/WebSite_createWebSiteAccount.py

@@ -71,6 +71,7 @@ else:
 shopping_cart = context.SaleOrder_getShoppingCart()
 if shopping_cart is not None:
   shopping_cart.manage_setLocalRoles(user_id, ['Owner'])
+  portal.portal_sessions[container.REQUEST['session_id']].update(shopping_cart=shopping_cart)
 
 """
 response = context.REQUEST.RESPONSE

bt5/erp5_commerce/SkinTemplateItem/portal_skins/erp5_commerce/Resource_addToShoppingCart.py

@@ -43,12 +43,14 @@ for order_line in shopping_cart_items:
     line_found=True
     break
 
-if line_found == False:
+if not line_found:
   ## new Resource so add it to shopping cart
   order_line = shopping_cart.newContent(portal_type='Sale Order Line')
   order_line.setResource(resource.getRelativeUrl())
   order_line.setQuantity(quantity)
 
+context.getPortalObject().portal_sessions[request['session_id']].update(shopping_cart=shopping_cart)
+
 if( context.getPortalType() == 'Product'):
   context.Base_redirect('Resource_viewAsShop',
                       keep_items={'portal_status_message':context.Base_translateString("Added to cart.")})

bt5/erp5_commerce/SkinTemplateItem/portal_skins/erp5_commerce/SaleOrder_deleteShoppingCartItem.py

 """
   Delete a shopping cart item.
 """
-
-translateString = context.Base_translateString
+portal = context.getPortalObject()
+translateString = portal.Base_translateString
 shopping_cart = context.SaleOrder_getShoppingCart()
 
 if field_my_order_line_id is not None:
@@ -11,5 +11,6 @@ if field_my_order_line_id is not None:
 else:
   portal_status_message = "Please select an item."
 
+portal.portal_sessions[container.REQUEST['session_id']].update(shopping_cart=shopping_cart)
 context.Base_redirect(form_id, \
                       keep_items={'portal_status_message': translateString(portal_status_message, mapping={})})

bt5/erp5_commerce/TestTemplateItem/portal_components/test.erp5.testERP5Commerce.py

@@ -197,6 +197,8 @@ class TestCommerce(ERP5TypeTestCase):
     self.clearModule(self.portal.currency_module)
     self.clearModule(self.portal.sale_trade_condition_module)
     self.portal.portal_caches.clearAllCache()
+    self.portal.portal_sessions.manage_delObjects([SESSION_ID])
+    self.commit()
 
   def createDefaultOrganisation(self):
     """

bt5/erp5_officejs_support_request_ui_test/TestTemplateItem/portal_components/test.erp5.testSupportRequest.py

@@ -64,6 +64,8 @@ class SupportRequestTestCase(ERP5TypeTestCase, object):
     self.portal.person_module.manage_delObjects(
         [self.user.getId()])
     self.assertEqual(self.portal.ERP5Site_cleanupSupportRequestUITestDataSet(), 'Done.')
+    self.portal.portal_sessions.manage_delObjects(
+      ['support_request_module/erp5_officejs_support_request_ui_test_support_reuqest_001.latest_comment'])
     self.tic()
 
   def createUserAndLogin(self):

product/ERP5/bootstrap/erp5_core/PathTemplateItem/portal_caches/erp5_session_cache/1.xml → product/ERP5/bootstrap/erp5_core/PathTemplateItem/portal_caches/erp5_session_cache/2.xml

@@ -2,35 +2,21 @@
 <ZopeData>
   <record id="1" aka="AAAAAAAAAAE=">
     <pickle>
-      <global name="Ram Cache" module="erp5.portal_type"/>
+      <global name="Distributed Ram Cache" module="erp5.portal_type"/>
     </pickle>
     <pickle>
       <dictionary>
         <item>
-            <key> <string>_count</string> </key>
+            <key> <string>categories</string> </key>
             <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+              <tuple>
+                <string>specialise/portal_memcached/default_memcached_plugin</string>
+              </tuple>
             </value>
         </item>
-        <item>
-            <key> <string>_mt_index</string> </key>
-            <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
-            </value>
-        </item>
-        <item>
-            <key> <string>_tree</string> </key>
-            <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
-            </value>
-        </item>
-        <item>
-            <key> <string>cache_expire_check_interval</string> </key>
-            <value> <int>1800</int> </value>
-        </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>1</string> </value>
+            <value> <string>2</string> </value>
         </item>
         <item>
             <key> <string>int_index</string> </key>
@@ -38,35 +24,13 @@
         </item>
         <item>
             <key> <string>portal_type</string> </key>
-            <value> <string>Ram Cache</string> </value>
+            <value> <string>Distributed Ram Cache</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>RAM Cache</string> </value>
+            <value> <string>Distributed Ram Cache</string> </value>
         </item>
       </dictionary>
     </pickle>
   </record>
-  <record id="2" aka="AAAAAAAAAAI=">
-    <pickle>
-      <global name="Length" module="BTrees.Length"/>
-    </pickle>
-    <pickle> <int>0</int> </pickle>
-  </record>
-  <record id="3" aka="AAAAAAAAAAM=">
-    <pickle>
-      <global name="OOBTree" module="BTrees.OOBTree"/>
-    </pickle>
-    <pickle>
-      <none/>
-    </pickle>
-  </record>
-  <record id="4" aka="AAAAAAAAAAQ=">
-    <pickle>
-      <global name="OOBTree" module="BTrees.OOBTree"/>
-    </pickle>
-    <pickle>
-      <none/>
-    </pickle>
-  </record>
 </ZopeData>

product/ERP5/bootstrap/erp5_core/ToolComponentTemplateItem/portal_components/tool.erp5.SessionTool.py

@@ -60,31 +60,20 @@ class Session(UserDict):
   # used to set duration of session
   session_duration = None
 
-  def _updatecontext(self, aq_context):
-    """ Update current aquisition context.
-         This makes only sense for local RAM Session."""
-    pass
-
-  def _updateSessionDuration(self, session_duration):
-    self.session_duration = int(session_duration)
-
-  def _updateSessionId(self, session_id):
-    self.session_id = session_id
-
-  def __str__(self):
-    return self.__repr__()
-
-  def edit(self, **kw):
-    """ Edit session object. """
-    for key, item in kw.items():
-      self.__setitem__(key, item)
-
-class RamSession(Session):
-  """ Local RAM Session dictionary """
-
   # a handle to current aquisition context
   _aq_context = None
 
+  def __getstate__(self):
+    """filter out acqusition wrappers when serializing.
+    """
+    state = {
+        'session_duration': self.session_duration,
+        'data': {k: aq_base(v) for k, v in self.data.iteritems()}
+    }
+    if 'session_id' in self.__dict__:
+      state['session_id']  = self.session_id
+    return state
+
   def _updatecontext(self, aq_context):
     """ Update current aquisition context. """
     self._aq_context = aq_context
@@ -98,9 +87,29 @@ class RamSession(Session):
       return value
     raise KeyError(key)
 
+  def _updateSessionDuration(self, session_duration):
+    self.session_duration = int(session_duration)
+
+  def _updateSessionId(self, session_id):
+    self.session_id = session_id
+
+  def __str__(self):
+    return self.__repr__()
+
+  def edit(self, **kw):
+    """ Edit session object. """
+    for key, item in kw.items():
+      self.__setitem__(key, item)
+
   def __setitem__(self, key, item):
     # save value without its acquisition context
-    Session.__setitem__(self, key, aq_base(item))
+    UserDict.__setitem__(self, key, aq_base(item))
+
+  def update(self, dict=None, **kwargs):  # pylint: disable=redefined-builtin
+    for k, v in (dict or kwargs).iteritems():
+      # make sure to use our __setitem__ which removes acquistion wrappers
+      self[k] = v
+
 
 class DistributedSession(Session):
   """ Distributed Session dictionary.
@@ -111,6 +120,7 @@ class DistributedSession(Session):
 
   def _updateStorage(self):
     """ Update backend storage. """
+    assert self.session_id
     storage_plugin.set(self.session_id, \
                        SESSION_SCOPE, \
                        value = self, \
@@ -174,13 +184,28 @@ class SessionTool(BaseTool):
       shopping_cart = session['shopping_cart']
 
       Please note that:
-       - developer is responsible for handling an unique sessiond_id (using cookies for example).
-       - it's not recommended to store in portal_sessions ZODB persistent objects because in order
-       to store them in Local RAM portal_sessions tool will remove aquisition wrapper. At "get"
-       request they'll be returend wrapped.
-       - developer can store temporary RAM based objects like 'TempOrder' but ONLY
-       when using Local RAM type of sessions. In a distributed environment one can use only
-       pickable types ue to the nature of memcached server.
+        - developer is responsible for handling an unique sessiond_id (using cookies for example).
+        - it's not recommended to store in portal_sessions ZODB persistent objects because in order
+      to store them in Local RAM portal_sessions tool will remove aquisition wrapper. At "get"
+      request they'll be returend wrapped.
+        - developer can store temporary ERP5 documents like 'TempOrder', but keep
+      in mind that after making changes to temporary documents they need to be
+      saved again in portal_sessions, so:
+
+      >>> shopping_cart = context.newContent(portal_type='Order', temp_object=True, id='987654321')
+      >>> shopping_cart.newContent(portal_type='Order Line', quantity=1, resource=...)
+      >>> session['shopping_cart'] = shopping_cart
+
+      # modifying a temp document from session is valid
+      >>> shopping_cart.getMovementList()[0].setQuantity(3)
+      # but the session still reference the documents as it was when saved to session:
+      >>> session['shopping_cart'].getMovementList()[0].getQuantity()
+      1
+      # to make the change persist in the session, the temp document has to be saved again:
+      >>> session['shopping_cart'] = shopping_cart
+      >>> session['shopping_cart'].getMovementList()[0].getQuantity()
+      3
+
       """
 
   id = 'portal_sessions'
@@ -206,7 +231,7 @@ class SessionTool(BaseTool):
       # init it in cache and use different Session types based on cache plugin type used as a storage
       storage_plugin_type = storage_plugin_.__class__.__name__
       if storage_plugin_type in ("RamCache",):
-        session = RamSession()
+        session = Session()
       elif storage_plugin_type in ("DistributedRamCache",):
         session = DistributedSession()
         session._updateSessionId(session_id)

product/ERP5Type/dynamic/portal_type_class.py

@@ -383,9 +383,8 @@ def loadTempPortalTypeClass(portal_type_name):
   """
   import erp5.portal_type
   klass = getattr(erp5.portal_type, portal_type_name)
+  return type(portal_type_name, (TemporaryDocumentMixin, klass), {})
 
-  return type("Temporary " + portal_type_name,
-              (TemporaryDocumentMixin, klass), {})
 
 last_sync = -1
 _bootstrapped = set()

product/ERP5Type/mixin/temporary.py

@@ -35,6 +35,7 @@ from Acquisition import aq_base
 from Products.ERP5Type.Accessor.Constant import PropertyGetter as \
   PropertyConstantGetter
 
+
 class TemporaryDocumentMixin(object):
   """
   Setters and attributes that are attached to temporary documents.
@@ -44,8 +45,10 @@ class TemporaryDocumentMixin(object):
   __roles__ = None
 
   def __getstate__(self):
-    # disallow persistent storage
-    raise PicklingError("Temporary objects can't be pickled")
+    if getattr(self, '_p_jar', None) is not None:
+      # disallow persistent storage
+      raise PicklingError("Temporary objects can't be pickled")
+    return super(TemporaryDocumentMixin, self).__getstate__()
 
   def reindexObject(self, *args, **kw):
     pass

product/ERP5Type/tests/testERP5Type.py

@@ -31,8 +31,10 @@ other Tests are in erp5_core_test:testERP5Type) which is deprecated in favor
 of Portal Type as Classes and ZODB Components
 """
 
+import pickle
 import unittest
 import warnings
+from Acquisition import aq_base
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from AccessControl.ZopeGuards import guarded_import
 from Products.ERP5Type.tests.utils import LogInterceptor
@@ -244,6 +246,19 @@ class TestERP5Type(ERP5TypeTestCase, LogInterceptor):
       self.assertTrue(o.isTempObject())
       self.assertTrue(guarded_import("Products.ERP5Type.Document", fromlist=["newTempBase"]))
 
+    def test_TempObjectPersistent(self):
+      # Temp objects can not be stored in ZODB
+      temp_object = self.portal.person_module.newContent(portal_type='Person', temp_object=True)
+      self.assertTrue(temp_object.isTempObject())
+
+      # they can be pickled
+      self.assertTrue(pickle.dumps(aq_base(temp_object)))
+
+      # but they can not be saved in ZODB accidentally
+      self.portal.person_module.oops = temp_object
+      self.assertRaisesRegexp(Exception, "Temporary objects can't be pickled", self.commit)
+      self.abort()
+
     def test_warnings_redirected_to_event_log(self):
       self._catch_log_errors()
       self.addCleanup(self._ignore_log_errors)