erp5_wendelin: Add Base_deleteZBigArray to manually GC ZBigArray

With the new extension method 'Base_deleteZBigArray' we can garbage
collect individual ZBigArray.

bt5/erp5_wendelin/ExtensionTemplateItem/portal_components/extension.erp5.Wendelin.py

@@ -4,6 +4,14 @@
 """
 from wendelin.bigarray.array_zodb import ZBigArray
 import numpy as np
+from ZODB.utils import u64
+try:  # duplication wrt neo/client/patch
+  from ZODB.Connection import TransactionMetaData
+except ImportError: # BBB: ZODB < 5
+  from ZODB.BaseStorage import TransactionRecord
+  TransactionMetaData = lambda user='', description='', extension=None: \
+      TransactionRecord(None, None, user, description, extension)
+
 
 def DataStream_copyCSVToDataArray(data_stream, chunk_list, start, end, \
                                   data_array_reference=None):
@@ -58,3 +66,32 @@ def DataStream_copyCSVToDataArray(data_stream, chunk_list, start, end, \
   zarray[-ndarray_shape[0]:] = ndarray
   
   return start, end
+
+
+def Base_deleteZBigArray(zbigarray):
+  """'Base_deleteZBigArray' explicitly garbage collects zbigarray.
+  In order to free storage space, the DB needs to be packed after GC.
+  """
+  conn = zbigarray._p_jar
+  storage = conn.db().storage
+
+  def rm(obj):
+    obj._p_activate()
+    storage.deleteObject(obj._p_oid, obj._p_serial, txn)
+    obj._p_deactivate()
+
+  txn = TransactionMetaData(description='Base_deleteZBigArray(0x%x)' % u64(zbigarray._p_oid))
+  try:
+    storage.tpc_begin(txn)
+    zfile = zbigarray.zfile
+    blktab = zfile.blktab
+    for b in blktab.values():
+      rm(b)
+    for obj in (blktab, zfile, zbigarray):
+      rm(obj)
+    storage.tpc_vote(txn)
+    storage.tpc_finish(txn)
+  except:
+    storage.tpc_abort(txn)
+    raise
+

bt5/erp5_wendelin/ExtensionTemplateItem/portal_components/extension.erp5.Wendelin.xml

@@ -6,12 +6,6 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>_recorded_property_dict</string> </key>
-            <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
-            </value>
-        </item>
         <item>
             <key> <string>default_reference</string> </key>
             <value> <string>Wendelin</string> </value>
@@ -55,28 +49,13 @@
         <item>
             <key> <string>workflow_history</string> </key>
             <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
             </value>
         </item>
       </dictionary>
     </pickle>
   </record>
   <record id="2" aka="AAAAAAAAAAI=">
-    <pickle>
-      <global name="PersistentMapping" module="Persistence.mapping"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>data</string> </key>
-            <value>
-              <dictionary/>
-            </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-  <record id="3" aka="AAAAAAAAAAM=">
     <pickle>
       <global name="PersistentMapping" module="Persistence.mapping"/>
     </pickle>
@@ -89,7 +68,7 @@
                 <item>
                     <key> <string>component_validation_workflow</string> </key>
                     <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
                     </value>
                 </item>
               </dictionary>
@@ -98,7 +77,7 @@
       </dictionary>
     </pickle>
   </record>
-  <record id="4" aka="AAAAAAAAAAQ=">
+  <record id="3" aka="AAAAAAAAAAM=">
     <pickle>
       <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
     </pickle>

bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/Base_deleteZBigArray.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>Base_deleteZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>Wendelin</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>Base_deleteZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>Base_deleteZBigArray</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.py

 ##############################################################################
 #
-# Copyright (c) 2002-2015 Nexedi SA and Contributors. All Rights Reserved.
+# Copyright (c) 2002-2024 Nexedi SA and Contributors. All Rights Reserved.
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -33,6 +33,9 @@ import textwrap
 import urllib
 import uuid
 from zExceptions import BadRequest
+from ZODB.POSException import POSKeyError
+
+from golang import defer, func
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.utils import createZODBPythonScript, removeZODBPythonScript
@@ -881,3 +884,46 @@ result = [x for x in data_bucket_stream.getBucketIndexKeySequenceByIndex()]
     self.assertRaises(ValueError,
                       portal.Base_wendelinTextToNumpy,
                       wendelin_text)
+
+  @func
+  def test_19_Base_deleteZBigArray(self):
+    """Test 'Base_deleteZBigArray' garbage collects ZBigArray & associated OIDs"""
+    conn = self.portal._p_jar
+    db = conn.db()
+
+    @func
+    def get(obj):
+      tempconn = db.open()
+      defer(tempconn.close)
+      return tempconn.get(obj._p_oid)
+
+    def assertInDB(obj):
+      self.assertNotEqual(None, get(obj))
+
+    def assertNotInDB(obj):
+      self.assertRaises(POSKeyError, get, obj)
+
+    root = conn.root()
+    arr = root.arr = ZBigArray(shape=(0, 1), dtype=int)
+    @defer
+    def _():
+      del root.arr
+      self.tic()
+    self.tic()
+
+    assertInDB(arr)
+    assertInDB(arr.zfile)
+    assertInDB(arr.zfile.blktab)
+
+    arr.append([[0]])
+    self.tic()
+
+    blk = arr.zfile.blktab[0]
+    assertInDB(blk)
+
+    self.portal.Base_deleteZBigArray(arr)
+
+    assertNotInDB(arr)
+    assertNotInDB(arr.zfile)
+    assertNotInDB(arr.zfile.blktab)
+    assertNotInDB(blk)