slapos_slap_tool: reduce number of sql queries

No need to query the catalog multiple times for every partitions

master/bt5/slapos_slap_tool/ToolComponentTemplateItem/portal_components/tool.erp5.SlapTool.py

@@ -181,12 +181,16 @@ class SlapTool(BaseTool):
     slap_computer._computer_partition_list = []
     slap_computer._software_release_list = \
        self._getSoftwareReleaseValueListForComputer(computer_id)
-    for computer_partition in self.getPortalObject().portal_catalog.unrestrictedSearchResults(
-                    parent_uid=parent_uid,
-                    validation_state="validated",
-                    portal_type="Computer Partition"):
-      slap_computer._computer_partition_list.append(
-          self._getSlapPartitionByPackingList(_assertACI(computer_partition.getObject())))
+
+    unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
+
+    computer_partition_list = unrestrictedSearchResults(
+      parent_uid=parent_uid,
+      validation_state="validated",
+      portal_type="Computer Partition"
+    )
+    self._calculateSlapComputerInformation(slap_computer, computer_partition_list)
+
     return dumps(slap_computer)
 
   def _fillComputerInformationCache(self, computer_id, user):
@@ -235,6 +239,40 @@ class SlapTool(BaseTool):
       self.activate(activity='SQLQueue', tag=tag)._fillComputerInformationCache(
         computer_id, user)
 
+  def _calculateSlapComputerInformation(self, slap_computer, computer_partition_list):
+    if len(computer_partition_list) == 0:
+      return
+
+    unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
+
+    computer_partition_uid_list = [x.uid for x in computer_partition_list]
+    grouped_software_instance_list = unrestrictedSearchResults(
+      portal_type="Software Instance",
+      default_aggregate_uid=computer_partition_uid_list,
+      validation_state="validated",
+      group_by_list=['default_aggregate_uid'],
+      select_list=['default_aggregate_uid', 'count(*)']
+    )
+    slave_software_instance_list = unrestrictedSearchResults(
+      default_aggregate_uid=computer_partition_uid_list,
+      portal_type='Slave Instance',
+      validation_state="validated",
+      select_list=['default_aggregate_uid'],
+      **{"slapos_item.slap_state": "start_requested"}
+    )
+
+    for computer_partition in computer_partition_list:
+      software_instance_list = [x for x in grouped_software_instance_list if (x.default_aggregate_uid == computer_partition.getUid())]
+      if (len(software_instance_list) == 1) and (software_instance_list[0]['count(*)'] > 1):
+        software_instance_list = software_instance_list + software_instance_list
+      slap_computer._computer_partition_list.append(
+        self._getSlapPartitionByPackingList(
+          _assertACI(computer_partition.getObject()),
+          software_instance_list,
+          [x for x in slave_software_instance_list if (x.default_aggregate_uid == computer_partition.getUid())]
+        )
+      )
+
   def _getComputerInformation(self, computer_id, user):
     user_document = _assertACI(self.getPortalObject().portal_catalog.unrestrictedGetResultValue(
       reference=user, portal_type=['Person', 'Computer', 'Software Instance']))
@@ -277,9 +315,8 @@ class SlapTool(BaseTool):
                     parent_uid=parent_uid,
                     validation_state="validated",
                     portal_type="Computer Partition")
-    for computer_partition in computer_partition_list:
-      slap_computer._computer_partition_list.append(
-          self._getSlapPartitionByPackingList(_assertACI(computer_partition.getObject())))
+
+    self._calculateSlapComputerInformation(slap_computer, computer_partition_list)
     return dumps(slap_computer)
 
   @UnrestrictedMethod
@@ -918,9 +955,10 @@ class SlapTool(BaseTool):
       LOG('SlapTool', INFO, 'Issue during parsing xml:', error=True)
     return result_dict
 
-  def _getSlapPartitionByPackingList(self, computer_partition_document):
+  def _getSlapPartitionByPackingList(self, computer_partition_document,
+                                     software_instance_list,
+                                     slave_instance_sql_list):
     computer = computer_partition_document
-    portal = self.getPortalObject()
     while computer.getPortalType() != 'Computer':
       computer = computer.getParentValue()
     computer_id = computer.getReference().decode("UTF-8")
@@ -934,20 +972,13 @@ class SlapTool(BaseTool):
     software_instance = None
 
     if computer_partition_document.getSlapState() == 'busy':
-      software_instance_list = portal.portal_catalog.unrestrictedSearchResults(
-          portal_type="Software Instance",
-          default_aggregate_uid=computer_partition_document.getUid(),
-          validation_state="validated",
-          limit=2,
-          )
       software_instance_count = len(software_instance_list)
       if software_instance_count == 1:
         software_instance = _assertACI(software_instance_list[0].getObject())
       elif software_instance_count > 1:
         # XXX do not prevent the system to work if one partition is broken
-        raise NotImplementedError, "Too many instances %s linked to %s" % \
-          ([x.path for x in software_instance_list],
-           computer_partition_document.getRelativeUrl())
+        raise NotImplementedError, "Too many instances linked to %s" % \
+           computer_partition_document.getRelativeUrl()
 
     if software_instance is not None:
       state = software_instance.getSlapState()
@@ -965,7 +996,9 @@ class SlapTool(BaseTool):
       slap_partition._need_modification = 1
 
       parameter_dict = self._getSoftwareInstanceAsParameterDict(
-                                                       software_instance)
+        software_instance,
+        slave_instance_sql_list=slave_instance_sql_list
+      )
       # software instance has to define an xml parameter
       slap_partition._parameter_dict = self._instanceXmlToDict(
         parameter_dict.pop('xml'))
@@ -1515,7 +1548,7 @@ class SlapTool(BaseTool):
         return software_instance
 
   @UnrestrictedMethod
-  def _getSoftwareInstanceAsParameterDict(self, software_instance):
+  def _getSoftwareInstanceAsParameterDict(self, software_instance, slave_instance_sql_list=None):
     portal = software_instance.getPortalObject()
     computer_partition = software_instance.getAggregateValue(portal_type="Computer Partition")
     timestamp = int(computer_partition.getModificationDate())
@@ -1546,12 +1579,13 @@ class SlapTool(BaseTool):
     slave_instance_list = []
     if (software_instance.getPortalType() == "Software Instance"):
       append = slave_instance_list.append
-      slave_instance_sql_list = portal.portal_catalog.unrestrictedSearchResults(
-        default_aggregate_uid=computer_partition.getUid(),
-        portal_type='Slave Instance',
-        validation_state="validated",
-        **{"slapos_item.slap_state": "start_requested"}
-      )
+      if slave_instance_sql_list is None:
+        slave_instance_sql_list = portal.portal_catalog.unrestrictedSearchResults(
+          default_aggregate_uid=computer_partition.getUid(),
+          portal_type='Slave Instance',
+          validation_state="validated",
+          **{"slapos_item.slap_state": "start_requested"}
+        )
       for slave_instance in slave_instance_sql_list:
         slave_instance = _assertACI(slave_instance.getObject())
         # XXX Use catalog to filter more efficiently

master/bt5/slapos_slap_tool/ToolComponentTemplateItem/portal_components/tool.erp5.SlapTool.xml

@@ -6,6 +6,12 @@
     </pickle>
     <pickle>
       <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
         <item>
             <key> <string>default_reference</string> </key>
             <value> <string>SlapTool</string> </value>
@@ -49,13 +55,28 @@
         <item>
             <key> <string>workflow_history</string> </key>
             <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
             </value>
         </item>
       </dictionary>
     </pickle>
   </record>
   <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
     <pickle>
       <global name="PersistentMapping" module="Persistence.mapping"/>
     </pickle>
@@ -68,7 +89,7 @@
                 <item>
                     <key> <string>component_validation_workflow</string> </key>
                     <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
                     </value>
                 </item>
               </dictionary>
@@ -77,7 +98,7 @@
       </dictionary>
     </pickle>
   </record>
-  <record id="3" aka="AAAAAAAAAAM=">
+  <record id="4" aka="AAAAAAAAAAQ=">
     <pickle>
       <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
     </pickle>