CMFActivity: better date ordering by using micro-precision

Originally, uids somehow sorted messages by date of insertion, in particular
for those that were created within the same second. But since random uids,
such messages became validated or processed in random order.

Note however that by default, messages created in the same transaction all have
exactly the same date, so commit a42da4de
("CMFActivity: Do not use offset for scanning messages to validate.")
forces us to keep the ordering on uids (in addition to priority/date).

Existing instances will upgrade automatically, using the already existing code
to upgrade tables in a generic way. You should see the following logs:

    INFO CMFActivity 'message_queue' table upgraded
    ALTER TABLE message_queue
      MODIFY COLUMN date datetime(6) NOT NULL AFTER uid,
      MODIFY COLUMN processing_date datetime(6) DEFAULT NULL AFTER processing
    INFO CMFActivity 'message_job' table upgraded
    ALTER TABLE message_job
      MODIFY COLUMN date datetime(6) NOT NULL AFTER uid,
      MODIFY COLUMN processing_date datetime(6) DEFAULT NULL AFTER processing
    INFO CMFActivity 'message' table upgraded
    ALTER TABLE message
      MODIFY COLUMN date datetime(6) NOT NULL AFTER uid,
      MODIFY COLUMN processing_date datetime(6) DEFAULT NULL AFTER processing


/reviewed-on nexedi/erp5!820

product/CMFActivity/Activity/SQLBase.py

@@ -75,6 +75,8 @@ _DequeueMessageException = Exception()
 def sqltest_dict():
   sqltest_dict = {}
   no_quote_type = int, float, long
+  def render_datetime(x):
+      return "%.4d-%.2d-%.2d %.2d:%.2d:%09.6f" % x.toZone('UTC').parts()[:6]
   def _(name, column=None, op="="):
     if column is None:
       column = name
@@ -83,18 +85,17 @@ def sqltest_dict():
       if isinstance(value, no_quote_type):
         return column_op + str(value)
       if isinstance(value, DateTime):
-        value = value.toZone('UTC').ISO()
+        value = render_datetime(value)
       if isinstance(value, basestring):
         return column_op + render_string(value)
       assert op == "=", value
       if value is None: # XXX: see comment in SQLBase._getMessageList
         return column + " IS NULL"
       for x in value:
-        if isinstance(x, no_quote_type):
-          render_string = str
-        elif isinstance(x, DateTime):
-          value = (x.toZone('UTC').ISO() for x in value)
-        return "%s IN (%s)" % (column, ', '.join(map(render_string, value)))
+        return "%s IN (%s)" % (column, ', '.join(map(
+          str if isinstance(x, no_quote_type) else
+          render_datetime if isinstance(x, DateTime) else
+          render_string, value)))
       return "0"
     sqltest_dict[name] = render
   _('active_process_uid')

product/CMFActivity/skins/activity/SQLBase_createMessageTable.zsql

@@ -10,13 +10,13 @@ class_file:
 <params>table</params>
 CREATE TABLE <dtml-var table> (
   `uid` BIGINT UNSIGNED NOT NULL,
-  `date` DATETIME NOT NULL,
+  `date` DATETIME(6) NOT NULL,
   `path` VARCHAR(255) NOT NULL,
   `active_process_uid` INT UNSIGNED NULL,
   `method_id` VARCHAR(255) NOT NULL,
   `processing_node` SMALLINT NOT NULL DEFAULT -1,
   `processing` TINYINT NOT NULL DEFAULT 0,
-  `processing_date` DATETIME,
+  `processing_date` DATETIME(6),
   `priority` TINYINT NOT NULL DEFAULT 0,
   `group_method_id` VARCHAR(255) NOT NULL DEFAULT '',
   `tag` VARCHAR(255) NOT NULL,

product/CMFActivity/skins/activity/SQLBase_getNow.zsql

@@ -8,4 +8,4 @@ class_name:
 class_file:
 </dtml-comment>
 <params></params>
-SELECT UTC_TIMESTAMP()
+SELECT UTC_TIMESTAMP(6)

product/CMFActivity/skins/activity/SQLBase_getPriority.zsql

@@ -13,6 +13,6 @@ SELECT `priority`, `date` FROM
   <dtml-var table>
 WHERE
   processing_node = 0
-  AND date <= UTC_TIMESTAMP()
+  AND date <= UTC_TIMESTAMP(6)
 ORDER BY priority, date
 LIMIT 1

product/CMFActivity/skins/activity/SQLBase_getReservableMessageList.zsql

@@ -19,7 +19,7 @@ FROM
   <dtml-var table>
 WHERE
   processing_node=0
-  AND date <= <dtml-sqlvar to_date type="datetime">
+  AND date <= <dtml-sqlvar to_date type="datetime(6)">
   <dtml-if expr="group_method_id is not None">
     AND group_method_id = <dtml-sqlvar group_method_id type="string">
   </dtml-if>

product/CMFActivity/skins/activity/SQLBase_processMessage.zsql

@@ -12,7 +12,7 @@ uid</params>
 UPDATE
   <dtml-var table>
 SET
-  processing_date = UTC_TIMESTAMP(),
+  processing_date = UTC_TIMESTAMP(6),
   processing = 1
 WHERE
   <dtml-sqltest uid type="int" multiple>

product/CMFActivity/skins/activity/SQLBase_reactivate.zsql

@@ -15,7 +15,7 @@ delay
 UPDATE
   <dtml-var table>
 SET
-  date = DATE_ADD(UTC_TIMESTAMP(), INTERVAL
+  date = DATE_ADD(UTC_TIMESTAMP(6), INTERVAL
     <dtml-sqlvar delay type="int"> SECOND)
 <dtml-if expr="retry is not None">
   , priority = priority + <dtml-sqlvar retry type="int">

product/CMFActivity/skins/activity/SQLBase_writeMessageList.zsql

@@ -29,7 +29,7 @@ VALUES
   <dtml-sqlvar expr="uid_list[loop_item]" type="int">,
   <dtml-sqlvar expr="path_list[loop_item]" type="string">,
   <dtml-sqlvar expr="active_process_uid_list[loop_item]" type="int" optional>,
-  <dtml-if expr="date_list is not None"><dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime"><dtml-else>UTC_TIMESTAMP()</dtml-if><dtml-else>UTC_TIMESTAMP()</dtml-if>,
+  <dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime(6)"><dtml-else>UTC_TIMESTAMP(6)</dtml-if>,
   <dtml-sqlvar expr="method_id_list[loop_item]" type="string">,
   <dtml-sqlvar expr="processing_node_list[loop_item]" type="int">,
   0,

product/CMFActivity/skins/activity/SQLJoblib_createMessageTable.zsql

@@ -10,13 +10,13 @@ class_file:
 <params></params>
 CREATE TABLE message_job (
   `uid` BIGINT UNSIGNED NOT NULL,
-  `date` DATETIME NOT NULL,
+  `date` DATETIME(6) NOT NULL,
   `path` VARCHAR(255) NOT NULL,
   `active_process_uid` INT UNSIGNED NULL,
   `method_id` VARCHAR(255) NOT NULL,
   `processing_node` SMALLINT NOT NULL DEFAULT -1,
   `processing` TINYINT NOT NULL DEFAULT 0,
-  `processing_date` DATETIME,
+  `processing_date` DATETIME(6),
   `priority` TINYINT NOT NULL DEFAULT 0,
   `group_method_id` VARCHAR(255) NOT NULL DEFAULT '',
   `tag` VARCHAR(255) NOT NULL,

product/CMFActivity/skins/activity/SQLJoblib_writeMessage.zsql

@@ -30,7 +30,7 @@ VALUES
   <dtml-sqlvar expr="uid_list[loop_item]" type="int">,
   <dtml-sqlvar expr="path_list[loop_item]" type="string">,
   <dtml-sqlvar expr="active_process_uid_list[loop_item]" type="int" optional>,
-  <dtml-if expr="date_list is not None"><dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime"><dtml-else>UTC_TIMESTAMP()</dtml-if><dtml-else>UTC_TIMESTAMP()</dtml-if>,
+  <dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime(6)"><dtml-else>UTC_TIMESTAMP(6)</dtml-if>,
   <dtml-sqlvar expr="method_id_list[loop_item]" type="string">,
   <dtml-sqlvar expr="processing_node_list[loop_item]" type="int">,
   0,