Commit 5d3ea021 authored by Vincent Pelletier's avatar Vincent Pelletier

Use REPLACE rather than INSERT for DELETE..INSERT indexation pattern.

When a document gets rows inserted in category table while there was none
before (typically: first document indexation), it may trigger
  IntegrityError: (1062, "Duplicate entry '...' for key 'PRIMARY'")
because in the DELETE..INSERT pattern, DELETE finds no matching rows and
does not gap-lock (because we enable innodb_locks_unsafe_for_binlog), then
the second INSERT happening, chronologically speaking, waits for the
transaction of the first to e committed, and on commit it causes such
duplicate key error.

A transient visible effect of this change is that if both competing
indexations see a different document state (because document got modified
in some 3rd transaction), the union of the resulting set of rows will be
visible until the reindexation which must have been triggered by the 3rd
transaction gets executed, at which point only the latest set will be
visible.

A similar issue exists before this change, with stricter conditions: it
needs the intersection of both sets to be empty, because a non-empty
intersection causes the duplicate key error solved here.

This change has been measured to improve scalability of an invoice building
test case (naturally triggering indexations) starting from ~12 activity
nodes:
   8 nodes:  +1.4% invoices/hour
  12 nodes:  +9.5% invoices/hour
  16 nodes: +12.3% invoices/hour
(values are the difference between DELETE..INSERT and DELETE..REPLACE)
parent 4ec66892
...@@ -82,7 +82,7 @@ WHERE\n ...@@ -82,7 +82,7 @@ WHERE\n
</dtml-if>\n </dtml-if>\n
</dtml-in>\n </dtml-in>\n
<dtml-if expr="_.len(movement_list) > 0">\n <dtml-if expr="_.len(movement_list) > 0">\n
INSERT INTO\n REPLACE INTO\n
item\n item\n
VALUES\n VALUES\n
<dtml-in prefix="loop" expr="movement_list">\n <dtml-in prefix="loop" expr="movement_list">\n
......
...@@ -66,7 +66,7 @@ DELETE FROM `measure` WHERE\n ...@@ -66,7 +66,7 @@ DELETE FROM `measure` WHERE\n
<dtml-if measure_list>\n <dtml-if measure_list>\n
<dtml-var sql_delimiter>\n <dtml-var sql_delimiter>\n
\n \n
INSERT INTO `measure`\n REPLACE INTO `measure`\n
VALUES\n VALUES\n
<dtml-in measure_list prefix="loop">\n <dtml-in measure_list prefix="loop">\n
(\n (\n
......
...@@ -72,7 +72,7 @@ WHERE\n ...@@ -72,7 +72,7 @@ WHERE\n
<dtml-call expr="category_list.append((uid[loop_item], uid_item[0], uid_item[1], uid_item[2]))">\n <dtml-call expr="category_list.append((uid[loop_item], uid_item[0], uid_item[1], uid_item[2]))">\n
</dtml-in></dtml-if></dtml-let></dtml-if></dtml-in>\n </dtml-in></dtml-if></dtml-let></dtml-if></dtml-in>\n
<dtml-if expr="category_list">\n <dtml-if expr="category_list">\n
INSERT INTO category VALUES\n REPLACE INTO category VALUES\n
<dtml-in prefix="loop" expr="category_list">\n <dtml-in prefix="loop" expr="category_list">\n
(<dtml-sqlvar expr="loop_item[0]" type="int">, <dtml-sqlvar expr="loop_item[1]" type="int">, <dtml-sqlvar expr="loop_item[2]" type="int">, <dtml-sqlvar expr="loop_item[3]" type="int">)<dtml-if sequence-end><dtml-else>,</dtml-if>\n (<dtml-sqlvar expr="loop_item[0]" type="int">, <dtml-sqlvar expr="loop_item[1]" type="int">, <dtml-sqlvar expr="loop_item[2]" type="int">, <dtml-sqlvar expr="loop_item[3]" type="int">)<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n </dtml-in>\n
......
...@@ -72,7 +72,7 @@ WHERE\n ...@@ -72,7 +72,7 @@ WHERE\n
<dtml-call expr="category_list.append((uid[loop_item], uid_item[0], uid_item[1], uid_item[2]))">\n <dtml-call expr="category_list.append((uid[loop_item], uid_item[0], uid_item[1], uid_item[2]))">\n
</dtml-in></dtml-if></dtml-let></dtml-if></dtml-in> \n </dtml-in></dtml-if></dtml-let></dtml-if></dtml-in> \n
<dtml-if expr="category_list">\n <dtml-if expr="category_list">\n
INSERT INTO category VALUES\n REPLACE INTO category VALUES\n
<dtml-in prefix="loop" expr="category_list">\n <dtml-in prefix="loop" expr="category_list">\n
(<dtml-sqlvar expr="loop_item[0]" type="int">, <dtml-sqlvar expr="loop_item[1]" type="int">, <dtml-sqlvar expr="loop_item[2]" type="int">, <dtml-sqlvar expr="loop_item[3]" type="int">)<dtml-if sequence-end><dtml-else>,</dtml-if>\n (<dtml-sqlvar expr="loop_item[0]" type="int">, <dtml-sqlvar expr="loop_item[1]" type="int">, <dtml-sqlvar expr="loop_item[2]" type="int">, <dtml-sqlvar expr="loop_item[3]" type="int">)<dtml-if sequence-end><dtml-else>,</dtml-if>\n
</dtml-in>\n </dtml-in>\n
......
...@@ -76,7 +76,7 @@ WHERE\n ...@@ -76,7 +76,7 @@ WHERE\n
</dtml-if>\n </dtml-if>\n
</dtml-in>\n </dtml-in>\n
<dtml-if expr="_.len(predicate_list) > 0">\n <dtml-if expr="_.len(predicate_list) > 0">\n
INSERT INTO predicate_category VALUES \n REPLACE INTO predicate_category VALUES \n
<dtml-in prefix="loop" expr="predicate_list">\n <dtml-in prefix="loop" expr="predicate_list">\n
<dtml-if sequence-start><dtml-else>,</dtml-if>\n <dtml-if sequence-start><dtml-else>,</dtml-if>\n
<dtml-if "predicate_property_dict[loop_item].has_key(\'membership_criterion_category_list\')">\n <dtml-if "predicate_property_dict[loop_item].has_key(\'membership_criterion_category_list\')">\n
......
...@@ -48,7 +48,7 @@ DELETE FROM `quantity_unit_conversion` WHERE\n ...@@ -48,7 +48,7 @@ DELETE FROM `quantity_unit_conversion` WHERE\n
<dtml-call "value_list.extend(loop_item.values())">\n <dtml-call "value_list.extend(loop_item.values())">\n
</dtml-in>\n </dtml-in>\n
\n \n
INSERT INTO `quantity_unit_conversion`\n REPLACE INTO `quantity_unit_conversion`\n
VALUES\n VALUES\n
<dtml-in "value_list" prefix="loop">\n <dtml-in "value_list" prefix="loop">\n
(\n (\n
......
...@@ -152,7 +152,7 @@ WHERE\n ...@@ -152,7 +152,7 @@ WHERE\n
</dtml-if>\n </dtml-if>\n
</dtml-in> \n </dtml-in> \n
<dtml-if "row_list">\n <dtml-if "row_list">\n
INSERT INTO\n REPLACE INTO\n
stock\n stock\n
VALUES\n VALUES\n
<dtml-in prefix="row" expr="row_list">\n <dtml-in prefix="row" expr="row_list">\n
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment