bug 839327 Crash in Gcalc_operation_reducer::end_couple with ST_UNION and MULTIPOLYGONs in 5.3-gis.

        When edges of a polygon coicide, it can form an pike, that is turned into a line after an operation.
        In this case a former polygon point can be an end of a single line, and that case wasn't properly handled.


per-file comments:
  mysql-test/r/gis-precise.result
bug 839327 Crash in Gcalc_operation_reducer::end_couple with ST_UNION and MULTIPOLYGONs in 5.3-gis.
        test result updated.

  mysql-test/t/gis-precise.test
bug 839327 Crash in Gcalc_operation_reducer::end_couple with ST_UNION and MULTIPOLYGONs in 5.3-gis.
        test case added.

  sql/gcalc_tools.cc
bug 839327 Crash in Gcalc_operation_reducer::end_couple with ST_UNION and MULTIPOLYGONs in 5.3-gis.
        in the scev_two_ends case check if we have single line ending on a polygon node.

mysql-test/r/gis-precise.result

@@ -328,3 +328,6 @@ SELECT ST_DISTANCE(POINTFROMTEXT('POINT(7 1)'),MULTILINESTRINGFROMTEXT('MULTILIN
 ST_DISTANCE(POINTFROMTEXT('POINT(7 1)'),MULTILINESTRINGFROMTEXT('MULTILINESTRING(
                 (4 7,9 7,6 1,3 4,1 1), (3 5, 2 5, 2 4, 3 4, 3 5))'))
 1
+SELECT AsText(ST_UNION(POLYGONFROMTEXT('POLYGON((12 9, 3 6, 3 0, 12 9))'), POLYGONFROMTEXT('POLYGON((2 2, 7 2, 4 2, 2 0, 2 2))')));
+AsText(ST_UNION(POLYGONFROMTEXT('POLYGON((12 9, 3 6, 3 0, 12 9))'), POLYGONFROMTEXT('POLYGON((2 2, 7 2, 4 2, 2 0, 2 2))')))
+GEOMETRYCOLLECTION(POLYGON((2 0,2 2,3 2,3 6,12 9,3 0,3 1,2 0)),LINESTRING(5 2,7 2))

mysql-test/t/gis-precise.test

@@ -211,3 +211,7 @@ SELECT AsText(ST_UNION(GEOMETRYFROMTEXT('POINT(8 1)') ,MULTILINESTRINGFROMTEXT('
 SELECT ST_DISTANCE(POINTFROMTEXT('POINT(7 1)'),MULTILINESTRINGFROMTEXT('MULTILINESTRING(
                 (4 7,9 7,6 1,3 4,1 1), (3 5, 2 5, 2 4, 3 4, 3 5))'));
 
+#bug 839327 Crash in Gcalc_operation_reducer::end_couple with ST_UNION and MULTIPOLYGONs in 5.3-gis
+SELECT AsText(ST_UNION(POLYGONFROMTEXT('POLYGON((12 9, 3 6, 3 0, 12 9))'), POLYGONFROMTEXT('POLYGON((2 2, 7 2, 4 2, 2 0, 2 2))')));
+
+

sql/gcalc_tools.cc

@@ -626,9 +626,19 @@ int Gcalc_operation_reducer::count_slice(Gcalc_scan_iterator *si)
       }
       case scev_two_ends:
       {
-        if (cur_t->enabled() &&
-            end_couple(cur_t, cur_t->get_next(), events->pi))
+        if (cur_t->enabled() && cur_t->get_next()->enabled())
+        {
+          /* When two threads are ended here */
+          if (end_couple(cur_t, cur_t->get_next(), events->pi))
+            return 1;
+        }
+        else if (cur_t->enabled() || cur_t->get_next()->enabled())
+        {
+          /* Rare case when edges of a polygon coincide */
+          if (end_line(cur_t->enabled() ? cur_t : cur_t->get_next(),
+                       events->pi, si))
             return 1;
+        }
         *cur_t_hook= cur_t->get_next()->get_next();
         free_item(cur_t->next);
         free_item(cur_t);