MDEV-4252 geometry query crashes server.

      The bug was found by Alyssa Milburn.
      If the number of points of a geometry feature read from
      binary representation is greater than 0x10000000, then
      the (uint32) (num_points * 16) will cut the higher byte,
      which leads to various errors.
      Fixed by additional check if (num_points > max_n_points).

mysql-test/r/gis.result

@@ -1087,4 +1087,7 @@ NULL
 #
 SELECT GEOMETRYCOLLECTION((SELECT @@OLD));
 ERROR 22007: Illegal non geometric '' value found during parsing
+select astext(0x0100000000030000000100000000000010);
+astext(0x0100000000030000000100000000000010)
+NULL
 End of 5.1 tests

mysql-test/t/gis.test

@@ -826,5 +826,6 @@ SELECT ISCLOSED(CONVERT(CONCAT('     ', 0x2), BINARY(20)));
 --error ER_ILLEGAL_VALUE_FOR_TYPE
 SELECT GEOMETRYCOLLECTION((SELECT @@OLD));
 
+select astext(0x0100000000030000000100000000000010);
 
 --echo End of 5.1 tests

sql/spatial.cc

@@ -556,7 +556,7 @@ bool Gis_line_string::get_data_as_wkt(String *txt, const char **end) const
   n_points= uint4korr(data);
   data += 4;
 
-  if (n_points < 1 ||
+  if (n_points < 1 || n_points > max_n_points ||
       no_data(data, SIZEOF_STORED_DOUBLE * 2 * n_points) ||
       txt->reserve(((MAX_DIGITS_IN_DOUBLE + 1)*2 + 1) * n_points))
     return 1;
@@ -594,7 +594,8 @@ int Gis_line_string::geom_length(double *len) const
     return 1;
   n_points= uint4korr(data);
   data+= 4;
-  if (n_points < 1 || no_data(data, SIZEOF_STORED_DOUBLE * 2 * n_points))
+  if (n_points < 1 || n_points > max_n_points ||
+      no_data(data, SIZEOF_STORED_DOUBLE * 2 * n_points))
     return 1;
 
   get_point(&prev_x, &prev_y, data);
@@ -628,7 +629,7 @@ int Gis_line_string::is_closed(int *closed) const
     return 0;
   }
   data+= 4;
-  if (n_points == 0 ||
+  if (n_points == 0 || n_points > max_n_points ||
       no_data(data, SIZEOF_STORED_DOUBLE * 2 * n_points))
     return 1;
 
@@ -798,7 +799,8 @@ bool Gis_polygon::get_data_as_wkt(String *txt, const char **end) const
       return 1;
     n_points= uint4korr(data);
     data+= 4;
-    if (no_data(data, (SIZEOF_STORED_DOUBLE*2) * n_points) ||
+    if (n_points > max_n_points ||
+        no_data(data, (SIZEOF_STORED_DOUBLE*2) * n_points) ||
 	txt->reserve(2 + ((MAX_DIGITS_IN_DOUBLE + 1) * 2 + 1) * n_points))
       return 1;
     txt->qs_append('(');
@@ -852,7 +854,8 @@ int Gis_polygon::area(double *ar, const char **end_of_data) const
     if (no_data(data, 4))
       return 1;
     n_points= uint4korr(data);
-    if (no_data(data, (SIZEOF_STORED_DOUBLE*2) * n_points))
+    if (n_points > max_n_points ||
+        no_data(data, (SIZEOF_STORED_DOUBLE*2) * n_points))
       return 1;
     get_point(&prev_x, &prev_y, data+4);
     data+= (4+SIZEOF_STORED_DOUBLE*2);
@@ -888,7 +891,8 @@ int Gis_polygon::exterior_ring(String *result) const
   n_points= uint4korr(data);
   data+= 4;
   length= n_points * POINT_DATA_SIZE;
-  if (no_data(data, length) || result->reserve(1+4+4+ length))
+  if (n_points > max_n_points ||
+      no_data(data, length) || result->reserve(1+4+4+ length))
     return 1;
 
   result->q_append((char) wkb_ndr);
@@ -973,7 +977,8 @@ int Gis_polygon::centroid_xy(double *x, double *y) const
       return 1;
     org_n_points= n_points= uint4korr(data);
     data+= 4;
-    if (no_data(data, (SIZEOF_STORED_DOUBLE*2) * n_points))
+    if (n_points > max_n_points ||
+        no_data(data, (SIZEOF_STORED_DOUBLE*2) * n_points))
       return 1;
     get_point(&prev_x, &prev_y, data);
     data+= (SIZEOF_STORED_DOUBLE*2);
@@ -1260,7 +1265,8 @@ bool Gis_multi_line_string::get_data_as_wkt(String *txt,
       return 1;
     n_points= uint4korr(data + WKB_HEADER_SIZE);
     data+= WKB_HEADER_SIZE + 4;
-    if (no_data(data, n_points * (SIZEOF_STORED_DOUBLE*2)) ||
+    if (n_points > max_n_points ||
+        no_data(data, n_points * (SIZEOF_STORED_DOUBLE*2)) ||
 	txt->reserve(2 + ((MAX_DIGITS_IN_DOUBLE + 1) * 2 + 1) * n_points))
       return 1;
     txt->qs_append('(');
@@ -1521,7 +1527,8 @@ bool Gis_multi_polygon::get_data_as_wkt(String *txt, const char **end) const
         return 1;
       uint32 n_points= uint4korr(data);
       data+= 4;
-      if (no_data(data, (SIZEOF_STORED_DOUBLE * 2) * n_points) ||
+      if (n_points > max_n_points ||
+          no_data(data, (SIZEOF_STORED_DOUBLE * 2) * n_points) ||
 	  txt->reserve(2 + ((MAX_DIGITS_IN_DOUBLE + 1) * 2 + 1) * n_points,
 		       512))
 	return 1;
@@ -1604,6 +1611,8 @@ int Gis_multi_polygon::geometry_n(uint32 num, String *result) const
       if (no_data(data, 4))
 	return 1;
       n_points= uint4korr(data);
+      if (n_points > max_n_points)
+        return 1;
       data+= 4 + POINT_DATA_SIZE * n_points;
     }
   } while (--num);

sql/spatial.h

@@ -196,6 +196,11 @@ struct Geometry_buffer;
 
 class Geometry
 {
+public:
+  // Maximum number of points in feature that can fit into String
+  static const uint32 max_n_points=
+    (uint32) (UINT_MAX32 - WKB_HEADER_SIZE - 4 /* n_points */) /
+    POINT_DATA_SIZE;
 public:
   Geometry() {}                               /* Remove gcc warning */
   virtual ~Geometry() {}                        /* Remove gcc warning */
@@ -379,10 +384,6 @@ class Gis_point: public Geometry
 
 class Gis_line_string: public Geometry
 {
-  // Maximum number of points in LineString that can fit into String
-  static const uint32 max_n_points=
-    (uint32) (UINT_MAX32 - WKB_HEADER_SIZE - 4 /* n_points */) /
-    POINT_DATA_SIZE;
 public:
   Gis_line_string() {}                        /* Remove gcc warning */
   virtual ~Gis_line_string() {}               /* Remove gcc warning */