MDEV-12136 SELECT COUNT(DISTINCT) returns the wrong value when tmp_table_size is limited

use the correct value for the merge_buffer size, max_in_memory_size
is too small and merge_walk() fails.

also: remove a cast.

mysql-test/r/count_distinct.result

@@ -94,3 +94,11 @@ count(distinct i)
 2
 drop table t1;
 drop view v1;
+create table t1 (user_id char(64) character set utf8);
+insert t1 values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17);
+set @@tmp_table_size = 1024;
+select count(distinct user_id) from t1;
+count(distinct user_id)
+17
+drop table t1;
+set @@tmp_table_size = default;

mysql-test/t/count_distinct.test

@@ -107,3 +107,17 @@ create view v1 as select * from t1;
 select count(distinct i) from v1;
 drop table t1;
 drop view v1;
+
+#
+# MDEV-12136 SELECT COUNT(DISTINCT) returns the wrong value when tmp_table_size is limited
+#
+create table t1 (user_id char(64) character set utf8);
+insert t1 values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17);
+set @@tmp_table_size = 1024;
+select count(distinct user_id) from t1;
+drop table t1;
+set @@tmp_table_size = default;
+
+#
+# End of 5.5 tests
+#

sql/uniques.cc

@@ -461,7 +461,7 @@ C_MODE_END
     <> 0  error
 */
 
-static bool merge_walk(uchar *merge_buffer, ulong merge_buffer_size,
+static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size,
                        uint key_length, BUFFPEK *begin, BUFFPEK *end,
                        tree_walk_action walk_action, void *walk_action_arg,
                        qsort_cmp2 compare, void *compare_arg,
@@ -470,7 +470,7 @@ static bool merge_walk(uchar *merge_buffer, ulong merge_buffer_size,
   BUFFPEK_COMPARE_CONTEXT compare_context = { compare, compare_arg };
   QUEUE queue;
   if (end <= begin ||
-      merge_buffer_size < (ulong) (key_length * (end - begin + 1)) ||
+      merge_buffer_size < (size_t) (key_length * (end - begin + 1)) ||
       init_queue(&queue, (uint) (end - begin), offsetof(BUFFPEK, key), 0,
                  buffpek_compare, &compare_context, 0, 0))
     return 1;
@@ -615,7 +615,7 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg)
   
   if (!res)
   {  
-    res= merge_walk(merge_buffer, (ulong) max_in_memory_size, full_size,
+    res= merge_walk(merge_buffer, buff_sz, full_size,
                     (BUFFPEK *) file_ptrs.buffer,
                     (BUFFPEK *) file_ptrs.buffer + file_ptrs.elements,
                     action, walk_action_arg,