Added rowid_filter support to Aria

This includes:
- cleanup and optimization of filtering and pushdown engine code.
- Adjusted costs for rowid filters (based on extensive testing
  and profiling).

This made a small two changes to the handler_rowid_filter_is_active()
API:
- One should not call it with a zero pointer!
- One does not need to call handler_rowid_filter_is_active() for every
  row anymore. It is enough to check if filter is active by calling it
  call it during index_init() or when handler::rowid_filter_changed()
  is called

The changes was to avoid unnecessary function calls and checks if
pushdown conditions and rowid_filter is not used.

Updated costs for rowid_filter_lookup() to be closer to reality.
The old cost was based only on rowid_compare_cost. This is now
changed to take into account the overhead in checking the rowid.

Changed the Range_rowid_filter class to use DYNAMIC_ARRAY directly
instead of Dynamic_array<>. This was done to be able to use the new
append_dynamic() functions which gives a notable speed improvment
compared to the old code.  Removing the abstraction also makes
the code easier to understand.

The cost of filtering is now slightly lower than before, which
is reflected in some test cases that is now using rowid filters.

include/my_compare.h

@@ -154,6 +154,5 @@ typedef enum check_result {
 
 typedef check_result_t (*index_cond_func_t)(void *param);
 typedef check_result_t (*rowid_filter_func_t)(void *param);
-typedef int (*rowid_filter_is_active_func_t)(void *param);
 
 #endif /* _my_compare_h */

include/my_sys.h

@@ -353,6 +353,14 @@ typedef struct st_dynamic_array
   myf malloc_flags;
 } DYNAMIC_ARRAY;
 
+
+typedef struct st_dynamic_array_append
+{
+  DYNAMIC_ARRAY *array;
+  uchar *pos, *end;
+} DYNAMIC_ARRAY_APPEND;
+
+
 typedef struct st_my_tmpdir
 {
   DYNAMIC_ARRAY full_list;
@@ -856,6 +864,10 @@ extern void freeze_size(DYNAMIC_ARRAY *array);
 #define push_dynamic(A,B) insert_dynamic((A),(B))
 #define reset_dynamic(array) ((array)->elements= 0)
 #define sort_dynamic(A,cmp) my_qsort((A)->buffer, (A)->elements, (A)->size_of_element, (cmp))
+extern void init_append_dynamic(DYNAMIC_ARRAY_APPEND *append,
+                                DYNAMIC_ARRAY *array);
+extern my_bool append_dynamic(DYNAMIC_ARRAY_APPEND *append,
+                              const void * element);
 
 extern my_bool init_dynamic_string(DYNAMIC_STRING *str, const char *init_str,
 				   size_t init_alloc,size_t alloc_increment);

mysql-test/main/opt_trace.result

@@ -12239,16 +12239,16 @@ explain select * from t1 where a<10 and b between 10 and 50 and c < 10	{
                             "rowid_filter_index": "b",
                             "index_only_cost": 0.001515222,
                             "filter_startup_cost": 3.004222e-4,
-                            "find_key_and_filter_lookup_cost": 6.445451e-4,
+                            "find_key_and_filter_lookup_cost": 7.827422e-4,
                             "filter_selectivity": 0.021,
                             "original_rows": 9,
                             "new_rows": 0.189,
                             "original_access_cost": 0.011516778,
-                            "with_filter_access_cost": 0.0023698,
+                            "with_filter_access_cost": 0.002507997,
                             "original_found_rows_cost": 0.010001556,
                             "with_filter_found_rows_cost": 2.100327e-4,
                             "org_cost": 0.011804778,
-                            "filter_cost": 0.00267627,
+                            "filter_cost": 0.002814467,
                             "filter_used": true
                           },
                           "access_type": "range",
@@ -12256,7 +12256,7 @@ explain select * from t1 where a<10 and b between 10 and 50 and c < 10	{
                           "rows": 9,
                           "rows_after_filter": 0.189,
                           "rows_out": 0.017766,
-                          "cost": 0.00267627,
+                          "cost": 0.002814467,
                           "chosen": true
                         }
                       ],
@@ -12264,7 +12264,7 @@ explain select * from t1 where a<10 and b between 10 and 50 and c < 10	{
                         "type": "range",
                         "rows_read": 0.189,
                         "rows_out": 0.017766,
-                        "cost": 0.00267627,
+                        "cost": 0.002814467,
                         "uses_join_buffering": false,
                         "rowid_filter_index": "b"
                       }
@@ -12276,7 +12276,7 @@ explain select * from t1 where a<10 and b between 10 and 50 and c < 10	{
                 "plan_prefix": "",
                 "table": "t1",
                 "rows_for_plan": 0.017766,
-                "cost_for_plan": 0.00267627,
+                "cost_for_plan": 0.002814467,
                 "pushdown_cond_selectivity": 0.094,
                 "filtered": 0.1974,
                 "rows_out": 0.017766
@@ -12286,7 +12286,7 @@ explain select * from t1 where a<10 and b between 10 and 50 and c < 10	{
           {
             "best_join_order": ["t1"],
             "rows": 0.017766,
-            "cost": 0.00267627
+            "cost": 0.002814467
           },
           {
             "table": "t1",
@@ -12674,20 +12674,20 @@ explain format=json select * from three, t1 where t1.a=three.a and t1.b<5000 and
                                 "rowid_filter_index": "b",
                                 "index_only_cost": 0.092006157,
                                 "filter_startup_cost": 0.149564727,
-                                "find_key_and_filter_lookup_cost": 0.085742374,
+                                "find_key_and_filter_lookup_cost": 0.129350121,
                                 "filter_selectivity": 0.4312,
                                 "original_rows": 1000,
                                 "new_rows": 431.2,
                                 "original_access_cost": 1.203290157,
-                                "with_filter_access_cost": 0.656934192,
+                                "with_filter_access_cost": 0.700541939,
                                 "original_found_rows_cost": 1.111284,
                                 "with_filter_found_rows_cost": 0.479185661,
                                 "org_cost": 3.705870471,
-                                "filter_cost": 2.161762502,
+                                "filter_cost": 2.292585745,
                                 "filter_used": true
                               },
                               "rows": 431.2,
-                              "cost": 2.161762502,
+                              "cost": 2.292585745,
                               "chosen": true
                             },
                             {

mysql-test/main/opt_trace_index_merge.result

@@ -329,7 +329,7 @@ set optimizer_trace='enabled=on';
 #  3-way ROR-intersection
 explain select key1,key2,key3 from t1 where key1=100 and key2=100 and key3=100;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	ref|filter	key1,key2,key3	key1|key2	5|5	const	2243 (3%)	Using where; Using rowid filter
+1	SIMPLE	t1	index_merge	key1,key2,key3	key1,key2	5,5	NULL	77	Using intersect(key1,key2); Using where
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives'))
 [
@@ -423,22 +423,6 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives'))
         },
         "analyzing_index_merge_union": 
         []
-    },
-    {
-        "range_scan_alternatives": 
-        [
-            {
-                "index": "key2",
-                "ranges": 
-                ["(100) <= (key2) <= (100)"],
-                "rowid_ordered": true,
-                "using_mrr": false,
-                "index_only": true,
-                "rows": 2243,
-                "cost": 0.312832109,
-                "chosen": true
-            }
-        ]
     }
 ]
 select JSON_DETAILED(JSON_EXTRACT(trace, '$**.chosen_range_access_summary')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
@@ -473,19 +457,6 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.chosen_range_access_summary'))
         "rows_for_plan": 77,
         "cost_for_plan": 0.572490756,
         "chosen": true
-    },
-    {
-        "range_access_plan": 
-        {
-            "type": "range_scan",
-            "index": "key2",
-            "rows": 2243,
-            "ranges": 
-            ["(100) <= (key2) <= (100)"]
-        },
-        "rows_for_plan": 2243,
-        "cost_for_plan": 0.312832109,
-        "chosen": true
     }
 ]
 # ROR-union(ROR-intersection, ROR-range)

mysql-test/main/optimizer_costs.result

@@ -31,8 +31,8 @@ OPTIMIZER_DISK_READ_RATIO	0.020000
 OPTIMIZER_ROW_COPY_COST	0.060866
 OPTIMIZER_ROW_LOOKUP_COST	0.130839
 OPTIMIZER_ROW_NEXT_FIND_COST	0.045916
-OPTIMIZER_ROWID_COMPARE_COST	0.001000
-OPTIMIZER_ROWID_COPY_COST	0.001000
+OPTIMIZER_ROWID_COMPARE_COST	0.002653
+OPTIMIZER_ROWID_COPY_COST	0.002653
 ENGINE	default
 OPTIMIZER_DISK_READ_COST	10.240000
 OPTIMIZER_INDEX_BLOCK_COPY_COST	0.035600

mysql-test/main/range.result

@@ -1967,7 +1967,7 @@ select count(*) from t2 left join t1
 on (t1.key1 < 3 or t1.key1 between 920 and 930) and t1.key2 < 1000;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	64	
-1	SIMPLE	t1	range|filter	i1,i2	i1|i2	4|4	NULL	12 (89%)	Using where; Using join buffer (flat, BNL join); Using rowid filter
+1	SIMPLE	t1	range	i1,i2	i1	4	NULL	12	Using where; Using join buffer (flat, BNL join)
 select count(*) from t2 left join t1
 on (t1.key1 < 3 or t1.key1 between 920 and 930) and t1.key2 < 1000;
 count(*)

mysql-test/main/rowid_filter_aria.test

+#
+# Test rowid filters with Aria
+#
+
+SET SESSION DEFAULT_STORAGE_ENGINE='Aria';
+
+#set global aria.optimizer_rowid_compare_cost=0.00001;
+#set global aria.optimizer_rowid_copy_cost=0.00001;
+--source rowid_filter.test

mysql-test/suite/maria/icp.result

@@ -409,7 +409,7 @@ WHERE (pk BETWEEN 4 AND 5 OR pk < 2) AND c1 < 240
 ORDER BY c1
 LIMIT 1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	PRIMARY,k1	k1	5	NULL	4	Using index condition; Using where
+1	SIMPLE	t1	range|filter	PRIMARY,k1	k1|PRIMARY	5|4	NULL	4 (38%)	Using index condition; Using where; Using rowid filter
 DROP TABLE t1;
 #
 #

mysys/array.c

@@ -92,7 +92,7 @@ my_bool init_dynamic_array2(PSI_memory_key psi_key, DYNAMIC_ARRAY *array,
 my_bool insert_dynamic(DYNAMIC_ARRAY *array, const void * element)
 {
   void *buffer;
-  if (array->elements == array->max_element)
+  if (unlikely(array->elements == array->max_element))
   {						/* Call only when necessary */
     if (!(buffer=alloc_dynamic(array)))
       return TRUE;
@@ -102,7 +102,42 @@ my_bool insert_dynamic(DYNAMIC_ARRAY *array, const void * element)
     buffer=array->buffer+(array->elements * array->size_of_element);
     array->elements++;
   }
-  memcpy(buffer,element,(size_t) array->size_of_element);
+  memcpy(buffer, element, array->size_of_element);
+  return FALSE;
+}
+
+
+/* Fast version of appending to dynamic array */
+
+void init_append_dynamic(DYNAMIC_ARRAY_APPEND *append,
+                         DYNAMIC_ARRAY *array)
+{
+  append->array= array;
+  append->pos= array->buffer + array->elements * array->size_of_element;
+  append->end= array->buffer + array->max_element * array->size_of_element;
+}
+
+
+my_bool append_dynamic(DYNAMIC_ARRAY_APPEND *append,
+                       const void *element)
+{
+  DYNAMIC_ARRAY *array= append->array;
+  size_t size_of_element= array->size_of_element;
+  if (unlikely(append->pos == append->end))
+  {
+    void *buffer;
+    if (!(buffer=alloc_dynamic(array)))
+      return TRUE;
+    append->pos= (uchar*)buffer + size_of_element;
+    append->end= array->buffer + array->max_element * size_of_element;
+    memcpy(buffer, element, size_of_element);
+  }
+  else
+  {
+    array->elements++;
+    memcpy(append->pos, element, size_of_element);
+    append->pos+= size_of_element;
+  }
   return FALSE;
 }
 
@@ -281,7 +316,7 @@ my_bool allocate_dynamic(DYNAMIC_ARRAY *array, size_t max_elements)
 
 void get_dynamic(DYNAMIC_ARRAY *array, void *element, size_t idx)
 {
-  if (idx >= array->elements)
+  if (unlikely(idx >= array->elements))
   {
     DBUG_PRINT("warning",("To big array idx: %d, array size is %d",
                           idx,array->elements));
@@ -306,7 +341,7 @@ void delete_dynamic(DYNAMIC_ARRAY *array)
   /*
     Just mark as empty if we are using a static buffer
   */
-  if (!(array->malloc_flags & MY_INIT_BUFFER_USED) && array->buffer)
+  if (array->buffer && !(array->malloc_flags & MY_INIT_BUFFER_USED))
     my_free(array->buffer);
 
   array->buffer= 0;

sql/handler.cc

@@ -6964,8 +6964,7 @@ check_result_t handler_rowid_filter_check(void *h_arg)
 
 extern "C" int handler_rowid_filter_is_active(void *h_arg)
 {
-  if (!h_arg)
-    return false;
+  DBUG_ASSERT(h_arg);
   handler *h= (handler*) h_arg;
   return h->rowid_filter_is_active;
 }

sql/handler.h

@@ -4665,10 +4665,14 @@ class handler :public Sql_alloc
    DBUG_ASSERT(pushed_rowid_filter != NULL &&
                save_pushed_rowid_filter == NULL);
    save_pushed_rowid_filter= pushed_rowid_filter;
-   if (rowid_filter_is_active)
-     save_rowid_filter_is_active= rowid_filter_is_active;
+   save_rowid_filter_is_active= rowid_filter_is_active;
    pushed_rowid_filter= NULL;
-   rowid_filter_is_active= false;
+
+   if (rowid_filter_is_active)
+   {
+     rowid_filter_is_active= false;
+     rowid_filter_changed();
+   }
  }
 
  virtual void enable_pushed_rowid_filter()
@@ -4676,12 +4680,17 @@ class handler :public Sql_alloc
    DBUG_ASSERT(save_pushed_rowid_filter != NULL &&
                pushed_rowid_filter == NULL);
    pushed_rowid_filter= save_pushed_rowid_filter;
+   save_pushed_rowid_filter= NULL;
    if (save_rowid_filter_is_active)
+   {
      rowid_filter_is_active= true;
-   save_pushed_rowid_filter= NULL;
+     rowid_filter_changed();
+   }
  }
 
  virtual bool rowid_filter_push(Rowid_filter *rowid_filter) { return true; }
+ /* Signal that rowid filter may have been enabled / disabled */
+ virtual void rowid_filter_changed() {}
 
  /* Needed for partition / spider */
   virtual TABLE_LIST *get_next_global_for_child() { return NULL; }