MDEV-15853: Assertion `tab->filesort_result == 0' failed

The issue here is that the window function execution is not called for the correct join tab, when we have GROUP BY
where we create extra temporary tables then we need to call window function execution for the last join tab. For doing
so the current code does not take into account the JOIN::aggr_tables.
Fixed by introducing a new function JOIN::total_join_tab_cnt that takes in account the temporary tables also.

mysql-test/r/win.result

@@ -3299,3 +3299,18 @@ ROW_NUMBER() OVER()	i
 SELECT ROW_NUMBER() OVER(), i FROM t1 WHERE 0;
 ROW_NUMBER() OVER()	i
 DROP TABLE t1;
+#
+# MDEV-15853: Assertion `tab->filesort_result == 0' failed
+#
+CREATE TABLE t1 ( a1 int);
+insert into t1 values (1),(2),(3);
+CREATE TABLE t2 (b1 int, a1 int, a2 int);
+insert into t2 values (1,2,3),(2,3,4),(3,4,5);
+SELECT COUNT(DISTINCT t2.a2),
+rank() OVER (ORDER BY t2.b1)
+FROM t2 ,t1 GROUP BY t2.b1 ORDER BY t1.a1;
+COUNT(DISTINCT t2.a2)	rank() OVER (ORDER BY t2.b1)
+1	1
+1	2
+1	3
+DROP TABLE t1,t2;

mysql-test/t/win.test

@@ -2067,3 +2067,18 @@ SELECT DISTINCT ROW_NUMBER() OVER(), i FROM t1 WHERE 0;
 SELECT ROW_NUMBER() OVER(), i FROM t1 WHERE 0;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-15853: Assertion `tab->filesort_result == 0' failed
+--echo #
+
+CREATE TABLE t1 ( a1 int);
+insert into t1 values (1),(2),(3);
+
+CREATE TABLE t2 (b1 int, a1 int, a2 int);
+insert into t2 values (1,2,3),(2,3,4),(3,4,5);
+
+--sorted_result
+SELECT COUNT(DISTINCT t2.a2),
+      rank() OVER (ORDER BY t2.b1)
+FROM t2 ,t1 GROUP BY t2.b1 ORDER BY t1.a1;
+DROP TABLE t1,t2;

sql/sql_select.cc

@@ -2851,7 +2851,7 @@ bool JOIN::make_aggr_tables_info()
     - duplicate value removal
     Both of these operations are done after window function computation step.
   */
-  curr_tab= join_tab + exec_join_tab_cnt() + aggr_tables - 1;
+  curr_tab= join_tab + total_join_tab_cnt();
   if (select_lex->window_funcs.elements)
   {
     curr_tab->window_funcs_step= new Window_funcs_computation;

sql/sql_select.h

@@ -1503,6 +1503,15 @@ class JOIN :public Sql_alloc
   /* Number of tables actually joined at the top level */
   uint exec_join_tab_cnt() { return tables_list ? top_join_tab_count : 0; }
 
+  /*
+    Number of tables in the join which also includes the temporary tables
+    created for GROUP BY, DISTINCT , WINDOW FUNCTION etc.
+  */
+  uint total_join_tab_cnt()
+  {
+    return exec_join_tab_cnt() + aggr_tables - 1;
+  }
+
   int prepare(TABLE_LIST *tables, uint wind_num,
 	      COND *conds, uint og_num, ORDER *order, bool skip_order_by,
               ORDER *group, Item *having, ORDER *proc_param, SELECT_LEX *select,

sql/sql_window.cc

@@ -2754,7 +2754,7 @@ bool Window_func_runner::exec(THD *thd, TABLE *tbl, SORT_INFO *filesort_result)
 bool Window_funcs_sort::exec(JOIN *join)
 {
   THD *thd= join->thd;
-  JOIN_TAB *join_tab= join->join_tab + join->exec_join_tab_cnt();
+  JOIN_TAB *join_tab= join->join_tab + join->total_join_tab_cnt();
 
   /* Sort the table based on the most specific sorting criteria of
      the window functions. */