revert the fix for MDEV-5898, restore the fix for MDEV-5549.

simplify test case for MDEV-5898

mysql-test/r/select_found.result

@@ -291,11 +291,7 @@ select found_rows();
 found_rows()
 1
 drop table t1, t2;
-create table t1 (
-a1 int auto_increment primary key,
-b1  datetime,
-c1 int
-);
+create table t1 (a1 int auto_increment primary key, c1 int);
 insert t1 (a1) values (null);
 insert t1 (a1) select null from t1;
 insert t1 (a1) select null from t1;
@@ -307,19 +303,14 @@ insert t1 (a1) select null from t1;
 insert t1 (a1) select null from t1;
 insert t1 (a1) select null from t1;
 update t1 set c1=a1 % 2;
-create table t2 (
-a2 int,
-b2 int,
-c2  char(16) default '',
-primary key (a2, b2)
-);
+create table t2 (a2 int, b2 int, c2  char(16) default '', primary key (a2, b2));
 insert t2 select a1, 1, 'ok' from t1;
 insert t2 select a1, 2, 'ko' from t1;
 insert t2 select a1, 3, 'ko' from t1;
 insert t2 select a1, 4, 'ok' from t1;
 insert t2 select a1, 5, 'ok' from t1;
-select sql_calc_found_rows distinct a1,b1,c2 from t1 join t2 on a2=a1 
-where a1 <= 256 and c1=0 and c2='ok' order by b1 desc, a1 desc limit 46;
+select sql_calc_found_rows distinct a1,c2 from t1 join t2 on a2=a1 
+where a1 <= 256 and c1=0 and c2='ok' order by a1 desc limit 46;
 select found_rows();
 found_rows()
 128

mysql-test/t/select_found.test

@@ -208,11 +208,10 @@ select f1 from t1,t2 where f1=f3 and f2=3 order by f1;
 select found_rows();
 drop table t1, t2;
 
-create table t1 (
-  a1 int auto_increment primary key,
-  b1  datetime,
-  c1 int
-);
+#
+# MDEV-5898 FOUND_ROWS() return incorrect value when using DISTINCT
+#
+create table t1 (a1 int auto_increment primary key, c1 int);
 
 insert t1 (a1) values (null);
 insert t1 (a1) select null from t1;
@@ -226,12 +225,7 @@ insert t1 (a1) select null from t1;
 insert t1 (a1) select null from t1;
 update t1 set c1=a1 % 2;
 
-create table t2 (
-  a2 int,
-  b2 int,
-  c2  char(16) default '',
-  primary key (a2, b2)
-);
+create table t2 (a2 int, b2 int, c2  char(16) default '', primary key (a2, b2));
 
 insert t2 select a1, 1, 'ok' from t1;
 insert t2 select a1, 2, 'ko' from t1;
@@ -239,10 +233,9 @@ insert t2 select a1, 3, 'ko' from t1;
 insert t2 select a1, 4, 'ok' from t1;
 insert t2 select a1, 5, 'ok' from t1;
 
---disable_result_log
-select sql_calc_found_rows distinct a1,b1,c2 from t1 join t2 on a2=a1 
-  where a1 <= 256 and c1=0 and c2='ok' order by b1 desc, a1 desc limit 46;
---enable_result_log
+--disable_result_log ONCE
+select sql_calc_found_rows distinct a1,c2 from t1 join t2 on a2=a1 
+  where a1 <= 256 and c1=0 and c2='ok' order by a1 desc limit 46;
 
 select found_rows();
 

sql/filesort.cc

@@ -686,7 +686,7 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
   ref_pos= ref_buff;
   quick_select=select && select->quick;
   record=0;
-  *found_rows= 0;
+  *found_rows= pq ? 0 : HA_POS_ERROR; // don't count unless pq is used
   flag= ((file->ha_table_flags() & HA_REC_NOT_IN_SEQ) || quick_select);
   if (flag)
     ref_pos= &file->ref[0];
@@ -806,9 +806,14 @@ static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
 
     if (write_record)
     {
-      (*found_rows)++;
       if (pq)
       {
+        /*
+          only count rows when pq is used - otherwise there might be
+          other filters *after* the filesort, we don't know the final row
+          count here
+        */
+        (*found_rows)++;
         pq->push(ref_pos);
         idx= pq->num_elements();
       }

sql/sql_select.cc

@@ -3074,8 +3074,7 @@ void JOIN::exec_inner()
                                    Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF);
   error= do_select(curr_join, curr_fields_list, NULL, procedure);
   thd->limit_found_rows= curr_join->send_records;
-  if (curr_join->order && curr_join->sortorder &&
-      curr_join->select_options & OPTION_FOUND_ROWS)
+  if (curr_join->order && curr_join->filesort_found_rows)
   {
     /* Use info provided by filesort. */
     DBUG_ASSERT(curr_join->table_count > curr_join->const_tables);
@@ -18702,7 +18701,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
         records are read. Because of optimization in some cases it can
         provide only select_limit_cnt+1 records.
       */
-      if (join->order && join->sortorder &&
+      if (join->order && join->filesort_found_rows &&
           join->select_options & OPTION_FOUND_ROWS)
       {
         DBUG_PRINT("info", ("filesort NESTED_LOOP_QUERY_LIMIT"));
@@ -18724,6 +18723,7 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
 	  /* Join over all rows in table;  Return number of found rows */
 	  TABLE *table=jt->table;
 
+          join->select_options ^= OPTION_FOUND_ROWS;
 	  if (table->sort.record_pointers ||
 	      (table->sort.io_cache && my_b_inited(table->sort.io_cache)))
 	  {
@@ -20556,7 +20556,11 @@ create_sort_index(THD *thd, JOIN *join, ORDER *order,
                             select, filesort_limit, 0,
                             &examined_rows, &found_rows);
   table->sort.found_records= filesort_retval;
+  if (found_rows != HA_POS_ERROR)
+  {
     tab->records= found_rows;                     // For SQL_CALC_ROWS
+    join->filesort_found_rows= true;
+  }
 
   if (quick_created)
   {

sql/sql_select.h

@@ -1120,6 +1120,12 @@ public:
   */
   bool no_rows_in_result_called;
 
+  /**
+    This is set if SQL_CALC_ROWS was calculated by filesort()
+    and should be taken from the appropriate JOIN_TAB
+  */
+  bool filesort_found_rows;
+  
   /**
     Copy of this JOIN to be used with temporary tables.
 
@@ -1335,6 +1341,7 @@ public:
     emb_sjm_nest= NULL;
     sjm_lookup_tables= 0;
 
+    filesort_found_rows= false;
     exec_saved_explain= false;
     /* 
       The following is needed because JOIN::cleanup(true) may be called for