MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly

The problem was in calculating of the mask to clear unused null bits in case of using full byte.

mysql-test/r/row-checksum-old.result

@@ -85,3 +85,19 @@ checksum table t1 extended;
 Table	Checksum
 test.t1	4108368782
 drop table t1;
+#
+# MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly
+#
+CREATE TABLE t1 ( c1 int NOT NULL, c2 int NOT NULL, c4 varchar(20), c5 varchar(20), c6 varchar(20), c7 varchar(20), c8 varchar(20), c9 varchar(20), c10 varchar(20), c11 varchar(20), c12 varchar(20), c13 varchar(20), c14 varchar(20), c15 varchar(20), c16 varchar(20), c19 int NOT NULL, c20 int NOT NULL, c21 varchar(20), c22 VARCHAR(20), c23 varchar(20));
+insert into t1 values (5,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,"dog",NULL,NULL);
+# Important is that checksum is different from following
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2514025256
+UPDATE t1 SET c21='cat' WHERE  c1=5;
+# Important is that checksum is different from above
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2326430205
+drop table t1;
+# End of 5.5 tests

mysql-test/r/row-checksum.result

@@ -85,3 +85,19 @@ checksum table t1 extended;
 Table	Checksum
 test.t1	3885665021
 drop table t1;
+#
+# MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly
+#
+CREATE TABLE t1 ( c1 int NOT NULL, c2 int NOT NULL, c4 varchar(20), c5 varchar(20), c6 varchar(20), c7 varchar(20), c8 varchar(20), c9 varchar(20), c10 varchar(20), c11 varchar(20), c12 varchar(20), c13 varchar(20), c14 varchar(20), c15 varchar(20), c16 varchar(20), c19 int NOT NULL, c20 int NOT NULL, c21 varchar(20), c22 VARCHAR(20), c23 varchar(20));
+insert into t1 values (5,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,"dog",NULL,NULL);
+# Important is that checksum is different from following
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2514025256
+UPDATE t1 SET c21='cat' WHERE  c1=5;
+# Important is that checksum is different from above
+CHECKSUM TABLE t1 EXTENDED;
+Table	Checksum
+test.t1	2326430205
+drop table t1;
+# End of 5.5 tests

mysql-test/t/row-checksum.test

@@ -60,3 +60,20 @@ checksum table t1;
 checksum table t1 quick;
 checksum table t1 extended;
 drop table t1;
+
+--echo #
+--echo # MDEV-17085: CHECKSUM TABLE EXTENDED does not work correctly
+--echo #
+
+CREATE TABLE t1 ( c1 int NOT NULL, c2 int NOT NULL, c4 varchar(20), c5 varchar(20), c6 varchar(20), c7 varchar(20), c8 varchar(20), c9 varchar(20), c10 varchar(20), c11 varchar(20), c12 varchar(20), c13 varchar(20), c14 varchar(20), c15 varchar(20), c16 varchar(20), c19 int NOT NULL, c20 int NOT NULL, c21 varchar(20), c22 VARCHAR(20), c23 varchar(20));
+
+insert into t1 values (5,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,0,"dog",NULL,NULL);
+--echo # Important is that checksum is different from following
+CHECKSUM TABLE t1 EXTENDED;
+UPDATE t1 SET c21='cat' WHERE  c1=5;
+--echo # Important is that checksum is different from above
+CHECKSUM TABLE t1 EXTENDED;
+
+drop table t1;
+
+--echo # End of 5.5 tests

sql/sql_table.cc

@@ -7844,7 +7844,10 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables,
       {
 	/* calculating table's checksum */
 	ha_checksum crc= 0;
-        uchar null_mask=256 -  (1 << t->s->last_null_bit_pos);
+        DBUG_ASSERT(t->s->last_null_bit_pos < 8);
+        uchar null_mask= (t->s->last_null_bit_pos ?
+                          (256 -  (1 << t->s->last_null_bit_pos)):
+                          0);
 
         t->use_all_columns();