MDEV-15563: Instantly change a column to NULL

Allow instant changes of columns in ROW_FORMAT=REDUNDANT
from NOT NULL to NULL.

Later, this may be implemented for ROW_FORMAT=COMPACT or DYNAMIC,
but in that case any indexes on the table must be rebuilt.

dict_table_t::prepare_instant(): Add some debug assertions,
and relax a debug assertion so that the number of fields is
allowed not to change.

dict_index_t::instant_add_field(): Relax a debug assertion,
allowing a column to change from NOT NULL to NULL.

dict_table_t::instant_column(): Add debug assertions.

instant_alter_column_possible(): Allow ALTER_COLUMN_NULLABLE
when applicable.

innodb_insert_sys_columns(): Add the parameter bool update=false
to run UPDATE instead of INSERT.

innobase_instant_add_col(): Remove; let the only caller invoke
innodb_insert_sys_columns() directly.

innobase_instant_try(): Update the SYS_COLUMNS record if the
column is changed. Only convert the table to the instant ALTER TABLE
format if necessary. For ALTER_COLUMN_NULLABLE in ROW_FORMAT=REDUNDANT,
there is no data format change.

mysql-test/suite/innodb/r/instant_alter.result

@@ -612,6 +612,9 @@ INSERT INTO t1 VALUES
 (1, 1, 'a', 1, 1, 'a', 'a', 'a', 1, 'a');
 ALTER TABLE t1 DROP COLUMN f1;
 DROP TABLE t1;
+CREATE TABLE t1 (f1 VARCHAR(1), f2 VARCHAR(2)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+ALTER TABLE t1 MODIFY f2 VARCHAR (8) FIRST;
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1170,6 +1173,9 @@ INSERT INTO t1 VALUES
 (1, 1, 'a', 1, 1, 'a', 'a', 'a', 1, 'a');
 ALTER TABLE t1 DROP COLUMN f1;
 DROP TABLE t1;
+CREATE TABLE t1 (f1 VARCHAR(1), f2 VARCHAR(2)) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+ALTER TABLE t1 MODIFY f2 VARCHAR (8) FIRST;
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1728,10 +1734,13 @@ INSERT INTO t1 VALUES
 (1, 1, 'a', 1, 1, 'a', 'a', 'a', 1, 'a');
 ALTER TABLE t1 DROP COLUMN f1;
 DROP TABLE t1;
+CREATE TABLE t1 (f1 VARCHAR(1), f2 VARCHAR(2)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+ALTER TABLE t1 MODIFY f2 VARCHAR (8) FIRST;
+DROP TABLE t1;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-117
+120
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;

mysql-test/suite/innodb/r/instant_alter_null.result

+create table t (a int NOT NULL) engine=innodb row_format=   compressed;
+alter table t modify a int NULL, algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE
+drop table t;
+create table t (a int NOT NULL) engine=innodb row_format=   dynamic;
+alter table t modify a int NULL, algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE
+drop table t;
+create table t (a int NOT NULL) engine=innodb row_format=   compact;
+alter table t modify a int NULL, algorithm=instant;
+ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE
+drop table t;
+create table t (
+id int primary key,
+a int NOT NULL default 0,
+b int NOT NULL default 0,
+c int NOT NULL default 0,
+index idx (a,b,c)
+) engine=innodb row_format=redundant;
+insert into t (id, a) values (0, NULL);
+ERROR 23000: Column 'a' cannot be null
+insert into t (id, b) values (0, NULL);
+ERROR 23000: Column 'b' cannot be null
+insert into t (id, c) values (0, NULL);
+ERROR 23000: Column 'c' cannot be null
+insert into t values (1,1,1,1);
+set @id = (select table_id from information_schema.innodb_sys_tables
+where name = 'test/t');
+select * from information_schema.innodb_sys_columns where table_id=@id;
+TABLE_ID	NAME	POS	MTYPE	PRTYPE	LEN
+TABLE_ID	id	0	6	1283	4
+TABLE_ID	a	1	6	1283	4
+TABLE_ID	b	2	6	1283	4
+TABLE_ID	c	3	6	1283	4
+alter table t modify a int NULL, algorithm=instant;
+insert into t values (2, NULL, 2, 2);
+alter table t modify b int NULL, algorithm=nocopy;
+insert into t values (3, NULL, NULL, 3);
+alter table t modify c int NULL, algorithm=inplace;
+insert into t values (4, NULL, NULL, NULL);
+select * from information_schema.innodb_sys_columns where table_id=@id;
+TABLE_ID	NAME	POS	MTYPE	PRTYPE	LEN
+TABLE_ID	id	0	6	1283	4
+TABLE_ID	a	1	6	1027	4
+TABLE_ID	b	2	6	1027	4
+TABLE_ID	c	3	6	1027	4
+select * from t;
+id	a	b	c
+4	NULL	NULL	NULL
+3	NULL	NULL	3
+2	NULL	2	2
+1	1	1	1
+check table t;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+drop table t;

mysql-test/suite/innodb/t/instant_alter.test

@@ -486,6 +486,10 @@ INSERT INTO t1 VALUES
 ALTER TABLE t1 DROP COLUMN f1;
 DROP TABLE t1;
 
+eval CREATE TABLE t1 (f1 VARCHAR(1), f2 VARCHAR(2)) $engine;
+ALTER TABLE t1 MODIFY f2 VARCHAR (8) FIRST;
+DROP TABLE t1;
+
 dec $format;
 }
 disconnect analyze;

mysql-test/suite/innodb/t/instant_alter_null.test

+--source include/have_innodb.inc
+
+create table t (a int NOT NULL) engine=innodb row_format=   compressed;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+alter table t modify a int NULL, algorithm=instant;
+drop table t;
+
+create table t (a int NOT NULL) engine=innodb row_format=   dynamic;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+alter table t modify a int NULL, algorithm=instant;
+drop table t;
+
+create table t (a int NOT NULL) engine=innodb row_format=   compact;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+alter table t modify a int NULL, algorithm=instant;
+drop table t;
+
+create table t (
+  id int primary key,
+  a int NOT NULL default 0,
+  b int NOT NULL default 0,
+  c int NOT NULL default 0,
+  index idx (a,b,c)
+) engine=innodb row_format=redundant;
+
+--error ER_BAD_NULL_ERROR
+insert into t (id, a) values (0, NULL);
+--error ER_BAD_NULL_ERROR
+insert into t (id, b) values (0, NULL);
+--error ER_BAD_NULL_ERROR
+insert into t (id, c) values (0, NULL);
+
+insert into t values (1,1,1,1);
+
+set @id = (select table_id from information_schema.innodb_sys_tables
+where name = 'test/t');
+
+--replace_column 1 TABLE_ID
+select * from information_schema.innodb_sys_columns where table_id=@id;
+
+alter table t modify a int NULL, algorithm=instant;
+insert into t values (2, NULL, 2, 2);
+
+alter table t modify b int NULL, algorithm=nocopy;
+insert into t values (3, NULL, NULL, 3);
+
+alter table t modify c int NULL, algorithm=inplace;
+insert into t values (4, NULL, NULL, NULL);
+
+--replace_column 1 TABLE_ID
+select * from information_schema.innodb_sys_columns where table_id=@id;
+
+select * from t;
+
+check table t;
+
+drop table t;

storage/innobase/handler/handler0alter.cc

@@ -187,12 +187,11 @@ inline void dict_table_t::prepare_instant(const dict_table_t& old,
 
 	mtr_t mtr;
 	mtr.start();
-	/* Prevent oindex.n_core_fields and others, so that
+	/* Protect oindex.n_core_fields and others, so that
 	purge cannot invoke dict_index_t::clear_instant_alter(). */
 	instant_metadata_lock(oindex, mtr);
 
-	for (unsigned i = 0; i + DATA_N_SYS_COLS < old.n_cols;
-	     i++) {
+	for (unsigned i = 0; i + DATA_N_SYS_COLS < old.n_cols; i++) {
 		if (col_map[i] != i) {
 			first_alter_pos = 1 + i;
 			goto add_metadata;
@@ -201,8 +200,11 @@ inline void dict_table_t::prepare_instant(const dict_table_t& old,
 
 	if (!old.instant) {
 		/* Columns were not dropped or reordered.
-		Therefore columns must have been added at the end. */
-		DBUG_ASSERT(index.n_fields > oindex.n_fields);
+		Therefore columns must have been added at the end,
+		or modified instantly in place. */
+		DBUG_ASSERT(index.n_fields >= oindex.n_fields);
+		DBUG_ASSERT(index.n_fields > oindex.n_fields
+			    || !not_redundant());
 set_core_fields:
 		index.n_core_fields = oindex.n_core_fields;
 		index.n_core_null_bytes = oindex.n_core_null_bytes;
@@ -387,8 +389,12 @@ inline void dict_index_t::instant_add_field(const dict_index_t& instant)
 #ifndef DBUG_OFF
 	for (unsigned i = 0; i < n_fields; i++) {
 		DBUG_ASSERT(fields[i].same(instant.fields[i]));
+		/* Instant conversion from NULL to NOT NULL is not allowed. */
+		DBUG_ASSERT(!fields[i].col->is_nullable()
+			    || instant.fields[i].col->is_nullable());
 		DBUG_ASSERT(fields[i].col->is_nullable()
-			    == instant.fields[i].col->is_nullable());
+			    == instant.fields[i].col->is_nullable()
+			    || !table->not_redundant());
 	}
 #endif
 	n_fields = instant.n_fields;
@@ -457,6 +463,10 @@ inline void dict_table_t::instant_column(const dict_table_t& table,
 
 		if (const dict_col_t* o = find(old_cols, col_map, n_cols, i)) {
 			c.def_val = o->def_val;
+			DBUG_ASSERT(!((c.prtype ^ o->prtype)
+				      & ~(DATA_NOT_NULL | DATA_VERSIONED)));
+			DBUG_ASSERT(c.mtype == o->mtype);
+			DBUG_ASSERT(c.len >= o->len);
 			continue;
 		}
 
@@ -1374,11 +1384,26 @@ instant_alter_column_possible(
 		return false;
 	}
 
-	if (!(ha_alter_info->handler_flags
-	      & (ALTER_ADD_STORED_BASE_COLUMN
-		 | ALTER_DROP_STORED_COLUMN
-		 | ALTER_STORED_COLUMN_ORDER))) {
-		return false;
+	static constexpr alter_table_operations avoid_rebuild
+		= ALTER_ADD_STORED_BASE_COLUMN
+		| ALTER_DROP_STORED_COLUMN
+		| ALTER_STORED_COLUMN_ORDER
+		| ALTER_COLUMN_NULLABLE;
+
+	if (!(ha_alter_info->handler_flags & avoid_rebuild)) {
+		alter_table_operations flags = ha_alter_info->handler_flags
+			& ~avoid_rebuild;
+		/* None of the flags are set that we can handle
+		specially to avoid rebuild. In this case, we can
+		allow ALGORITHM=INSTANT, except if some requested
+		operation requires that the table be rebuilt. */
+		if (flags & INNOBASE_ALTER_REBUILD) {
+			return false;
+		}
+		if ((flags & ALTER_OPTIONS)
+		    && alter_options_need_rebuild(ha_alter_info, table)) {
+			return false;
+		}
 	}
 
 	/* At the moment, we disallow ADD [UNIQUE] INDEX together with
@@ -1402,12 +1427,24 @@ instant_alter_column_possible(
 	    & ((INNOBASE_ALTER_REBUILD | INNOBASE_ONLINE_CREATE)
 	       & ~ALTER_DROP_STORED_COLUMN
 	       & ~ALTER_STORED_COLUMN_ORDER
-	       & ~ALTER_ADD_STORED_BASE_COLUMN & ~ALTER_OPTIONS)) {
+	       & ~ALTER_ADD_STORED_BASE_COLUMN
+	       & ~ALTER_COLUMN_NULLABLE
+	       & ~ALTER_OPTIONS)) {
+		return false;
+	}
+
+	if ((ha_alter_info->handler_flags & ALTER_OPTIONS)
+	    && alter_options_need_rebuild(ha_alter_info, table)) {
 		return false;
 	}
 
-	return !(ha_alter_info->handler_flags & ALTER_OPTIONS)
-		|| !alter_options_need_rebuild(ha_alter_info, table);
+	if ((ha_alter_info->handler_flags
+	     & ALTER_COLUMN_NULLABLE)
+	    && ib_table.not_redundant()) {
+		return false;
+	}
+
+	return true;
 }
 
 /** Check whether the non-const default value for the field
@@ -2003,9 +2040,7 @@ ha_innobase::check_if_supported_inplace_alter(
 		af++;
 	}
 
-	if (supports_instant
-	    || !(ha_alter_info->handler_flags
-		 & ~(INNOBASE_ALTER_INSTANT | INNOBASE_INPLACE_IGNORE))) {
+	if (supports_instant) {
 		DBUG_RETURN(HA_ALTER_INPLACE_INSTANT);
 	}
 
@@ -4797,6 +4832,7 @@ static bool innobase_insert_sys_virtual(
 @param[in]	prtype		precise type
 @param[in]	len		fixed length in bytes, or 0
 @param[in]	n_base		number of base columns of virtual columns, or 0
+@param[in]	update		whether to update instead of inserting
 @retval	false	on success
 @retval	true	on failure (my_error() will have been called) */
 static bool innodb_insert_sys_columns(
@@ -4807,7 +4843,8 @@ static bool innodb_insert_sys_columns(
 	ulint		prtype,
 	ulint		len,
 	ulint		n_base,
-	trx_t*		trx)
+	trx_t*		trx,
+	bool		update = false)
 {
 	pars_info_t*	info = pars_info_create();
 	pars_info_add_ull_literal(info, "id", table_id);
@@ -4818,6 +4855,24 @@ static bool innodb_insert_sys_columns(
 	pars_info_add_int4_literal(info, "len", len);
 	pars_info_add_int4_literal(info, "base", n_base);
 
+	if (update) {
+		if (DB_SUCCESS != que_eval_sql(
+			    info,
+			    "PROCEDURE UPD_COL () IS\n"
+			    "BEGIN\n"
+			    "UPDATE SYS_COLUMNS SET\n"
+			    "NAME=:name, MTYPE=:mtype, PRTYPE=:prtype, "
+			    "LEN=:len, PREC=:base\n"
+			    "WHERE TABLE_ID=:id AND POS=:pos;\n"
+			    "END;\n", FALSE, trx)) {
+			my_error(ER_INTERNAL_ERROR, MYF(0),
+				 "InnoDB: Updating SYS_COLUMNS failed");
+			return true;
+		}
+
+		return false;
+	}
+
 	if (DB_SUCCESS != que_eval_sql(
 		    info,
 		    "PROCEDURE ADD_COL () IS\n"
@@ -4918,25 +4973,6 @@ innobase_add_virtual_try(
 	return false;
 }
 
-/** Add the newly added column in the sys_column system table.
-@param[in]	table_id	table id
-@param[in]	pos		position of the column
-@param[in]	field_name	field name
-@param[in]	type		data type
-@retval true Failure
-@retval false Success. */
-static bool innobase_instant_add_col(
-	table_id_t	table_id,
-	ulint		pos,
-	const char*	field_name,
-	const dtype_t&	type,
-	trx_t*		trx)
-{
-	return innodb_insert_sys_columns(table_id, pos, field_name,
-					 type.mtype, type.prtype, type.len, 0,
-					 trx);
-}
-
 /** Delete metadata from SYS_COLUMNS and SYS_VIRTUAL.
 @param[in]	id	table id
 @param[in]	pos	first SYS_COLUMNS.POS
@@ -5401,12 +5437,21 @@ static bool innobase_instant_try(
 		If it is NULL, the column was added by this ALTER TABLE. */
 		ut_ad(!new_field->field == !old);
 
-		if (old && (!ctx->first_alter_pos
-			    || i < ctx->first_alter_pos - 1)) {
+		bool update = old && (!ctx->first_alter_pos
+				      || i < ctx->first_alter_pos - 1);
+		DBUG_ASSERT(!old || !((old->prtype ^ col->prtype)
+				      & ~(DATA_NOT_NULL | DATA_VERSIONED)));
+		if (update
+		    && old->prtype == d->type.prtype) {
 			/* The record is already present in SYS_COLUMNS. */
-		} else if (innobase_instant_add_col(user_table->id, i,
-						    (*af)->field_name.str,
-						    d->type, trx)) {
+			DBUG_ASSERT(old->mtype == col->mtype);
+			DBUG_ASSERT(old->len == col->len);
+		} else if (innodb_insert_sys_columns(user_table->id, i,
+						     (*af)->field_name.str,
+						     d->type.mtype,
+						     d->type.prtype,
+						     d->type.len, 0, trx,
+						     update)) {
 			return true;
 		}
 
@@ -5485,7 +5530,7 @@ static bool innobase_instant_try(
 	que_thr_t* thr = pars_complete_graph_for_exec(
 		NULL, trx, ctx->heap, NULL);
 
-	dberr_t err;
+	dberr_t err = DB_SUCCESS;
 	if (rec_is_metadata(rec, *index)) {
 		ut_ad(page_rec_is_user_rec(rec));
 		if (!page_has_next(block->frame)
@@ -5572,12 +5617,13 @@ static bool innobase_instant_try(
 		/* MDEV-17383: free metadata BLOBs! */
 		btr_page_empty(block, NULL, index, 0, &mtr);
 		index->clear_instant_alter();
-		err = DB_SUCCESS;
+		goto func_exit;
+	} else if (!user_table->is_instant()) {
+		ut_ad(!user_table->not_redundant());
 		goto func_exit;
 	}
 
 	/* Convert the table to the instant ALTER TABLE format. */
-	ut_ad(user_table->is_instant());
 	mtr.commit();
 	mtr.start();
 	index->set_modified(mtr);