Include dropped columns in ctx->instant_table and dict_index_t

row_metadata_to_tuple(): Convert a metadata record to a data tuple,
based on the new info_bits of the metadata record.

btr_cur_pessimistic_update(): Invoke row_metadata_to_tuple() if needed.

dict_index_t::instant_add_field(): Append fields at the end, if any.
The fields for dropped columns are preserved.

dict_table_t::instant_column(): Renamed from instant_add_column().
Add the parameter col_map so that columns can be reordered.
FIXME: Also support ADD/DROP/reorder/rename of VIRTUAL COLUMN.

ha_innobase_inplace_ctx::prepare_instant(): Preserve any instantly
dropped columns in instant_table and its first index.
FIXME: Support instant ALTER TABLE even if hidden FTS_DOC_ID exists.

innobase_instant_try(): Simplify the logic for detecting dropped
or reordered columns. If any drop or reorder is needed, construct
an update vector with the metadata blob.

dtype_t::metadata_blob_init(): Initialize the metadata BLOB data type.

dict_table_t::find(): Find an old column based on a new column number.

rec_convert_dtuple_to_metadata_comp(): Convert an alter metadata tuple
into a record.

row_rec_to_index_entry_impl(): Add the template parameter mblob
and the optional parameter info_bits for specifying the desired new
info bits. For the metadata tuple, allow conversion between the original
format (ADD COLUMN only) and the generic format (with hidden BLOB).

row_upd_index_replace_metadata(): Apply an update vector to an
alter_metadata tuple.

mysql-test/suite/innodb/r/instant_alter.result

@@ -182,8 +182,8 @@ affected rows: 0
 info: Records: 0  Duplicates: 0  Warnings: 0
 INSERT INTO t1 SET id=9;
 ALTER TABLE t1 DROP c3;
-affected rows: 9
-info: Records: 9  Duplicates: 0  Warnings: 0
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -599,8 +599,8 @@ affected rows: 0
 info: Records: 0  Duplicates: 0  Warnings: 0
 INSERT INTO t1 SET id=9;
 ALTER TABLE t1 DROP c3;
-affected rows: 9
-info: Records: 9  Duplicates: 0  Warnings: 0
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -1016,8 +1016,8 @@ affected rows: 0
 info: Records: 0  Duplicates: 0  Warnings: 0
 INSERT INTO t1 SET id=9;
 ALTER TABLE t1 DROP c3;
-affected rows: 9
-info: Records: 9  Duplicates: 0  Warnings: 0
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -1310,5 +1310,5 @@ SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-54
+57
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;

storage/innobase/btr/btr0cur.cc

@@ -4726,8 +4726,14 @@ btr_cur_pessimistic_update(
 		rec, index, *offsets, page_is_leaf(page),
 		ULINT_UNDEFINED, offsets_heap);
 
-	dtuple_t*	new_entry = row_rec_to_index_entry(
-		rec, index, *offsets, &n_ext, entry_heap);
+	const bool is_metadata = rec_is_metadata(rec, index);
+	ut_ad(!is_metadata || (flags & BTR_NO_LOCKING_FLAG));
+
+	dtuple_t* new_entry = is_metadata
+		? row_metadata_to_tuple(rec, index, *offsets, &n_ext,
+					entry_heap, update->info_bits)
+		: row_rec_to_index_entry(rec, index, *offsets, &n_ext,
+					 entry_heap);
 
 	/* The page containing the clustered index record
 	corresponding to new_entry is latched in mtr.  If the
@@ -4739,9 +4745,6 @@ btr_cur_pessimistic_update(
 						     entry_heap);
 	btr_cur_trim(new_entry, index, update, thr);
 
-	const bool is_metadata = new_entry->info_bits
-		& REC_INFO_MIN_REC_FLAG;
-
 	/* We have to set appropriate extern storage bits in the new
 	record to be inserted: we have to remember which fields were such */
 
@@ -4769,10 +4772,10 @@ btr_cur_pessimistic_update(
 	}
 
 	if (page_zip_rec_needs_ext(
-			rec_get_converted_size(index, new_entry, n_ext),
-			page_is_comp(page),
-			dict_index_get_n_fields(index),
-			block->page.size)
+		    rec_get_converted_size(index, new_entry, n_ext),
+		    page_is_comp(page),
+		    dict_index_get_n_fields(index),
+		    block->page.size)
 	    || UNIV_UNLIKELY(rec_is_alter_metadata(rec, index))) {
 
 		big_rec_vec = dtuple_convert_big_rec(index, update, new_entry, &n_ext);
@@ -4838,7 +4841,7 @@ btr_cur_pessimistic_update(
 		ut_ad(new_entry->is_metadata());
 		ut_ad(index->is_instant());
 		/* This can be innobase_add_instant_try() performing a
-		subsequent instant ADD COLUMN, or its rollback by
+		subsequent instant ALTER TABLE, or its rollback by
 		row_undo_mod_clust_low(). */
 		ut_ad(flags & BTR_NO_LOCKING_FLAG);
 	} else {

storage/innobase/include/data0data.h

@@ -646,9 +646,11 @@ struct dtuple_t {
 	@param[in]	index	index possibly with instantly added columns */
 	void trim(const dict_index_t& index);
 
-	/** @return whether this is a hidden metadata record
-	for instant ALTER TABLE (not only ADD COLUMN) */
-	bool is_alter_metadata() const
+	/**
+	@param info_bits	the info_bits of a data tuple
+	@return whether this is a hidden metadata record
+	for instant ADD COLUMN or ALTER TABLE */
+	static bool is_alter_metadata(ulint info_bits)
 	{
 		return UNIV_UNLIKELY(info_bits == REC_INFO_METADATA_ALTER);
 	}
@@ -663,6 +665,10 @@ struct dtuple_t {
 				     == REC_INFO_METADATA_ADD);
 	}
 
+	/** @return whether this is a hidden metadata record
+	for instant ALTER TABLE (not only ADD COLUMN) */
+	bool is_alter_metadata() const { return is_alter_metadata(info_bits); }
+
 	/** @return whether this is a hidden metadata record
 	for instant ADD COLUMN or ALTER TABLE */
 	bool is_metadata() const { return is_metadata(info_bits); }

storage/innobase/include/data0type.h

@@ -554,6 +554,16 @@ struct dtype_t{
 	{
 		return (prtype & DATA_VERSIONED) == DATA_VERS_END;
 	}
+
+	/** Set the type of the BLOB in the hidden metadata record. */
+	void metadata_blob_init()
+	{
+		prtype = DATA_NOT_NULL;
+		mtype = DATA_BLOB;
+		len = 0;
+		mbminlen = 0;
+		mbmaxlen = 0;
+	}
 };
 
 /** The DB_TRX_ID,DB_ROLL_PTR values for "no history is available" */

storage/innobase/include/dict0mem.h

@@ -1156,18 +1156,9 @@ struct dict_index_t {
 		return fields[n].col->instant_value(len);
 	}
 
-	/** Adjust clustered index metadata for instant ADD/DROP COLUMN.
-	@param[in]	instant	clustered index definition
-					after instant ADD/DROP COLUM
-	@param[in]	n_newly_add	number of newly added columns
-	@param[in]	n_newly_drop	number of newly dropped columns
-	@param[in]	col_map	mapping of old table cols
-					to new table cols */
-	inline void instant_op_field(
-		const dict_index_t&	instant,
-		ulint			n_newly_add,
-		ulint			n_newly_drop,
-		const ulint*		col_map);
+	/** Adjust index metadata for instant ADD/DROP/reorder COLUMN.
+	@param[in]	clustered index definition after instant ALTER TABLE */
+	inline void instant_add_field(const dict_index_t& instant);
 
 	/** Remove instant ALTER TABLE metadata.
 	Protected by index root page x-latch or table X-lock. */
@@ -1649,6 +1640,25 @@ struct dict_table_t {
 	/** @return the number of instantly dropped columns */
 	unsigned n_dropped() const { return instant ? instant->n_dropped : 0; }
 
+	/** Look up an old column.
+	@param[in]	cols	the old columns of the table
+	@param[in]	col_map	map from old table columns to altered ones
+	@param[in]	n_cols	number of old columns
+	@param[in]	i	the number of the new column
+	@return	old column
+	@retval	NULL	if column i was added to the table */
+	static const dict_col_t* find(const dict_col_t* cols,
+				      const ulint* col_map, ulint n_cols,
+				      ulint i)
+	{
+		for (ulint o = n_cols; o--; ) {
+			if (col_map[o] == i) {
+				return &cols[o];
+			}
+		}
+		return NULL;
+	}
+
 	/** Read the metadata blob and fill the non primary fields,
 	non-drop nullable fields and fill the drop columns in the
 	vector.
@@ -1675,29 +1685,12 @@ struct dict_table_t {
 	@param[in]	data	metadata blob data. */
 	void construct_dropped_columns(const byte* data);
 
-	/** Fill the dropped column in dropped_cols
-	@param[in]	table		instant table
-	@param[in]	col_map		mapping of cols from old table
-					to new table
-	@param[in]	n_newly_drop	number of newly drop column */
-	inline void fill_dropped_column(
-		const dict_table_t&	table,
-		const ulint*		col_map,
-		ulint			n_newly_drop);
-
-	/** Adjust table metadat for instant operation.
-	@param[in]	table		instant table
-	@param[in]	col_map		mapping of cols from old table
-					to new table
-	@param[in]	n_newly_add	number of newly added column
-	@param[in]	n_newly_drop	number of newly drop column */
-	void instant_op_column(
-		const dict_table_t&	table,
-		const ulint*		col_map,
-		ulint			n_newly_add,
-		ulint			n_newly_drop);
-
-	/** Roll back instant_op_column().
+	/** Adjust table metadata for instant ADD/DROP/reorder COLUMN.
+	@param[in]	table		altered table (with dropped columns)
+	@param[in]	map		mapping from cols[] to table.cols[] */
+	void instant_column(const dict_table_t& table, const ulint* map);
+
+	/** Roll back instant_column().
 	@param[in]	old_n_cols		original n_cols
 	@param[in]	old_cols		original cols
 	@param[in]	old_col_names		original col_names

storage/innobase/include/row0row.h

@@ -243,6 +243,24 @@ row_rec_to_index_entry(
 	mem_heap_t*		heap)	/*!< in: memory heap from which
 					the memory needed is allocated */
 	MY_ATTRIBUTE((warn_unused_result));
+
+/** Convert a metadata record to a data tuple.
+@param[in]	rec		metadata record
+@param[in]	index		clustered index after instant ALTER TABLE
+@param[in]	offsets		rec_get_offsets(rec)
+@param[out]	n_ext		number of externally stored fields
+@param[in,out]	heap		memory heap for allocations
+@param[in]	info_bits	the info_bits after an update */
+dtuple_t*
+row_metadata_to_tuple(
+	const rec_t*		rec,
+	const dict_index_t*	index,
+	const ulint*		offsets,
+	ulint*			n_ext,
+	mem_heap_t*		heap,
+	ulint			info_bits)
+	MY_ATTRIBUTE((nonnull,warn_unused_result));
+
 /*******************************************************************//**
 Builds from a secondary index record a row reference with which we can
 search the clustered index record.

storage/innobase/include/row0upd.h

@@ -493,6 +493,10 @@ struct upd_t{
 	/** @return whether this is for a hidden metadata record
 	for instant ALTER TABLE */
 	bool is_metadata() const { return dtuple_t::is_metadata(info_bits); }
+	/** @return whether this is for a hidden metadata record
+	for instant ALTER TABLE (not only ADD COLUMN) */
+	bool is_alter_metadata() const
+	{ return dtuple_t::is_alter_metadata(info_bits); }
 
 #ifdef UNIV_DEBUG
         bool validate() const

storage/innobase/rem/rem0rec.cc

@@ -1087,8 +1087,7 @@ rec_get_offsets_func(
 		      || n == n_fields /* btr_pcur_restore_position() */
 		      || (n + (index->id == DICT_INDEXES_ID)
 			  >= index->n_core_fields && n <= index->n_fields
-			  + (index->table->instant
-			     ? index->table->instant->n_dropped : 0)));
+			  + unsigned(rec_is_alter_metadata(rec, false))));
 
 		if (is_user_rec && leaf && n < index->n_fields) {
 			ut_ad(!index->is_dummy);
@@ -1782,33 +1781,30 @@ rec_convert_dtuple_to_rec_old(
 	return(rec);
 }
 
-/** Convert a data tuple into a default ROW_FORMAT=COMPACT record
+/** Convert a data tuple into a ROW_FORMAT=COMPACT metadata record
 for instant alter operation.
 @param[out]	rec		converted record
 @param[in]	clust_index	clustered index
 @param[in]	fields		data fields to convert
 @param[in]	n_fields	number of data fields */
-static
+ATTRIBUTE_COLD static
 void
-rec_convert_dtuple_to_default_rec_comp(
+rec_convert_dtuple_to_metadata_comp(
 	rec_t*			rec,
 	const dict_index_t*	clust_index,
 	const dfield_t*		fields,
 	ulint			n_fields)
 {
-	const dfield_t*	field;
-	const dtype_t*	type;
 	byte*		end;
 	byte*		nulls = rec - (REC_N_NEW_EXTRA_BYTES + 1);
 	byte*		UNINIT_VAR(lens);
-	ulint		len;
-	ulint		fixed_len;
 	ulint		null_mask	= 1;
 
 	/* Nullable non-dropped fields + number of dropped fields. */
 	unsigned	n_null_bytes = UT_BITS_IN_BYTES(
 		(clust_index->get_n_non_drop_nullable_fields()
 				 + clust_index->table->instant->n_dropped));
+	ut_ad(n_fields == ulint(clust_index->n_fields) + 1);
 
 	rec_set_n_add_field(nulls, n_null_bytes);
 
@@ -1822,41 +1818,24 @@ rec_convert_dtuple_to_default_rec_comp(
 	/* clear the SQL-null flags */
 	memset(lens + 1, 0, ulint(nulls - lens));
 
-	ulint 	i;
-	ulint	field_no = 0;
 	/* Store the data and the offsets */
-	for (i = 0; i < n_fields; i++) {
-		const dict_field_t*	ifield;
-		dict_col_t*		col = NULL;
-		bool			is_dropped = false;
-		bool			is_nullable = false;
+	for (ulint i = 0, j = 0; i < n_fields; i++) {
+		const dfield_t*	field = &fields[i];
+		ulint len = dfield_get_len(field);
+		ulint fixed_len;
 
-		field = &fields[i];
-		type = dfield_get_type(field);
-		len = dfield_get_len(field);
-
-		if (i == unsigned(clust_index->n_uniq + DATA_ROLL_PTR)) {
-			/* Default row blob. */
+		if (i == unsigned(clust_index->n_uniq + 2)) {
+			/* Metadata blob */
 			fixed_len = 0;
+			ut_ad(dfield_is_ext(field));
+			ut_ad(dtype_get_prtype(&field->type) & DATA_NOT_NULL);
 			goto data_write;
 		} else {
-			ifield = dict_index_get_nth_field(clust_index, field_no++);
-			fixed_len = ifield->fixed_len;
-			col = ifield->col;
-			is_dropped = col->is_dropped();
+			fixed_len = dict_index_get_nth_field(
+				clust_index, j++)->fixed_len;
 		}
 
-		is_nullable = !(dtype_get_prtype(type) & DATA_NOT_NULL);
-
-		if (is_dropped) {
-			if (is_nullable) {
-				/* Set the nullable flag for
-				dropped column */
-				*nulls |= null_mask;
-			}
-
-			null_mask <<= 1;
-		} else if (is_nullable) {
+		if (!(dtype_get_prtype(&field->type) & DATA_NOT_NULL)) {
 			if (dfield_is_null(field)) {
 				*nulls |= null_mask;
 				null_mask <<= 1;
@@ -1866,19 +1845,13 @@ rec_convert_dtuple_to_default_rec_comp(
 			null_mask <<= 1;
 		}
 
-		ut_ad(!dfield_is_null(field) || is_dropped);
-
-		/* Set the length for the dropped column. */
-		if (fixed_len && !is_dropped) {
-			ut_ad(len <= fixed_len);
-			ut_ad(!dfield_is_ext(field));
+		if (fixed_len) {
 		} else if (dfield_is_ext(field)) {
 			*lens-- = (byte) (len >> 8) | 0xc0;
 			*lens-- = (byte) len;
 		} else {
 			if (len < 128 || !DATA_BIG_LEN_MTYPE(
-				dtype_get_len(type), dtype_get_mtype(type))) {
-
+				    field->type.len, field->type.mtype)) {
 				*lens-- = (byte) len;
 			} else {
 				ut_ad(len < 16384);
@@ -1888,8 +1861,9 @@ rec_convert_dtuple_to_default_rec_comp(
 		}
 
 data_write:
-		/* Store the record only non-dropped column. */
-		if (len && !is_dropped) {
+		ut_ad(!fixed_len || len <= fixed_len);
+		ut_ad(!fixed_len || !dfield_is_ext(field));
+		if (len) {
 			memcpy(end, dfield_get_data(field), len);
 			end += len;
 		}
@@ -2113,7 +2087,7 @@ rec_convert_dtuple_to_rec_new(
 			  && dict_table_is_comp(index->table))) {
 		rec_get_metadata_converted_size(index, dtuple, &extra_size);
 		rec = buf + extra_size;
-		rec_convert_dtuple_to_default_rec_comp(
+		rec_convert_dtuple_to_metadata_comp(
 			rec, index, dtuple->fields, dtuple->n_fields);
 		rec_set_info_bits_new(
 			rec, dtuple->info_bits & ~REC_NEW_STATUS_MASK);

storage/innobase/row/row0upd.cc

@@ -1323,43 +1323,46 @@ row_upd_index_replace_new_col_val(
 }
 
 /** Apply an update vector to an metadata entry.
-@param[in,out]	entry	index entry to be updated; the clustered index default
-			row record
+@param[in,out]	entry	clustered index metadata record to be updated
 @param[in]	index	index of the entry
 @param[in]	update	update vector built for the entry
 @param[in,out]	heap	memory heap for copying off-page columns */
 static
 void
-row_upd_index_replace_metadata_pos(
+row_upd_index_replace_metadata(
 	dtuple_t*		entry,
 	const dict_index_t*	index,
 	const upd_t*		update,
 	mem_heap_t*		heap)
 {
 	ut_ad(!index->table->skip_alter_undo);
-
-	const page_size_t&	page_size = dict_table_page_size(index->table);
-
-	dtuple_set_info_bits(entry, update->info_bits);
-
-	for (unsigned i = entry->n_fields;
-	     i >= unsigned(index->n_uniq + DATA_ROLL_PTR); i--) {
-		const dict_field_t*	field = NULL;
-		const dict_col_t*	col = NULL;
-		const upd_field_t*	uf;
-
-		uf = upd_get_field_by_field_no(update, i -1, false);
-
-		if (uf) {
-			if (i > unsigned(index->n_uniq + DATA_ROLL_PTR)) {
-				field = dict_index_get_nth_field(index, i - 1);
-				col = dict_field_get_col(field);
-			}
-
-			row_upd_index_replace_new_col_val(
-				dtuple_get_nth_field(entry, i),
-				field, col, uf, heap, page_size);
+	ut_ad(update->is_alter_metadata());
+	ut_ad(entry->info_bits == update->info_bits);
+	ut_ad(entry->n_fields == ulint(index->n_fields) + 1);
+	const page_size_t& page_size = dict_table_page_size(index->table);
+	const ulint first = index->n_uniq + 2;
+
+	for (ulint i = upd_get_n_fields(update); i--; ) {
+		const upd_field_t* uf = upd_get_nth_field(update, i);
+		ut_ad(!upd_fld_is_virtual_col(uf));
+		ut_ad(uf->field_no >= first);
+		ulint f = uf->field_no;
+		dfield_t* dfield = dtuple_get_nth_field(entry, f);
+
+		if (f-- == first) {
+			ut_ad(dfield_is_ext(&uf->new_val));
+			ut_ad(dfield_get_len(&uf->new_val) == FIELD_REF_SIZE);
+			ut_ad(!dfield_is_null(&uf->new_val));
+			ut_ad(dfield_is_ext(dfield));
+			ut_ad(dfield_get_len(dfield) == FIELD_REF_SIZE);
+			ut_ad(!dfield_is_null(dfield));
+			dfield->data = uf->new_val.data;
+			continue;
 		}
+
+		const dict_field_t* field = dict_index_get_nth_field(index, f);
+		row_upd_index_replace_new_col_val(dfield, field, field->col,
+						  uf, heap, page_size);
 	}
 }
 
@@ -1378,16 +1381,17 @@ row_upd_index_replace_new_col_vals_index_pos(
 	mem_heap_t*		heap)
 {
 	ut_ad(!index->table->skip_alter_undo);
-
-	const page_size_t&	page_size = dict_table_page_size(index->table);
-
-	dtuple_set_info_bits(entry, update->info_bits);
+	ut_ad(!entry->is_metadata() || entry->info_bits == update->info_bits);
 
 	if (UNIV_UNLIKELY(entry->is_alter_metadata())) {
-		row_upd_index_replace_metadata_pos(entry, index, update, heap);
+		row_upd_index_replace_metadata(entry, index, update, heap);
 		return;
 	}
 
+	const page_size_t&	page_size = dict_table_page_size(index->table);
+
+	dtuple_set_info_bits(entry, update->info_bits);
+
 	for (unsigned i = index->n_fields; i--; ) {
 		const dict_field_t*	field;
 		const dict_col_t*	col;