ha_innodb.cc, row0sel.c:

  Fix Bug #9526 in 5.0: MySQL ENUM and SET columns are internally actually unsigned integer types; we must take care that old tables still treat ENUM and SET (incorrectly) as a character string, while new created tables treat it correctly as an unsigned integer


innobase/row/row0sel.c:
  Fix Bug #9526 in 5.0: MySQL ENUM and SET columns are internally actually unsigned integer types; we must take care that old tables still treat ENUM and SET (incorrectly) as a character string, while new created tables treat it correctly as an unsigned integer
sql/ha_innodb.cc:
  Fix Bug #9526 in 5.0: MySQL ENUM and SET columns are internally actually unsigned integer types; we must take care that old tables still treat ENUM and SET (incorrectly) as a character string, while new created tables treat it correctly as an unsigned integer

innobase/row/row0sel.c

@@ -2145,12 +2145,16 @@ row_sel_convert_mysql_key_to_innobase(
 		}
 
  		if (dtype_get_mysql_type(dfield_get_type(dfield))
-					== DATA_MYSQL_TRUE_VARCHAR) {
+					== DATA_MYSQL_TRUE_VARCHAR
+		    && dfield_get_type(dfield)->mtype != DATA_INT) {
 			/* In a MySQL key value format, a true VARCHAR is
 			always preceded by 2 bytes of a length field.
 			dfield_get_type(dfield)->len returns the maximum
 			'payload' len in bytes. That does not include the
-			2 bytes that tell the actual data length. */
+			2 bytes that tell the actual data length.
+
+			We added the check != DATA_INT to make sure we do
+			not treat MySQL ENUM or SET as a true VARCHAR! */
 
 			data_len += 2;
 			data_field_len += 2;

sql/ha_innodb.cc

@@ -2272,12 +2272,42 @@ ulint
 get_innobase_type_from_mysql_type(
 /*==============================*/
 				/* out: DATA_BINARY, DATA_VARCHAR, ... */
+	ulint*	unsigned_flag,	/* out: DATA_UNSIGNED if an 'unsigned type';
+				at least ENUM and SET, and unsigned integer
+				types are 'unsigned types' */
 	Field*	field)		/* in: MySQL field */
 {
 	/* The following asserts try to check that the MySQL type code fits in
 	8 bits: this is used in ibuf and also when DATA_NOT_NULL is ORed to
 	the type */
 
+	DBUG_ASSERT((ulint)FIELD_TYPE_STRING < 256);
+	DBUG_ASSERT((ulint)FIELD_TYPE_VAR_STRING < 256);
+	DBUG_ASSERT((ulint)FIELD_TYPE_DOUBLE < 256);
+	DBUG_ASSERT((ulint)FIELD_TYPE_FLOAT < 256);
+	DBUG_ASSERT((ulint)FIELD_TYPE_DECIMAL < 256);
+
+	if (field->flags & UNSIGNED_FLAG) {
+
+		*unsigned_flag = DATA_UNSIGNED;
+	} else {
+		*unsigned_flag = 0;
+	}
+
+	if (field->real_type() == FIELD_TYPE_ENUM
+	    || field->real_type() == FIELD_TYPE_SET) {
+
+		/* MySQL has field->type() a string type for these, but the
+		data is actually internally stored as an unsigned integer
+		code! */
+
+		*unsigned_flag = DATA_UNSIGNED; /* MySQL has its own unsigned
+						flag set to zero, even though
+						internally this is an unsigned
+						integer type */
+		return(DATA_INT);
+	}
+
 	switch (field->type()) {
 	        /* NOTE that we only allow string types in DATA_MYSQL
 		and DATA_VARMYSQL */
@@ -2313,8 +2343,6 @@ get_innobase_type_from_mysql_type(
 		case FIELD_TYPE_DATETIME:
 		case FIELD_TYPE_YEAR:
 		case FIELD_TYPE_NEWDATE:
-		case FIELD_TYPE_ENUM:
-		case FIELD_TYPE_SET:
 		case FIELD_TYPE_TIME:
 		case FIELD_TYPE_TIMESTAMP:
 					return(DATA_INT);
@@ -2686,7 +2714,7 @@ build_template(
 					get_field_offset(table, field);
 
 		templ->mysql_col_len = (ulint) field->pack_length();
-		templ->type = get_innobase_type_from_mysql_type(field);
+		templ->type = index->table->cols[i].type.mtype;
 		templ->mysql_type = (ulint)field->type();
 
 		if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) {
@@ -2698,8 +2726,8 @@ build_template(
 				index->table->cols[i].type.prtype);
 		templ->mbminlen = index->table->cols[i].type.mbminlen;
 		templ->mbmaxlen = index->table->cols[i].type.mbmaxlen;
-		templ->is_unsigned = (ulint) (field->flags & UNSIGNED_FLAG);
-
+		templ->is_unsigned = index->table->cols[i].type.prtype
+							& DATA_UNSIGNED;
 		if (templ->type == DATA_BLOB) {
 			prebuilt->templ_contains_blob = TRUE;
 		}
@@ -2962,7 +2990,6 @@ calc_row_difference(
         byte*	        buf;
 	upd_field_t*	ufield;
 	ulint		col_type;
-	ulint		is_unsigned;
 	ulint		n_changed = 0;
 	dfield_t	dfield;
 	uint		i;
@@ -2998,8 +3025,7 @@ calc_row_difference(
 
 		field_mysql_type = field->type();
 	
-		col_type = get_innobase_type_from_mysql_type(field);
-		is_unsigned = (ulint) (field->flags & UNSIGNED_FLAG);
+		col_type = prebuilt->table->cols[i].type.mtype;
 
 		switch (col_type) {
 
@@ -3072,8 +3098,7 @@ calc_row_difference(
 			}
 
 			ufield->exp = NULL;
-			ufield->field_no =
-					(prebuilt->table->cols + i)->clust_pos;
+			ufield->field_no = prebuilt->table->cols[i].clust_pos;
 			n_changed++;
 		}
 	}
@@ -3932,19 +3957,14 @@ create_table_def(
 	for (i = 0; i < n_cols; i++) {
 		field = form->field[i];
 
-		col_type = get_innobase_type_from_mysql_type(field);
+		col_type = get_innobase_type_from_mysql_type(&unsigned_type,
+									field);
 		if (field->null_ptr) {
 			nulls_allowed = 0;
 		} else {
 			nulls_allowed = DATA_NOT_NULL;
 		}
 
-		if (field->flags & UNSIGNED_FLAG) {
-			unsigned_type = DATA_UNSIGNED;
-		} else {
-			unsigned_type = 0;
-		}
-
 		if (field->binary()) {
 			binary_type = DATA_BINARY_TYPE;
 		} else {
@@ -4021,6 +4041,7 @@ create_index(
 	ulint		ind_type;
 	ulint		col_type;
 	ulint		prefix_len;
+	ulint		is_unsigned;
   	ulint		i;
   	ulint		j;
 
@@ -4070,7 +4091,8 @@ create_index(
 
 		ut_a(j < form->s->fields);
 
-		col_type = get_innobase_type_from_mysql_type(key_part->field);
+		col_type = get_innobase_type_from_mysql_type(
+					&is_unsigned, key_part->field);
 
 		if (DATA_BLOB == col_type
 		    || (key_part->length < field->pack_length()