Implementation of rows-in-block

- Fixes some things missed in myisam->maria port
- Moved variables that holds the state for the current row to 'cur_row'
- Changed most uchar * to byte * to be able to remove a lot of casts
- Removed RAID support
- Added CHECK for rows-in-block
- Added allocate_dynamic() for easier usage of dynamic rows when we know how many entries we will need
- Reorder columns after CREATE for more optimal row storage (for rows-in-block)
- Removed flag 'RRND_PRESERVER_LASTINX' (not needed)
- Extended ma_test_all.sh to test more completely all row formats
- New structs and variables to hold rows-in-block and bitmap information
- Added org_data_file_type in header to allow easy restore of old record format when doing maria_pack / maria_chk -u
- More virtual functions to handle different row types
- Pointer to row is now MARIA_RECORD_POS instead of my_off_t
- New header signature for MARIA index files
- Fixed bugs in ma_test1.c and ma_test2.c
- All key and row blocks are now of same size
- We now only have one link chain for deleted key blocks


include/m_string.h:
  Define bzero_if_purify
include/maria.h:
  Implementation of rows-in-block
include/my_base.h:
  Implementation of rows-in-block
include/my_handler.h:
  Cleanup macros
  Added size_to_store_key_length()
include/my_sys.h:
  Added 'allocate_dynamic()'
include/myisamchk.h:
  Implementation of rows-in-block
mysys/array.c:
  Added allocate_dynamic()
mysys/mf_keycache.c:
  Moved DBUG_ENTER to it's right position
mysys/my_pread.c:
  Ensure my_errno is always set
sql/filesort.cc:
  Fixed some compiler warnings
sql/gen_lex_hash.cc:
  Removed not needed 'inline'
sql/ha_maria.cc:
  Implementation of rows-in-block
  Fixed compiler warnings
sql/mysqld.cc:
  Fixed setting of wrong variable
sql/uniques.cc:
  Fixed compiler warnings
storage/maria/Makefile.am:
  Implementation of rows-in-block
storage/maria/ma_check.c:
  Removed RAID functions
  Added support for CHECK of rows-in-blocks rows
storage/maria/ma_checksum.c:
  Implementation of rows-in-block
storage/maria/ma_close.c:
  Implementation of rows-in-block
storage/maria/ma_create.c:
  Implementation of rows-in-block:
  - Reorder columns
  - All key blocks are now of same size
  - Removed old RAID support
storage/maria/ma_dbug.c:
  Implementation of rows-in-block
storage/maria/ma_delete.c:
  Implementation of rows-in-block
storage/maria/ma_delete_all.c:
  Implementation of rows-in-block
storage/maria/ma_dynrec.c:
  info->rec_buff is now allocated through _ma_alloc_buffer()
  Use new info->cur_row structure
storage/maria/ma_extra.c:
  Implementation of rows-in-block
storage/maria/ma_ft_boolean_search.c:
  Removed compiler warnings
  Indentation fixes
storage/maria/ma_ft_nlq_search.c:
  Removed compiler warnings
  Indentation fixes
storage/maria/ma_ft_update.c:
  Removed some casts
storage/maria/ma_fulltext.h:
  Changed pointer type
storage/maria/ma_info.c:
  Implementation of rows-in-block
  More general _ma_report_error()
storage/maria/ma_init.c:
  Implementation of rows-in-block
storage/maria/ma_key.c:
  Implementation of rows-in-block
  Removed some casts
storage/maria/ma_keycache.c:
  Fixed DBUG entry
storage/maria/ma_locking.c:
  Implementation of rows-in-block
storage/maria/ma_open.c:
  Implementation of rows-in-block
storage/maria/ma_packrec.c:
  Indentation fixes
  Changed uchar * to byte * to make it possible to remove some casts
storage/maria/ma_page.c:
  Implementation of rows-in-block
storage/maria/ma_range.c:
  Implementation of rows-in-block
storage/maria/ma_rfirst.c:
  Implementation of rows-in-block
storage/maria/ma_rkey.c:
  Implementation of rows-in-block
  Indentation fixes
storage/maria/ma_rlast.c:
  Implementation of rows-in-block
storage/maria/ma_rnext.c:
  Implementation of rows-in-block
storage/maria/ma_rnext_same.c:
  Implementation of rows-in-block
storage/maria/ma_rprev.c:
  Implementation of rows-in-block
storage/maria/ma_rrnd.c:
  Implementation of rows-in-block
  Removed flag 'RRND_PRESERVER_LASTINX', by not resetting lastinx (This is reset by maria_scan_init())
storage/maria/ma_rsame.c:
  Implementation of rows-in-block
storage/maria/ma_rsamepos.c:
  Implementation of rows-in-block
storage/maria/ma_rt_index.c:
  Implementation of rows-in-block
storage/maria/ma_rt_index.h:
  Implementation of rows-in-block
storage/maria/ma_rt_key.c:
  Implementation of rows-in-block
storage/maria/ma_rt_key.h:
  Implementation of rows-in-block
storage/maria/ma_rt_mbr.c:
  Implementation of rows-in-block
storage/maria/ma_rt_mbr.h:
  Implementation of rows-in-block
storage/maria/ma_rt_split.c:
  Implementation of rows-in-block
storage/maria/ma_rt_test.c:
  Indentation fix
storage/maria/ma_scan.c:
  Implementation of rows-in-block
  Added 'maria_scan_end()'
storage/maria/ma_search.c:
  Implementation of rows-in-block
storage/maria/ma_sort.c:
  Indentation fixes
  uchar -> byte to be able to remove some casts
storage/maria/ma_sp_defs.h:
  uchar * -> byte *
storage/maria/ma_sp_key.c:
  uchar * -> byte *
storage/maria/ma_sp_test.c:
  Indentation fixes
storage/maria/ma_static.c:
  New header signature for MARIA
storage/maria/ma_statrec.c:
  int -> my_bool functions
  my_off_t -> MARIA_RECORD_POS
  Fixed argument order for _ma_read_static_record()
storage/maria/ma_test1.c:
  Implementation of rows-in-block
  Fixed some bugs in VARCHAR and BLOB testing
storage/maria/ma_test2.c:
  Implementation of rows-in-block
  Fixed bug in BLOB testing
storage/maria/ma_test3.c:
  Implementation of rows-in-block
storage/maria/ma_test_all.sh:
  Run all tests with dynamic, static and block row formats
  (For the moment we skip REPAIR test of rows-in-block as this is not yet implemented)
storage/maria/ma_unique.c:
  Implementation of rows-in-block
storage/maria/ma_update.c:
  Implementation of rows-in-block
storage/maria/ma_write.c:
  Implementation of rows-in-block
  Write of row is split into two parts, as rows-in-block format require us to do write of row before keys (to get row position) in contrast to all other row formats
storage/maria/maria_chk.c:
  Implementation of rows-in-block
storage/maria/maria_def.h:
  Implementation of rows-in-block
  - New structs and variables to hold rows-in-block and bitmap information
  - Added org_data_file_type in header to allow easy restore of old record format when doing maria_pack / maria_chk -u
  - More virtual functions to handle different row types
  - Pointer to row is now MARIA_RECORD_POS instead of my_off_t
  - uchar -> byte for many parameters to avoid casts
storage/maria/maria_ftdump.c:
  Implementation of rows-in-block
storage/maria/maria_pack.c:
  Implementation of rows-in-block
storage/myisam/mi_check.c:
  Added new row types into switch to avoid compiler warnings
  Added some casts to avoid warnings after changing type of lastkey and buff
storage/myisam/mi_create.c:
  Fix that 'pack_fields' is calculated correctly
storage/myisam/mi_rsamepos.c:
  Implementation of rows-in-block
storage/myisam/mi_test2.c:
  Fixed wrong printf
storage/myisam/sort.c:
  uchar * -> byte *
support-files/magic:
  Added support for Maria files
  Fided wrong entry's for MyISAM files
storage/maria/ma_bitmap.c:
  New BitKeeper file ``storage/maria/ma_bitmap.c''
storage/maria/ma_blockrec.c:
  New BitKeeper file ``storage/maria/ma_blockrec.c''
storage/maria/ma_blockrec.h:
  New BitKeeper file ``storage/maria/ma_blockrec.h''

include/m_string.h

@@ -127,7 +127,10 @@ extern	int bcmp(const char *s1,const char *s2,uint len);
 extern	int my_bcmp(const char *s1,const char *s2,uint len);
 #undef bcmp
 #define bcmp(A,B,C) my_bcmp((A),(B),(C))
-#endif
+#define bzero_if_purify(A,B) bzero(A,B)
+#else
+#define bzero_if_purify(A,B)
+#endif /* HAVE_purify */
 
 #ifndef bmove512
 extern	void bmove512(gptr dst,const gptr src,uint len);

include/my_base.h

@@ -48,7 +48,7 @@
 #define HA_OPEN_FOR_REPAIR		32	/* open even if crashed */
 #define HA_OPEN_FROM_SQL_LAYER          64
 
-	/* The following is parameter to ha_rkey() how to use key */
+/* The following is parameter to ha_rkey() how to use key */
 
 /*
   We define a complete-field prefix of a key value as a prefix where
@@ -433,7 +433,7 @@ enum en_fieldtype {
 };
 
 enum data_file_type {
-  STATIC_RECORD,DYNAMIC_RECORD,COMPRESSED_RECORD
+  STATIC_RECORD, DYNAMIC_RECORD, COMPRESSED_RECORD, BLOCK_RECORD
 };
 
 /* For key ranges */

include/my_handler.h

@@ -58,33 +58,35 @@ typedef struct st_HA_KEYSEG		/* Key-portion */
 } HA_KEYSEG;
 
 #define get_key_length(length,key) \
-{ if ((uchar) *(key) != 255) \
-    length= (uint) (uchar) *((key)++); \
+{ if (*(uchar*) (key) != 255) \
+    length= (uint) *(uchar*) ((key)++); \
   else \
-  { length=mi_uint2korr((key)+1); (key)+=3; } \
+  { length= mi_uint2korr((key)+1); (key)+=3; } \
 }
 
 #define get_key_length_rdonly(length,key) \
-{ if ((uchar) *(key) != 255) \
-    length= ((uint) (uchar) *((key))); \
+{ if (*(uchar*) (key) != 255) \
+    length= ((uint) *(uchar*) ((key))); \
   else \
-  { length=mi_uint2korr((key)+1); } \
+  { length= mi_uint2korr((key)+1); } \
 }
 
 #define get_key_pack_length(length,length_pack,key) \
-{ if ((uchar) *(key) != 255) \
-  { length= (uint) (uchar) *((key)++); length_pack=1; }\
+{ if (*(uchar*) (key) != 255) \
+  { length= (uint) *(uchar*) ((key)++); length_pack= 1; }\
   else \
-  { length=mi_uint2korr((key)+1); (key)+=3; length_pack=3; } \
+  { length=mi_uint2korr((key)+1); (key)+= 3; length_pack= 3; } \
 }
 
 #define store_key_length_inc(key,length) \
 { if ((length) < 255) \
-  { *(key)++=(length); } \
+  { *(key)++= (length); } \
   else \
   { *(key)=255; mi_int2store((key)+1,(length)); (key)+=3; } \
 }
 
+#define size_to_store_key_length(length) ((length) < 255 ? 1 : 3)
+
 #define get_rec_bits(bit_ptr, bit_ofs, bit_len) \
   (((((uint16) (bit_ptr)[1] << 8) | (uint16) (bit_ptr)[0]) >> (bit_ofs)) & \
    ((1 << (bit_len)) - 1))

include/my_sys.h

@@ -760,6 +760,7 @@ extern my_bool insert_dynamic(DYNAMIC_ARRAY *array,gptr element);
 extern byte *alloc_dynamic(DYNAMIC_ARRAY *array);
 extern byte *pop_dynamic(DYNAMIC_ARRAY*);
 extern my_bool set_dynamic(DYNAMIC_ARRAY *array,gptr element,uint array_index);
+extern my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements);
 extern void get_dynamic(DYNAMIC_ARRAY *array,gptr element,uint array_index);
 extern void delete_dynamic(DYNAMIC_ARRAY *array);
 extern void delete_dynamic_element(DYNAMIC_ARRAY *array, uint array_index);

include/myisamchk.h

@@ -76,8 +76,8 @@
 
 typedef struct st_sort_key_blocks		/* Used when sorting */
 {
-  uchar *buff, *end_pos;
-  uchar lastkey[HA_MAX_POSSIBLE_KEY_BUFF];
+  byte *buff, *end_pos;
+  byte lastkey[HA_MAX_POSSIBLE_KEY_BUFF];
   uint last_length;
   int inited;
 } SORT_KEY_BLOCKS;
@@ -121,20 +121,24 @@ typedef struct st_handler_check_param
   my_off_t search_after_block;
   my_off_t new_file_pos, key_file_blocks;
   my_off_t keydata, totaldata, key_blocks, start_check_pos;
-  ha_rows total_records, total_deleted;
+  my_off_t used, empty, splits, del_length, link_used;
+  ha_rows total_records, total_deleted, records,del_blocks;
+  ha_rows full_page_count, tail_count;
   ha_checksum record_checksum, glob_crc;
   ha_checksum key_crc[HA_MAX_POSSIBLE_KEY];
+  ha_checksum tmp_key_crc[HA_MAX_POSSIBLE_KEY];
+  ha_checksum tmp_record_checksum;
   ulong use_buffers, read_buffer_length, write_buffer_length;
   ulong sort_buffer_length, sort_key_blocks;
   ulong rec_per_key_part[HA_MAX_KEY_SEG * HA_MAX_POSSIBLE_KEY];
   uint out_flag, warning_printed, error_printed, verbose;
   uint opt_sort_key, total_files, max_level;
   uint testflag, key_cache_block_size;
-  int tmpfile_createflag;
+  int tmpfile_createflag, err_count;
   myf myf_rw;
   uint8 language;
   my_bool using_global_keycache, opt_lock_memory, opt_follow_links;
-  my_bool retry_repair, force_sort, calc_checksum;
+  my_bool retry_repair, force_sort, calc_checksum, static_row_size;
   char temp_filename[FN_REFLEN];
   IO_CACHE read_cache;
   enum_handler_stats_method stats_method;
@@ -143,15 +147,15 @@ typedef struct st_handler_check_param
 
 typedef struct st_sort_ftbuf
 {
-  uchar *buf, *end;
+  byte *buf, *end;
   int count;
-  uchar lastkey[HA_MAX_KEY_BUFF];
+  byte lastkey[HA_MAX_KEY_BUFF];
 } SORT_FT_BUF;
 
 
 typedef struct st_buffpek {
   my_off_t file_pos;                    /* Where we are in the sort file */
-  uchar *base,*key;                     /* Key pointers */
+  byte *base, *key;                     /* Key pointers */
   ha_rows count;                        /* Number of rows in table */
   ulong mem_count;                      /* numbers of keys in memory */
   ulong max_keys;                       /* Max keys in buffert */

mysys/array.c

@@ -61,7 +61,8 @@ my_bool init_dynamic_array(DYNAMIC_ARRAY *array, uint element_size,
   array->max_element=init_alloc;
   array->alloc_increment=alloc_increment;
   array->size_of_element=element_size;
-  if (!(array->buffer=(char*) my_malloc_ci(element_size*init_alloc,MYF(MY_WME))))
+  if (!(array->buffer=(char*) my_malloc_ci(element_size*init_alloc,
+                                           MYF(MY_WME))))
   {
     array->max_element=0;
     DBUG_RETURN(TRUE);
@@ -154,7 +155,7 @@ byte *pop_dynamic(DYNAMIC_ARRAY *array)
 }
 
 /*
-  Replace elemnent in array with given element and index
+  Replace element in array with given element and index
 
   SYNOPSIS
     set_dynamic()
@@ -175,28 +176,53 @@ my_bool set_dynamic(DYNAMIC_ARRAY *array, gptr element, uint idx)
 {
   if (idx >= array->elements)
   {
-    if (idx >= array->max_element)
+    if (idx >= array->max_element && allocate_dynamic(array, idx))
+      return TRUE;
+    bzero((gptr) (array->buffer+array->elements*array->size_of_element),
+	  (idx - array->elements)*array->size_of_element);
+    array->elements=idx+1;
+  }
+  memcpy(array->buffer+(idx * array->size_of_element),element,
+	 (size_t) array->size_of_element);
+  return FALSE;
+}
+
+
+/*
+  Ensure that dynamic array has enough elements
+
+  SYNOPSIS
+    allocate_dynamic()
+    array
+    max_elements        Numbers of elements that is needed
+
+  NOTES
+   Any new allocated element are NOT initialized
+
+  RETURN VALUE
+    FALSE	Ok
+    TRUE	Allocation of new memory failed
+*/
+
+my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements)
+{
+  if (max_elements >= array->max_element)
   {
     uint size;
     char *new_ptr;
-      size=(idx+array->alloc_increment)/array->alloc_increment;
+    size= (max_elements + array->alloc_increment)/array->alloc_increment;
     size*= array->alloc_increment;
-      if (!(new_ptr=(char*) my_realloc(array->buffer,size*
+    if (!(new_ptr= (char*) my_realloc(array->buffer,size*
                                       array->size_of_element,
                                       MYF(MY_WME | MY_ALLOW_ZERO_PTR))))
       return TRUE;
-      array->buffer=new_ptr;
-      array->max_element=size;
+    array->buffer= new_ptr;
+    array->max_element= size;
   }
-    bzero((gptr) (array->buffer+array->elements*array->size_of_element),
-	  (idx - array->elements)*array->size_of_element);
-    array->elements=idx+1;
-  }
-  memcpy(array->buffer+(idx * array->size_of_element),element,
-	 (size_t) array->size_of_element);
   return FALSE;
 }
 
+
 /*
   Get an element from array by given index
 

mysys/mf_keycache.c

@@ -1791,8 +1791,6 @@ byte *key_cache_read(KEY_CACHE *keycache,
   uint offset= 0;
   byte *start= buff;
   DBUG_ENTER("key_cache_read");
-  DBUG_PRINT("enter", ("fd: %u  pos: %lu  length: %u",
-               (uint) file, (ulong) filepos, length));
 
   if (keycache->can_be_used)
   {
@@ -1802,6 +1800,11 @@ byte *key_cache_read(KEY_CACHE *keycache,
     uint status;
     int page_st;
 
+    DBUG_PRINT("enter", ("fd: %u  pos: %lu  page: %lu  length: %u",
+                         (uint) file, (ulong) filepos,
+                         (ulong) (filepos / keycache->key_cache_block_size),
+                         length));
+
     offset= (uint) (filepos & (keycache->key_cache_block_size-1));
     /* Read data in key_cache_block_size increments */
     do
@@ -2053,10 +2056,6 @@ int key_cache_write(KEY_CACHE *keycache,
   reg1 BLOCK_LINK *block;
   int error=0;
   DBUG_ENTER("key_cache_write");
-  DBUG_PRINT("enter",
-	     ("fd: %u  pos: %lu  length: %u  block_length: %u  key_block_length: %u",
-	      (uint) file, (ulong) filepos, length, block_length,
-	      keycache ? keycache->key_cache_block_size : 0));
 
   if (!dont_write)
   {
@@ -2078,6 +2077,12 @@ int key_cache_write(KEY_CACHE *keycache,
     int page_st;
     uint offset;
 
+    DBUG_PRINT("enter",
+               ("fd: %u  pos: %lu  page: %lu  length: %u  block_length: %u",
+                (uint) file, (ulong) filepos,
+                (ulong) (filepos / keycache->key_cache_block_size),
+                length, block_length));
+
     offset= (uint) (filepos & (keycache->key_cache_block_size-1));
     do
     {

mysys/my_pread.c

@@ -52,7 +52,7 @@ uint my_pread(File Filedes, byte *Buffer, uint Count, my_off_t offset,
     if (!error)                                 /* Seek was successful */
     {
       if ((readbytes = (uint) read(Filedes, Buffer, Count)) == -1L)
-        my_errno= errno;
+        my_errno= errno ? errno : -1;
 
       /*
         We should seek back, even if read failed. If this fails,
@@ -68,7 +68,7 @@ uint my_pread(File Filedes, byte *Buffer, uint Count, my_off_t offset,
 #else
     if ((error= ((readbytes =
                   (uint) pread(Filedes, Buffer, Count, offset)) != Count)))
-      my_errno= errno;
+      my_errno= errno ? errno : -1;
 #endif
     if (error || readbytes != Count)
     {
@@ -84,9 +84,11 @@ uint my_pread(File Filedes, byte *Buffer, uint Count, my_off_t offset,
 	  my_error(EE_READ, MYF(ME_BELL+ME_WAITTANG),
 		   my_filename(Filedes),my_errno);
 	else if (MyFlags & (MY_NABP | MY_FNABP))
+        {
 	  my_error(EE_EOFERR, MYF(ME_BELL+ME_WAITTANG),
 		   my_filename(Filedes),my_errno);
         }
+      }
       if ((int) readbytes == -1 || (MyFlags & (MY_FNABP | MY_NABP)))
 	DBUG_RETURN(MY_FILE_ERROR);		/* Return with error */
     }

sql/filesort.cc

@@ -1032,7 +1032,7 @@ uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
 
 void reuse_freed_buff(QUEUE *queue, BUFFPEK *reuse, uint key_length)
 {
-  uchar *reuse_end= reuse->base + reuse->max_keys * key_length;
+  byte *reuse_end= reuse->base + reuse->max_keys * key_length;
   for (uint i= 0; i < queue->elements; ++i)
   {
     BUFFPEK *bp= (BUFFPEK *) queue_element(queue, i);
@@ -1103,7 +1103,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
   offset= rec_length-res_length;
   maxcount= (ulong) (param->keys/((uint) (Tb-Fb) +1));
   to_start_filepos= my_b_tell(to_file);
-  strpos= (uchar*) sort_buffer;
+  strpos= sort_buffer;
   org_max_rows=max_rows= param->max_rows;
 
   /* The following will fire if there is not enough space in sort_buffer */
@@ -1115,7 +1115,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
     DBUG_RETURN(1);                                /* purecov: inspected */
   for (buffpek= Fb ; buffpek <= Tb ; buffpek++)
   {
-    buffpek->base= strpos;
+    buffpek->base= (byte*) strpos;
     buffpek->max_keys= maxcount;
     strpos+= (uint) (error= (int) read_to_buffer(from_file, buffpek,
                                                  rec_length));
@@ -1207,7 +1207,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
     }
   }
   buffpek= (BUFFPEK*) queue_top(&queue);
-  buffpek->base= sort_buffer;
+  buffpek->base= (byte*) sort_buffer;
   buffpek->max_keys= param->keys;
 
   /*
@@ -1242,7 +1242,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
     else
     {
       register uchar *end;
-      strpos= buffpek->key+offset;
+      strpos= (uchar*) buffpek->key+offset;
       for (end= strpos+buffpek->mem_count*rec_length ;
            strpos != end ;
            strpos+= rec_length)

sql/gen_lex_hash.cc

@@ -463,7 +463,7 @@ int main(int argc,char **argv)
   printf("\nstatic unsigned int symbols_max_len=%d;\n\n", max_len2);
 
   printf("\
-static inline SYMBOL *get_hash_symbol(const char *s,\n\
+static SYMBOL *get_hash_symbol(const char *s,\n\
                                unsigned int len,bool function)\n\
 {\n\
   register uchar *hash_map;\n\

sql/ha_maria.cc

@@ -220,10 +220,10 @@ int ha_maria::dump(THD * thd, int fd)
 {
   MARIA_SHARE *share= file->s;
   NET *net= &thd->net;
-  uint blocksize= share->blocksize;
+  uint block_size= share->block_size;
   my_off_t bytes_to_read= share->state.state.data_file_length;
   int data_fd= file->dfile;
-  byte *buf= (byte *) my_malloc(blocksize, MYF(MY_WME));
+  byte *buf= (byte *) my_malloc(block_size, MYF(MY_WME));
   if (!buf)
     return ENOMEM;
 
@@ -231,7 +231,7 @@ int ha_maria::dump(THD * thd, int fd)
   my_seek(data_fd, 0L, MY_SEEK_SET, MYF(MY_WME));
   for (; bytes_to_read > 0;)
   {
-    uint bytes= my_read(data_fd, buf, blocksize, MYF(MY_WME));
+    uint bytes= my_read(data_fd, buf, block_size, MYF(MY_WME));
     if (bytes == MY_FILE_ERROR)
     {
       error= errno;
@@ -364,7 +364,7 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt)
   const char *old_proc_info= thd->proc_info;
 
   thd->proc_info= "Checking table";
-  mariachk_init(&param);
+  maria_chk_init(&param);
   param.thd= thd;
   param.op_name= "check";
   param.db_name= table->s->db.str;
@@ -454,7 +454,7 @@ int ha_maria::analyze(THD *thd, HA_CHECK_OPT * check_opt)
   HA_CHECK param;
   MARIA_SHARE *share= file->s;
 
-  mariachk_init(&param);
+  maria_chk_init(&param);
   param.thd= thd;
   param.op_name= "analyze";
   param.db_name= table->s->db.str;
@@ -510,7 +510,7 @@ int ha_maria::restore(THD * thd, HA_CHECK_OPT *check_opt)
 err:
   {
     HA_CHECK param;
-    mariachk_init(&param);
+    maria_chk_init(&param);
     param.thd= thd;
     param.op_name= "restore";
     param.db_name= table->s->db.str;
@@ -570,7 +570,7 @@ int ha_maria::backup(THD * thd, HA_CHECK_OPT *check_opt)
 err:
   {
     HA_CHECK param;
-    mariachk_init(&param);
+    maria_chk_init(&param);
     param.thd= thd;
     param.op_name= "backup";
     param.db_name= table->s->db.str;
@@ -591,7 +591,7 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt)
   if (!file)
     return HA_ADMIN_INTERNAL_ERROR;
 
-  mariachk_init(&param);
+  maria_chk_init(&param);
   param.thd= thd;
   param.op_name= "repair";
   param.testflag= ((check_opt->flags & ~(T_EXTEND)) |
@@ -638,7 +638,7 @@ int ha_maria::optimize(THD * thd, HA_CHECK_OPT *check_opt)
     return HA_ADMIN_INTERNAL_ERROR;
   HA_CHECK param;
 
-  mariachk_init(&param);
+  maria_chk_init(&param);
   param.thd= thd;
   param.op_name= "optimize";
   param.testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE |
@@ -833,7 +833,7 @@ err:
   {
     /* Send error to user */
     HA_CHECK param;
-    mariachk_init(&param);
+    maria_chk_init(&param);
     param.thd= thd;
     param.op_name= "assign_to_keycache";
     param.db_name= table->s->db.str;
@@ -901,7 +901,7 @@ int ha_maria::preload_keys(THD * thd, HA_CHECK_OPT *check_opt)
 err:
   {
     HA_CHECK param;
-    mariachk_init(&param);
+    maria_chk_init(&param);
     param.thd= thd;
     param.op_name= "preload_keys";
     param.db_name= table->s->db.str;
@@ -1010,7 +1010,7 @@ int ha_maria::enable_indexes(uint mode)
     HA_CHECK param;
     const char *save_proc_info= thd->proc_info;
     thd->proc_info= "Creating index";
-    mariachk_init(&param);
+    maria_chk_init(&param);
     param.op_name= "recreating_index";
     param.testflag= (T_SILENT | T_REP_BY_SORT | T_QUICK |
                      T_CREATE_MISSING_KEYS);
@@ -1395,7 +1395,7 @@ void ha_maria::info(uint flag)
   if (flag & HA_STATUS_ERRKEY)
   {
     errkey= info.errkey;
-    my_store_ptr(dup_ref, ref_length, info.dupp_key_pos);
+    my_store_ptr(dup_ref, ref_length, info.dup_key_pos);
   }
   /* Faster to always update, than to do it based on flag */
   stats.update_time= info.update_time;
@@ -1482,6 +1482,7 @@ int ha_maria::create(const char *name, register TABLE *table_arg,
   HA_KEYSEG *keyseg;
   TABLE_SHARE *share= table_arg->s;
   uint options= share->db_options_in_use;
+  enum data_file_type row_type;
   DBUG_ENTER("ha_maria::create");
 
   type= HA_KEYTYPE_BINARY;                      // Keep compiler happy
@@ -1607,22 +1608,22 @@ int ha_maria::create(const char *name, register TABLE *table_arg,
     if (recpos != minpos)
     {                                           // Reserved space (Null bits?)
       bzero((char*) recinfo_pos, sizeof(*recinfo_pos));
-      recinfo_pos->type= (int) FIELD_NORMAL;
+      recinfo_pos->type= FIELD_NORMAL;
       recinfo_pos++->length= (uint16) (minpos - recpos);
     }
     if (!found)
       break;
 
     if (found->flags & BLOB_FLAG)
-      recinfo_pos->type= (int) FIELD_BLOB;
+      recinfo_pos->type= FIELD_BLOB;
     else if (found->type() == MYSQL_TYPE_VARCHAR)
       recinfo_pos->type= FIELD_VARCHAR;
     else if (!(options & HA_OPTION_PACK_RECORD))
-      recinfo_pos->type= (int) FIELD_NORMAL;
+      recinfo_pos->type= FIELD_NORMAL;
     else if (found->zero_pack())
-      recinfo_pos->type= (int) FIELD_SKIP_ZERO;
+      recinfo_pos->type= FIELD_SKIP_ZERO;
     else
-      recinfo_pos->type= (int) ((length <= 3 ||
+      recinfo_pos->type= ((length <= 3 ||
                            (found->flags & ZEROFILL_FLAG)) ?
                           FIELD_NORMAL :
                           found->type() == MYSQL_TYPE_STRING ||
@@ -1666,12 +1667,26 @@ int ha_maria::create(const char *name, register TABLE *table_arg,
   if (options & HA_OPTION_DELAY_KEY_WRITE)
     create_flags |= HA_CREATE_DELAY_KEY_WRITE;
 
+  switch (info->row_type) {
+  case ROW_TYPE_FIXED:
+    row_type= STATIC_RECORD;
+    break;
+  case ROW_TYPE_DYNAMIC:
+    row_type= DYNAMIC_RECORD;
+    break;
+  default:
+  case ROW_TYPE_PAGES:
+    row_type= BLOCK_RECORD;
+    break;
+  }
+
   /* TODO: Check that the following fn_format is really needed */
   error=
-    maria_create(fn_format
-                 (buff, name, "", "", MY_UNPACK_FILENAME | MY_APPEND_EXT),
-                 share->keys, keydef, (uint) (recinfo_pos - recinfo), recinfo,
-                 0, (MARIA_UNIQUEDEF *) 0, &create_info, create_flags);
+    maria_create(fn_format(buff, name, "", "",
+                           MY_UNPACK_FILENAME | MY_APPEND_EXT),
+                 row_type, share->keys, keydef, (uint) (recinfo_pos - recinfo),
+                 recinfo, 0, (MARIA_UNIQUEDEF *) 0, &create_info,
+                 create_flags);
 
   my_free((gptr) recinfo, MYF(0));
   DBUG_RETURN(error);

sql/mysqld.cc

@@ -7133,6 +7133,7 @@ static void mysql_init_variables(void)
     when collecting index statistics for MyISAM tables.
   */
   global_system_variables.myisam_stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
+  global_system_variables.maria_stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
 
   /* Variables that depends on compile options */
 #ifndef DBUG_OFF

sql/uniques.cc

@@ -446,7 +446,7 @@ static bool merge_walk(uchar *merge_buffer, uint merge_buffer_size,
   */
   for (top= begin; top != end; ++top)
   {
-    top->base= merge_buffer + (top - begin) * piece_size;
+    top->base= (byte*) (merge_buffer + (top - begin) * piece_size);
     top->max_keys= max_key_count_per_piece;
     bytes_read= read_to_buffer(file, top, key_length);
     if (bytes_read == (uint) (-1))

storage/maria/Makefile.am

@@ -28,7 +28,7 @@ bin_PROGRAMS =		maria_chk maria_pack maria_ftdump
 maria_chk_DEPENDENCIES=	$(LIBRARIES)
 maria_pack_DEPENDENCIES=$(LIBRARIES)
 noinst_PROGRAMS =	ma_test1 ma_test2 ma_test3 ma_rt_test ma_sp_test
-noinst_HEADERS =	maria_def.h ma_rt_index.h ma_rt_key.h ma_rt_mbr.h ma_sp_defs.h ma_fulltext.h ma_ftdefs.h ma_ft_test1.h ma_ft_eval.h
+noinst_HEADERS =	maria_def.h ma_blockrec.h ma_rt_index.h ma_rt_key.h ma_rt_mbr.h ma_sp_defs.h ma_fulltext.h ma_ftdefs.h ma_ft_test1.h ma_ft_eval.h
 ma_test1_DEPENDENCIES=	$(LIBRARIES)
 ma_test2_DEPENDENCIES=	$(LIBRARIES)
 ma_test3_DEPENDENCIES=	$(LIBRARIES)
@@ -42,6 +42,7 @@ libmaria_a_SOURCES =	ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c \
 			ma_search.c ma_page.c ma_key.c ma_locking.c \
 			ma_rrnd.c ma_scan.c ma_cache.c \
 			ma_statrec.c ma_packrec.c ma_dynrec.c \
+			ma_blockrec.c ma_bitmap.c \
 			ma_update.c ma_write.c ma_unique.c \
 			ma_delete.c \
 			ma_rprev.c ma_rfirst.c ma_rlast.c ma_rsame.c \

storage/maria/ma_blockrec.h

+/* Copyright (C) 2007 Michael Widenius
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+/*
+  Storage of records in block
+*/
+
+#define LSN_SIZE		7
+#define DIRCOUNT_SIZE		1	/* Stores number of rows on page */
+#define EMPTY_SPACE_SIZE	2	/* Stores empty space on page */
+#define PAGE_TYPE_SIZE		1
+#define PAGE_SUFFIX_SIZE	0	/* Bytes for page suffix */
+#define PAGE_HEADER_SIZE	(LSN_SIZE + DIRCOUNT_SIZE + EMPTY_SPACE_SIZE +\
+                                 PAGE_TYPE_SIZE)
+#define PAGE_OVERHEAD_SIZE	(PAGE_HEADER_SIZE + DIR_ENTRY_SIZE + \
+                                 PAGE_SUFFIX_SIZE)
+#define BLOCK_RECORD_POINTER_SIZE	6
+
+#define FULL_PAGE_SIZE(block_size) ((block_size) - LSN_SIZE - PAGE_TYPE_SIZE)
+
+#define ROW_EXTENT_PAGE_SIZE	5
+#define ROW_EXTENT_COUNT_SIZE   2
+#define ROW_EXTENT_SIZE		(ROW_EXTENT_PAGE_SIZE + ROW_EXTENT_COUNT_SIZE)
+#define TAIL_BIT		0x8000	/* Bit in page_count to signify tail */
+#define ELEMENTS_RESERVED_FOR_MAIN_PART 4
+#define EXTRA_LENGTH_FIELDS		3
+
+#define FLAG_SIZE		1
+#define TRANSID_SIZE		6
+#define VERPTR_SIZE		7
+#define DIR_ENTRY_SIZE		4
+#define FIELD_OFFSET_SIZE	2
+
+/* Minimum header size needed for a new row */
+#define BASE_ROW_HEADER_SIZE FLAG_SIZE
+#define TRANS_ROW_EXTRA_HEADER_SIZE TRANSID_SIZE
+
+#define PAGE_TYPE_MASK 127
+enum en_page_type { UNALLOCATED_PAGE, HEAD_PAGE, TAIL_PAGE, BLOB_PAGE, MAX_PAGE_TYPE };
+
+#define PAGE_TYPE_OFFSET        LSN_SIZE
+#define DIR_ENTRY_OFFSET        LSN_SIZE+PAGE_TYPE_SIZE
+#define EMPTY_SPACE_OFFSET      (DIR_ENTRY_OFFSET + DIRCOUNT_SIZE)
+
+#define PAGE_CAN_BE_COMPACTED   128             /* Bit in PAGE_TYPE */
+
+/* Bits used for flag byte (one byte, first in record) */
+#define ROW_FLAG_TRANSID                1
+#define ROW_FLAG_VER_PTR                2
+#define ROW_FLAG_DELETE_TRANSID         4
+#define ROW_FLAG_NULLS_EXTENDED         8
+#define ROW_FLAG_EXTENTS                128
+#define ROW_FLAG_ALL			(1+2+4+8+128)
+
+/* Variables that affects how data pages are utilized */
+#define MIN_TAIL_SIZE           32
+
+/* Fixed part of Max possible header size; See table in ma_blockrec.c */
+#define MAX_FIXED_HEADER_SIZE (FLAG_SIZE + 3 + ROW_EXTENT_SIZE + 3)
+#define TRANS_MAX_FIXED_HEADER_SIZE (MAX_FIXED_HEADER_SIZE + \
+                                     FLAG_SIZE + TRANSID_SIZE + VERPTR_SIZE + \
+                                     TRANSID_SIZE)
+
+/* We use 1 byte in record header to store number of directory entries */
+#define MAX_ROWS_PER_PAGE	255
+
+/* Bits for MARIA_BITMAP_BLOCKS->used */
+#define BLOCKUSED_USED		 1
+#define BLOCKUSED_USE_ORG_BITMAP 2
+#define BLOCKUSED_TAIL		 4
+
+/* defines that affects allocation (density) of data */
+
+/* If we fill up a block to 75 %, don't create a new tail page for it */
+#define MAX_TAIL_SIZE(block_size) ((block_size) *3 / 4)
+
+/* Functions to convert MARIA_RECORD_POS to/from page:offset */
+
+static inline MARIA_RECORD_POS ma_recordpos(ulonglong page, uint offset)
+{
+  return (MARIA_RECORD_POS) ((page << 8) | offset);
+}
+
+static inline my_off_t ma_recordpos_to_page(MARIA_RECORD_POS record_pos)
+{
+  return record_pos >> 8;
+}
+
+static inline my_off_t ma_recordpos_to_offset(MARIA_RECORD_POS record_pos)
+{
+  return record_pos & 255;
+}
+
+/* ma_blockrec.c */
+void _ma_init_block_record_data(void);
+my_bool _ma_once_init_block_row(MARIA_SHARE *share, File dfile);
+my_bool _ma_once_end_block_row(MARIA_SHARE *share);
+my_bool _ma_init_block_row(MARIA_HA *info);
+void _ma_end_block_row(MARIA_HA *info);
+
+my_bool _ma_update_block_record(MARIA_HA *info, MARIA_RECORD_POS pos,
+                                const byte *record);
+my_bool _ma_delete_block_record(MARIA_HA *info);
+int     _ma_read_block_record(MARIA_HA *info, byte *record,
+                              MARIA_RECORD_POS record_pos);
+int _ma_read_block_record2(MARIA_HA *info, byte *record,
+                           byte *data, byte *end_of_data);
+int     _ma_scan_block_record(MARIA_HA *info, byte *record,
+                              MARIA_RECORD_POS, my_bool);
+my_bool _ma_cmp_block_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def,
+                             const byte *record, MARIA_RECORD_POS pos);
+my_bool _ma_scan_init_block_record(MARIA_HA *info);
+void _ma_scan_end_block_record(MARIA_HA *info);
+
+MARIA_RECORD_POS _ma_write_init_block_record(MARIA_HA *info,
+                                             const byte *record);
+my_bool _ma_write_block_record(MARIA_HA *info, const byte *record);
+my_bool _ma_write_abort_block_record(MARIA_HA *info);
+my_bool _ma_compare_block_record(register MARIA_HA *info,
+                                 register const byte *record);
+
+/* ma_bitmap.c */
+my_bool _ma_bitmap_init(MARIA_SHARE *share, File file);
+my_bool _ma_bitmap_end(MARIA_SHARE *share);
+my_bool _ma_flush_bitmap(MARIA_SHARE *share);
+my_bool _ma_read_bitmap_page(MARIA_SHARE *share, MARIA_FILE_BITMAP *bitmap,
+                             ulonglong page);
+my_bool _ma_bitmap_find_place(MARIA_HA *info, MARIA_ROW *row,
+                              MARIA_BITMAP_BLOCKS *result_blocks);
+my_bool _ma_bitmap_release_unused(MARIA_HA *info, MARIA_BITMAP_BLOCKS *blocks);
+my_bool _ma_bitmap_free_full_pages(MARIA_HA *info, const byte *extents,
+                                   uint count);
+my_bool _ma_bitmap_set(MARIA_HA *info, ulonglong pos, my_bool head,
+                       uint empty_space);
+my_bool _ma_reset_full_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap,
+                                 ulonglong page, uint page_count);
+uint _ma_free_size_to_head_pattern(MARIA_FILE_BITMAP *bitmap, uint size);
+my_bool _ma_bitmap_find_new_place(MARIA_HA *info, MARIA_ROW *new_row,
+                                  ulonglong page, uint free_size,
+                                  MARIA_BITMAP_BLOCKS *result_blocks);
+my_bool _ma_check_bitmap_data(MARIA_HA *info,
+                              enum en_page_type page_type, ulonglong page,
+                              uint empty_space, uint *bitmap_pattern);
+my_bool _ma_check_if_right_bitmap_type(MARIA_HA *info,
+                                       enum en_page_type page_type,
+                                       ulonglong page,
+                                       uint *bitmap_pattern);

storage/maria/ma_checksum.c

@@ -18,23 +18,27 @@
 
 #include "maria_def.h"
 
-ha_checksum _ma_checksum(MARIA_HA *info, const byte *buf)
+ha_checksum _ma_checksum(MARIA_HA *info, const byte *record)
 {
   uint i;
   ha_checksum crc=0;
   MARIA_COLUMNDEF *rec=info->s->rec;
 
-  for (i=info->s->base.fields ; i-- ; buf+=(rec++)->length)
+  if (info->s->base.null_bytes)
+    crc= my_checksum(crc, record, info->s->base.null_bytes);
+
+  for (i=info->s->base.fields ; i-- ; )
   {
-    const byte *pos;
+    const byte *pos= record + rec->offset;
     ulong length;
+
     switch (rec->type) {
     case FIELD_BLOB:
     {
       length= _ma_calc_blob_length(rec->length-
 					maria_portable_sizeof_char_ptr,
-					buf);
-      memcpy((char*) &pos, buf+rec->length- maria_portable_sizeof_char_ptr,
+					pos);
+      memcpy((char*) &pos, pos+rec->length- maria_portable_sizeof_char_ptr,
 	     sizeof(char*));
       break;
     }
@@ -42,18 +46,17 @@ ha_checksum _ma_checksum(MARIA_HA *info, const byte *buf)
     {
       uint pack_length= HA_VARCHAR_PACKLENGTH(rec->length-1);
       if (pack_length == 1)
-        length= (ulong) *(uchar*) buf;
+        length= (ulong) *(uchar*) pos;
       else
-        length= uint2korr(buf);
-      pos= buf+pack_length;
+        length= uint2korr(pos);
+      pos+= pack_length;
       break;
     }
     default:
-      length=rec->length;
-      pos=buf;
+      length= rec->length;
       break;
     }
-    crc=my_checksum(crc, pos ? pos : "", length);
+    crc= my_checksum(crc, pos ? pos : "", length);
   }
   return crc;
 }

storage/maria/ma_close.c

@@ -28,7 +28,7 @@ int maria_close(register MARIA_HA *info)
   int error=0,flag;
   MARIA_SHARE *share=info->s;
   DBUG_ENTER("maria_close");
-  DBUG_PRINT("enter",("base: %lx  reopen: %u  locks: %u",
+  DBUG_PRINT("enter",("base: 0x%lx  reopen: %u  locks: %u",
 		      info,(uint) share->reopen, (uint) share->tot_locks));
 
   pthread_mutex_lock(&THR_LOCK_maria);
@@ -60,16 +60,22 @@ int maria_close(register MARIA_HA *info)
   maria_open_list=list_delete(maria_open_list,&info->open_list);
   pthread_mutex_unlock(&share->intern_lock);
 
-  my_free(_ma_get_rec_buff_ptr(info, info->rec_buff), MYF(MY_ALLOW_ZERO_PTR));
+  my_free(info->rec_buff, MYF(MY_ALLOW_ZERO_PTR));
+  (share->end)(info);
+
+  if (info->s->data_file_type == BLOCK_RECORD)
+    info->dfile= -1;                /* Closed in ma_end_once_block_row */
   if (flag)
   {
-    if (share->kfile >= 0 &&
-	flush_key_blocks(share->key_cache, share->kfile,
-			 share->temporary ? FLUSH_IGNORE_CHANGED :
-			 FLUSH_RELEASE))
-      error=my_errno;
     if (share->kfile >= 0)
     {
+      if ((*share->once_end)(share))
+        error= my_errno;
+      if (flush_key_blocks(share->key_cache, share->kfile,
+                           share->temporary ? FLUSH_IGNORE_CHANGED :
+                           FLUSH_RELEASE))
+        error= my_errno;
+
       /*
         If we are crashed, we can safely flush the current state as it will
         not change the crashed state.
@@ -78,18 +84,13 @@ int maria_close(register MARIA_HA *info)
       */
       if (share->mode != O_RDONLY && maria_is_crashed(info))
 	_ma_state_info_write(share->kfile, &share->state, 1);
-      if (my_close(share->kfile,MYF(0)))
-        error = my_errno;
+      if (my_close(share->kfile, MYF(0)))
+        error= my_errno;
     }
 #ifdef HAVE_MMAP
     if (share->file_map)
       _ma_unmap_file(info);
 #endif
-    if (share->decode_trees)
-    {
-      my_free((gptr) share->decode_trees,MYF(0));
-      my_free((gptr) share->decode_tables,MYF(0));
-    }
 #ifdef THREAD
     thr_lock_delete(&share->lock);
     VOID(pthread_mutex_destroy(&share->intern_lock));

storage/maria/ma_fulltext.h

@@ -25,4 +25,4 @@ int  _ma_ft_cmp(MARIA_HA *, uint, const byte *, const byte *);
 int  _ma_ft_add(MARIA_HA *, uint, byte *, const byte *, my_off_t);
 int  _ma_ft_del(MARIA_HA *, uint, byte *, const byte *, my_off_t);
 
-uint _ma_ft_convert_to_ft2(MARIA_HA *, uint, uchar *);
+uint _ma_ft_convert_to_ft2(MARIA_HA *, uint, byte *);