postreview changes for page cache and pre review commit for loghandler

storage/maria/unittest/test_file.c:
  Rename: unittest/mysys/test_file.c -> storage/maria/unittest/test_file.c
storage/maria/unittest/test_file.h:
  Rename: unittest/mysys/test_file.h -> storage/maria/unittest/test_file.h
include/pagecache.h:
  A waiting queue mechanism moved to separate file wqueue.*
  Pointer name changed for compatibility
mysys/Makefile.am:
  A waiting queue mechanism moved to separate file wqueue.*
mysys/mf_keycache.c:
  fixed unsigned comparison
mysys/mf_pagecache.c:
  A waiting queue mechanism moved to separate file wqueue.*
  Fixed bug in unregistering block during write
storage/maria/Makefile.am:
  The loghandler files added
storage/maria/ma_control_file.h:
  Now we have loghandler and can compile control file
storage/maria/maria_def.h:
  Including files need for compilation of maria
storage/maria/unittest/Makefile.am:
  unit tests of loghandler
storage/maria/unittest/ma_control_file-t.c:
  Used maria def
storage/maria/unittest/mf_pagecache_consist.c:
  fixed memory overrun
storage/maria/unittest/mf_pagecache_single.c:
  fixed used uninitialized memory
unittest/mysys/Makefile.am:
  unittests of pagecache moved to maria becase pagecache need loghandler
include/wqueue.h:
  New BitKeeper file ``include/wqueue.h''
mysys/wqueue.c:
  New BitKeeper file ``mysys/wqueue.c''
storage/maria/ma_loghandler.c:
  New BitKeeper file ``storage/maria/ma_loghandler.c''
storage/maria/ma_loghandler.h:
  New BitKeeper file ``storage/maria/ma_loghandler.h''
storage/maria/ma_loghandler_lsn.h:
  New BitKeeper file ``storage/maria/ma_loghandler_lsn.h''
storage/maria/unittest/ma_test_loghandler-t.c:
  New BitKeeper file ``storage/maria/unittest/ma_test_loghandler-t.c''
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
  New BitKeeper file ``storage/maria/unittest/ma_test_loghandler_multigroup-t.c''
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
  New BitKeeper file ``storage/maria/unittest/ma_test_loghandler_multithread-t.c''
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
  New BitKeeper file ``storage/maria/unittest/ma_test_loghandler_pagecache-t.c''

include/pagecache.h

@@ -20,11 +20,13 @@
 #define _pagecache_h
 C_MODE_START
 
+#include "../storage/maria/ma_loghandler_lsn.h"
+
 /* Type of the page */
 enum pagecache_page_type
 {
 #ifndef DBUG_OFF
-  /* used only for control page type chenging during debugging */
+  /* used only for control page type changing during debugging */
   PAGECACHE_EMPTY_PAGE,
 #endif
   /* the page does not contain LSN */
@@ -34,7 +36,7 @@ enum pagecache_page_type
 };
 
 /*
-  This enum describe lock status changing. every typr of page cache will
+  This enum describe lock status changing. every type of page cache will
   interpret WRITE/READ lock as it need.
 */
 enum pagecache_page_lock
@@ -71,9 +73,7 @@ enum pagecache_write_mode
 
 typedef void *PAGECACHE_PAGE_LINK;
 
-/* TODO: move to loghandler emulator */
-typedef void LOG_HANDLER;
-typedef void *LSN;
+typedef void *LSN_PTR;
 
 /* file descriptor for Maria */
 typedef struct st_pagecache_file
@@ -82,7 +82,7 @@ typedef struct st_pagecache_file
 } PAGECACHE_FILE;
 
 /* page number for maria */
-typedef uint32 maria_page_no_t;
+typedef uint32 pgcache_page_no_t;
 
 /* declare structures that is used by  st_pagecache */
 
@@ -93,11 +93,9 @@ typedef struct st_pagecache_page PAGECACHE_PAGE;
 struct st_pagecache_hash_link;
 typedef struct st_pagecache_hash_link PAGECACHE_HASH_LINK;
 
-/* info about requests in a waiting queue */
-typedef struct st_pagecache_wqueue
-{
-  struct st_my_thread_var *last_thread;  /* circular list of waiting threads */
-} PAGECACHE_WQUEUE;
+#include <wqueue.h>
+
+typedef my_bool (*pagecache_disk_read_validator)(byte *page, gptr data);
 
 #define PAGECACHE_CHANGED_BLOCKS_HASH 128  /* must be power of 2 */
 
@@ -136,16 +134,14 @@ typedef struct st_pagecache
   PAGECACHE_BLOCK_LINK *used_last;/* ptr to the last block of the LRU chain  */
   PAGECACHE_BLOCK_LINK *used_ins;/* ptr to the insertion block in LRU chain  */
   pthread_mutex_t cache_lock;    /* to lock access to the cache structure    */
-  PAGECACHE_WQUEUE resize_queue; /* threads waiting during resize operation  */
-  PAGECACHE_WQUEUE waiting_for_hash_link;/* waiting for a free hash link     */
-  PAGECACHE_WQUEUE waiting_for_block;   /* requests waiting for a free block */
+  WQUEUE resize_queue; /* threads waiting during resize operation  */
+  WQUEUE waiting_for_hash_link;/* waiting for a free hash link     */
+  WQUEUE waiting_for_block;   /* requests waiting for a free block */
   /* hash for dirty file bl.*/
   PAGECACHE_BLOCK_LINK *changed_blocks[PAGECACHE_CHANGED_BLOCKS_HASH];
   /* hash for other file bl.*/
   PAGECACHE_BLOCK_LINK *file_blocks[PAGECACHE_CHANGED_BLOCKS_HASH];
 
-  LOG_HANDLER *loghandler;       /* loghandler structure */
-
   /*
     The following variables are and variables used to hold parameters for
     initializing the key cache.
@@ -169,24 +165,29 @@ typedef struct st_pagecache
 
 extern int init_pagecache(PAGECACHE *pagecache, my_size_t use_mem,
                           uint division_limit, uint age_threshold,
-                          uint block_size,
-                          LOG_HANDLER *loghandler);
+                          uint block_size);
 extern int resize_pagecache(PAGECACHE *pagecache,
                             my_size_t use_mem, uint division_limit,
                             uint age_threshold);
 extern void change_pagecache_param(PAGECACHE *pagecache, uint division_limit,
                                    uint age_threshold);
-extern byte *pagecache_read(PAGECACHE *pagecache,
-                            PAGECACHE_FILE *file,
-                            maria_page_no_t pageno,
-                            uint level,
-                            byte *buff,
-                            enum pagecache_page_type type,
-                            enum pagecache_page_lock lock,
-                            PAGECACHE_PAGE_LINK *link);
+
+#define pagecache_read(P,F,N,L,B,T,K,I) \
+  pagecache_valid_read(P,F,N,L,B,T,K,I,0,0)
+
+extern byte *pagecache_valid_read(PAGECACHE *pagecache,
+                                  PAGECACHE_FILE *file,
+                                  pgcache_page_no_t pageno,
+                                  uint level,
+                                  byte *buff,
+                                  enum pagecache_page_type type,
+                                  enum pagecache_page_lock lock,
+                                  PAGECACHE_PAGE_LINK *link,
+                                  pagecache_disk_read_validator validator,
+                                  gptr validator_data);
 extern my_bool pagecache_write(PAGECACHE *pagecache,
                                PAGECACHE_FILE *file,
-                               maria_page_no_t pageno,
+                               pgcache_page_no_t pageno,
                                uint level,
                                byte *buff,
                                enum pagecache_page_type type,
@@ -196,20 +197,20 @@ extern my_bool pagecache_write(PAGECACHE *pagecache,
                                PAGECACHE_PAGE_LINK *link);
 extern void pagecache_unlock_page(PAGECACHE *pagecache,
                                   PAGECACHE_FILE *file,
-                                  maria_page_no_t pageno,
+                                  pgcache_page_no_t pageno,
                                   enum pagecache_page_lock lock,
                                   enum pagecache_page_pin pin,
                                   my_bool stamp_this_page,
-                                  LSN first_REDO_LSN_for_page);
+                                  LSN_PTR first_REDO_LSN_for_page);
 extern void pagecache_unlock(PAGECACHE *pagecache,
                              PAGECACHE_PAGE_LINK *link,
                              enum pagecache_page_lock lock,
                              enum pagecache_page_pin pin,
                              my_bool stamp_this_page,
-                             LSN first_REDO_LSN_for_page);
+                             LSN_PTR first_REDO_LSN_for_page);
 extern void pagecache_unpin_page(PAGECACHE *pagecache,
                                  PAGECACHE_FILE *file,
-                                 maria_page_no_t pageno);
+                                 pgcache_page_no_t pageno);
 extern void pagecache_unpin(PAGECACHE *pagecache,
                             PAGECACHE_PAGE_LINK *link);
 extern int flush_pagecache_blocks(PAGECACHE *keycache,
@@ -217,7 +218,7 @@ extern int flush_pagecache_blocks(PAGECACHE *keycache,
                                   enum flush_type type);
 extern my_bool pagecache_delete_page(PAGECACHE *pagecache,
                                      PAGECACHE_FILE *file,
-                                     maria_page_no_t pageno,
+                                     pgcache_page_no_t pageno,
                                      enum pagecache_page_lock lock,
                                      my_bool flush);
 extern void end_pagecache(PAGECACHE *keycache, my_bool cleanup);

include/wqueue.h

+
+#ifndef _wqueue_h
+#define _wqueue_h
+
+#include <my_global.h>
+#include <my_pthread.h>
+
+/* info about requests in a waiting queue */
+typedef struct st_pagecache_wqueue
+{
+  struct st_my_thread_var *last_thread;         /* circular list of waiting
+                                                   threads */
+} WQUEUE;
+
+#ifdef THREAD
+void wqueue_link_into_queue(WQUEUE *wqueue, struct st_my_thread_var *thread);
+void wqueue_unlink_from_queue(WQUEUE *wqueue, struct st_my_thread_var *thread);
+void wqueue_add_to_queue(WQUEUE *wqueue, struct st_my_thread_var *thread);
+void wqueue_add_and_wait(WQUEUE *wqueue,
+                         struct st_my_thread_var *thread,
+                         pthread_mutex_t *lock);
+void wqueue_release_queue(WQUEUE *wqueue);
+
+#endif
+
+#endif

mysys/Makefile.am

@@ -56,7 +56,7 @@ libmysys_a_SOURCES =    my_init.c my_getwd.c mf_getdate.c my_mmap.c \
 			my_handler.c my_netware.c my_largepage.c \
 			my_memmem.c \
 			my_windac.c my_access.c base64.c my_libwrap.c \
-		        mf_pagecache.c
+		        mf_pagecache.c wqueue.c
 EXTRA_DIST =		thr_alarm.c thr_lock.c my_pthread.c my_thr_init.c \
 			thr_mutex.c thr_rwlock.c \
 			CMakeLists.txt mf_soundex.c \

mysys/mf_keycache.c

@@ -1008,12 +1008,12 @@ static void unlink_block(KEY_CACHE *keycache, BLOCK_LINK *block)
 
   KEYCACHE_THREAD_TRACE("unlink_block");
 #if defined(KEYCACHE_DEBUG)
+  KEYCACHE_DBUG_ASSERT(keycache->blocks_available != 0);
   keycache->blocks_available--;
   KEYCACHE_DBUG_PRINT("unlink_block",
     ("unlinked block %u  status=%x   #requests=%u  #available=%u",
      BLOCK_NUMBER(block), block->status,
      block->requests, keycache->blocks_available));
-  KEYCACHE_DBUG_ASSERT(keycache->blocks_available >= 0);
 #endif
 }
 

mysys/wqueue.c

+
+#include <wqueue.h>
+
+#define STRUCT_PTR(TYPE, MEMBER, a)                                           \
+          (TYPE *) ((char *) (a) - offsetof(TYPE, MEMBER))
+/*
+  Link a thread into double-linked queue of waiting threads.
+
+  SYNOPSIS
+    wqueue_link_into_queue()
+      wqueue              pointer to the queue structure
+      thread              pointer to the thread to be added to the queue
+
+  RETURN VALUE
+    none
+
+  NOTES.
+    Queue is represented by a circular list of the thread structures
+    The list is double-linked of the type (**prev,*next), accessed by
+    a pointer to the last element.
+*/
+
+void wqueue_link_into_queue(WQUEUE *wqueue, struct st_my_thread_var *thread)
+{
+  struct st_my_thread_var *last;
+  if (!(last= wqueue->last_thread))
+  {
+    /* Queue is empty */
+    thread->next= thread;
+    thread->prev= &thread->next;
+  }
+  else
+  {
+    thread->prev= last->next->prev;
+    last->next->prev= &thread->next;
+    thread->next= last->next;
+    last->next= thread;
+  }
+  wqueue->last_thread= thread;
+}
+
+
+/*
+  Add a thread to single-linked queue of waiting threads
+
+  SYNOPSIS
+    wqueue_add_to_queue()
+      wqueue              pointer to the queue structure
+      thread              pointer to the thread to be added to the queue
+
+  RETURN VALUE
+    none
+
+  NOTES.
+    Queue is represented by a circular list of the thread structures
+    The list is single-linked of the type (*next), accessed by a pointer
+    to the last element.
+*/
+
+void wqueue_add_to_queue(WQUEUE *wqueue, struct st_my_thread_var *thread)
+{
+  struct st_my_thread_var *last;
+  if (!(last= wqueue->last_thread))
+    thread->next= thread;
+  else
+  {
+    thread->next= last->next;
+    last->next= thread;
+  }
+  wqueue->last_thread= thread;
+}
+
+/*
+  Unlink a thread from double-linked queue of waiting threads
+
+  SYNOPSIS
+    wqueue_unlink_from_queue()
+      wqueue              pointer to the queue structure
+      thread              pointer to the thread to be removed from the queue
+
+  RETURN VALUE
+    none
+
+  NOTES.
+    See NOTES for link_into_queue
+*/
+
+void wqueue_unlink_from_queue(WQUEUE *wqueue, struct st_my_thread_var *thread)
+{
+  if (thread->next == thread)
+    /* The queue contains only one member */
+    wqueue->last_thread= NULL;
+  else
+  {
+    thread->next->prev= thread->prev;
+    *thread->prev= thread->next;
+    if (wqueue->last_thread == thread)
+      wqueue->last_thread= STRUCT_PTR(struct st_my_thread_var, next,
+                                      thread->prev);
+  }
+  thread->next= NULL;
+}
+
+
+/*
+  Remove all threads from queue signaling them to proceed
+
+  SYNOPSIS
+    wqueue_realease_queue()
+      wqueue              pointer to the queue structure
+      thread              pointer to the thread to be added to the queue
+
+  RETURN VALUE
+    none
+
+  NOTES.
+    See notes for add_to_queue
+    When removed from the queue each thread is signaled via condition
+    variable thread->suspend.
+*/
+
+void wqueue_release_queue(WQUEUE *wqueue)
+{
+  struct st_my_thread_var *last= wqueue->last_thread;
+  struct st_my_thread_var *next= last->next;
+  struct st_my_thread_var *thread;
+  do
+  {
+    thread= next;
+    pthread_cond_signal(&thread->suspend);
+    next= thread->next;
+    thread->next= NULL;
+  }
+  while (thread != last);
+  wqueue->last_thread= NULL;
+}
+
+
+/*
+  Add thread and wait
+
+  SYNOPSYS
+    wqueue_add_and_wait()
+    wqueue               queue to add to
+    thread               thread which is waiting
+    lock                 mutex need for the operation
+*/
+
+void wqueue_add_and_wait(WQUEUE *wqueue,
+                         struct st_my_thread_var *thread, pthread_mutex_t *lock)
+{
+  DBUG_ENTER("wqueue_add_and_wait");
+  DBUG_PRINT("enter", ("thread ox%lxcond 0x%lx, mutex 0x%lx",
+                       (ulong) thread, (ulong) &thread->suspend, (ulong) lock));
+  wqueue_add_to_queue(wqueue, thread);
+  do
+  {
+    DBUG_PRINT("info", ("wait... cond 0x%lx, mutex 0x%lx",
+                        (ulong) &thread->suspend, (ulong) lock));
+    pthread_cond_wait(&thread->suspend, lock);
+    DBUG_PRINT("info", ("wait done cond 0x%lx, mutex 0x%lx, next 0x%lx",
+                        (ulong) &thread->suspend, (ulong) lock,
+                        (ulong) thread->next));
+  }
+  while (thread->next);
+  DBUG_VOID_RETURN;
+}

storage/maria/Makefile.am

@@ -110,7 +110,7 @@ libmaria_a_SOURCES =	ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c \
 			ma_ft_nlq_search.c ft_maria.c ma_sort.c \
 			ha_maria.cc trnman.c lockman.c tablockman.c \
 			ma_rt_index.c ma_rt_key.c ma_rt_mbr.c ma_rt_split.c \
-			ma_sp_key.c ma_control_file.c
+			ma_sp_key.c ma_control_file.c ma_loghandler.c
 CLEANFILES =		test?.MA? FT?.MA? isam.log ma_test_all ma_rt_test.MA? sp_test.MA?
 
 SUFFIXES = .sh

storage/maria/ma_control_file.h

@@ -17,28 +17,10 @@
 /*
   WL#3234 Maria control file
   First version written by Guilhem Bichot on 2006-04-27.
-  Does not compile yet.
 */
 
-#ifndef _control_file_h
-#define _control_file_h
-
-/*
-  Not everybody needs to call the control file that's why control_file.h is
-  not in maria_def.h. However, policy or habit may want to change this.
-*/
-
-#ifndef REMOVE_WHEN_SANJA_PUSHES_LOG_HANDLER
-/*
-  this is to get the control file to compile, until Sanja pushes the log
-  handler which will supersede those definitions.
-*/
-typedef struct st_lsn {
-  uint32 file_no;
-  uint32 rec_offset;
-} LSN;
-#define maria_data_root "."
-#endif
+#ifndef _ma_control_file_h
+#define _ma_control_file_h
 
 #define CONTROL_FILE_BASE_NAME "maria_control"
 /*

storage/maria/ma_loghandler_lsn.h

+#ifndef _ma_loghandler_lsn_h
+#define _ma_loghandler_lsn_h
+
+/* Transaction log record address (file_no is int24 on the disk) */
+typedef struct st_translog_address
+{
+  uint32 file_no;
+  uint32 rec_offset;
+} TRANSLOG_ADDRESS;
+
+/*
+  Compare addresses
+    A1 >  A2 -> result  > 0
+    A1 == A2 -> 0
+    A1 <  A2 -> result < 0
+*/
+#define cmp_translog_addr(A1,A2) \
+  ((A1).file_no == (A2).file_no ? \
+   ((int64)(A1).rec_offset) - (int64)(A2).rec_offset : \
+   ((int64)(A1).file_no - (int64)(A2).file_no))
+
+/* LSN type (address of certain log record chank */
+typedef TRANSLOG_ADDRESS LSN;
+
+/* Puts LSN into buffer (dst) */
+#define lsn7store(dst, lsn) \
+  do { \
+    int3store((dst), (lsn)->file_no); \
+    int4store((dst) + 3, (lsn)->rec_offset); \
+  } while (0)
+
+/* Unpacks LSN from the buffer (P) */
+#define lsn7korr(lsn, P) \
+  do { \
+    (lsn)->file_no= uint3korr(P); \
+    (lsn)->rec_offset= uint4korr((P) + 3); \
+  } while (0)
+
+#endif

storage/maria/maria_def.h

@@ -26,6 +26,10 @@
 #include <my_no_pthread.h>
 #endif
 
+#include <pagecache.h>
+#include "ma_loghandler.h"
+#include "ma_control_file.h"
+
 /* undef map from my_nosys; We need test-if-disk full */
 #undef my_write	
 
@@ -438,6 +442,7 @@ extern LIST *maria_open_list;
 extern uchar NEAR maria_file_magic[], NEAR maria_pack_file_magic[];
 extern uint NEAR maria_read_vec[], NEAR maria_readnext_vec[];
 extern uint maria_quick_table_bits;
+extern const char *maria_data_root;
 extern my_bool maria_inited;
 
 	/* This is used by _ma_calc_xxx_key_length och _ma_store_key */

storage/maria/unittest/Makefile.am

@@ -14,8 +14,10 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-AM_CPPFLAGS      = @ZLIB_INCLUDES@ -I$(top_builddir)/include 
-AM_CPPFLAGS     += -I$(top_srcdir)/include -I$(top_srcdir)/unittest/mytap
+AM_CPPFLAGS      = @ZLIB_INCLUDES@ -I$(top_builddir)/include \
+                  -I$(top_srcdir)/include -I$(top_srcdir)/unittest/mytap
+INCLUDES         = @ZLIB_INCLUDES@ -I$(top_builddir)/include \
+                  -I$(top_srcdir)/include -I$(top_srcdir)/unittest/mytap
 
 # Only reason to link with libmyisam.a here is that it's where some fulltext
 # pieces are (but soon we'll remove fulltext dependencies from Maria).
@@ -24,6 +26,54 @@ LDADD=			$(top_builddir)/unittest/mytap/libmytap.a \
                         $(top_builddir)/storage/myisam/libmyisam.a \
 			$(top_builddir)/mysys/libmysys.a \
 			$(top_builddir)/dbug/libdbug.a \
-			$(top_builddir)/strings/libmystrings.a @ZLIB_LIBS@
-noinst_PROGRAMS =	ma_control_file-t trnman-t lockman-t lockman1-t lockman2-t
-CLEANFILES =		maria_control
+			$(top_builddir)/strings/libmystrings.a @ZLIB_LIBS@ \
+			$(top_builddir)/storage/maria/ma_loghandler.o
+noinst_PROGRAMS =	ma_control_file-t trnman-t lockman-t lockman1-t \
+                        lockman2-t \
+			mf_pagecache_single_1k-t mf_pagecache_single_8k-t \
+			mf_pagecache_single_64k-t \
+			mf_pagecache_consist_1k-t mf_pagecache_consist_64k-t \
+			mf_pagecache_consist_1kHC-t \
+			mf_pagecache_consist_64kHC-t \
+			mf_pagecache_consist_1kRD-t \
+			mf_pagecache_consist_64kRD-t \
+			mf_pagecache_consist_1kWR-t \
+			mf_pagecache_consist_64kWR-t \
+			ma_test_loghandler-t \
+                        ma_test_loghandler_multigroup-t \
+			ma_test_loghandler_multithread-t \
+			ma_test_loghandler_pagecache-t
+
+mf_pagecache_single_src =	mf_pagecache_single.c $(top_srcdir)/mysys/mf_pagecache.c test_file.c
+mf_pagecache_consist_src =	mf_pagecache_consist.c $(top_srcdir)/mysys/mf_pagecache.c test_file.c
+mf_pagecache_common_cppflags =	-DEXTRA_DEBUG -DPAGECACHE_DEBUG -DMAIN
+
+mf_pagecache_single_1k_t_SOURCES =	$(mf_pagecache_single_src)
+mf_pagecache_single_8k_t_SOURCES =	$(mf_pagecache_single_src)
+mf_pagecache_single_64k_t_SOURCES =	$(mf_pagecache_single_src)
+mf_pagecache_single_1k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024
+mf_pagecache_single_8k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=8192
+mf_pagecache_single_64k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536
+
+mf_pagecache_consist_1k_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_1k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024
+mf_pagecache_consist_64k_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_64k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536
+
+mf_pagecache_consist_1kHC_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_1kHC_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -DTEST_HIGH_CONCURENCY
+mf_pagecache_consist_64kHC_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_64kHC_t_CPPFLAGS =	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_HIGH_CONCURENCY
+
+mf_pagecache_consist_1kRD_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_1kRD_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -DTEST_READERS
+mf_pagecache_consist_64kRD_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_64kRD_t_CPPFLAGS =	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_READERS
+
+mf_pagecache_consist_1kWR_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_1kWR_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -DTEST_WRITERS
+mf_pagecache_consist_64kWR_t_SOURCES =	$(mf_pagecache_consist_src)
+mf_pagecache_consist_64kWR_t_CPPFLAGS =	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_WRITERS
+
+CLEANFILES =		maria_control page_cache_test_file_1 \
+			maria_log.???????? maria_control

storage/maria/unittest/ma_control_file-t.c

@@ -33,7 +33,7 @@
 #endif
 
 #include "maria.h"
-#include "../../../storage/maria/ma_control_file.h"
+#include "../../../storage/maria/maria_def.h"
 #include <my_getopt.h>
 
 char file_name[FN_REFLEN];

storage/maria/unittest/ma_test_loghandler_pagecache-t.c

+#include "../maria_def.h"
+#include <stdio.h>
+#include <errno.h>
+
+#ifndef DBUG_OFF
+static const char *default_dbug_option;
+#endif
+
+#define PCACHE_SIZE (1024*1024*10)
+#define PCACHE_PAGE TRANSLOG_PAGE_SIZE
+#define LOG_FILE_SIZE (1024L*1024L*1024L + 1024L*1024L*512)
+#define LOG_FLAGS 0
+
+static char *first_translog_file= (char*)"maria_log.00000001";
+static char *file1_name= (char*)"page_cache_test_file_1";
+static PAGECACHE_FILE file1;
+
+int main(int argc, char *argv[])
+{
+  uint pagen;
+  uchar long_tr_id[6];
+  PAGECACHE pagecache;
+  LSN lsn;
+  MY_STAT st, *stat;
+
+  MY_INIT(argv[0]);
+
+  bzero(&pagecache, sizeof(pagecache));
+  maria_data_root= ".";
+  /* be sure that we have no logs in the directory*/
+  if (my_stat(CONTROL_FILE_BASE_NAME, &st,  MYF(0)))
+    my_delete(CONTROL_FILE_BASE_NAME, MYF(0));
+  if (my_stat(first_translog_file, &st,  MYF(0)))
+    my_delete(first_translog_file, MYF(0));
+
+  bzero(long_tr_id, 6);
+#ifndef DBUG_OFF
+#if defined(__WIN__)
+  default_dbug_option= "d:t:i:O,\\ma_test_loghandler_pagecache.trace";
+#else
+  default_dbug_option= "d:t:i:o,/tmp/ma_test_loghandler_pagecache.trace";
+#endif
+  if (argc > 1)
+  {
+    DBUG_SET(default_dbug_option);
+    DBUG_SET_INITIAL(default_dbug_option);
+  }
+#endif
+
+  if (ma_control_file_create_or_open())
+  {
+    fprintf(stderr, "Can't init control file (%d)\n", errno);
+    exit(1);
+  }
+  if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
+                             PCACHE_PAGE)) == 0)
+  {
+    fprintf(stderr, "Got error: init_pagecache() (errno: %d)\n", errno);
+    exit(1);
+  }
+  if (translog_init(".", LOG_FILE_SIZE, 50112, 0, &pagecache, LOG_FLAGS))
+  {
+    fprintf(stderr, "Can't init loghandler (%d)\n", errno);
+    translog_destroy();
+    exit(1);
+  }
+
+  if ((stat= my_stat(first_translog_file, &st,  MYF(0))) == 0)
+  {
+    fprintf(stderr, "There is no %s (%d)\n", first_translog_file, errno);
+    exit(1);
+  }
+  if (st.st_size != TRANSLOG_PAGE_SIZE)
+  {
+    fprintf(stderr,
+            "incorrect initial size of %s: %ld instead of %ld\n",
+            first_translog_file, (long)st.st_size, (long)TRANSLOG_PAGE_SIZE);
+    exit(1);
+  }
+  int4store(long_tr_id, 0);
+  if (translog_write_record(&lsn,
+                            LOGREC_LONG_TRANSACTION_ID,
+                            0, NULL, 6, long_tr_id, 0))
+  {
+    fprintf(stderr, "Can't write record #%lu\n", (ulong) 0);
+    translog_destroy();
+    exit(1);
+  }
+
+  if ((file1.file= my_open(file1_name,
+                           O_CREAT | O_TRUNC | O_RDWR, MYF(0))) == -1)
+  {
+    fprintf(stderr, "Got error during file1 creation from open() (errno: %d)\n",
+	    errno);
+    exit(1);
+  }
+  if (chmod(file1_name, S_IRWXU | S_IRWXG | S_IRWXO) != 0)
+  {
+    fprintf(stderr, "Got error during file1 chmod() (errno: %d)\n",
+	    errno);
+    exit(1);
+  }
+
+  {
+    uchar page[PCACHE_PAGE];
+
+    bzero(page, PCACHE_PAGE);
+#define PAGE_LSN_OFFSET 0
+    lsn7store(page + PAGE_LSN_OFFSET, &lsn);
+    pagecache_write(&pagecache, &file1, 0, 3, (char*)page,
+                    PAGECACHE_LSN_PAGE,
+                    PAGECACHE_LOCK_LEFT_UNLOCKED,
+                    PAGECACHE_PIN_LEFT_UNPINNED,
+                    PAGECACHE_WRITE_DELAY,
+                    0);
+    flush_pagecache_blocks(&pagecache, &file1, FLUSH_FORCE_WRITE);
+  }
+  if ((stat= my_stat(first_translog_file, &st,  MYF(0))) == 0)
+  {
+    fprintf(stderr, "can't stat %s (%d)\n", first_translog_file, errno);
+    exit(1);
+  }
+  if (st.st_size != TRANSLOG_PAGE_SIZE * 2)
+  {
+    fprintf(stderr,
+            "incorrect initial size of %s: %ld instead of %ld\n",
+            first_translog_file,
+            (long)st.st_size, (long)(TRANSLOG_PAGE_SIZE * 2));
+    exit(1);
+  }
+
+  translog_destroy();
+  end_pagecache(&pagecache, 1);
+  ma_control_file_end();
+  my_delete(CONTROL_FILE_BASE_NAME, MYF(0));
+  my_delete(first_translog_file, MYF(0));
+  my_delete(file1_name, MYF(0));
+
+  exit(0);
+}

unittest/mysys/mf_pagecache_consist.c → storage/maria/unittest/mf_pagecache_consist.c

@@ -57,6 +57,29 @@ static uint flush_divider= 1000;
 #endif /*TEST_HIGH_CONCURENCY*/
 
 
+/*
+  Get pseudo-random length of the field in (0;limit)
+
+  SYNOPSYS
+    get_len()
+    limit                limit for generated value
+
+  RETURN
+    length where length >= 0 & length < limit
+*/
+
+static uint get_len(uint limit)
+{
+  uint32 rec_len;
+  do
+  {
+    rec_len= random() /
+      (RAND_MAX / limit);
+  } while (rec_len >= limit || rec_len == 0);
+  return rec_len;
+}
+
+
 /* check page consistency */
 uint check_page(uchar *buff, ulong offset, int page_locked, int page_no,
                 int tag)
@@ -70,7 +93,7 @@ uint check_page(uchar *buff, ulong offset, int page_locked, int page_no,
   {
     uint len= *((uint *)(buff + end));
     uint j;
-    end+= sizeof(uint)+ sizeof(uint);
+    end+= sizeof(uint) + sizeof(uint);
     if (len + end > PAGE_SIZE)
     {
       diag("incorrect field header #%u by offset %lu\n", i, offset + end + j);
@@ -178,7 +201,7 @@ void reader(int num)
 
   for (i= 0; i < number_of_tests; i++)
   {
-    uint page= rand()/(RAND_MAX/number_of_pages);
+    uint page= get_len(number_of_pages);
     pagecache_read(&pagecache, &file1, page, 3, (char*)buffr,
                    PAGECACHE_PLAIN_PAGE,
                    PAGECACHE_LOCK_LEFT_UNLOCKED,
@@ -201,13 +224,13 @@ void writer(int num)
   for (i= 0; i < number_of_tests; i++)
   {
     uint end;
-    uint page= rand()/(RAND_MAX/number_of_pages);
+    uint page= get_len(number_of_pages);
     pagecache_read(&pagecache, &file1, page, 3, (char*)buffr,
                    PAGECACHE_PLAIN_PAGE,
                    PAGECACHE_LOCK_WRITE,
                    0);
     end= check_page(buffr, page * PAGE_SIZE, 1, page, num);
-    put_rec(buffr, end, rand()/(RAND_MAX/record_length_limit), num);
+    put_rec(buffr, end, get_len(record_length_limit), num);
     pagecache_write(&pagecache, &file1, page, 3, (char*)buffr,
                     PAGECACHE_PLAIN_PAGE,
                     PAGECACHE_LOCK_WRITE_UNLOCK,
@@ -348,7 +371,7 @@ int main(int argc, char **argv __attribute__((unused)))
 
 
   if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
-                             PAGE_SIZE, 0)) == 0)
+                             PAGE_SIZE)) == 0)
   {
     fprintf(stderr,"Got error: init_pagecache() (errno: %d)\n",
             errno);

unittest/mysys/mf_pagecache_single.c → storage/maria/unittest/mf_pagecache_single.c

@@ -346,7 +346,7 @@ int simple_big_test()
   unsigned char *buffw= (unsigned char *)malloc(PAGE_SIZE);
   unsigned char *buffr= (unsigned char *)malloc(PAGE_SIZE);
   struct file_desc *desc=
-    (struct file_desc *)malloc((PCACHE_SIZE/(PAGE_SIZE/2)) *
+    (struct file_desc *)malloc((PCACHE_SIZE/(PAGE_SIZE/2) + 1) *
                                sizeof(struct file_desc));
   int res, i;
   DBUG_ENTER("simple_big_test");
@@ -363,6 +363,8 @@ int simple_big_test()
                     PAGECACHE_WRITE_DELAY,
                     0);
   }
+  desc[i].length= 0;
+  desc[i].content= NULL;
   ok(1, "Simple big file write");
   /* check written pages sequentally read */
   for (i= 0; i < PCACHE_SIZE/(PAGE_SIZE/2); i++)
@@ -528,7 +530,7 @@ int main(int argc, char **argv __attribute__((unused)))
   plan(12);
 
   if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0,
-                             PAGE_SIZE, 0)) == 0)
+                             PAGE_SIZE)) == 0)
   {
     fprintf(stderr,"Got error: init_pagecache() (errno: %d)\n",
             errno);

unittest/mysys/Makefile.am

@@ -7,45 +7,4 @@ LDADD 		= $(top_builddir)/unittest/mytap/libmytap.a \
 		  $(top_builddir)/dbug/libdbug.a \
 		  $(top_builddir)/strings/libmystrings.a
 
-noinst_PROGRAMS  =	bitmap-t base64-t my_atomic-t \
-			mf_pagecache_single_1k-t mf_pagecache_single_8k-t \
-			mf_pagecache_single_64k-t \
-			mf_pagecache_consist_1k-t mf_pagecache_consist_64k-t \
-			mf_pagecache_consist_1kHC-t mf_pagecache_consist_64kHC-t \
-			mf_pagecache_consist_1kRD-t mf_pagecache_consist_64kRD-t \
-			mf_pagecache_consist_1kWR-t mf_pagecache_consist_64kWR-t
-
-# tests for mysys/mf_pagecache.c
-
-mf_pagecache_single_src =	mf_pagecache_single.c $(top_srcdir)/mysys/mf_pagecache.c test_file.c
-mf_pagecache_consist_src =	mf_pagecache_consist.c $(top_srcdir)/mysys/mf_pagecache.c test_file.c
-mf_pagecache_common_cppflags =	-DEXTRA_DEBUG -DPAGECACHE_DEBUG -DMAIN
-
-mf_pagecache_single_1k_t_SOURCES =	$(mf_pagecache_single_src)
-mf_pagecache_single_8k_t_SOURCES =	$(mf_pagecache_single_src)
-mf_pagecache_single_64k_t_SOURCES =	$(mf_pagecache_single_src)
-mf_pagecache_single_1k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024
-mf_pagecache_single_8k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=8192
-mf_pagecache_single_64k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536
-
-mf_pagecache_consist_1k_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_1k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024
-mf_pagecache_consist_64k_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_64k_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536
-
-mf_pagecache_consist_1kHC_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_1kHC_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -DTEST_HIGH_CONCURENCY
-mf_pagecache_consist_64kHC_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_64kHC_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_HIGH_CONCURENCY
-
-mf_pagecache_consist_1kRD_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_1kRD_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -DTEST_READERS
-mf_pagecache_consist_64kRD_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_64kRD_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_READERS
-
-mf_pagecache_consist_1kWR_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_1kWR_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=1024 -DTEST_WRITERS
-mf_pagecache_consist_64kWR_t_SOURCES =	$(mf_pagecache_consist_src)
-mf_pagecache_consist_64kWR_t_CPPFLAGS = 	$(mf_pagecache_common_cppflags) -DPAGE_SIZE=65536 -DTEST_WRITERS
-
-CLEANFILES = 	my_pagecache_debug.log page_cache_test_file_1
+noinst_PROGRAMS  =	bitmap-t base64-t my_atomic-t