Fix for Bug #1952

"SHOW TABLE STATUS very slow w/large number of tables"
Replaced old algorithm which were used in my_dir() and stored
all information about directory entries in one chunk of memory
with new one which stores file names and MY_STAT structures in
separate memroot, so now we don't need to copy this data during
reallocation of dir_entry array.


include/my_dir.h:
  Changed mystat member of FILEINFO structure to pointer since 
  this prevents unneeded memory allocation and initialization.
  Added comment about new hidden members of MY_DIR structure.
mysys/my_lib.c:
  Replaced old algorithm in my_dir() which stored all information
  about directory entries in one chunk of memory with new one 
  which stores file names and MY_STAT structures in separate
  memroot. Now we don't copy this data during reallocation of 
  array with FILEINFO structures.
  Also tuned sizes of memory chunks during first-other
  reallocations (we suppose that we either have < 100 files 
  in the directory or > 1000 of them).
sql/sql_show.cc:
  Updated only place in code where mystat member
  of FILEINFO structure is used.

include/my_dir.h

@@ -74,14 +74,21 @@ typedef struct my_stat
 
 #endif /* USE_MY_STAT_STRUCT */
 
-typedef struct fileinfo		/* Struct returned from my_dir & my_stat */
+/* Struct describing one file returned from my_dir */
+typedef struct fileinfo
 {
   char			*name;
-  MY_STAT		mystat;
+  MY_STAT		*mystat;
 } FILEINFO;
 
 typedef struct st_my_dir	/* Struct returned from my_dir */
 {
+  /*
+    These members are just copies of parts of DYNAMIC_ARRAY structure, 
+    which is allocated right after the end of MY_DIR structure (MEM_ROOT
+    for storing names is also resides there). We've left them here because
+    we don't want to change code that uses my_dir.
+  */
   struct fileinfo	*dir_entry;
   uint			number_off_files;
 } MY_DIR;

mysys/my_lib.c

@@ -62,8 +62,15 @@
 #define READDIR(A,B,C) (!(C=readdir(A)))
 #endif
 
+/*
+  We are assuming that directory we are reading is either has less than 
+  100 files and so can be read in one initial chunk or has more than 1000
+  files and so big increment are suitable.
+*/
+#define ENTRIES_START_SIZE (8192/sizeof(FILEINFO))
+#define ENTRIES_INCREMENT  (65536/sizeof(FILEINFO))
+#define NAMES_START_SIZE   32768
 
-#define STARTSIZE	ONCE_ALLOC_INIT*8  /* some mallocmargin */
 
 static int	comp_names(struct fileinfo *a,struct fileinfo *b);
 
@@ -74,7 +81,13 @@ void my_dirend(MY_DIR *buffer)
 {
   DBUG_ENTER("my_dirend");
   if (buffer)
+  {
+    delete_dynamic((DYNAMIC_ARRAY*)((char*)buffer + 
+                                    ALIGN_SIZE(sizeof(MY_DIR))));
+    free_root((MEM_ROOT*)((char*)buffer + ALIGN_SIZE(sizeof(MY_DIR)) + 
+                          ALIGN_SIZE(sizeof(DYNAMIC_ARRAY))), MYF(0));
     my_free((gptr) buffer,MYF(0));
+  }
   DBUG_VOID_RETURN;
 } /* my_dirend */
 
@@ -91,14 +104,14 @@ static int comp_names(struct fileinfo *a, struct fileinfo *b)
 
 MY_DIR	*my_dir(const char *path, myf MyFlags)
 {
+  char          *buffer;
+  MY_DIR        *result= 0;
+  FILEINFO      finfo;
+  DYNAMIC_ARRAY *dir_entries_storage;
+  MEM_ROOT      *names_storage;
   DIR		*dirp;
   struct dirent *dp;
-  struct fileinfo *fnames;
-  char	       *buffer, *obuffer, *tempptr;
-  uint		fcnt,i,size,firstfcnt, maxfcnt,length;
   char		tmp_path[FN_REFLEN+1],*tmp_file;
-  my_ptrdiff_t	diff;
-  bool		eof;
 #ifdef THREAD
   char	dirent_tmp[sizeof(struct dirent)+_POSIX_PATH_MAX+1];
 #endif
@@ -110,74 +123,72 @@ MY_DIR	*my_dir(const char *path, myf MyFlags)
 #endif
 
   dirp = opendir(directory_file_name(tmp_path,(my_string) path));
-  size = STARTSIZE;
 #if defined(__amiga__)
   if ((dirp->dd_fd) < 0)			/* Directory doesn't exists */
     goto error;
 #endif
-  if (dirp == NULL || ! (buffer = (char *) my_malloc(size, MyFlags)))
+  if (dirp == NULL || 
+      ! (buffer= my_malloc(ALIGN_SIZE(sizeof(MY_DIR)) + 
+                           ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)) +
+                           sizeof(MEM_ROOT), MyFlags)))
+    goto error;
+
+  dir_entries_storage= (DYNAMIC_ARRAY*)(buffer + ALIGN_SIZE(sizeof(MY_DIR))); 
+  names_storage= (MEM_ROOT*)(buffer + ALIGN_SIZE(sizeof(MY_DIR)) +
+                             ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)));
+  
+  if (my_init_dynamic_array(dir_entries_storage, sizeof(FILEINFO),
+                            ENTRIES_START_SIZE, ENTRIES_INCREMENT))
+  {
+    my_free((gptr) buffer,MYF(0));
     goto error;
+  }
+  init_alloc_root(names_storage, NAMES_START_SIZE, NAMES_START_SIZE);
+  
+  /* MY_DIR structure is allocated and completly initialized at this point */
+  result= (MY_DIR*)buffer;
 
-  fcnt = 0;
   tmp_file=strend(tmp_path);
-  firstfcnt = maxfcnt = (size - sizeof(MY_DIR)) /
-    (sizeof(struct fileinfo) + FN_LEN);
-  fnames=   (struct fileinfo *) (buffer + sizeof(MY_DIR));
-  tempptr = (char *) (fnames + maxfcnt);
 
 #ifdef THREAD
   dp= (struct dirent*) dirent_tmp;
 #else
   dp=0;
 #endif
-  eof=0;
-  for (;;)
+  
+  while (!(READDIR(dirp,(struct dirent*) dirent_tmp,dp)))
   {
-    while (fcnt < maxfcnt &&
-	   !(eof= READDIR(dirp,(struct dirent*) dirent_tmp,dp)))
+    if (!(finfo.name= strdup_root(names_storage, dp->d_name)))
+      goto error;
+    
+    if (MyFlags & MY_WANT_STAT)
     {
-      bzero((gptr) (fnames+fcnt),sizeof(fnames[0])); /* for purify */
-      fnames[fcnt].name = tempptr;
-      tempptr = strmov(tempptr,dp->d_name) + 1;
-      if (MyFlags & MY_WANT_STAT)
-      {
-	VOID(strmov(tmp_file,dp->d_name));
-	VOID(my_stat(tmp_path, &fnames[fcnt].mystat, MyFlags));
-      }
-      ++fcnt;
+      if (!(finfo.mystat= (MY_STAT*)alloc_root(names_storage, 
+                                               sizeof(MY_STAT))))
+        goto error;
+      
+      bzero(finfo.mystat, sizeof(MY_STAT));
+      VOID(strmov(tmp_file,dp->d_name));
+      VOID(my_stat(tmp_path, finfo.mystat, MyFlags));
     }
-    if (eof)
-      break;
-    size += STARTSIZE; obuffer = buffer;
-    if (!(buffer = (char *) my_realloc((gptr) buffer, size,
-				       MyFlags | MY_FREE_ON_ERROR)))
-      goto error;			/* No memory */
-    length= (uint) (sizeof(struct fileinfo ) * firstfcnt);
-    diff=    PTR_BYTE_DIFF(buffer , obuffer) + (int) length;
-    fnames=  (struct fileinfo *) (buffer + sizeof(MY_DIR));
-    tempptr= ADD_TO_PTR(tempptr,diff,char*);
-    for (i = 0; i < maxfcnt; i++)
-      fnames[i].name = ADD_TO_PTR(fnames[i].name,diff,char*);
-
-    /* move filenames upp a bit */
-    maxfcnt += firstfcnt;
-    bmove_upp(tempptr,tempptr-length,
-	      (uint) (tempptr- (char*) (fnames+maxfcnt)));
+    else
+      finfo.mystat= NULL;
+
+    if (push_dynamic(dir_entries_storage, (gptr)&finfo))
+      goto error;
   }
 
   (void) closedir(dirp);
-  {
-    MY_DIR * s = (MY_DIR *) buffer;
-    s->number_off_files = (uint) fcnt;
-    s->dir_entry = fnames;
-  }
-  if (!(MyFlags & MY_DONT_SORT))
-    qsort((void *) fnames, (size_s) fcnt, sizeof(struct fileinfo),
-	  (qsort_cmp) comp_names);
 #if defined(THREAD) && !defined(HAVE_READDIR_R)
   pthread_mutex_unlock(&THR_LOCK_open);
 #endif
-  DBUG_RETURN((MY_DIR *) buffer);
+  result->dir_entry= (FILEINFO *)dir_entries_storage->buffer;
+  result->number_off_files= dir_entries_storage->elements;
+  
+  if (!(MyFlags & MY_DONT_SORT))
+    qsort((void *) result->dir_entry, result->number_off_files,
+          sizeof(FILEINFO), (qsort_cmp) comp_names);
+  DBUG_RETURN(result);
 
  error:
 #if defined(THREAD) && !defined(HAVE_READDIR_R)
@@ -186,6 +197,7 @@ MY_DIR	*my_dir(const char *path, myf MyFlags)
   my_errno=errno;
   if (dirp)
     (void) closedir(dirp);
+  my_dirend(result);
   if (MyFlags & (MY_FAE | MY_WME))
     my_error(EE_DIR,MYF(ME_BELL+ME_WAITTANG),path,my_errno);
   DBUG_RETURN((MY_DIR *) NULL);
@@ -349,10 +361,11 @@ my_string directory_file_name (my_string dst, const char *src)
 
 MY_DIR	*my_dir(const char *path, myf MyFlags)
 {
-  struct fileinfo *fnames;
-  char	       *buffer, *obuffer, *tempptr;
-  int		eof,i,fcnt,firstfcnt,length,maxfcnt;
-  uint		size;
+  char          *buffer;
+  MY_DIR        *result= 0;
+  FILEINFO      finfo;
+  DYNAMIC_ARRAY *dir_entries_storage;
+  MEM_ROOT      *names_storage;
 #ifdef __BORLANDC__
   struct ffblk       find;
 #else
@@ -360,7 +373,6 @@ MY_DIR	*my_dir(const char *path, myf MyFlags)
 #endif
   ushort	mode;
   char		tmp_path[FN_REFLEN],*tmp_file,attrib;
-  my_ptrdiff_t	diff;
 #ifdef _WIN64
   __int64       handle;
 #else
@@ -392,85 +404,88 @@ MY_DIR	*my_dir(const char *path, myf MyFlags)
     goto error;
 #endif
 
-  size = STARTSIZE;
-  firstfcnt = maxfcnt = (size - sizeof(MY_DIR)) /
-    (sizeof(struct fileinfo) + FN_LEN);
-  if ((buffer = (char *) my_malloc(size, MyFlags)) == 0)
+  if (!(buffer= my_malloc(ALIGN_SIZE(sizeof(MY_DIR)) + 
+                          ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)) +
+                          sizeof(MEM_ROOT), MyFlags)))
     goto error;
-  fnames=   (struct fileinfo *) (buffer + sizeof(MY_DIR));
-  tempptr = (char *) (fnames + maxfcnt);
 
-  fcnt = 0;
-  for (;;)
+  dir_entries_storage= (DYNAMIC_ARRAY*)(buffer + ALIGN_SIZE(sizeof(MY_DIR))); 
+  names_storage= (MEM_ROOT*)(buffer + ALIGN_SIZE(sizeof(MY_DIR)) +
+                             ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)));
+  
+  if (my_init_dynamic_array(dir_entries_storage, sizeof(FILEINFO),
+                            ENTRIES_START_SIZE, ENTRIES_INCREMENT))
   {
-    do
+    my_free((gptr) buffer,MYF(0));
+    goto error;
+  }
+  init_alloc_root(names_storage, NAMES_START_SIZE, NAMES_START_SIZE);
+  
+  /* MY_DIR structure is allocated and completly initialized at this point */
+  result= (MY_DIR*)buffer;
+ 
+  do
+  {
+#ifdef __BORLANDC__
+    if (!(finfo.name= strdup_root(names_storage, find.ff_name)))
+      goto error;
+#else
+    if (!(finfo.name= strdup_root(names_storage, find.name)))
+      goto error;
+#endif    
+    if (MyFlags & MY_WANT_STAT)
     {
-      fnames[fcnt].name = tempptr;
+      if (!(finfo.mystat= (MY_STAT*)alloc_root(names_storage, 
+                                               sizeof(MY_STAT))))
+        goto error;
+      
+      bzero(finfo.mystat, sizeof(MY_STAT));
 #ifdef __BORLANDC__
-      tempptr = strmov(tempptr,find.ff_name) + 1;
-      fnames[fcnt].mystat.st_size=find.ff_fsize;
-      fnames[fcnt].mystat.st_uid=fnames[fcnt].mystat.st_gid=0;
+      finfo.mystat->st_size=find.ff_fsize;
       mode=MY_S_IREAD; attrib=find.ff_attrib;
 #else
-      tempptr = strmov(tempptr,find.name) + 1;
-      fnames[fcnt].mystat.st_size=find.size;
-      fnames[fcnt].mystat.st_uid=fnames[fcnt].mystat.st_gid=0;
+      finfo.mystat->st_size=find.size;
       mode=MY_S_IREAD; attrib=find.attrib;
 #endif
       if (!(attrib & _A_RDONLY))
 	mode|=MY_S_IWRITE;
       if (attrib & _A_SUBDIR)
 	mode|=MY_S_IFDIR;
-      fnames[fcnt].mystat.st_mode=mode;
+      finfo.mystat->st_mode=mode;
 #ifdef __BORLANDC__
-      fnames[fcnt].mystat.st_mtime=((uint32) find.ff_ftime);
+      finfo.mystat->st_mtime=((uint32) find.ff_ftime);
 #else
-      fnames[fcnt].mystat.st_mtime=((uint32) find.time_write);
-#endif
-      ++fcnt;
-#ifdef __BORLANDC__
-    } while ((eof= findnext(&find)) == 0 && fcnt < maxfcnt);
-#else
-    } while ((eof= _findnext(handle,&find)) == 0 && fcnt < maxfcnt);
+      finfo.mystat->st_mtime=((uint32) find.time_write);
 #endif
+    }
+    else
+      finfo.mystat= NULL;
 
-    DBUG_PRINT("test",("eof: %d  errno: %d",eof,errno));
-    if (eof)
-      break;
-    size += STARTSIZE; obuffer = buffer;
-    if (!(buffer = (char *) my_realloc((gptr) buffer, size,
-				       MyFlags | MY_FREE_ON_ERROR)))
+    if (push_dynamic(dir_entries_storage, (gptr)&finfo))
       goto error;
-    length= sizeof(struct fileinfo ) * firstfcnt;
-    diff=    PTR_BYTE_DIFF(buffer , obuffer) +length;
-    fnames=  (struct fileinfo *) (buffer + sizeof(MY_DIR));
-    tempptr= ADD_TO_PTR(tempptr,diff,char*);
-    for (i = 0; i < maxfcnt; i++)
-      fnames[i].name = ADD_TO_PTR(fnames[i].name,diff,char*);
-
-    /* move filenames upp a bit */
-    maxfcnt += firstfcnt;
-    bmove_upp(tempptr,ADD_TO_PTR(tempptr,-length,char*),
-	      (int) PTR_BYTE_DIFF(tempptr,fnames+maxfcnt));
-  }
-  {
-    MY_DIR * s = (MY_DIR *) buffer;
-    s->number_off_files = (uint) fcnt;
-    s->dir_entry = fnames;
-  }
-  if (!(MyFlags & MY_DONT_SORT))
-    qsort(fnames,fcnt,sizeof(struct fileinfo),(qsort_cmp) comp_names);
-#ifndef __BORLANDC__
+    
+#ifdef __BORLANDC__
+  } while (findnext(&find) == 0);
+#else
+  } while (_findnext(handle,&find) == 0);
+  
   _findclose(handle);
 #endif
-  DBUG_RETURN((MY_DIR *) buffer);
 
+  result->dir_entry= (FILEINFO *)dir_entries_storage->buffer;
+  result->number_off_files= dir_entries_storage->elements;
+  
+  if (!(MyFlags & MY_DONT_SORT))
+    qsort((void *) result->dir_entry, result->number_off_files,
+          sizeof(FILEINFO), (qsort_cmp) comp_names);
+  DBUG_RETURN(result);
 error:
   my_errno=errno;
 #ifndef __BORLANDC__
   if (handle != -1)
       _findclose(handle);
 #endif
+  my_dirend(result);
   if (MyFlags & MY_FAE+MY_WME)
     my_error(EE_DIR,MYF(ME_BELL+ME_WAITTANG),path,errno);
   DBUG_RETURN((MY_DIR *) NULL);
@@ -485,14 +500,14 @@ MY_DIR	*my_dir(const char *path, myf MyFlags)
 
 MY_DIR	*my_dir(const char* path, myf MyFlags)
 {
-  struct fileinfo *fnames;
-  char	       *buffer, *obuffer, *tempptr;
-  int		eof,i,fcnt,firstfcnt,length,maxfcnt;
-  uint		size;
+  char          *buffer;
+  MY_DIR        *result= 0;
+  FILEINFO      finfo;
+  DYNAMIC_ARRAY *dir_entries_storage;
+  MEM_ROOT      *names_storage;
   struct find_t find;
   ushort	mode;
   char		tmp_path[FN_REFLEN],*tmp_file,attrib;
-  my_ptrdiff_t	diff;
   DBUG_ENTER("my_dir");
   DBUG_PRINT("my",("path: '%s' stat: %d  MyFlags: %d",path,MyFlags));
 
@@ -514,63 +529,65 @@ MY_DIR	*my_dir(const char* path, myf MyFlags)
   if (_dos_findfirst(tmp_path,_A_NORMAL | _A_SUBDIR, &find))
     goto error;
 
-  size = STARTSIZE;
-  firstfcnt = maxfcnt = (size - sizeof(MY_DIR)) /
-    (sizeof(struct fileinfo) + FN_LEN);
-  if ((buffer = (char *) my_malloc(size, MyFlags)) == 0)
+  if (!(buffer= my_malloc(ALIGN_SIZE(sizeof(MY_DIR)) + 
+                          ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)) +
+                          sizeof(MEM_ROOT), MyFlags)))
     goto error;
-  fnames=   (struct fileinfo *) (buffer + sizeof(MY_DIR));
-  tempptr = (char *) (fnames + maxfcnt);
 
-  fcnt = 0;
-  for (;;)
+  dir_entries_storage= (DYNAMIC_ARRAY*)(buffer + ALIGN_SIZE(sizeof(MY_DIR))); 
+  names_storage= (MEM_ROOT*)(buffer + ALIGN_SIZE(sizeof(MY_DIR)) +
+                             ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)));
+  
+  if (my_init_dynamic_array(dir_entries_storage, sizeof(FILEINFO),
+                            ENTRIES_START_SIZE, ENTRIES_INCREMENT))
   {
-    do
+    my_free((gptr) buffer,MYF(0));
+    goto error;
+  }
+  init_alloc_root(names_storage, NAMES_START_SIZE, NAMES_START_SIZE);
+  
+  /* MY_DIR structure is allocated and completly initialized at this point */
+  result= (MY_DIR*)buffer;
+ 
+  do
+  {
+    if (!(finfo.name= strdup_root(names_storage, find.name)))
+      goto error;
+    
+    if (MyFlags & MY_WANT_STAT)
     {
-      fnames[fcnt].name = tempptr;
-      tempptr = strmov(tempptr,find.name) + 1;
-      fnames[fcnt].mystat.st_size=find.size;
-      fnames[fcnt].mystat.st_uid=fnames[fcnt].mystat.st_gid=0;
-      mode=MY_S_IREAD; attrib=find.attrib;
+      if (!(finfo.mystat= (MY_STAT*)alloc_root(names_storage, 
+                                               sizeof(MY_STAT))))
+        goto error;
+      
+      bzero(finfo.mystat, sizeof(MY_STAT));
+      finfo.mystat->st_size= find.size;
+      mode= MY_S_IREAD; attrib= find.attrib;
       if (!(attrib & _A_RDONLY))
-	mode|=MY_S_IWRITE;
+	mode|= MY_S_IWRITE;
       if (attrib & _A_SUBDIR)
-	mode|=MY_S_IFDIR;
-      fnames[fcnt].mystat.st_mode=mode;
-      fnames[fcnt].mystat.st_mtime=((uint32) find.wr_date << 16) +
-					     find.wr_time;
-      ++fcnt;
-    } while ((eof= _dos_findnext(&find)) == 0 && fcnt < maxfcnt);
-
-    DBUG_PRINT("test",("eof: %d  errno: %d",eof,errno));
-    if (eof)
-      break;
-    size += STARTSIZE; obuffer = buffer;
-    if (!(buffer = (char *) my_realloc((gptr) buffer, size,
-				       MyFlags | MY_FREE_ON_ERROR)))
+	mode|= MY_S_IFDIR;
+      finfo.mystat->st_mode= mode;
+      finfo.mystat->st_mtime= ((uint32) find.wr_date << 16) + find.wr_time;
+    }
+    else
+      finfo.mystat= NULL;
+
+    if (push_dynamic(dir_entries_storage, (gptr)&finfo))
       goto error;
-    length= sizeof(struct fileinfo ) * firstfcnt;
-    diff=    PTR_BYTE_DIFF(buffer , obuffer) +length;
-    fnames=  (struct fileinfo *) (buffer + sizeof(MY_DIR));
-    tempptr= ADD_TO_PTR(tempptr,diff,char*);
-    for (i = 0; i < maxfcnt; i++)
-      fnames[i].name = ADD_TO_PTR(fnames[i].name,diff,char*);
-
-    /* move filenames upp a bit */
-    maxfcnt += firstfcnt;
-    bmove_upp(tempptr,ADD_TO_PTR(tempptr,-length,char*),
-	      (int) PTR_BYTE_DIFF(tempptr,fnames+maxfcnt));
-  }
-  {
-    MY_DIR * s = (MY_DIR *) buffer;
-    s->number_off_files = (uint) fcnt;
-    s->dir_entry = fnames;
-  }
+  
+  } while (_dos_findnext(&find) == 0);
+  
+  result->dir_entry= (FILEINFO *)dir_entries_storage->buffer;
+  result->number_off_files= dir_entries_storage->elements;
+  
   if (!(MyFlags & MY_DONT_SORT))
-    qsort(fnames,fcnt,sizeof(struct fileinfo),(qsort_cmp) comp_names);
-  DBUG_RETURN((MY_DIR *) buffer);
+    qsort((void *) result->dir_entry, result->number_off_files,
+          sizeof(FILEINFO), (qsort_cmp) comp_names);
+  DBUG_RETURN(result);
 
 error:
+  my_dirend(result);
   if (MyFlags & MY_FAE+MY_WME)
     my_error(EE_DIR,MYF(ME_BELL+ME_WAITTANG),path,errno);
   DBUG_RETURN((MY_DIR *) NULL);

sql/sql_show.cc

@@ -213,7 +213,7 @@ mysql_find_files(THD *thd,List<char> *files, const char *db,const char *path,
       else
 #endif
       {
-        if (file->name[0] == '.' || !MY_S_ISDIR(file->mystat.st_mode) ||
+        if (file->name[0] == '.' || !MY_S_ISDIR(file->mystat->st_mode) ||
             (wild && wild_compare(file->name,wild)))
           continue;
       }