Fixed problems with original gzio() functions not being able to correctly...

Fixed problems with original gzio() functions not being able to correctly return lengths of long rows. 

storage/archive/archive_test.c

 #include "azlib.h"
+#include <string.h>
+#include <assert.h>
 #include <stdio.h>
 
-#define TEST_STRING "This is a test"
+#define TEST_FILENAME "test.az"
+#define TEST_STRING "YOU don't know about me without you have read a book by the name of The Adventures of Tom Sawyer; but that ain't no matter.  That book was made by Mr. Mark Twain, and he told the truth, mainly.  There was things which he stretched, but mainly he told the truth.  That is nothing.  I never seen anybody but lied one time or another, without it was Aunt Polly, or the widow, or maybe Mary.  Aunt Polly--Tom's Aunt Polly, she is--and Mary, and the Widow Douglas is all told about in that book, which is mostly a true book, with some stretchers, as I said before.  Now the way that the book winds up is this:  Tom and me found the money that the robbers hid in the cave, and it made us rich.  We got six thousand dollars apiece--all gold.  It was an awful sight of money when it was piled up.  Well, Judge Thatcher he took it and put it out at interest, and it fetched us a dollar a day apiece all the year round --more than a body could tell what to do with.  The Widow Douglas she took me for her son, and allowed she would..."
 #define BUFFER_LEN 1024
+#define TWOGIG 2147483648
+#define FOURGIG 4294967296
 
 int main(int argc __attribute__((unused)), char *argv[])
 {
-  int ret;
-  azio_stream foo, foo1;
+  unsigned long ret;
+  int error;
+  azio_stream writer_handle, reader_handle;
   char buffer[BUFFER_LEN];
+  unsigned long write_length;
+  unsigned long read_length= 0;
 
   MY_INIT(argv[0]);
 
-  if (!(ret= azopen(&foo, "test", O_CREAT|O_WRONLY|O_TRUNC|O_BINARY)))
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_CREAT|O_WRONLY|O_TRUNC|O_BINARY)))
   {
     printf("Could not create test file\n");
     return 0;
   }
-  azwrite(&foo, TEST_STRING, sizeof(TEST_STRING));
-  azflush(&foo, Z_FINISH);
+  ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+  assert(ret == BUFFER_LEN);
+  azflush(&writer_handle, Z_FINISH);
 
-  if (!(ret= azopen(&foo1, "test", O_RDONLY|O_BINARY)))
+  if (!(ret= azopen(&reader_handle, TEST_FILENAME, O_RDONLY|O_BINARY)))
   {
     printf("Could not open test file\n");
     return 0;
   }
-  ret= azread(&foo1, buffer, BUFFER_LEN);
-  printf("Read %d bytes\n", ret);
-  printf("%s\n", buffer);
-  azrewind(&foo1);
-  azclose(&foo);
-  if (!(ret= azopen(&foo, "test", O_APPEND|O_WRONLY|O_BINARY)))
+  ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+  printf("Read %lu bytes, expected %d\n", ret, BUFFER_LEN);
+
+  azrewind(&reader_handle);
+  azclose(&writer_handle);
+
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_APPEND|O_WRONLY|O_BINARY)))
+  {
+    printf("Could not open file (%s) for appending\n", TEST_FILENAME);
+    return 0;
+  }
+  ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+  assert(ret == BUFFER_LEN);
+  azflush(&writer_handle, Z_FINISH);
+
+  /* Read the original data */
+  ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+  printf("Read %lu bytes, expected %d\n", ret, BUFFER_LEN);
+  assert(ret == BUFFER_LEN);
+  assert(!error);
+
+  /* Read the new data */
+  ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+  printf("Read %lu bytes, expected %d\n", ret, BUFFER_LEN);
+  assert(ret == BUFFER_LEN);
+  assert(!error);
+
+  azclose(&writer_handle);
+  azclose(&reader_handle);
+  unlink(TEST_FILENAME);
+
+  /* Start size tests */
+  printf("About to run 2gig and 4gig test now, you may want to hit CTRL-C\n");
+
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_CREAT|O_WRONLY|O_TRUNC|O_BINARY)))
   {
     printf("Could not create test file\n");
     return 0;
   }
-  azwrite(&foo, TEST_STRING, sizeof(TEST_STRING));
-  azflush(&foo, Z_FINISH);
-  ret= azread(&foo1, buffer, BUFFER_LEN);
-  printf("Read %d bytes\n", ret);
-  printf("%s\n", buffer);
-  azclose(&foo);
-  azclose(&foo1);
 
-  /* unlink("test"); */
+  for (write_length= 0; write_length < TWOGIG ; write_length+= ret)
+  {
+    ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+    assert(!error);
+    if (ret != BUFFER_LEN)
+    {
+      printf("Size %lu\n", ret);
+      assert(ret != BUFFER_LEN);
+    }
+  }
+  assert(write_length == TWOGIG);
+  printf("Read %lu bytes, expected %lu\n", write_length, TWOGIG);
+  azflush(&writer_handle, Z_FINISH);
+
+  printf("Reading back data\n");
+
+  if (!(ret= azopen(&reader_handle, TEST_FILENAME, O_RDONLY|O_BINARY)))
+  {
+    printf("Could not open test file\n");
+    return 0;
+  }
+
+  while ((ret= azread(&reader_handle, buffer, BUFFER_LEN, &error)))
+  {
+    read_length+= ret;
+    assert(!memcmp(buffer, TEST_STRING, ret));
+    if (ret != BUFFER_LEN)
+    {
+      printf("Size %lu\n", ret);
+      assert(ret != BUFFER_LEN);
+    }
+  }
+
+  assert(read_length == TWOGIG);
+  azclose(&writer_handle);
+  azclose(&reader_handle);
+  unlink(TEST_FILENAME);
+
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_CREAT|O_WRONLY|O_TRUNC|O_BINARY)))
+  {
+    printf("Could not create test file\n");
+    return 0;
+  }
+
+  for (write_length= 0; write_length < FOURGIG ; write_length+= ret)
+  {
+    ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+    assert(ret == BUFFER_LEN);
+  }
+  assert(write_length == FOURGIG);
+  printf("Read %lu bytes, expected %lu\n", write_length, FOURGIG);
+  azclose(&writer_handle);
+  azclose(&reader_handle);
+  unlink(TEST_FILENAME);
+
   return 0;
 }

storage/archive/azio.c

 /*
   azio is a modified version of gzio. It  makes use of mysys and removes mallocs.
+    -Brian Aker
 */
 
 /* gzio.c -- IO on .gz files
@@ -292,15 +293,28 @@ int destroy (s)
   Reads the given number of uncompressed bytes from the compressed file.
   azread returns the number of bytes actually read (0 for end of file).
 */
-int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
+unsigned long ZEXPORT azread ( azio_stream *s, voidp buf, unsigned long len, int *error)
 {
   Bytef *start = (Bytef*)buf; /* starting point for crc computation */
   Byte  *next_out; /* == stream.next_out but not forced far (for MSDOS) */
+  *error= 0;
 
-  if (s->mode != 'r') return Z_STREAM_ERROR;
+  if (s->mode != 'r')
+  { 
+    *error= Z_STREAM_ERROR;
+    return 0;
+  }
 
-  if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1;
-  if (s->z_err == Z_STREAM_END) return 0;  /* EOF */
+  if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO)
+  { 
+    *error= s->z_err;
+    return 0;
+  }
+
+  if (s->z_err == Z_STREAM_END)  /* EOF */
+  { 
+    return 0;
+  }
 
   next_out = (Byte*)buf;
   s->stream.next_out = (Bytef*)buf;
@@ -315,7 +329,9 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
     start++;
     if (s->last) {
       s->z_err = Z_STREAM_END;
-      return 1;
+      { 
+        return 1;
+      }
     }
   }
 
@@ -342,7 +358,9 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
       s->in  += len;
       s->out += len;
       if (len == 0) s->z_eof = 1;
-      return (int)len;
+      { 
+        return len;
+      }
     }
     if (s->stream.avail_in == 0 && !s->z_eof) {
 
@@ -386,8 +404,13 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
 
   if (len == s->stream.avail_out &&
       (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO))
-    return -1;
-  return (int)(len - s->stream.avail_out);
+  {
+    *error= s->z_err;
+
+    return 0;
+  }
+
+  return (len - s->stream.avail_out);
 }
 
 
@@ -396,7 +419,7 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
   Writes the given number of uncompressed bytes into the compressed file.
   azwrite returns the number of bytes actually written (0 in case of error).
 */
-int azwrite (azio_stream *s, voidpc buf, unsigned len)
+unsigned long azwrite (azio_stream *s, voidpc buf, unsigned long len)
 {
 
   s->stream.next_in = (Bytef*)buf;
@@ -424,7 +447,7 @@ int azwrite (azio_stream *s, voidpc buf, unsigned len)
   }
   s->crc = crc32(s->crc, (const Bytef *)buf, len);
 
-  return (int)(len - s->stream.avail_in);
+  return (unsigned long)(len - s->stream.avail_in);
 }
 
 #endif
@@ -580,11 +603,12 @@ my_off_t azseek (s, offset, whence)
     if (s->last) s->z_err = Z_STREAM_END;
   }
   while (offset > 0)  {
-    int size = Z_BUFSIZE;
+    int error;
+    unsigned long size = Z_BUFSIZE;
     if (offset < Z_BUFSIZE) size = (int)offset;
 
-    size = azread(s, s->outbuf, (uInt)size);
-    if (size <= 0) return -1L;
+    size = azread(s, s->outbuf, size, &error);
+    if (error <= 0) return -1L;
     offset -= size;
   }
   return s->out;

storage/archive/azlib.h

@@ -206,7 +206,7 @@ int azdopen(azio_stream *s,File fd, int Flags);
 */
 
 
-extern int azread(azio_stream *file, voidp buf, unsigned len);
+extern unsigned long azread ( azio_stream *s, voidp buf, unsigned long len, int *error);
 /*
      Reads the given number of uncompressed bytes from the compressed file.
    If the input file was not in gzip format, gzread copies the given number
@@ -214,7 +214,7 @@ extern int azread(azio_stream *file, voidp buf, unsigned len);
      gzread returns the number of uncompressed bytes actually read (0 for
    end of file, -1 for error). */
 
-extern int azwrite (azio_stream *file, voidpc buf, unsigned len);
+extern unsigned long azwrite (azio_stream *s, voidpc buf, unsigned long len);
 /*
      Writes the given number of uncompressed bytes into the compressed file.
    gzwrite returns the number of uncompressed bytes actually written

storage/archive/ha_archive.cc

@@ -242,19 +242,34 @@ ha_archive::ha_archive(handlerton *hton, TABLE_SHARE *table_arg)
 */
 int ha_archive::read_data_header(azio_stream *file_to_read)
 {
+  int error;
+  unsigned long ret;
   uchar data_buffer[DATA_BUFFER_SIZE];
   DBUG_ENTER("ha_archive::read_data_header");
 
   if (azrewind(file_to_read) == -1)
     DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
 
-  if (azread(file_to_read, data_buffer, DATA_BUFFER_SIZE) != DATA_BUFFER_SIZE)
-    DBUG_RETURN(errno ? errno : -1);
+  ret= azread(file_to_read, data_buffer, DATA_BUFFER_SIZE, &error);
+
+  if (ret != DATA_BUFFER_SIZE)
+  {
+    DBUG_PRINT("ha_archive", ("Reading, expected %lu got %lu", 
+                              DATA_BUFFER_SIZE, ret));
+    DBUG_RETURN(1);
+  }
+
+  if (error)
+  {
+    DBUG_PRINT("ha_archive", ("Compression error (%d)", error));
+    DBUG_RETURN(1);
+  }
   
-  DBUG_PRINT("ha_archive::read_data_header", ("Check %u", data_buffer[0]));
-  DBUG_PRINT("ha_archive::read_data_header", ("Version %u", data_buffer[1]));
+  DBUG_PRINT("ha_archive", ("Check %u", data_buffer[0]));
+  DBUG_PRINT("ha_archive", ("Version %u", data_buffer[1]));
 
   share->data_version= (uchar)data_buffer[1];
+  DBUG_PRINT("ha_archive", ("Set Version %u", share->data_version));
   
   if ((data_buffer[0] != (uchar)ARCHIVE_CHECK_HEADER) &&  
       (data_buffer[1] != (uchar)ARCHIVE_VERSION))
@@ -277,11 +292,12 @@ int ha_archive::write_data_header(azio_stream *file_to_write)
   if (azwrite(file_to_write, &data_buffer, DATA_BUFFER_SIZE) != 
       DATA_BUFFER_SIZE)
     goto error;
-  DBUG_PRINT("ha_archive::write_data_header", ("Check %u", (uint)data_buffer[0]));
-  DBUG_PRINT("ha_archive::write_data_header", ("Version %u", (uint)data_buffer[1]));
+  DBUG_PRINT("ha_archive", ("Check %u", (uint)data_buffer[0]));
+  DBUG_PRINT("ha_archive", ("Version %u", (uint)data_buffer[1]));
 
   DBUG_RETURN(0);
 error:
+  DBUG_PRINT("ha_archive", ("Could not write full data header"));
   DBUG_RETURN(errno);
 }
 
@@ -783,30 +799,37 @@ int ha_archive::real_write_row(byte *buf, azio_stream *writer)
 {
   my_off_t written;
   uint *ptr, *end;
-  int r_pack_length;
+  unsigned long r_pack_length;
   byte size_buffer[ARCHIVE_ROW_HEADER_SIZE]; // Longest possible row length with blobs
   DBUG_ENTER("ha_archive::real_write_row");
 
   // We pack the row for writing
   r_pack_length= pack_row(buf);
-  DBUG_PRINT("ha_archive",("Pack row length %d", r_pack_length));
+  DBUG_PRINT("ha_archive",("Pack row length %lu", r_pack_length));
 
   // Store the size of the row before the row
   bzero(size_buffer, ARCHIVE_ROW_HEADER_SIZE);
   int4store(size_buffer, (int)r_pack_length); 
-  DBUG_PRINT("ha_archive",("Pack %d %d %d %d", size_buffer[0], size_buffer[1], size_buffer[2], size_buffer[3]));
-  azwrite(writer, size_buffer, ARCHIVE_ROW_HEADER_SIZE);
+  written= azwrite(writer, size_buffer, ARCHIVE_ROW_HEADER_SIZE);
+
+  if (written != ARCHIVE_ROW_HEADER_SIZE)
+  {
+    DBUG_PRINT("ha_archive", ("Died writing row header"));
+    DBUG_RETURN(-1);
+  }
 
   written= azwrite(writer, record_buffer->buffer, r_pack_length);
-  DBUG_PRINT("ha_archive::real_write_row", ("Wrote %d bytes expected %d", 
-                                            (uint32)written, 
-                                            (uint32)r_pack_length));
+  if (written != r_pack_length)
+  {
+    DBUG_PRINT("ha_archive", ("Wrote %llu bytes expected %lu", 
+                                              (unsigned long long) written, 
+                                               r_pack_length));
+    DBUG_RETURN(-1);
+  }
+
   if (!delayed_insert || !bulk_insert)
     share->dirty= TRUE;
 
-  if (written != (my_off_t)r_pack_length)
-    DBUG_RETURN(errno ? errno : -1);
-
   DBUG_RETURN(0);
 }
 
@@ -830,7 +853,7 @@ int ha_archive::max_row_length(const byte *buf)
 }
 
 
-int ha_archive::pack_row(const byte *record)
+unsigned long ha_archive::pack_row(const byte *record)
 {
   byte *ptr;
 
@@ -850,7 +873,7 @@ int ha_archive::pack_row(const byte *record)
     ptr=(byte*) (*field)->pack((char*) ptr,
 			       (char*) record + (*field)->offset());
 
-  DBUG_RETURN((size_t) (ptr - record_buffer->buffer));
+  DBUG_RETURN((unsigned long) (ptr - record_buffer->buffer));
 }
 
 
@@ -1116,6 +1139,8 @@ int ha_archive::get_row(azio_stream *file_to_read, byte *buf)
 {
   int rc;
   DBUG_ENTER("ha_archive::get_row");
+  DBUG_PRINT("ha_archive", ("Picking version for get_row() %d -> %d", 
+                            share->data_version, ARCHIVE_VERSION));
   if (share->data_version == ARCHIVE_VERSION)
     rc= get_row_version3(file_to_read, buf);
   else
@@ -1145,14 +1170,15 @@ int ha_archive::unpack_row(azio_stream *file_to_read, char *record)
 {
   DBUG_ENTER("ha_archive::unpack_row");
 
-  int read; // Bytes read, azread() returns int
+  unsigned long read;
+  int error;
   byte size_buffer[ARCHIVE_ROW_HEADER_SIZE];
-  int row_len;
+  unsigned long row_len;
 
   /* First we grab the length stored */
-  read= azread(file_to_read, (byte *)size_buffer, ARCHIVE_ROW_HEADER_SIZE);
+  read= azread(file_to_read, (byte *)size_buffer, ARCHIVE_ROW_HEADER_SIZE, &error);
 
-  if (read == Z_STREAM_ERROR)
+  if (error == Z_STREAM_ERROR ||  (read && read < ARCHIVE_ROW_HEADER_SIZE))
     DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
 
   /* If we read nothing we are at the end of the file */
@@ -1160,12 +1186,16 @@ int ha_archive::unpack_row(azio_stream *file_to_read, char *record)
     DBUG_RETURN(HA_ERR_END_OF_FILE);
 
   row_len= sint4korr(size_buffer);
-  DBUG_PRINT("ha_archive",("Unpack row length %d -> %llu", row_len, 
-                           (unsigned long long)table->s->reclength));
+  DBUG_PRINT("ha_archive",("Unpack row length %lu -> %lu", row_len, 
+                           (unsigned long)table->s->reclength));
   fix_rec_buff(row_len);
 
-  if (azread(file_to_read, record_buffer->buffer, row_len) != row_len)
+  read= azread(file_to_read, record_buffer->buffer, row_len, &error);
+
+  if (read != row_len || error)
+  {
     DBUG_RETURN(-1);
+  }
 
   /* Copy null bits */
   const char *ptr= (const char*) record_buffer->buffer;
@@ -1188,18 +1218,25 @@ int ha_archive::get_row_version3(azio_stream *file_to_read, byte *buf)
 
 int ha_archive::get_row_version2(azio_stream *file_to_read, byte *buf)
 {
-  int read; // Bytes read, azread() returns int
+  unsigned long read;
+  int error;
   uint *ptr, *end;
   char *last;
   size_t total_blob_length= 0;
   MY_BITMAP *read_set= table->read_set;
   DBUG_ENTER("ha_archive::get_row_version2");
 
-  read= azread(file_to_read, buf, table->s->reclength);
-  DBUG_PRINT("ha_archive::get_row_version2", ("Read %d bytes expected %lu", read, 
-                                     (unsigned long)table->s->reclength));
+  read= azread(file_to_read, buf, table->s->reclength, &error);
 
-  if (read == Z_STREAM_ERROR)
+  if (read != table->s->reclength)
+  {
+    DBUG_PRINT("ha_archive::get_row_version2", ("Read %lu bytes expected %lu", 
+                                                read, 
+                                                (unsigned long)table->s->reclength));
+    DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+  }
+
+  if (error == Z_STREAM_ERROR || error == Z_DATA_ERROR )
     DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
 
   /* If we read nothing we are at the end of the file */
@@ -1238,7 +1275,11 @@ int ha_archive::get_row_version2(azio_stream *file_to_read, byte *buf)
       if (bitmap_is_set(read_set,
                         ((Field_blob*) table->field[*ptr])->field_index))
       {
-        read= azread(file_to_read, last, size);
+        read= azread(file_to_read, last, size, &error);
+
+        if (error)
+          DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+
         if ((size_t) read != size)
           DBUG_RETURN(HA_ERR_END_OF_FILE);
         ((Field_blob*) table->field[*ptr])->set_ptr(size, last);
@@ -1448,17 +1489,42 @@ int ha_archive::optimize(THD* thd, HA_CHECK_OPT* check_opt)
     /* 
       The quick method is to just read the data raw, and then compress it directly.
     */
-    int read; // Bytes read, azread() returns int
+    unsigned long read, written;
+    int error;
     char block[IO_SIZE];
     if (azrewind(&archive) == -1)
     {
       rc= HA_ERR_CRASHED_ON_USAGE;
-      DBUG_PRINT("ha_archive", ("archive HA_ERR_CRASHED_ON_USAGE"));
+      DBUG_PRINT("ha_archive", ("crashed on rewinding file"));
       goto error;
     }
 
-    while ((read= azread(&archive, block, IO_SIZE)) > 0)
-      azwrite(&writer, block, read);
+    while ((read= azread(&archive, block, IO_SIZE, &error)) > 0)
+    {
+      if (error)
+      {
+        rc= HA_ERR_CRASHED_ON_USAGE;
+        DBUG_PRINT("ha_archive", ("azread error on read"));
+        goto error;
+      }
+
+      written= azwrite(&writer, block, read);
+      if (written != read)
+      {
+        rc= HA_ERR_CRASHED_ON_USAGE;
+        DBUG_PRINT("ha_archive::real_write_row", 
+                   ("Crashed wrote %lu bytes expected %lu", 
+                    written, read));
+        goto error;
+      }
+    }
+
+    if (error)
+    {
+      rc= HA_ERR_CRASHED_ON_USAGE;
+      DBUG_PRINT("ha_archive", ("retrieved zero blocks and error'ed"));
+      goto error;
+    }
   }
 
   azclose(&writer);

storage/archive/ha_archive.h

@@ -158,6 +158,6 @@ class ha_archive: public handler
   int max_row_length(const byte *buf);
   bool fix_rec_buff(int length);
   int unpack_row(azio_stream *file_to_read, char *record);
-  int pack_row(const byte *record);
+  unsigned long pack_row(const byte *record);
 };