Merge romeo.(none):/home/bkroot/mysql-5.1-new-rpl

into  romeo.(none):/home/bk/b19459-mysql-5.1-new

client/mysqlbinlog.cc

@@ -476,6 +476,30 @@ static bool check_database(const char *log_dbname)
 }
 
 
+
+static int
+write_event_header_and_base64(Log_event *ev, FILE *result_file,
+                              PRINT_EVENT_INFO *print_event_info)
+{
+  DBUG_ENTER("write_event_header_and_base64");
+  /* Write header and base64 output to cache */
+  IO_CACHE result_cache;
+  if (init_io_cache(&result_cache, -1, 0, WRITE_CACHE, 0L, FALSE,
+                    MYF(MY_WME | MY_NABP)))
+  {
+    return 1;
+  }
+
+  ev->print_header(&result_cache, print_event_info, FALSE);
+  ev->print_base64(&result_cache, print_event_info, FALSE);
+
+  /* Read data from cache and write to result file */
+  my_b_copy_to_file(&result_cache, result_file);
+  end_io_cache(&result_cache);
+  DBUG_RETURN(0);
+}
+
+
 /*
   Process an event
 
@@ -538,18 +562,18 @@ int process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
 
     print_event_info->base64_output= opt_base64_output;
 
+    DBUG_PRINT("debug", ("event_type: %s", ev->get_type_str()));
+
     switch (ev_type) {
     case QUERY_EVENT:
       if (check_database(((Query_log_event*)ev)->db))
         goto end;
       if (opt_base64_output)
-      {
-        ev->print_header(result_file, print_event_info);
-        ev->print_base64(result_file, print_event_info);
-      }
+        write_event_header_and_base64(ev, result_file, print_event_info);
       else
         ev->print(result_file, print_event_info);
       break;
+
     case CREATE_FILE_EVENT:
     {
       Create_file_log_event* ce= (Create_file_log_event*)ev;
@@ -570,8 +594,7 @@ int process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
       */
       if (opt_base64_output)
       {
-        ce->print_header(result_file, print_event_info);
-        ce->print_base64(result_file, print_event_info);
+        write_event_header_and_base64(ce, result_file, print_event_info);
       }
       else
         ce->print(result_file, print_event_info, TRUE);

include/base64.h

@@ -39,7 +39,8 @@ int base64_encode(const void *src, size_t src_len, char *dst);
 /*
   Decode a base64 string into data
 */
-int base64_decode(const char *src, size_t src_len, void *dst);
+int base64_decode(const char *src, size_t src_len,
+                  void *dst, const char **end_ptr);
 
 
 #ifdef __cplusplus

include/my_sys.h

@@ -517,6 +517,7 @@ typedef int (*qsort2_cmp)(const void *, const void *, const void *);
 			 (uint) (*(info)->current_pos - (info)->request_pos))
 
 /* tell write offset in the SEQ_APPEND cache */
+int      my_b_copy_to_file(IO_CACHE *cache, FILE *file);
 my_off_t my_b_append_tell(IO_CACHE* info);
 my_off_t my_b_safe_tell(IO_CACHE* info); /* picks the correct tell() */
 

mysys/base64.c

@@ -125,44 +125,69 @@ pos(unsigned char c)
 /*
   Decode a base64 string
 
-  Note: We require that dst is pre-allocated to correct size.
-        See base64_needed_decoded_length().
-
-  RETURN  Number of bytes produced in dst or -1 in case of failure
+  SYNOPSIS
+    base64_decode()
+    src      Pointer to base64-encoded string
+    len      Length of string at 'src'
+    dst      Pointer to location where decoded data will be stored
+    end_ptr  Pointer to variable that will refer to the character
+             after the end of the encoded data that were decoded. Can
+             be NULL.
+
+  DESCRIPTION
+
+    The base64-encoded data in the range ['src','*end_ptr') will be
+    decoded and stored starting at 'dst'.  The decoding will stop
+    after 'len' characters have been read from 'src', or when padding
+    occurs in the base64-encoded data. In either case: if 'end_ptr' is
+    non-null, '*end_ptr' will be set to point to the character after
+    the last read character, even in the presence of error.
+
+  NOTE
+    We require that 'dst' is pre-allocated to correct size.
+
+  SEE ALSO
+    base64_needed_decoded_length().
+
+  RETURN VALUE
+    Number of bytes written at 'dst' or -1 in case of failure
 */
 int
-base64_decode(const char *src, size_t size, void *dst)
+base64_decode(const char *const src_base, size_t const len,
+              void *dst, const char **end_ptr)
 {
   char b[3];
   size_t i= 0;
   char *dst_base= (char *)dst;
+  char const *src= src_base;
   char *d= dst_base;
   size_t j;
 
-  while (i < size)
+  while (i < len)
   {
     unsigned c= 0;
     size_t mark= 0;
 
-    SKIP_SPACE(src, i, size);
+    SKIP_SPACE(src, i, len);
 
     c += pos(*src++);
     c <<= 6;
     i++;
 
-    SKIP_SPACE(src, i, size);
+    SKIP_SPACE(src, i, len);
 
     c += pos(*src++);
     c <<= 6;
     i++;
 
-    SKIP_SPACE(src, i, size);
+    SKIP_SPACE(src, i, len);
 
-    if (* src != '=')
+    if (*src != '=')
       c += pos(*src++);
     else
     {
-      i= size;
+      src += 2;                /* There should be two bytes padding */
+      i= len;
       mark= 2;
       c <<= 6;
       goto end;
@@ -170,13 +195,14 @@ base64_decode(const char *src, size_t size, void *dst)
     c <<= 6;
     i++;
 
-    SKIP_SPACE(src, i, size);
+    SKIP_SPACE(src, i, len);
 
     if (*src != '=')
       c += pos(*src++);
     else
     {
-      i= size;
+      src += 1;                 /* There should be one byte padding */
+      i= len;
       mark= 1;
       goto end;
     }
@@ -191,11 +217,14 @@ base64_decode(const char *src, size_t size, void *dst)
       *d++= b[j];
   }
 
-  if (i != size)
-  {
-    return -1;
-  }
-  return d - dst_base;
+  if (end_ptr != NULL)
+    *end_ptr= src;
+
+  /*
+    The variable 'i' is set to 'len' when padding has been read, so it
+    does not actually reflect the number of bytes read from 'src'.
+   */
+  return i != len ? -1 : d - dst_base;
 }
 
 

mysys/mf_iocache2.c

@@ -24,6 +24,53 @@
 #include <stdarg.h>
 #include <m_ctype.h>
 
+/*
+  Copy contents of an IO_CACHE to a file.
+
+  SYNOPSIS
+    my_b_copy_to_file()
+    cache  IO_CACHE to copy from
+    file   File to copy to
+
+  DESCRIPTION
+    Copy the contents of the cache to the file. The cache will be
+    re-inited to a read cache and will read from the beginning of the
+    cache.
+
+    If a failure to write fully occurs, the cache is only copied
+    partially.
+
+  TODO
+    Make this function solid by handling partial reads from the cache
+    in a correct manner: it should be atomic.
+
+  RETURN VALUE
+    0  All OK
+    1  An error occured
+*/
+int
+my_b_copy_to_file(IO_CACHE *cache, FILE *file)
+{
+  byte buf[IO_SIZE];
+  DBUG_ENTER("my_b_copy_to_file");
+
+  /* Reinit the cache to read from the beginning of the cache */
+  if (reinit_io_cache(cache, READ_CACHE, 0L, FALSE, FALSE))
+    DBUG_RETURN(1);
+  uint bytes_in_cache= my_b_bytes_in_cache(cache);
+  while (bytes_in_cache > 0) {
+    uint const read_bytes= min(bytes_in_cache, sizeof(buf));
+    DBUG_PRINT("debug", ("Remaining %u bytes - Reading %u bytes",
+                         bytes_in_cache, read_bytes));
+    if (my_b_read(cache, buf, read_bytes))
+      DBUG_RETURN(1);
+    if (my_fwrite(file, buf, read_bytes, MYF(MY_WME | MY_NABP)) == (uint) -1)
+      DBUG_RETURN(1);
+    bytes_in_cache -= read_bytes;
+  }
+  DBUG_RETURN(0);
+}
+
 my_off_t my_b_append_tell(IO_CACHE* info)
 {
   /*

sql/log_event.h

@@ -519,14 +519,30 @@ typedef struct st_print_event_info
       bzero(db, sizeof(db));
       bzero(charset, sizeof(charset));
       bzero(time_zone_str, sizeof(time_zone_str));
+      uint const flags = MYF(MY_WME | MY_NABP);
+      init_io_cache(&head_cache, -1, 0, WRITE_CACHE, 0L, FALSE, flags);
+      init_io_cache(&body_cache, -1, 0, WRITE_CACHE, 0L, FALSE, flags);
     }
 
+  ~st_print_event_info() {
+    end_io_cache(&head_cache);
+    end_io_cache(&body_cache);
+  }
+
+
   /* Settings on how to print the events */
   bool short_form;
   bool base64_output;
   my_off_t hexdump_from;
   uint8 common_header_len;
 
+  /*
+     These two caches are used by the row-based replication events to
+     collect the header information and the main body of the events
+     making up a statement.
+   */
+  IO_CACHE head_cache;
+  IO_CACHE body_cache;
 } PRINT_EVENT_INFO;
 #endif
 
@@ -637,9 +653,11 @@ class Log_event
                                    const Format_description_log_event *description_event);
   /* print*() functions are used by mysqlbinlog */
   virtual void print(FILE* file, PRINT_EVENT_INFO* print_event_info) = 0;
-  void print_timestamp(FILE* file, time_t *ts = 0);
-  void print_header(FILE* file, PRINT_EVENT_INFO* print_event_info);
-  void print_base64(FILE* file, PRINT_EVENT_INFO* print_event_info);
+  void print_timestamp(IO_CACHE* file, time_t *ts = 0);
+  void print_header(IO_CACHE* file, PRINT_EVENT_INFO* print_event_info,
+                    bool is_more);
+  void print_base64(IO_CACHE* file, PRINT_EVENT_INFO* print_event_info,
+                    bool is_more);
 #endif
 
   static void *operator new(size_t size)
@@ -804,7 +822,7 @@ class Query_log_event: public Log_event
                  uint32 q_len_arg);
 #endif /* HAVE_REPLICATION */
 #else
-  void print_query_header(FILE* file, PRINT_EVENT_INFO* print_event_info);
+  void print_query_header(IO_CACHE* file, PRINT_EVENT_INFO* print_event_info);
   void print(FILE* file, PRINT_EVENT_INFO* print_event_info);
 #endif
 
@@ -1864,6 +1882,10 @@ class Rows_log_event : public Log_event
 		 Log_event_type event_type,
 		 const Format_description_log_event *description_event);
 
+#ifdef MYSQL_CLIENT
+  void print_helper(FILE *, PRINT_EVENT_INFO *, char const *const name);
+#endif
+
 #ifndef MYSQL_CLIENT
   virtual int do_add_row_data(byte *data, my_size_t length);
 #endif

sql/share/errmsg.txt

@@ -5968,6 +5968,9 @@ ER_CANT_ACTIVATE_LOG
         ger "Kann Logdatei '%-.64s' nicht aktivieren"
 ER_RBR_NOT_AVAILABLE
         eng "The server was not built with row-based replication"
+ER_BASE64_DECODE_ERROR
+        eng "Decoding of base64 string failed"
+        swe "Avkodning av base64 strng misslyckades"
         ger "Der Server hat keine zeilenbasierte Replikation"
 ER_NO_TRIGGERS_ON_SYSTEM_SCHEMA
         eng "Triggers can not be created on system tables"

sql/sql_binlog.cc

@@ -31,6 +31,7 @@
 
 void mysql_client_binlog_statement(THD* thd)
 {
+  DBUG_ENTER("mysql_client_binlog_statement");
   DBUG_PRINT("info",("binlog base64: '%*s'",
                      (thd->lex->comment.length < 2048 ?
                       thd->lex->comment.length : 2048),
@@ -43,8 +44,8 @@ void mysql_client_binlog_statement(THD* thd)
   my_bool nsok= thd->net.no_send_ok;
   thd->net.no_send_ok= TRUE;
 
-  const my_size_t coded_len= thd->lex->comment.length + 1;
-  const my_size_t event_len= base64_needed_decoded_length(coded_len);
+  my_size_t coded_len= thd->lex->comment.length + 1;
+  my_size_t decoded_len= base64_needed_decoded_length(coded_len);
   DBUG_ASSERT(coded_len > 0);
 
   /*
@@ -57,9 +58,8 @@ void mysql_client_binlog_statement(THD* thd)
     new Format_description_log_event(4);
 
   const char *error= 0;
-  char *buf= (char *) my_malloc(event_len, MYF(MY_WME));
+  char *buf= (char *) my_malloc(decoded_len, MYF(MY_WME));
   Log_event *ev = 0;
-  int res;
 
   /*
     Out of memory check
@@ -73,43 +73,97 @@ void mysql_client_binlog_statement(THD* thd)
   thd->rli_fake->sql_thd= thd;
   thd->rli_fake->no_storage= TRUE;
 
-  res= base64_decode(thd->lex->comment.str, coded_len, buf);
-
-  DBUG_PRINT("info",("binlog base64 decoded_len=%d, event_len=%d\n",
-                     res, uint4korr(buf + EVENT_LEN_OFFSET)));
-  /*
-    Note that 'res' is the correct event length, 'event_len' was
-    calculated based on the base64-string that possibly contained
-    extra spaces, so it can be longer than the real event.
-  */
-  if (res < EVENT_LEN_OFFSET
-      || (uint) res != uint4korr(buf+EVENT_LEN_OFFSET))
+  for (char const *strptr= thd->lex->comment.str ;
+       strptr < thd->lex->comment.str + thd->lex->comment.length ; )
   {
-    my_error(ER_SYNTAX_ERROR, MYF(0));
-    goto end;
-  }
-
-  ev= Log_event::read_log_event(buf, res, &error, desc);
+    char const *endptr= 0;
+    int bytes_decoded= base64_decode(strptr, coded_len, buf, &endptr);
+
+    DBUG_PRINT("info",
+               ("bytes_decoded=%d; strptr=0x%lu; endptr=0x%lu ('%c':%d)",
+                bytes_decoded, strptr, endptr, *endptr, *endptr));
+
+    if (bytes_decoded < 0)
+    {
+      my_error(ER_BASE64_DECODE_ERROR, MYF(0));
+      goto end;
+    }
+    else if (bytes_decoded == 0)
+      break; // If no bytes where read, the string contained only whitespace
+
+    DBUG_ASSERT(bytes_decoded > 0);
+    DBUG_ASSERT(endptr > strptr);
+    coded_len-= endptr - strptr;
+    strptr= endptr;
 
-  DBUG_PRINT("info",("binlog base64 err=%s", error));
-  if (!ev)
-  {
     /*
-      This could actually be an out-of-memory, but it is more
-      likely causes by a bad statement
+      Now we have one or more events stored in the buffer. The size of
+      the buffer is computed based on how much base64-encoded data
+      there were, so there should be ample space for the data (maybe
+      even too much, since a statement can consist of a considerable
+      number of events).
+
+      TODO: Switch to use a stream-based base64 encoder/decoder in
+      order to be able to read exactly what is necessary.
     */
-    my_error(ER_SYNTAX_ERROR, MYF(0));
-    goto end;
-  }
 
-  DBUG_PRINT("info",("ev->get_type_code()=%d", ev->get_type_code()));
-  DBUG_PRINT("info",("buf+EVENT_TYPE_OFFSET=%d", buf+EVENT_TYPE_OFFSET));
+    DBUG_PRINT("info",("binlog base64 decoded_len=%d, bytes_decoded=%d",
+                       decoded_len, bytes_decoded));
 
-  ev->thd= thd;
-  if (ev->exec_event(thd->rli_fake))
-  {
-    my_error(ER_UNKNOWN_ERROR, MYF(0), "Error executing BINLOG statement");
-    goto end;
+    /*
+      Now we start to read events of the buffer, until there are no
+      more.
+    */
+    for (char *bufptr= buf ; bytes_decoded > 0 ; )
+    {
+      /*
+        Checking that the first event in the buffer is not truncated.
+      */
+      ulong event_len= uint4korr(bufptr + EVENT_LEN_OFFSET);
+      DBUG_PRINT("info", ("event_len=%lu, bytes_decoded=%d",
+                          event_len, bytes_decoded));
+      if (bytes_decoded < EVENT_LEN_OFFSET || (uint) bytes_decoded < event_len)
+      {
+        my_error(ER_SYNTAX_ERROR, MYF(0));
+        goto end;
+      }
+
+      ev= Log_event::read_log_event(bufptr, event_len, &error, desc);
+
+      DBUG_PRINT("info",("binlog base64 err=%s", error));
+      if (!ev)
+      {
+        /*
+          This could actually be an out-of-memory, but it is more likely
+          causes by a bad statement
+        */
+        my_error(ER_SYNTAX_ERROR, MYF(0));
+        goto end;
+      }
+
+      bytes_decoded -= event_len;
+      bufptr += event_len;
+
+      DBUG_PRINT("info",("ev->get_type_code()=%d", ev->get_type_code()));
+      DBUG_PRINT("info",("bufptr+EVENT_TYPE_OFFSET=0x%lx",
+                         bufptr+EVENT_TYPE_OFFSET));
+      DBUG_PRINT("info", ("bytes_decoded=%d; bufptr=0x%lx; buf[EVENT_LEN_OFFSET]=%u",
+                          bytes_decoded, bufptr, uint4korr(bufptr+EVENT_LEN_OFFSET)));
+      ev->thd= thd;
+      if (int err= ev->exec_event(thd->rli_fake))
+      {
+        DBUG_PRINT("info", ("exec_event() - error=%d", error));
+        /*
+          TODO: Maybe a better error message since the BINLOG statement
+          now contains several events.
+        */
+        my_error(ER_UNKNOWN_ERROR, MYF(0), "Error executing BINLOG statement");
+        goto end;
+      }
+
+      delete ev;
+      ev= 0;
+    }
   }
 
   /*
@@ -126,10 +180,7 @@ void mysql_client_binlog_statement(THD* thd)
   */
   thd->net.no_send_ok= nsok;
 
-  if (ev)
-    delete ev;
-  if (desc)
-    delete desc;
-  if (buf)
-    my_free(buf, MYF(0));
+  delete desc;
+  my_free(buf, MYF(MY_ALLOW_ZERO_PTR));
+  DBUG_VOID_RETURN;
 }

storage/ndb/src/mgmapi/mgmapi.cpp

@@ -1906,7 +1906,7 @@ ndb_mgm_get_configuration(NdbMgmHandle handle, unsigned int version) {
       break;
 
     void *tmp_data = malloc(base64_needed_decoded_length((size_t) (len - 1)));
-    const int res = base64_decode(buf64, len-1, tmp_data);
+    const int res = base64_decode(buf64, len-1, tmp_data, NULL);
     delete[] buf64;
     UtilBuffer tmp;
     tmp.append((void *) tmp_data, res);

unittest/mysys/base64-t.c

@@ -54,7 +54,7 @@ main(void)
 
     /* Decode */
     dst= (char *) malloc(base64_needed_decoded_length(strlen(str)));
-    dst_len= base64_decode(str, strlen(str), dst);
+    dst_len= base64_decode(str, strlen(str), dst, NULL);
     ok(dst_len == src_len, "Comparing lengths");
 
     cmp= memcmp(src, dst, src_len);