Separate rbuf and wbuf so that log.c can use it. Check in a perliminary version of log.c

git-svn-id: file:///svn/tokudb@162 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/Makefile

@@ -48,7 +48,7 @@ hashtable.o: hashtable.h brttypes.h memory.h key.h yerror.h ../include/ydb-const
 memory.o: memory.h
 primes.o: primes.h
 hashtest: hashtable.o memory.o primes.o
-brt-serialize.o: brt.h cachetable.h memory.h mdict.h pma.h brttypes.h brt-internal.h hashtable.h
+brt-serialize.o: brt.h cachetable.h memory.h mdict.h pma.h brttypes.h brt-internal.h hashtable.h wbuf.h rbuf.h
 header-io.o: brttypes.h brt-internal.h memory.h
 mdict-test: hashtable.o pma.o memory.o
 brt-bigtest: memory.o ybt.o brt.o pma.o cachetable.o key.o hashtable.o brt-serialize.o

newbrt/brt-internal.h

@@ -70,7 +70,7 @@ struct brt {
 };
 
 /* serialization code */
-void serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node);
+int serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node);
 int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesize);
 unsigned int serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
 int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);

newbrt/brt-serialize-test.c

@@ -34,7 +34,7 @@ void test_serialize(void) {
 
     deserialize_brtnode_from(fd, nodesize*20, &dn, nodesize);
 
-    serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn);
+    r = serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn);  assert(r==0);
 
 
     assert(dn->thisnodename==nodesize*20);

newbrt/brt-serialize.c

@@ -5,88 +5,14 @@
 //#include "pma.h"
 #include "brt-internal.h"
 #include "key.h"
+#include "rbuf.h"
+#include "wbuf.h"
+
 
 #include <assert.h>
 #include <unistd.h>
-#include <string.h>
 #include <stdio.h>
 #include <arpa/inet.h>
-#include <errno.h>
-
-struct cursor {
-    unsigned char *buf;
-    unsigned int  size;
-    unsigned int  ndone;
-};
-
-void wbuf_char (struct cursor *w, int ch) {
-    assert(w->ndone<w->size);
-    w->buf[w->ndone++]=ch;
-}
-
-void wbuf_int (struct cursor *w, unsigned int i) {
-#if 0
-    wbuf_char(w, i>>24);
-    wbuf_char(w, i>>16);
-    wbuf_char(w, i>>8);
-    wbuf_char(w, i>>0);
-#else
-    assert(w->ndone + 4 <= w->size);
-    w->buf[w->ndone+0] = i>>24;
-    w->buf[w->ndone+1] = i>>16;
-    w->buf[w->ndone+2] = i>>8;
-    w->buf[w->ndone+3] = i>>0;
-    w->ndone += 4;
-#endif
-}
-
-void wbuf_bytes (struct cursor *w, bytevec bytes_bv, int nbytes) {
-    const unsigned char *bytes=bytes_bv; 
-    wbuf_int(w, nbytes);
-#if 0
-    { int i; for (i=0; i<nbytes; i++) wbuf_char(w, bytes[i]); }
-#else
-    assert(w->ndone + nbytes <= w->size);
-    memcpy(w->buf + w->ndone, bytes, nbytes);
-    w->ndone += nbytes;
-#endif
-}
-
-void wbuf_diskoff (struct cursor *w, diskoff off) {
-    wbuf_int(w, off>>32);
-    wbuf_int(w, off&0xFFFFFFFF);
-}
-
-unsigned int rbuf_char (struct cursor *r) {
-    assert(r->ndone<r->size);
-    return r->buf[r->ndone++];
-}
-
-unsigned int rbuf_int (struct cursor *r) {
-    unsigned char c0 = rbuf_char(r);
-    unsigned char c1 = rbuf_char(r);
-    unsigned char c2 = rbuf_char(r);
-    unsigned char c3 = rbuf_char(r);
-    return ((c0<<24)|
-	    (c1<<16)|
-	    (c2<<8)|
-	    (c3<<0));
-}
-
-/* Return a pointer into the middle of the buffer. */
-void rbuf_bytes (struct cursor *r, bytevec *bytes, unsigned int *n_bytes)
-{
-    *n_bytes = rbuf_int(r);
-    *bytes =   &r->buf[r->ndone];
-    r->ndone+=*n_bytes;
-    assert(r->ndone<=r->size);
-}
-
-diskoff rbuf_diskoff (struct cursor *r) {
-    unsigned i0 = rbuf_int(r);  
-    unsigned i1 = rbuf_int(r);
-    return ((unsigned long long)(i0)<<32) | ((unsigned long long)(i1));
-}
 
 static unsigned int serialize_brtnode_size_slow(BRTNODE node) {
     unsigned int size=4+4; /* size+height */
@@ -147,14 +73,13 @@ unsigned int serialize_brtnode_size (BRTNODE node) {
     return result;
 }
 
-void serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node) {
-    struct cursor w;
+int serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node) {
+    struct wbuf w;
     int i;
     unsigned int calculated_size = serialize_brtnode_size(node);
+    int r;
     assert(size>0);
-    w.buf=toku_malloc(size);
-    w.size=size;
-    w.ndone=0;
+    if ((r=wbuf_init(&w, size))) return r;
     //printf("%s:%d serializing %lld w height=%d p0=%p\n", __FILE__, __LINE__, off, node->height, node->mdicts[0]);
     wbuf_int(&w, calculated_size);
     wbuf_int(&w, node->height);
@@ -200,11 +125,12 @@ void serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node) {
     //printf("%s:%d wrote %d bytes for %lld size=%lld\n", __FILE__, __LINE__, w.ndone, off, size);
     assert(w.ndone<=size);
     toku_free(w.buf);
+    return 0;
 }
 
 int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesize) {
     TAGMALLOC(BRTNODE, result);
-    struct cursor rc;
+    struct rbuf rc;
     int i;
     uint32_t datasize;
     int r;
@@ -373,7 +299,7 @@ void verify_counts (BRTNODE node) {
 }
     
 int serialize_brt_header_to (int fd, struct brt_header *h) {
-    struct cursor w;
+    struct wbuf w;
     int i;
     unsigned int size=0; /* I don't want to mess around calculating it exactly. */ 
     size += 4+4+8+8+4; /* this size, the tree's nodesize, freelist, unused_memory, nnamed_rootse. */
@@ -416,7 +342,7 @@ int serialize_brt_header_to (int fd, struct brt_header *h) {
 int deserialize_brtheader_from (int fd, diskoff off, struct brt_header **brth) {
     printf("%s:%d calling MALLOC\n", __FILE__, __LINE__);
     struct brt_header *MALLOC(h);
-    struct cursor rc;
+    struct rbuf rc;
     int size;
     int sizeagain;
     assert(off==0);

newbrt/brt.c

@@ -69,7 +69,8 @@ void brtnode_flush_callback (CACHEFILE cachefile, diskoff nodename, void *brtnod
     assert(brtnode->thisnodename==nodename);
     //printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, brtnode, brtnode->mdicts[0]);
     if (write_me) {
-	serialize_brtnode_to(cachefile_fd(cachefile), brtnode->thisnodename, brtnode->nodesize, brtnode);
+	int r=serialize_brtnode_to(cachefile_fd(cachefile), brtnode->thisnodename, brtnode->nodesize, brtnode);
+	assert(r==0); // ?????
     }
     //printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, brtnode, brtnode->mdicts[0]);
     if (!keep_me) {

newbrt/log.c

@@ -77,3 +77,37 @@ int tokulogger_log_bytes(TOKULOGGER logger, int nbytes, char *bytes) {
     }
     return 0;
 }
+
+enum { LT_INSERT_WITH_NO_OVERWITE = 'I' }; 
+
+// Log an insertion of a key-value pair into a particular node of the tree.
+int tokulogger_log_brt_insert_with_no_overwrite (TOKULOGGER logger,
+						 TXNID txnid,
+						 diskoff diskoff,
+						 unsigned char *key,
+						 int keylen,
+						 unsigned char *val,
+						 int vallen) {
+    int buflen=30+keylen+vallen;
+    char buf[buflen];
+    WBUF wbuf;
+    int r;
+    r = wbuf_create(&wbuf, buf, buflen) ;
+    if (r!=0) return r;
+    wbuf_byte(&wbuf, LT_INSERT_WITH_NO_OVERWRITE);
+    wbuf_txnid(&wbuf, txnind);
+    wbuf_diskoff(&wbuf, diskoff);
+    wbuf_bytes(&wbuf, key, keylen);
+    wbuf_bytes(&wbuf, val, vallen);
+    return tokulogger_log_wbuf(logger, &wbuf);
+}
+
+int tokulogger_log_brt_remove (TOKULOGGER logger,
+			       TXNID txnid,
+			       diskoff diskoff,
+			       unsigned char *key,
+			       int keylen,
+			       unsigned char *val,
+			       int vallen) {
+    
+}

newbrt/rbuf.h

+#ifndef RBUF_H
+#define RBUF_H
+
+#include <assert.h>
+
+struct rbuf {
+    unsigned char *buf;
+    unsigned int  size;
+    unsigned int  ndone;
+};
+
+static unsigned int rbuf_char (struct rbuf *r) {
+    assert(r->ndone<r->size);
+    return r->buf[r->ndone++];
+}
+
+static unsigned int rbuf_int (struct rbuf *r) {
+    unsigned char c0 = rbuf_char(r);
+    unsigned char c1 = rbuf_char(r);
+    unsigned char c2 = rbuf_char(r);
+    unsigned char c3 = rbuf_char(r);
+    return ((c0<<24)|
+	    (c1<<16)|
+	    (c2<<8)|
+	    (c3<<0));
+}
+
+/* Return a pointer into the middle of the buffer. */
+static void rbuf_bytes (struct rbuf *r, bytevec *bytes, unsigned int *n_bytes)
+{
+    *n_bytes = rbuf_int(r);
+    *bytes =   &r->buf[r->ndone];
+    r->ndone+=*n_bytes;
+    assert(r->ndone<=r->size);
+}
+
+static diskoff rbuf_diskoff (struct rbuf *r) {
+    unsigned i0 = rbuf_int(r);  
+    unsigned i1 = rbuf_int(r);
+    return ((unsigned long long)(i0)<<32) | ((unsigned long long)(i1));
+}
+#endif

newbrt/wbuf.h

+#ifndef WBUF_H
+#define WBUF_H
+
+#include <assert.h>
+#include <string.h>
+#include <errno.h>
+
+/* When serializing a value, write it into a buffer. */
+/* This code requires that the buffer be big enough to hold whatever you put into it. */ 
+/* This abstraction doesn't do a good job of hiding its internals.
+ * Why?  The performance of this code is important, and we want to inline stuff */
+struct wbuf {
+    unsigned char *buf;
+    unsigned int  size;
+    unsigned int  ndone;
+};
+
+static int wbuf_init (struct wbuf *w, diskoff size) {
+    w->buf=toku_malloc(size);
+    w->size=size;
+    w->ndone=0;
+    return errno;
+}
+
+/* Write a character. */
+static inline void wbuf_char (struct wbuf *w, int ch) {
+    assert(w->ndone<w->size);
+    w->buf[w->ndone++]=ch;
+}
+
+static void wbuf_int (struct wbuf *w, unsigned int i) {
+#if 0
+    wbuf_char(w, i>>24);
+    wbuf_char(w, i>>16);
+    wbuf_char(w, i>>8);
+    wbuf_char(w, i>>0);
+#else
+    assert(w->ndone + 4 <= w->size);
+    w->buf[w->ndone+0] = i>>24;
+    w->buf[w->ndone+1] = i>>16;
+    w->buf[w->ndone+2] = i>>8;
+    w->buf[w->ndone+3] = i>>0;
+    w->ndone += 4;
+#endif
+}
+
+static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, int nbytes) {
+    const unsigned char *bytes=bytes_bv; 
+    wbuf_int(w, nbytes);
+#if 0
+    { int i; for (i=0; i<nbytes; i++) wbuf_char(w, bytes[i]); }
+#else
+    assert(w->ndone + nbytes <= w->size);
+    memcpy(w->buf + w->ndone, bytes, nbytes);
+    w->ndone += nbytes;
+#endif
+}
+
+static void wbuf_diskoff (struct wbuf *w, diskoff off) {
+    wbuf_int(w, off>>32);
+    wbuf_int(w, off&0xFFFFFFFF);
+}
+#endif