Addresses #1617 Add byte-order verification to header. version/size of header...

Addresses #1617 Add byte-order verification to header.  version/size of header is always stored in network order
version of log is always stored in network order
cleaned up toku_htod

git-svn-id: file:///svn/toku/tokudb@10723 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt-serialize.c

@@ -903,6 +903,7 @@ int toku_serialize_brt_header_size (struct brt_header *h) {
     unsigned int size = (+8 // "tokudata"
 			 +4 // size
 			 +4 // version
+                         +8 // byte order verification
 			 +4 // tree's nodesize
 			 +8 // free blocks
 			 +8 // unused blocks
@@ -930,8 +931,9 @@ int toku_serialize_brt_header_size (struct brt_header *h) {
 int toku_serialize_brt_header_to_wbuf (struct wbuf *wbuf, struct brt_header *h) {
     unsigned int size = toku_serialize_brt_header_size (h); // !!! seems silly to recompute the size when the caller knew it.  Do we really need the size?
     wbuf_literal_bytes(wbuf, "tokudata", 8);
-    wbuf_int    (wbuf, size);
-    wbuf_int    (wbuf, h->layout_version);
+    wbuf_network_int  (wbuf, size); //MUST be in network order regardless of disk order
+    wbuf_network_int  (wbuf, h->layout_version); //MUST be in network order regardless of disk order
+    wbuf_literal_bytes(wbuf, &toku_byte_order_host, 8); //Must not translate byte order
     wbuf_int    (wbuf, h->nodesize);
     //TODO: Use 'prelocked/unlocked' versions to make this atomic
 //TODO: #1463 START
@@ -1044,9 +1046,15 @@ deserialize_brtheader (u_int32_t size, int fd, DISKOFF off, struct brt_header **
     h->dirty=0;
     h->panic = 0;
     h->panic_string = 0;
-    h->layout_version = rbuf_int(&rc);
+    //version MUST be in network order on disk regardless of disk order
+    h->layout_version = rbuf_network_int(&rc);
+    assert(h->layout_version==BRT_LAYOUT_VERSION_10);
+    bytevec tmp_byte_order_check;
+    rbuf_literal_bytes(&rc, &tmp_byte_order_check, 8); //Must not translate byte order
+    int64_t byte_order_stored = *(int64_t*)tmp_byte_order_check;
+    assert(byte_order_stored == toku_byte_order_host);
+
     h->nodesize      = rbuf_int(&rc);
-    assert(h->layout_version==BRT_LAYOUT_VERSION_9 || h->layout_version==BRT_LAYOUT_VERSION_10);
     BLOCKNUM free_blocks = rbuf_blocknum(&rc);
     BLOCKNUM unused_blocks = rbuf_blocknum(&rc);
     h->n_named_roots = rbuf_int(&rc);
@@ -1142,7 +1150,10 @@ int toku_deserialize_brtheader_from (int fd, BLOCKNUM blocknum, struct brt_heade
     if (r!=12) return EINVAL;
     assert(memcmp(magic,"tokudata",8)==0);
     // It's version 7 or later, and the magi clooks OK
-    return deserialize_brtheader(toku_dtoh32(*(int*)(&magic[8])), fd, offset, brth);
+
+    //size MUST be in network order on disk regardless of disk order
+    u_int32_t size = toku_ntohl(*(int*)(&magic[8]));
+    return deserialize_brtheader(size, fd, offset, brth);
 }
 
 unsigned int toku_brt_pivot_key_len (BRT brt, struct kv_pair *pk) {

newbrt/log.c

@@ -155,8 +155,8 @@ static int open_logfile (TOKULOGGER logger) {
         if (logger->fd==-1) return errno;
     }
     logger->next_log_file_number++;
-    int version_l = toku_htod32(log_format_version);
     r = write_it(logger->fd, "tokulogg", 8);             if (r!=8) return errno;
+    int version_l = toku_htonl(log_format_version); //version MUST be in network byte order regardless of disk order
     r = write_it(logger->fd, &version_l, 4);             if (r!=4) return errno;
     logger->fsynced_lsn = logger->written_lsn;
     logger->n_in_file = 12;
@@ -832,7 +832,8 @@ int toku_read_and_print_logmagic (FILE *f, u_int32_t *versionp) {
 	    return DB_BADFORMAT;
 	}
 	//printf("tokulog v.%d\n", toku_dtoh32(version));
-	*versionp=toku_dtoh32(version);
+        //version MUST be in network order regardless of disk order
+	*versionp=toku_ntohl(version);
     }
     return 0;
 }

newbrt/rbuf.h

@@ -8,6 +8,7 @@
 #include "toku_assert.h"
 #include "brttypes.h"
 #include "memory.h"
+#include "toku_htonl.h"
 
 struct rbuf {
     unsigned char *buf;
@@ -20,6 +21,15 @@ static inline unsigned int rbuf_char (struct rbuf *r) {
     return r->buf[r->ndone++];
 }
 
+//Read an int that MUST be in network order regardless of disk order
+static unsigned int rbuf_network_int (struct rbuf *r) __attribute__((__unused__));
+static unsigned int rbuf_network_int (struct rbuf *r) {
+    assert(r->ndone+4 <= r->size);
+    u_int32_t result = toku_ntohl(*(u_int32_t*)(r->buf+r->ndone)); // This only works on machines where unaligned loads are OK.
+    r->ndone+=4;
+    return result;
+}
+
 static unsigned int rbuf_int (struct rbuf *r) {
 #if 1
     assert(r->ndone+4 <= r->size);

newbrt/tests/test.h

@@ -5,6 +5,7 @@
 
 #include "toku_portability.h"
 #include "brt.h"
+#include "toku_htonl.h"
 
 
 #define CKERR(r) do { if (r!=0) fprintf(stderr, "%s:%d error %d %s\n", __FILE__, __LINE__, r, strerror(r)); assert(r==0); } while (0)

newbrt/wbuf.h

@@ -37,6 +37,15 @@ static inline void wbuf_char (struct wbuf *w, unsigned char ch) {
     x1764_add(&w->checksum, &w->buf[w->ndone-1], 1);
 }
 
+//Write an int that MUST be in network order regardless of disk order
+static void wbuf_network_int (struct wbuf *w, int32_t i) __attribute__((__unused__));
+static void wbuf_network_int (struct wbuf *w, int32_t i) {
+    assert(w->ndone + 4 <= w->size);
+    *(u_int32_t*)(&w->buf[w->ndone]) = toku_htonl(i);
+    x1764_add(&w->checksum, &w->buf[w->ndone], 4);
+    w->ndone += 4;
+}
+
 static void wbuf_int (struct wbuf *w, int32_t i) {
 #if 0
     wbuf_char(w, i>>24);
@@ -57,6 +66,7 @@ static void wbuf_int (struct wbuf *w, int32_t i) {
     w->ndone += 4;
 #endif
 }
+
 static void wbuf_uint (struct wbuf *w, u_int32_t i) {
     wbuf_int(w, (int32_t)i);
 }

src/tests/test.h

@@ -9,6 +9,7 @@
 #include <db.h>
 #include <assert.h>
 #include <limits.h>
+#include "toku_htonl.h"
 #if defined(USE_TDB)
 #include "ydb.h"
 //TDB uses DB_NOTFOUND for c_del and DB_CURRENT errors.

toku_include/toku_htod.h

@@ -22,8 +22,9 @@
 #ifndef HTOD_H
 #define HTOD_H
 
-#include "toku_htonl.h"
+static const int64_t toku_byte_order_host = 0x0102030405060708LL;
 
+#include <endian.h>
 #if !defined(__BYTE_ORDER) || \
     !defined(__LITTLE_ENDIAN) || \
     !defined(__BIG_ENDIAN)
@@ -34,26 +35,18 @@
 #define INTEL_BYTE_ORDER    (__LITTLE_ENDIAN)
 #define HOST_BYTE_ORDER     (__BYTE_ORDER)
 
-//Switch DISK_BYTE_ORDER to INTEL_BYTE_ORDER to speed up intel.
-//#define DISK_BYTE_ORDER     (NETWORK_BYTE_ORDER)
+//DISK_BYTE_ORDER is the byte ordering for integers written to disk.
+//If DISK_BYTE_ORDER is the same as HOST_BYTE_ORDER no conversions are necessary.
+//Otherwise some structures require conversion to HOST_BYTE_ORDER on loading from disk (HOST_BYTE_ORDER in memory), and
+//others require conversion to HOST_BYTE_ORDER on every access/mutate (DISK_BYTE_ORDER in memory).
 #define DISK_BYTE_ORDER     (INTEL_BYTE_ORDER)
 
-#if defined(__PDP_ENDIAN) && (HOST_BYTE_ORDER==__PDP_ENDIAN)
-#error "Are we in ancient Rome?  You REALLY want support for PDP_ENDIAN?"
-#elif (HOST_BYTE_ORDER!=__BIG_ENDIAN) && (HOST_BYTE_ORDER!=__LITTLE_ENDIAN)
-#error HOST_BYTE_ORDER not well defined
+#if HOST_BYTE_ORDER!=INTEL_BYTE_ORDER
+//Even though the functions are noops if DISK==HOST, we do not have the logic to test whether the file was moved from another BYTE_ORDER machine.
+#error Only intel byte order supported so far.
 #endif
 
-#if HOST_BYTE_ORDER==DISK_BYTE_ORDER
-#define HTOD_NEED_SWAP 0
-#else
-#define HTOD_NEED_SWAP 1
-#if (HOST_BYTE_ORDER==__BIG_ENDIAN)
-#error Byte swapping on Big Endian is not coded in htod.c
-#endif
-#endif
-
-#if !HTOD_NEED_SWAP
+#if DISK_BYTE_ORDER == HOST_BYTE_ORDER
 static inline uint32_t
 toku_dtoh32(uint32_t i) {
     return i;
@@ -63,23 +56,8 @@ static inline uint32_t
 toku_htod32(uint32_t i) {
     return i;
 }
-
-#elif HOST_BYTE_ORDER == __LITTLE_ENDIAN //HTOD_NEED_SWAP
-
-static inline uint32_t
-toku_dtoh32(uint32_t i) {
-    return ntohl(i);
-}
-
-static inline uint32_t
-toku_htod32(uint32_t i) {
-    return htonl(i);
-}
-
-#elif HOST_BYTE_ORDER == __BIG_ENDIAN //!HTOD_NEED_SWAP
-
-#error Byte swapping in big endian not yet supported
-
+#else
+#error Not supported
 #endif
 
 #endif