CIL gets upset if includes are not in the same order, because int64_t gets...

CIL gets upset if includes are not in the same order, because int64_t gets typedefed to different things depending on what you include.  So do all the includes in a standard way.  Addresses #1185.

git-svn-id: file:///svn/tokudb.1131b+1080a+1185@6439 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/Makefile

@@ -6,6 +6,9 @@
 # For very verbose output do 
 #   make VERBOSE=2
 
+# For CIL do
+#   make CIL=1
+
 # GCOV_FLAGS = -fprofile-arcs -ftest-coverage
 # PROF_FLAGS = -pg
 OPTFLAGS = -O3
@@ -58,6 +61,9 @@ ifdef BRT_FANOUT
 CPPFLAGS += -DBRT_FANOUT=$(BRT_FANOUT)
 endif
 
+ifeq ($(CIL),1)
+	CC=../../cil/cil-1.3.6/bin/cilly --merge --keepmerged
+endif
 
 # When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test
 
@@ -94,14 +100,19 @@ BRT_SOURCES = \
   threadpool \
 # keep this line so I can ha vea \ on the previous line
 
-OFILES = newbrt.o $(CYG_ADD_LIBZ)
+ifeq ($(CIL),1)
+ OFILES = $(patsubst %,%.o,$(BRT_SOURCES))
+else
+ OFILES = newbrt.o
+endif
 TEST_OFILES = brt-test-helpers.o
 
 HFILES = $(wildcard *.h)
 
 BRT_C_FILES = $(patsubst %,%.c,$(BRT_SOURCES))
 
-ifeq ($(CC),icc)
+ifeq ($(CIL),1)
+else ifeq ($(CC),icc)
 newbrt.o: $(BRT_C_FILES) $(HFILES)
 	$(CC) -ipo-c $(CFLAGS) $(CPPFLAGS) $(BRT_C_FILES) -o $@
 else
@@ -110,10 +121,10 @@ newbrt.o: $(BRT_C_FILES) $(HFILES)
 endif
 
 tdb_logprint.o: log-internal.h brttypes.h log.h kv-pair.h log_header.h
-tdb_logprint: $(OFILES)
+tdb_logprint: $(OFILES) $(CYG_ADD_LIBZ)
 
 recover.o: log_header.h log-internal.h log.h brttypes.h kv-pair.h memory.h key.h cachetable.h
-tdb-recover: $(OFILES)
+tdb-recover: $(OFILES) $(CYG_ADD_LIBZ)
 
 roll.o: log_header.h log-internal.h log.h brttypes.h kv-pair.h memory.h key.h cachetable.h omt.h bread.h
 
@@ -123,7 +134,7 @@ log_header.h: log_code.c
 log_code.c: logformat
 	./logformat
 
-libs: newbrt.o
+libs: $(OFILES) $(CYG_ADD_LIBZ)
 bins: $(BINS)
 # Put the benchmarktest_256 first since it takes the longest (and we want to use parallelism in the make)
 
@@ -142,23 +153,19 @@ BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h omt.h brt.h brt-sea
 brt-test-helpers.o: $(BRT_INTERNAL_H_INCLUDES) toku_assert.h
 logformat: logformat.c toku_assert.c
 brt-serialize-test.o: $(BRT_INTERNAL_H_INCLUDES)
-fifo-test: newbrt.o
-log-test6 log-test5 log-test4 log-test3 log-test2 log-test: $(OFILES)
-omt-test.o: toku_assert.h memory.h toku_assert.h ../include/db.h brttypes.h
-omt-test: omt-test.o newbrt.o
 
 test_toku_malloc_plain_free: newbrt.o
 
 
 cachetable-test.o: cachetable.h memory.h 
-cachetable-test: $(OFILES)
+cachetable-test: $(OFILES) $(CYG_ADD_LIBZ)
 
 cachetable-test2.o: cachetable.h memory.h
-cachetable-test2: $(OFILES)
+cachetable-test2: $(OFILES) $(CYG_ADD_LIBZ)
 
 test-assert: newbrt.o
 
-brtdump: $(OFILES)
+brtdump: $(OFILES) $(CYG_ADD_LIBZ)
 
 test_oexcl: test_oexcl.o newbrt.o
 

newbrt/block_allocator.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
-#include "block_allocator.h"
-#include "memory.h"
-#include "toku_assert.h"
 
-#include <errno.h>
-#include <string.h>
+#include "includes.h"
 
 // Here's a very simple implementation.
 // It's not very fast at allocating or freeing.

newbrt/bread.c

 /* Buffered read. */
-#include <assert.h>
-#include <string.h>
-#include <unistd.h>
-#include <sys/types.h>
 
-#include "bread.h"
-#include "memory.h"
+#include "includes.h"
 
 struct bread {
     off_t current_offset;   // The current offset to be read (in the file).  That offset includes anything that is unread in the buffer.

newbrt/brt-serialize.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include "toku_assert.h"
-#include "block_allocator.h"
-#include "brt-internal.h"
-#include "key.h"
-#include "rbuf.h"
-#include "wbuf.h"
-#include "kv-pair.h"
-#include "mempool.h"
-
-#include <arpa/inet.h>
-#include <inttypes.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <zlib.h>
+#include "includes.h"
 
 #if 0
 static u_int64_t ntohll(u_int64_t v) {
@@ -177,13 +163,14 @@ wbufwriteleafentry (OMTVALUE lev, u_int32_t UU(idx), void *v) {
     return 0;
 }
 
-const int uncompressed_magic_len = (8 // tokuleaf or tokunode
-				    +4 // version
-				    +8 // lsn
-				    );
+enum { uncompressed_magic_len = (8 // tokuleaf or tokunode
+				 +4 // version
+				 +8 // lsn
+				 ) };
 
-const int compression_header_len = (4 // compressed_len
-				    +4); // uncompressed_len
+enum { compression_header_len = (4   // compressed_len
+				 +4 // uncompressed_len
+				 ) };
 
 void toku_serialize_brtnode_to (int fd, BLOCKNUM blocknum, BRTNODE node, struct brt_header *h) {
     //printf("%s:%d serializing\n", __FILE__, __LINE__);
@@ -885,7 +872,7 @@ int toku_serialize_fifo_at (int fd, off_t freeoff, FIFO fifo) {
     //printf("%s:%d Serializing fifo at %" PRId64 " (count=%d)\n", __FILE__, __LINE__, freeoff, toku_fifo_n_entries(fifo));
     lock_for_pwrite();
     {
-	int size=4;
+	enum { size=4 };
 	char buf[size];
 	struct wbuf w;
 	wbuf_init(&w, buf, size);

newbrt/brt-verify.c

@@ -15,9 +15,7 @@
  * Note: We don't yet have DUP trees, so thee checks on duplicate trees are unimplemented. (Nov 1 2007)
  */
 
-#include "brt-internal.h"
-#include "toku_assert.h"
-#include "kv-pair.h"
+#include "includes.h"
 
 static void verify_local_fingerprint (BRTNODE node) {
     u_int32_t fp=0;

newbrt/brt.c

@@ -24,23 +24,7 @@
  *     
  */
 
-#include <arpa/inet.h>
-#include <errno.h>
-#include <inttypes.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <unistd.h>
-
-#include "block_allocator.h"
-#include "toku_assert.h"
-#include "brt-internal.h"
-#include "key.h"
-#include "log_header.h"
-#include "kv-pair.h"
-#include "mempool.h"
-#include "leafentry.h"
+#include "includes.h"
 
 //#define SLOW
 #ifdef SLOW

newbrt/brtdump.c

 /* Tell me the diff between two brt files. */
-#include <stdio.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <fcntl.h>
-#include <inttypes.h>
 
-#include "toku_assert.h"
-#include "key.h"
-#include "brt-internal.h"
+#include "includes.h"
 
 static int dump_data = 1;
 
@@ -252,12 +245,12 @@ int main (int argc, const char *argv[]) {
     if (interactive) {
         while (1) {
             printf("brtdump>"); fflush(stdout);
-            const int maxline = 64;
+	    enum { maxline = 64};
             char line[maxline+1];
             readline(line, maxline);
             if (strcmp(line, "") == 0) 
                 break;
-            const int maxfields = 2;
+            enum { maxfields = 2 };
             char *fields[maxfields];
             int nfields = split_fields(line, fields, maxfields);
             if (nfields == 0) 

newbrt/cachetable.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include <errno.h>
-#include <malloc.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <unistd.h>
-
-#include "cachetable.h"
-#include "hashfun.h"
-#include "memory.h"
-#include "toku_assert.h"
-#include "brt-internal.h"
-#include "log_header.h"
-#include "threadpool.h"
-#include "cachetable-rwlock.h"
-#include <malloc.h>
+#include "includes.h"
 
 // execute the cachetable callbacks using a writer thread 0->no 1->yes
 #define DO_WRITER_THREAD 1

newbrt/fifo.c

-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-
-#include "toku_assert.h"
-#include "memory.h"
-#include "fifo.h"
-#include "ybt.h"
+#include "includes.h"
 
 struct fifo {
     int n_items_in_fifo;

newbrt/includes.h

+#ifndef SYSINCLUDES_H
+#define SYSINCLUDES_H
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 500
+#endif
+#define _FILE_OFFSET_BITS 64
+
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <malloc.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/file.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <zlib.h>
+
+#include "block_allocator.h"
+#include "brt.h"
+#include "brt-internal.h"
+#include "cachetable.h"
+#include "cachetable-rwlock.h"
+#include "fifo.h"
+#include "list.h"
+#include "key.h"
+#include "kv-pair.h"
+#include "leafentry.h"
+#include "log-internal.h"
+#include "log_header.h"
+#include "mempool.h"
+#include "rbuf.h"
+#include "threadpool.h"
+#include "toku_assert.h"
+#include "wbuf.h"
+
+#include "../include/db.h"
+
+#endif

newbrt/key.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include "brt-internal.h"
-#include "toku_assert.h"
-#include "key.h"
-#include <string.h>
+#include "includes.h"
 
 #if 0
 int toku_keycompare (bytevec key1b, ITEMLEN key1len, bytevec key2b, ITEMLEN key2len) {

newbrt/leafentry.c

 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include "brttypes.h"
-#include "leafentry.h"
-#include "memory.h"
-#include "toku_assert.h"
-#include "log.h"
-#include "wbuf.h"
-
-#include <arpa/inet.h>
-#include <inttypes.h>
-#include <stdlib.h>
-#include <string.h>
+#include "includes.h"
 
 u_int32_t toku_le_crc(LEAFENTRY v) {
     return x1764_memory(v, leafentry_memsize(v));

newbrt/log-internal.h

+#ifndef LOG_INTERNAL_H
+#define LOG_INTERNAL_H
+
 #ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
 
 #include "brt-internal.h"
@@ -156,3 +159,4 @@ static inline char *fixup_fname(BYTESTRING *f) {
 }
 
 int toku_read_rollback_backwards(BREAD, struct roll_entry **item, MEMARENA);
+#endif

newbrt/logformat.c

@@ -565,14 +565,13 @@ int main (int argc __attribute__((__unused__)), char *argv[]  __attribute__((__u
     unlink(headerpath);
     cf = fopen(codepath, "w");      assert(cf!=0);
     hf = fopen(headerpath, "w");     assert(hf!=0);
+    fprintf(hf, "#ifndef LOG_HEADER_H\n");
+    fprintf(hf, "#define  LOG_HEADER_H\n");
     fprintf2(cf, hf, "/* Do not edit this file.  This code generated by logformat.c.  Copyright 2007, 2008 Tokutek.    */\n");
     fprintf2(cf, hf, "#ident \"Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved.\"\n");
-    fprintf(cf, "#include <stdio.h>\n");
+    fprintf(cf, "#include \"includes.h\"\n");
     fprintf(hf, "#include \"brt-internal.h\"\n");
     fprintf(hf, "#include \"memarena.h\"\n");
-    fprintf(cf, "#include \"log_header.h\"\n");
-    fprintf(cf, "#include \"wbuf.h\"\n");
-    fprintf(cf, "#include \"log-internal.h\"\n");
     generate_enum();
     generate_log_struct();
     generate_dispatch();
@@ -580,6 +579,7 @@ int main (int argc __attribute__((__unused__)), char *argv[]  __attribute__((__u
     generate_log_reader();
     generate_logprint();
     generate_rollbacks();
+    fprintf(hf, "#endif\n");
     {
 	int r=fclose(hf);
 	assert(r==0);

newbrt/memarena.c

-#include <string.h>
-#include <stdio.h>
-
-#include "memarena.h"
-#include "memory.h"
-#include "toku_assert.h"
+#include "includes.h"
 
 struct memarena {
     char *buf;

newbrt/memory.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include "memory.h"
-#include "toku_assert.h"
-#include <string.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <stdio.h>
+#include "includes.h"
 
 int toku_memory_check=0;
 

newbrt/mempool.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include "mempool.h"
-#include "toku_assert.h"
-#include <stdio.h>
+#include "includes.h"
 
 void toku_mempool_init(struct mempool *mp, void *base, size_t size) {
     // printf("mempool_init %p %p %d\n", mp, base, size);

newbrt/omt.c

 #ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
 
-#include <errno.h>
-#include <sys/types.h>
-#include <stdint.h>
-#include <string.h>
-
-typedef void *OMTVALUE;
-#include "omt.h"
-#include "../newbrt/memory.h"
-#include "../newbrt/toku_assert.h"
-#include "../include/db.h"
-#include "../newbrt/brttypes.h"
+#include "includes.h"
 
 typedef u_int32_t node_idx;
 static const node_idx NODE_NULL = UINT32_MAX;

newbrt/rbuf.h

@@ -73,14 +73,14 @@ static inline void rbuf_TXNID (struct rbuf *r, TXNID *txnid) {
     *txnid = rbuf_ulonglong(r);
 }
 static inline void rbuf_ma_TXNID (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), TXNID *txnid) {
-    return rbuf_TXNID(r, txnid);
+    rbuf_TXNID(r, txnid);
 }
 
 static inline void rbuf_FILENUM (struct rbuf *r, FILENUM *filenum) {
     filenum->fileid = rbuf_int(r);
 }
 static inline void rbuf_ma_FILENUM (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), FILENUM *filenum) {
-    return rbuf_FILENUM(r, filenum);
+    rbuf_FILENUM(r, filenum);
 }
 
 // Don't try to use the same space, malloc it

newbrt/recover.c

@@ -7,20 +7,7 @@
 //    cd ../src/tests/tmpdir
 //    ../../../newbrt/recover ../dir.test_log2.c.tdb
 
-#include "cachetable.h"
-#include "key.h"
-#include "brt-internal.h"
-#include "log-internal.h"
-#include "log_header.h"
-#include "toku_assert.h"
-#include "kv-pair.h"
-
-
-#include <fcntl.h>
-#include <stdlib.h>
-#include <sys/file.h>
-#include <sys/stat.h>
-#include <unistd.h>
+#include "includes.h"
 
 //#define DO_VERIFY_COUNTS
 #ifdef DO_VERIFY_COUNTS

newbrt/roll.c

@@ -3,16 +3,7 @@
 
 /* rollback and rollforward routines. */
 
-#include <stdlib.h>
-#include <inttypes.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include "log_header.h"
-#include "log-internal.h"
-#include "cachetable.h"
-#include "key.h"
-#include "bread.h"
+#include "includes.h"
 
 // these flags control whether or not we send commit messages for
 // various operations

newbrt/tests/Makefile

@@ -158,7 +158,37 @@ check_%: %
 	$(VGRIND)  ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD)
 
 benchmark-test.o: ../brt.h ../brt-search.h ../../include/db.h
+ifeq ($(CIL),1)
+
+BRT_SOURCES = \
+  block_allocator \
+  bread \
+  brt-serialize \
+  brt-verify \
+  brt \
+  cachetable \
+  fifo \
+  fingerprint \
+  key \
+  leafentry \
+  log \
+  log_code \
+  memory \
+  memarena \
+  mempool \
+  omt \
+  recover \
+  roll \
+  toku_assert \
+  ybt \
+  x1764 \
+  trace_mem \
+  threadpool \
+# keep this line so I can ha vea \ on the previous line
+$(BINS): $(patsubst %,../%.o,$(BRT_SOURCES)) $(CYG_ADD_LIBZ)
+else
 $(BINS): ../newbrt.o $(CYG_ADD_LIBZ)
+endif
 
 test-inc-split test-del-inorder: ../brt-test-helpers.o
 

newbrt/threadpool.c

-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <malloc.h>
-#include <pthread.h>
-#include <errno.h>
-
-#include "threadpool.h"
+#include "includes.h"
 
 // use gcc builtin fetch_and_add 0->no 1->yes
 #define DO_ATOMIC_FETCH_AND_ADD 0

newbrt/threadpool.h

+#ifndef THREADPOOL_H
+#define THREADPOOL_H
+
 // A threadpool is a limited set of threads that can be used to apply a 
 // function to work contained in a work queue.  The work queue is outside
 // of the scope of the threadpool; the threadpool merely provides 
@@ -44,3 +47,5 @@ void threadpool_set_thread_idle(THREADPOOL);
 // get the current number of threads
 
 int threadpool_get_current_threads(THREADPOOL);
+
+#endif

newbrt/toku_assert.c

-#include "toku_assert.h"
-#include <stdio.h>
-#include <stdlib.h>
+#include "includes.h"
 
 int toku_continue_on_assert_failure=0;
 

newbrt/x1764.c

-#include "x1764.h"
-#include "toku_assert.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
+#include "includes.h"
 
 #define PRINT 0
 

newbrt/ybt.c

-#define _FILE_OFFSET_BITS 64
-#include "ybt.h"
-#include "memory.h"
-#include <errno.h>
-#include <string.h>
+#include "includes.h"
 
 DBT *toku_init_dbt (DBT *ybt) {
     memset(ybt, 0, sizeof(*ybt));