Merge in the double buffered io abstraction. It's not in the loader yet. Refs #2571. [t:2571]

{{{
svn merge -r 20089:20155 https://svn.tokutek.com/tokudb/toku/tokudb.2571
}}}
.


git-svn-id: file:///svn/toku/tokudb@20156 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/Makefile

@@ -52,6 +52,7 @@ BRT_SOURCES = \
   brt-test-helpers \
   cachetable \
   checkpoint \
+  dbufio \
   fifo \
   fingerprint \
   key \

newbrt/brtloader-callback.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
-#ident "$Id: brtloader.c 19913 2010-05-07 01:39:12Z prohaska $"
+#ident "$Id$"
 #ident "Copyright (c) 2007-2010 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 

newbrt/dbufio.h

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ifndef TOKU_DBUFIO_H
+#define TOKU_DBUFIO_H
+#ident "$Id: queue.c 20104 2010-05-12 17:22:40Z bkuszmaul $"
+#ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
+
+#include <toku_portability.h>
+#include <toku_pthread.h>
+
+#if defined(__cplusplus) || defined(__cilkplusplus)
+extern "C" {
+#endif
+
+/* Maintain a set of files for reading, with double buffering for the reads. */
+
+/* A DBUFIO_FILESET is a set of files.  The files are indexed from 0 to N-1, where N is specified when the set is created (and the files are also provided when the set is creaed). */
+/* An implementation would typically use a separate thread or asynchronous I/O to fetch ahead data for each file.  The system will typically fill two buffers of size M for each file.  One buffer is being read out of using dbuf_read(), and the other buffer is either empty (waiting on the asynchronous I/O to start), being filled in by the asynchronous I/O mechanism, or is waiting for the caller to read data from it. */
+typedef struct dbufio_fileset *DBUFIO_FILESET; 
+
+int create_dbufio_fileset (DBUFIO_FILESET *bfsp, int N, int fds[/*N*/], size_t bufsize);
+
+int destroy_dbufio_fileset(DBUFIO_FILESET);
+
+int dbufio_fileset_read (DBUFIO_FILESET bfs, int filenum, void *buf_v, size_t count, size_t *n_read);
+
+#if defined(__cplusplus) || defined(__cilkplusplus)
+};
+#endif
+
+
+#endif

newbrt/queue.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "$Id$"
+#ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
+#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
+
 #include <toku_portability.h>
 #include "toku_os.h"
 #include <errno.h>

newbrt/tests/dbufio-test.c

+#include "dbufio.h"
+#include <stdio.h>
+#include <fcntl.h>
+#include <toku_assert.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+enum { N  = 5 };
+enum { M = 10 };
+static void test1 (size_t chars_per_file, size_t bytes_per_read) {
+    int fds[N];
+    char fnames[N][100];
+    size_t n_read[N];
+    int still_live[N];
+    int n_live=N;
+    for (int i=0; i<N; i++) {
+	snprintf(fnames[i], 100, "dbufio-test-file%d.data", i);
+	unlink(fnames[i]);
+	fds[i] = open(fnames[i], O_CREAT|O_RDWR, S_IRWXU);
+	//printf("fds[%d]=%d is %s\n", i, fds[i], fnames[i]);
+	assert(fds[i]>=0);
+	n_read[i]=0;
+	still_live[i]=i;
+	for (size_t j=0; j<chars_per_file; j++) {
+	    unsigned char c = (i+j)%256;
+	    int r = toku_os_write(fds[i], &c, 1);
+	    if (r!=0) printf("fds[%d]=%d r=%d errno=%d (%s)\n", i, fds[i], r, errno, strerror(errno));
+	    assert(r==0);
+	}
+	{
+	    int r = lseek(fds[i], 0, SEEK_SET);
+	    assert(r==0);
+	}
+	
+    }
+    DBUFIO_FILESET bfs;
+    {
+	int r = create_dbufio_fileset(&bfs, N, fds, M);
+	assert(r==0);
+    }
+    while (n_live>0) {
+	int indirectnum = random()%n_live;
+	int filenum = still_live[indirectnum];
+	char buf[bytes_per_read];
+	size_t n_read_here=0;
+	int r = dbufio_fileset_read(bfs, filenum, buf, bytes_per_read, &n_read_here);
+	//printf("read(%d) -> %d (%ld) (old n_read=%ld)\n", filenum, r, n_read_here, n_read[filenum]);
+	if (r==0) {
+	    // did read something
+	    assert(n_read_here==bytes_per_read);
+	    n_read[filenum]+=n_read_here;
+	    //printf(" new n_read=%ld\n", n_read[filenum]);
+	    assert(n_read[filenum]<=chars_per_file);
+	} else {
+	    assert(r==EOF);
+	    assert(n_read[filenum]==chars_per_file);
+	    still_live[indirectnum] = still_live[n_live-1];
+	    n_live--;
+	}
+    }
+    {
+	int r = destroy_dbufio_fileset(bfs);
+	assert(r==0);
+    }
+    for (int i=0; i<N; i++) {
+	{
+	    int r = unlink(fnames[i]);
+	    assert(r==0);
+	}
+	{
+	    int r = close(fds[i]);
+	    assert(r==0);
+	}
+	assert(n_read[i]==chars_per_file);
+    }
+}
+
+				  
+
+int main (int argc __attribute__((__unused__)), char *argv[]__attribute__((__unused__))) {
+//    test1(0, 1);
+//    test1(1, 1);
+//    test1(15, 1);
+//    test1(100, 1);
+    test1(30, 3); // 3 and M are relatively prime.  But 3 divides the file size.
+    return 0;
+}