#2840 refs[t:2840] add debug prints to the loader to help fix the dbufio read...

#2840 refs[t:2840] add debug prints to the loader to help fix the dbufio read row problem, also remove the assert on this error path

git-svn-id: file:///svn/toku/tokudb@22673 c7de825b-a66e-492c-adef-691d508d4ae1

linux/file.c

@@ -106,6 +106,7 @@ static FILE *  (*t_fdopen)(int, const char *) = 0;
 static FILE *  (*t_fopen)(const char *, const char *) = 0;
 static int     (*t_open)(const char *, int, int) = 0;  // no implementation of variadic form until needed
 static int     (*t_fclose)(FILE *) = 0;
+static ssize_t (*t_read)(int, void *, size_t) = 0;
 
 int 
 toku_set_func_write (ssize_t (*write_fun)(int, const void *, size_t)) {
@@ -157,7 +158,11 @@ toku_set_func_fclose(int (*fclose_fun)(FILE*)) {
     return 0;
 }
 
-
+int 
+toku_set_func_read (ssize_t (*read_fun)(int, void *, size_t)) {
+    t_read = read_fun;
+    return 0;
+}
 
 void
 toku_os_full_write (int fd, const void *buf, size_t len) {
@@ -305,6 +310,15 @@ toku_os_close (int fd) {  // if EINTR, retry until success
     return r;
 }
 
+ssize_t 
+toku_os_read(int fd, void *buf, size_t count) {
+    ssize_t r;
+    if (t_read)
+        r = t_read(fd, buf, count);
+    else
+        r = read(fd, buf, count);
+    return r;
+}
 
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // fsync logic:

newbrt/brtloader.c

@@ -1666,7 +1666,7 @@ int toku_merge_some_files_using_dbufio (const BOOL to_q, FIDX dest_data, QUEUE q
             int r = pqueue_pop(pq, &node);
             if (r!=0) {
 		result = r;
-		lazy_assert(0);
+		invariant(0);
 		break;
             }
             mini = node->i;
@@ -1718,7 +1718,9 @@ int toku_merge_some_files_using_dbufio (const BOOL to_q, FIDX dest_data, QUEUE q
 		    toku_free(keys[mini].data);  keys[mini].data = NULL;
 		    toku_free(vals[mini].data);  vals[mini].data = NULL;
 		} else {
-                    lazy_assert(0);
+                    fprintf(stderr, "%s:%d r=%d errno=%d bfs=%p mini=%d\n", __FILE__, __LINE__, r, errno, bfs, mini);
+                    dbufio_print(bfs);
+                    // lazy_assert(0);
                     result = r;
                     break;
 		}
@@ -2301,7 +2303,7 @@ static int toku_loader_write_brt_from_q (BRTLOADER bl,
     seek_align(&out);
     int64_t lblock;
     result = allocate_block(&out, &lblock);
-    lazy_assert(result == 0); // can not fail since translations reserved above
+    invariant(result == 0); // can not fail since translations reserved above
     struct leaf_buf *lbuf = start_leaf(&out, descriptor, lblock);
     u_int64_t n_rows_remaining = bl->n_rows;
     u_int64_t old_n_rows_remaining = bl->n_rows;

newbrt/dbufio.c

@@ -29,7 +29,6 @@ struct dbufio_file {
     int    error_code[2]; // includes errno or eof. [0] is the error code associated with buf[0], [1] is the code for buf[1]
 
     BOOL io_done;
-
 };
 
 
@@ -128,11 +127,13 @@ static void* io_thread (void *v)
 	    }
 	    //printf("%s:%d Doing read fd=%d\n", __FILE__, __LINE__, dbf->fd);
 	    {
-		ssize_t readcode = read(dbf->fd, dbf->buf[1], bfs->bufsize);
+		ssize_t readcode = toku_os_read(dbf->fd, dbf->buf[1], bfs->bufsize);
 		//printf("%s:%d readcode=%ld\n", __FILE__, __LINE__, readcode);
 		if (readcode==-1) {
 		    // a real error.  Save the real error.
-		    dbf->error_code[1] = errno;
+                    int the_errno = errno;
+                    fprintf(stderr, "%s:%d dbf=%p fd=%d errno=%d\n", __FILE__, __LINE__, dbf, dbf->fd, the_errno);
+		    dbf->error_code[1] = the_errno;
 		    dbf->n_in_buf[1] = 0;
 		} else if (readcode==0) {
 		    // End of file.  Save it.
@@ -219,7 +220,7 @@ int create_dbufio_fileset (DBUFIO_FILESET *bfsp, int N, int fds[/*N*/], size_t b
 	    }
 	    bfs->files[i].io_done = FALSE;
 	    {
-		ssize_t r = read(bfs->files[i].fd, bfs->files[i].buf[0], bufsize);
+		ssize_t r = toku_os_read(bfs->files[i].fd, bfs->files[i].buf[0], bufsize);
 		if (r<0) {
 		    result=errno;
 		    break;
@@ -371,3 +372,18 @@ int dbufio_fileset_read (DBUFIO_FILESET bfs, int filenum, void *buf_v, size_t co
 	assert(0); // cannot get here.
     }
 }
+
+void
+dbufio_print(DBUFIO_FILESET bfs) {
+    fprintf(stderr, "%s:%d bfs=%p", __FILE__, __LINE__, bfs);
+    if (bfs->panic)
+        fprintf(stderr, " panic=%d", bfs->panic_errno);
+    fprintf(stderr, " N=%d %d %"PRIuMAX, bfs->N, bfs->n_not_done, bfs->bufsize);
+    for (int i = 0; i < bfs->N; i++) {
+        struct dbufio_file *dbf = &bfs->files[i];
+        if (dbf->error_code[0] || dbf->error_code[1])
+            fprintf(stderr, " %d=[%d,%d]", i, dbf->error_code[0], dbf->error_code[1]);
+    }
+    fprintf(stderr, "\n");
+                
+}

newbrt/dbufio.h

@@ -24,6 +24,8 @@ int dbufio_fileset_read (DBUFIO_FILESET bfs, int filenum, void *buf_v, size_t co
 
 int panic_dbufio_fileset(DBUFIO_FILESET, int error);
 
+void dbufio_print(DBUFIO_FILESET);
+
 C_END
 
 #endif

newbrt/tests/brtloader-test-merge-files-dbufio.c

@@ -140,7 +140,21 @@ bad_fclose(FILE * stream) {
     return rval;
 }
 
+int bad_read_errno = 0;
 
+static ssize_t
+bad_read(int fd, void *buf, size_t count) {
+    ssize_t rval;
+    event_count++;
+    if (event_count_trigger == event_count) {
+        event_hit();
+        if (verbose) printf("%s %d\n", __FUNCTION__, event_count);
+        errno = bad_read_errno;
+        rval = -1;
+    } else
+        rval = read(fd, buf, count);
+    return rval;
+}
 
 static int my_malloc_event = 1;
 static int my_malloc_count = 0, my_big_malloc_count = 0;
@@ -366,7 +380,7 @@ static void test (const char *directory, BOOL is_error) {
     toku_set_func_fopen(bad_fopen);
     toku_set_func_open(bad_open);
     toku_set_func_fclose(bad_fclose);
-
+    if (bad_read_errno) toku_set_func_read(bad_read);
 
     int result = 0;
     {
@@ -392,6 +406,7 @@ static void test (const char *directory, BOOL is_error) {
     toku_set_func_fopen(NULL);
     toku_set_func_open(NULL);
     toku_set_func_fclose(NULL);
+    toku_set_func_read(NULL);
     do_assert_hook = my_assert_hook;
 
     {
@@ -445,6 +460,7 @@ static int usage(const char *progname, int n) {
     fprintf(stderr, "[-s] set the small loader size factor\n");
     fprintf(stderr, "[-m] inject big malloc failures\n");
     fprintf(stderr, "[-tend NEVENTS] stop testing after N events\n");
+    fprintf(stderr, "[-bad_read_errno ERRNO]\n");
     return 1;
 }
 
@@ -467,12 +483,15 @@ int test_main (int argc, const char *argv[]) {
             toku_brtloader_set_size_factor(1);
         } else if (strcmp(argv[0],"-m") == 0) {
             my_malloc_event = 1;
-	} else if (strcmp(argv[0],"-tend") == 0) {
+	} else if (strcmp(argv[0],"-tend") == 0 && argc > 1) {
             argc--; argv++;
 	    tend = atoi(argv[0]);
-	} else if (strcmp(argv[0],"-tstart") == 0) {
+	} else if (strcmp(argv[0],"-tstart") == 0 && argc > 1) {
             argc--; argv++;
 	    tstart = atoi(argv[0]);
+        } else if (strcmp(argv[0], "-bad_read_errno") == 0 && argc > 1) {
+            argc--; argv++;
+            bad_read_errno = atoi(argv[0]);
 	} else if (argc!=1) {
             return usage(progname, N_RECORDS);
 	}
@@ -514,7 +533,7 @@ int test_main (int argc, const char *argv[]) {
         event_count_trigger = i;
         r = system(unlink_all); CKERR(r);
         r = toku_os_mkdir(directory, 0755); CKERR(r);
-
+        if (verbose) printf("event=%d\n", i);
 	test(directory, TRUE);
     }
     }

toku_include/toku_portability.h

@@ -170,7 +170,7 @@ FILE * toku_os_fopen(const char *filename, const char *mode);
 int toku_os_open(const char *path, int oflag, int mode);
 int toku_os_close(int fd);
 int toku_os_fclose(FILE * stream);
-
+ssize_t toku_os_read(int fd, void *buf, size_t count);
 
 // wrapper around fsync
 int toku_file_fsync_without_accounting(int fd);
@@ -198,7 +198,7 @@ int toku_set_func_fdopen (FILE * (*)(int, const char *));
 int toku_set_func_fopen (FILE * (*)(const char *, const char *));
 int toku_set_func_open (int (*)(const char *, int, int));  // variadic form not implemented until needed
 int toku_set_func_fclose(int (*)(FILE*));
-
+int toku_set_func_read(ssize_t (*)(int, void *, size_t));
 
 int toku_portability_init    (void);
 int toku_portability_destroy (void);