Use the filenums slower. Fixes #722.

git-svn-id: file:///svn/tokudb@3513 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt.c

@@ -458,10 +458,9 @@ static int brtleaf_split (TOKULOGGER logger, FILENUM filenum, BRT t, BRTNODE nod
 		     assert((char*)node->u.l.buffer_mempool.base<= p && p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
 		 }));
     r = toku_gpma_split(node->u.l.buffer, B->u.l.buffer, PMA_ITEM_OVERHEAD,
-			move_between_mempools,
-			note_move_items_within,
-			note_move_items_between,
-			&ms);
+			move_between_mempools,	 &ms,
+			note_move_items_within,  &ms,
+			note_move_items_between, &ms);
     GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
 		 ({
 		     char *p=vdata;

newbrt/cachetable.c

@@ -113,7 +113,7 @@ int toku_cachefile_of_filenum (CACHETABLE t, FILENUM filenum, CACHEFILE *cf) {
 int toku_cachetable_openfd (CACHEFILE *cf, CACHETABLE t, int fd, const char *fname) {
     int r;
     CACHEFILE extant;
-    FILENUM max_filenum_in_use={0};
+    static FILENUM next_filenum_to_use={0};
     struct stat statbuf;
     struct fileid fileid;
     memset(&fileid, 0, sizeof(fileid));
@@ -122,7 +122,6 @@ int toku_cachetable_openfd (CACHEFILE *cf, CACHETABLE t, int fd, const char *fna
     fileid.st_dev = statbuf.st_dev;
     fileid.st_ino = statbuf.st_ino;
     for (extant = t->cachefiles; extant; extant=extant->next) {
-	if (max_filenum_in_use.fileid<extant->filenum.fileid) max_filenum_in_use=extant->filenum;
 	if (memcmp(&extant->fileid, &fileid, sizeof(fileid))==0) {
 	    r = close(fd);
             assert(r == 0);
@@ -131,9 +130,16 @@ int toku_cachetable_openfd (CACHEFILE *cf, CACHETABLE t, int fd, const char *fna
 	    return 0;
 	}
     }
+ try_again:
+    for (extant = t->cachefiles; extant; extant=extant->next) {
+	if (next_filenum_to_use.fileid==extant->filenum.fileid) {
+	    next_filenum_to_use.fileid++;
+	    goto try_again;
+	}
+    }
     {
 	CACHEFILE MALLOC(newcf);
-	newcf->filenum.fileid = 1+max_filenum_in_use.fileid;
+	newcf->filenum.fileid = next_filenum_to_use.fileid++;
 	newcf->next = t->cachefiles;
 	newcf->refcount = 1;
 	newcf->fd = fd;

newbrt/gpma.c

@@ -623,9 +623,11 @@ int toku_gpma_lookup_bessel(GPMA pma, gpma_besselfun_t besf, int direction, void
 // If an error code is returned, then the pmas are likely to be all messed up.  Probably all you can do is close them.
 int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
 		     int (*realloc_data)(u_int32_t olen, void *odata, void **ndata, void *extra),
+		     void *extra_realloc,
 		     gpma_renumber_callback_t rcall,
+		     void *extra_rcall,
 		     gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved
-		     void *extra) {
+		     void *extra_rcall_across) {
     unsigned long totalweight=0;
     u_int32_t old_N = pma->N;
     {
@@ -686,7 +688,7 @@ int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
     for (i=0; i<n_right; i++) {
 	void *ndata;
 	//printf("%s:%d len=%d\n", __FILE__, __LINE__, rightitems[i].len);
-	r = realloc_data (rightitems[i].len, rightitems[i].data, &ndata, extra);
+	r = realloc_data (rightitems[i].len, rightitems[i].data, &ndata, extra_realloc);
 	if (r!=0) { goto L6; } // At this point the PMA is all messed up, and there is no easy way to put it all back together again.
 	rightitems[i].data=ndata;
     }
@@ -709,10 +711,11 @@ int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
     //toku_verify_gpma(pma);
     //toku_verify_gpma(newpma);
 
-    r = rcall_across_pmas(n_right, rightfroms, righttos, rightitems, 0, newpma->N, extra);
+    r = rcall_across_pmas(n_right, rightfroms, righttos, rightitems, old_N, newpma->N, extra_rcall_across);
     if (r!=0) { goto L6; }
-    r = rcall(n_left, leftfroms, lefttos, leftitems, old_N, pma->N, extra);
+    r = rcall(n_left, leftfroms, lefttos, leftitems, old_N, pma->N, extra_rcall);
     if (r!=0) { goto L6; }
+
     r=0;
     goto L6; // free all that stuff
 }

newbrt/gpma.h

@@ -112,9 +112,11 @@ int toku_gpma_move_inside_pma_by_renumbering (GPMA,
 
 int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
 		     int (*realloc_data)(u_int32_t len, void *odata, void **ndata, void *extra),
+		     void *extra_realloc,
 		     gpma_renumber_callback_t rcall,
-		     gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved.  It is called first (before the rcall)
-		     void *extra);
+		     void *extra_rcall,
+		     gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved.  It is called first (before the rcall).  The old_N is the size of pma before resizing.
+		     void *extra_rcall_across);
 
 void toku_verify_gpma (GPMA pma);
 

newbrt/recover.c

@@ -681,7 +681,7 @@ int move_indices (GPMA from, struct mempool *from_mempool,
     return 0;
 }
 
-void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto, u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N) {
+void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto, u_int32_t old_N, u_int32_t new_N) {
     struct cf_pair *pair = NULL;
     int r = find_cachefile(filenum, &pair);
     assert(r==0);

newbrt/test-gpma-blackbox.c

@@ -147,6 +147,33 @@ static void test_insert_A (void) {
     assert(count_frees==3);
 }
 
+struct rcall1_struct {
+    u_int32_t expect_n_right;
+    u_int32_t *expect_froms_right, *expect_tos_right;
+    u_int32_t  did_n_right;
+};
+
+int rcall1 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos,  struct gitem *items, u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N __attribute__((__unused__)), void *extra) {
+    //assert(old_N==expect_old_N);
+    struct rcall1_struct *s = extra;
+    //printf("old_N=%d new_N=%d\n", old_N, new_N);
+    u_int32_t j;
+    assert(nitems==s->expect_n_right);
+    //printf("outer moved:"); for (j=0; j<nitems; j++) printf(" %d->%d", froms[j], tos[j]); printf("\n");
+    for (j=0; j<nitems; j++) {
+	if (s->expect_froms_right) assert(s->expect_froms_right[j]==froms[j]);
+	if (s->expect_tos_right)   assert(s->expect_tos_right  [j]==tos[j]);
+	assert(items[j].len==1+strlen(items[j].data));
+	if (j>0) {
+	    assert(froms[j-1]<froms[j] && tos[j-1]<tos[j]);
+	    assert(strcmp(items[j-1].data, items[j].data)<0);
+	}
+    }
+    s->did_n_right = nitems;
+    return 0;
+}
+
+
 void test_split_internal (const char *strings[],
 			  int expect_n_left,
 			  u_int32_t *expect_froms_left,
@@ -194,7 +221,7 @@ void test_split_internal (const char *strings[],
 	    *ndata = data; // Don't have to do anything
 	    return 0;
 	}
-	int did_n_left=-1, did_n_right=-1;
+	int did_n_left=-1;
 	int rcall0 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos,  struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra) {
 	    //printf("%s:%d old_N=%d new_N=%d\n", __FILE__, __LINE__, old_N, new_N);
 	    assert(old_N==current_estimate_of_N);
@@ -217,26 +244,10 @@ void test_split_internal (const char *strings[],
 	    }
 	    return 0;
 	}
-	int rcall1 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos,  struct gitem *items, u_int32_t old_N, u_int32_t new_N __attribute__((__unused__)), void *extra) {
-	    assert(old_N==0);
-	    //printf("new_N=%d\n", new_N);
-	    assert(extra==0);
-	    u_int32_t j;
-	    if (expect_n_right>=0) assert(nitems==(u_int32_t)expect_n_right);
-	    //printf("outer moved:"); for (j=0; j<nitems; j++) printf(" %d->%d", froms[j], tos[j]); printf("\n");
-	    for (j=0; j<nitems; j++) {
-		if (expect_froms_right) assert(expect_froms_right[j]==froms[j]);
-		if (expect_tos_right)   assert(expect_tos_right  [j]==tos[j]);
-		assert(items[j].len==1+strlen(items[j].data));
-		if (j>0) {
-		    assert(froms[j-1]<froms[j] && tos[j-1]<tos[j]);
-		    assert(strcmp(items[j-1].data, items[j].data)<0);
-		}
-	    }
-	    did_n_right = nitems;
-	    return 0;
-	}
-	r = toku_gpma_split(pma1, pma2, 1, do_realloc, rcall0, rcall1, 0);
+	//u_int32_t expect_old_N = toku_gpma_index_limit(pma1);
+
+	struct rcall1_struct r1s = {expect_n_right, expect_froms_right, expect_tos_right, -1};
+	r = toku_gpma_split(pma1, pma2, 1, do_realloc, 0, rcall0, 0, rcall1, &r1s);
 	toku_verify_gpma(pma1);
 	toku_verify_gpma(pma2);
 	assert (r==0);