Addresses #583

We were reading error codes of malloc incorrectly.
errno is not valid unless you know you had an error.

git-svn-id: file:///svn/tokudb@3138 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/ybt.c

@@ -17,26 +17,34 @@ DBT *toku_fill_dbt(DBT *dbt, bytevec k, ITEMLEN len) {
 }
 
 int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp) {
+    int r = ENOSYS;
     if (ybt->flags==DB_DBT_MALLOC) {
     domalloc:
 	ybt->data = toku_malloc(vallen);
-	if (errno!=0) return errno;
+	if (!ybt->data && vallen > 0) { r = errno; goto cleanup; }
     } else if (ybt->flags==DB_DBT_REALLOC) {
 	if (ybt->data==0) goto domalloc;
-	ybt->data = toku_realloc(ybt->data, vallen);
-	if (errno!=0) return errno;
+	/* tmp is used to prevent a memory leak if realloc fails */
+        void* tmp = toku_realloc(ybt->data, vallen);
+	if (!tmp && vallen > 0) { r = errno; goto cleanup; }
+        ybt->data = tmp;
     } else if (ybt->flags==DB_DBT_USERMEM) {
         ybt->size = vallen;
-        if (ybt->ulen < vallen) return DB_BUFFER_SMALL;
+        if (ybt->ulen < vallen) { r = DB_BUFFER_SMALL; goto cleanup; }
     } else {
 	if (staticptrp==0) return -1;
 	void *staticptr=*staticptrp;
 	//void *old=staticptr;
-	if (staticptr==0) 
+	if (staticptr==0) { 
 	    staticptr = toku_malloc(vallen);
-	else
-	    staticptr = toku_realloc(staticptr, vallen);
-	if (errno!=0) return errno;
+            if (!staticptr && vallen > 0) { r = errno; goto cleanup; }
+        }
+	else {
+	    /* tmp is used to prevent a memory leak if realloc fails */
+            void* tmp = toku_realloc(staticptr, vallen);
+            if (!tmp && vallen > 0) { r = errno; goto cleanup; }
+            staticptr = tmp;
+        }
 	//if (old!=staticptr) printf("%s:%d MALLOC --> %p\n", __FILE__, __LINE__, staticptr);
 	*staticptrp = staticptr;
 	ybt->data = vallen > 0 ? staticptr : 0;
@@ -45,5 +53,8 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
     if (ybt->size>0) {
 	memcpy(ybt->data, val, vallen);
     }
-    return 0;
+    r = 0;
+cleanup:
+    return r;
 }
+