Commit 1f796663 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul Committed by Yoni Fogel

Check in the code for compressing the rolltmp file.

This was a tricky merge.  In the 1332a directory I did:
{{{
svn merge https://svn.tokutek.com/tokudb/toku/tokudb.1032b+1332@8415 https://svn.tokutek.com/tokudb/toku/tokudb.1032b+1332@8416
}}}
Then I was able to resolve the conflicts.

Then in the main line I did:
{{{
svn merge -r9042:9046 https://svn.tokutek.com/tokudb/toku/tokudb.1332a
}}}

Fixes #1332.


git-svn-id: file:///svn/toku/tokudb@9047 c7de825b-a66e-492c-adef-691d508d4ae1
parent d51bc82a
...@@ -3,20 +3,19 @@ ...@@ -3,20 +3,19 @@
#include "includes.h" #include "includes.h"
struct bread { struct bread {
toku_off_t current_offset; // The current offset to be read (in the file). That offset includes anything that is unread in the buffer. int64_t fileoff; // The byte before this offset is the next byte we will read (since we are reading backward)
int fd; int fd;
size_t bufsize; int bufoff; // The current offset in the buf. The next byte we will read is buf[bufoff-1] (assuming that bufoff>0).
char *buf; // A buffer of size bufsize; char *buf; // A buffer with at least bufoff bytes in it.
int bufoff; // The current offset buf.
}; };
BREAD create_bread_from_fd_initialize_at(int fd, toku_off_t filesize, size_t bufsize) { BREAD create_bread_from_fd_initialize_at(int fd) {
BREAD MALLOC(result); BREAD XMALLOC(result);
result->current_offset=filesize; int r = toku_os_get_file_size(fd, &result->fileoff);
assert(r==0);
result->fd=fd; result->fd=fd;
result->bufoff=0; result->bufoff=0;
result->bufsize=bufsize; result->buf = 0;
MALLOC_N(bufsize, result->buf);
return result; return result;
} }
...@@ -30,29 +29,48 @@ int close_bread_without_closing_fd(BREAD br) { ...@@ -30,29 +29,48 @@ int close_bread_without_closing_fd(BREAD br) {
ssize_t bread_backwards(BREAD br, void *vbuf, size_t nbytes) { ssize_t bread_backwards(BREAD br, void *vbuf, size_t nbytes) {
char *buf=vbuf; char *buf=vbuf;
ssize_t result=0; ssize_t result=0;
const int i4 = sizeof(u_int32_t);
while (nbytes > 0) { while (nbytes > 0) {
assert(br->current_offset >= (toku_off_t)nbytes);
// read whatever we can out of the buffer. // read whatever we can out of the buffer.
{ if (br->bufoff>0) {
size_t to_copy = ((size_t)br->bufoff >= nbytes) ? nbytes : (size_t)br->bufoff; size_t to_copy = ((size_t)br->bufoff >= nbytes) ? nbytes : (size_t)br->bufoff;
memcpy(buf+nbytes-to_copy, &br->buf[br->bufoff-to_copy], to_copy); memcpy(buf+nbytes-to_copy, &br->buf[br->bufoff-to_copy], to_copy);
nbytes -= to_copy; nbytes -= to_copy;
br->current_offset -= to_copy;
result += to_copy; result += to_copy;
br->bufoff -= to_copy; br->bufoff -= to_copy;
} }
if (nbytes>0) { if (nbytes>0) {
assert(br->bufoff==0); assert(br->bufoff==0);
size_t to_read = ((u_int64_t)br->current_offset >= (u_int64_t)br->bufsize) ? br->bufsize : (size_t)br->current_offset; u_int32_t compressed_length_n, uncompressed_length_n;
assert(to_read>0 && to_read<=br->bufsize); assert(br->fileoff>=i4); // there better be the three lengths plus the compressed data.
ssize_t r = pread(br->fd, br->buf, to_read, br->current_offset-to_read); { ssize_t r = pread(br->fd, &compressed_length_n, i4, br->fileoff- i4); assert(r==i4); }
assert(r==(ssize_t)to_read); u_int32_t compressed_length = ntohl(compressed_length_n);
br->bufoff = to_read; assert(br->fileoff >= compressed_length + 3*i4);
{ ssize_t r = pread(br->fd, &uncompressed_length_n, i4, br->fileoff-2*i4); assert(r==i4); }
u_int32_t uncompressed_length = ntohl(uncompressed_length_n);
char *XMALLOC_N(compressed_length, zbuf);
{
ssize_t r = pread(br->fd, zbuf, compressed_length, br->fileoff- compressed_length -2*i4);
assert(r==(ssize_t)compressed_length);
}
{
u_int32_t compressed_length_n_again;
ssize_t r = pread(br->fd, &compressed_length_n_again, i4, br->fileoff-compressed_length-3*i4); assert(r==i4);
assert(compressed_length_n_again == compressed_length_n);
}
uLongf destlen = uncompressed_length;
XREALLOC_N(uncompressed_length, br->buf);
uncompress((Bytef*)br->buf, &destlen, (Bytef*)zbuf, compressed_length);
assert(destlen==uncompressed_length);
toku_free(zbuf);
br->bufoff = uncompressed_length;
br->fileoff -= (compressed_length + 3*i4);
} }
} }
return result; return result;
} }
int bread_has_more(BREAD br) { int bread_has_more(BREAD br) {
return br->current_offset>0; return (br->fileoff>0) || (br->bufoff>0);
} }
...@@ -4,14 +4,16 @@ ...@@ -4,14 +4,16 @@
// A BREAD reads a file backwards using buffered I/O. BREAD stands for Buffered Read or Backwards Read. // A BREAD reads a file backwards using buffered I/O. BREAD stands for Buffered Read or Backwards Read.
// Conceivably, we could read forward too. // Conceivably, we could read forward too.
// The buffered I/O is buffered using a large buffer (e.g., something like a megabyte). // The buffered I/O is buffered using a large buffer (e.g., something like a megabyte).
// If not for the large-buffer requirement, we could have used a FILE. // Furthermore, data is compressed into blocks. Each block is a 4-byte compressed length (in network order), followed by compressed data, followed by a 4-byte uncompressed-length (in network order), followed by a 4-byte compressed length
// The compressed-length appears twice so that the file can be read backward or forward.
// If not for the large-buffer requirement, as well as compression, as well as reading backward, we could have used a FILE.
#include <sys/types.h> #include <sys/types.h>
typedef struct bread *BREAD; typedef struct bread *BREAD;
BREAD create_bread_from_fd_initialize_at(int fd, toku_off_t filesize, size_t bufsize); BREAD create_bread_from_fd_initialize_at(int fd);
// Effect: Given a file descriptor, fd, that points at a file of size filesize, create a BREAD. // Effect: Given a file descriptor, fd, create a BREAD.
// Requires: The filesize must be correct. The fd must be an open fd. // Requires: The fd must be an open fd.
int close_bread_without_closing_fd(BREAD); int close_bread_without_closing_fd(BREAD);
// Effect: Close the BREAD, but don't close the underlying fd. // Effect: Close the BREAD, but don't close the underlying fd.
......
...@@ -100,7 +100,7 @@ struct tokutxn { ...@@ -100,7 +100,7 @@ struct tokutxn {
size_t rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory. size_t rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
char *rollentry_filename; char *rollentry_filename;
int rollentry_fd; // If we spill the roll_entries, we write them into this fd. int rollentry_fd; // If we spill the roll_entries, we write them into this fd.
toku_off_t rollentry_filesize; // How many bytes are in the rollentry file (compressed) toku_off_t rollentry_filesize; // How many bytes are in the rollentry file (this is the uncompressed bytes. If the file is compressed it may actually be smaller (or even larger with header information))
u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children. u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children.
OMT open_brts; // a collection of the brts that we touched. Indexed by filenum. OMT open_brts; // a collection of the brts that we touched. Indexed by filenum.
}; };
......
...@@ -501,7 +501,7 @@ int toku_logger_commit (TOKUTXN txn, int nosync, void(*yield)(void*yieldv), void ...@@ -501,7 +501,7 @@ int toku_logger_commit (TOKUTXN txn, int nosync, void(*yield)(void*yieldv), void
// Read stuff out of the file and execute it. // Read stuff out of the file and execute it.
if (txn->rollentry_filename) { if (txn->rollentry_filename) {
r = toku_commit_fileentries(txn->rollentry_fd, txn->rollentry_filesize, txn, yield, yieldv); r = toku_commit_fileentries(txn->rollentry_fd, txn, yield, yieldv);
} }
} }
} }
...@@ -872,7 +872,7 @@ int toku_logger_abort(TOKUTXN txn, void (*yield)(void*), void*yieldv) { ...@@ -872,7 +872,7 @@ int toku_logger_abort(TOKUTXN txn, void (*yield)(void*), void*yieldv) {
list_remove(&txn->live_txns_link); list_remove(&txn->live_txns_link);
// Read stuff out of the file and roll it back. // Read stuff out of the file and roll it back.
if (txn->rollentry_filename) { if (txn->rollentry_filename) {
int r = toku_rollback_fileentries(txn->rollentry_fd, txn->rollentry_filesize, txn, yield, yieldv); int r = toku_rollback_fileentries(txn->rollentry_fd, txn, yield, yieldv);
assert(r==0); assert(r==0);
} }
return 0; return 0;
...@@ -1027,9 +1027,32 @@ int toku_maybe_spill_rollbacks (TOKUTXN txn) { ...@@ -1027,9 +1027,32 @@ int toku_maybe_spill_rollbacks (TOKUTXN txn) {
txn->rollentry_fd = open(txn->rollentry_filename, O_CREAT+O_RDWR+O_EXCL+O_BINARY, 0600); txn->rollentry_fd = open(txn->rollentry_filename, O_CREAT+O_RDWR+O_EXCL+O_BINARY, 0600);
if (txn->rollentry_fd==-1) return errno; if (txn->rollentry_fd==-1) return errno;
} }
ssize_t r = write_it(txn->rollentry_fd, buf, w.ndone); uLongf compressed_len = compressBound(w.ndone);
if (r<0) return r; char *MALLOC_N(compressed_len, compressed_buf);
assert(r==(ssize_t)w.ndone); {
int r = compress2((Bytef*)compressed_buf, &compressed_len,
(Bytef*)buf, w.ndone,
1);
assert(r==Z_OK);
}
{
u_int32_t v = htonl(compressed_len);
ssize_t r = write_it(txn->rollentry_fd, &v, sizeof(v)); assert(r==sizeof(v));
}
{
ssize_t r = write_it(txn->rollentry_fd, compressed_buf, compressed_len);
if (r<0) return r;
assert(r==(ssize_t)compressed_len);
}
{
u_int32_t v = htonl(w.ndone);
ssize_t r = write_it(txn->rollentry_fd, &v, sizeof(v)); assert(r==sizeof(v));
}
{
u_int32_t v = htonl(compressed_len);
ssize_t r = write_it(txn->rollentry_fd, &v, sizeof(v)); assert(r==sizeof(v));
}
toku_free(compressed_buf);
txn->rollentry_filesize+=w.ndone; txn->rollentry_filesize+=w.ndone;
toku_free(buf); toku_free(buf);
......
...@@ -179,12 +179,10 @@ int toku_maybe_spill_rollbacks (TOKUTXN txn); ...@@ -179,12 +179,10 @@ int toku_maybe_spill_rollbacks (TOKUTXN txn);
struct roll_entry; struct roll_entry;
int toku_rollback_fileentries (int fd, int toku_rollback_fileentries (int fd,
toku_off_t filesize,
TOKUTXN txn, TOKUTXN txn,
void (*yield)(void*yieldv), void (*yield)(void*yieldv),
void * yieldv); void * yieldv);
int toku_commit_fileentries (int fd, int toku_commit_fileentries (int fd,
toku_off_t filesize,
TOKUTXN txn, TOKUTXN txn,
void (*yield)(void*yieldv), void (*yield)(void*yieldv),
void * yieldv); void * yieldv);
......
...@@ -203,12 +203,11 @@ toku_rollback_cmddelete (TXNID xid, ...@@ -203,12 +203,11 @@ toku_rollback_cmddelete (TXNID xid,
int int
toku_commit_fileentries (int fd, toku_commit_fileentries (int fd,
toku_off_t filesize,
TOKUTXN txn, TOKUTXN txn,
YIELDF yield, YIELDF yield,
void * yieldv) void * yieldv)
{ {
BREAD f = create_bread_from_fd_initialize_at(fd, filesize, 1<<20); BREAD f = create_bread_from_fd_initialize_at(fd);
int r=0; int r=0;
MEMARENA ma = memarena_create(); MEMARENA ma = memarena_create();
int count=0; int count=0;
...@@ -230,12 +229,11 @@ toku_commit_fileentries (int fd, ...@@ -230,12 +229,11 @@ toku_commit_fileentries (int fd,
int int
toku_rollback_fileentries (int fd, toku_rollback_fileentries (int fd,
toku_off_t filesize,
TOKUTXN txn, TOKUTXN txn,
YIELDF yield, YIELDF yield,
void * yieldv) void * yieldv)
{ {
BREAD f = create_bread_from_fd_initialize_at(fd, filesize, 1<<20); BREAD f = create_bread_from_fd_initialize_at(fd);
assert(f); assert(f);
int r=0; int r=0;
MEMARENA ma = memarena_create(); MEMARENA ma = memarena_create();
...@@ -265,11 +263,7 @@ toku_commit_rollinclude (BYTESTRING bs, ...@@ -265,11 +263,7 @@ toku_commit_rollinclude (BYTESTRING bs,
char *fname = fixup_fname(&bs); char *fname = fixup_fname(&bs);
int fd = open(fname, O_RDONLY+O_BINARY); int fd = open(fname, O_RDONLY+O_BINARY);
assert(fd>=0); assert(fd>=0);
r = toku_commit_fileentries(fd, txn, yield, yieldv);
int64_t fsize = 0;
r = toku_os_get_file_size(fd, &fsize);
assert(r==0);
r = toku_commit_fileentries(fd, fsize, txn, yield, yieldv);
assert(r==0); assert(r==0);
r = close(fd); r = close(fd);
assert(r==0); assert(r==0);
...@@ -288,10 +282,7 @@ toku_rollback_rollinclude (BYTESTRING bs, ...@@ -288,10 +282,7 @@ toku_rollback_rollinclude (BYTESTRING bs,
char *fname = fixup_fname(&bs); char *fname = fixup_fname(&bs);
int fd = open(fname, O_RDONLY+O_BINARY); int fd = open(fname, O_RDONLY+O_BINARY);
assert(fd>=0); assert(fd>=0);
int64_t fsize = 0; r = toku_rollback_fileentries(fd, txn, yield, yieldv);
r = toku_os_get_file_size(fd, &fsize);
assert(r==0);
r = toku_rollback_fileentries(fd, fsize, txn, yield, yieldv);
assert(r==0); assert(r==0);
r = close(fd); r = close(fd);
assert(r==0); assert(r==0);
......
...@@ -10,61 +10,63 @@ ...@@ -10,61 +10,63 @@
#include <unistd.h> #include <unistd.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <zlib.h>
#include "../brttypes.h" #include "../brttypes.h"
#include "../bread.h" #include "../bread.h"
#define FNAME "bread-test.data" #define FNAME "bread-test.data"
#define RECORDS 2 #define RECORDS 20
#define RECORDLEN 100
char buf[RECORDS][RECORDLEN];
int sizes[RECORDS];
int sizesn[RECORDS];
int nwrote=0;
char wrotedata[RECORDS*RECORDLEN];
static void static void
test (int seed) { test (int seed) {
srandom(seed); srandom(seed);
unlink(FNAME); unlink(FNAME);
int i; int i;
char buf[RECORDS][100];
int sizes[RECORDS];
int sizesn[RECORDS];
toku_off_t off = 0;
{ {
int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777); int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
assert(fd>=0); assert(fd>=0);
for (i=0; i<RECORDS; i++) { for (i=0; i<RECORDS; i++) {
sizes[i] = random()%100; sizes[i] = 1+ random()%RECORDLEN;
sizesn[i] = toku_htonl(sizes[i]); sizesn[i] = toku_htonl(sizes[i]);
int j; int j;
for (j=0; j<sizes[i]; j++) { for (j=0; j<sizes[i]; j++) {
buf[i][j]=(char)random(); buf[i][j] = wrotedata[nwrote++] = (char)random();
} }
int r = write(fd, buf[i], sizes[i]); uLongf compressed_size = compressBound(sizes[i]);
assert(r==sizes[i]); Bytef compressed_buf[compressed_size];
off+=r; { int r = compress2(compressed_buf, &compressed_size, (Bytef*)(buf[i]), sizes[i], 1); assert(r==Z_OK); }
r = write(fd, &sizesn[i], 4); u_int32_t compressed_size_n = htonl(compressed_size);
assert(r==4); { int r = write(fd, &compressed_size_n, 4); assert(r==4); }
off+=4; { int r = write(fd, compressed_buf, compressed_size); assert(r==(int)compressed_size); }
{ int r = write(fd, &sizesn[i], 4); assert(r==4); } // the uncompressed size
{ int r = write(fd, &compressed_size_n, 4); assert(r==4); }
} }
{ int r=close(fd); assert(r==0); } { int r=close(fd); assert(r==0); }
} }
int fd = open(FNAME, O_RDONLY+O_BINARY); assert(fd>=0); int fd = open(FNAME, O_RDONLY+O_BINARY); assert(fd>=0);
// Now read it all backward // Now read it all backward
BREAD br = create_bread_from_fd_initialize_at(fd, off, 50); BREAD br = create_bread_from_fd_initialize_at(fd);
while (bread_has_more(br)) { while (bread_has_more(br)) {
assert(i>0); assert(nwrote>0);
i--; int to_read = 1+(random()%RECORDLEN); // read from 1 to 100 (if RECORDLEN is 100)
int sizen; if (to_read>nwrote) to_read=nwrote;
{ int r = bread_backwards(br, &sizen, 4); assert(r==4); } char rbuf[to_read];
int sizeh=toku_ntohl(sizen); int r = bread_backwards(br, rbuf, to_read);
assert(sizeh==sizes[i]); assert(r==to_read);
assert(0<=sizeh && sizeh<100); assert(memcmp(rbuf, &wrotedata[nwrote-to_read], to_read)==0);
{ nwrote-=to_read;
char rbuf[100];
int r = bread_backwards(br, rbuf,sizeh);
assert(r==sizeh);
assert(memcmp(rbuf, &buf[i][0], sizes[i])==0);
}
} }
assert(i==0); assert(nwrote==0);
{ int r=close_bread_without_closing_fd(br); assert(r==0); } { int r=close_bread_without_closing_fd(br); assert(r==0); }
{ int r=close(fd); assert(r==0); } { int r=close(fd); assert(r==0); }
unlink(FNAME); unlink(FNAME);
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <unistd.h> #include <unistd.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <zlib.h>
#include "../brttypes.h" #include "../brttypes.h"
#include "../bread.h" #include "../bread.h"
...@@ -32,6 +33,8 @@ test (u_int64_t fsize) { ...@@ -32,6 +33,8 @@ test (u_int64_t fsize) {
int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777); int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
assert(fd>=0); assert(fd>=0);
static u_int64_t buf[N_BIGINTS]; //windows cannot handle this on the stack static u_int64_t buf[N_BIGINTS]; //windows cannot handle this on the stack
static char compressed_buf[N_BIGINTS*2 + 1000]; // this is more than compressbound returns
uLongf compressed_len;
while (i*BIGINT_SIZE < fsize) { while (i*BIGINT_SIZE < fsize) {
if (verbose>0 && i % (1<<25) == 0) { if (verbose>0 && i % (1<<25) == 0) {
printf(" %s:test (%"PRIu64") forwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize); printf(" %s:test (%"PRIu64") forwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize);
...@@ -42,8 +45,31 @@ test (u_int64_t fsize) { ...@@ -42,8 +45,31 @@ test (u_int64_t fsize) {
for (j=0; j<N_BIGINTS; j++) { for (j=0; j<N_BIGINTS; j++) {
buf[j] = i++; buf[j] = i++;
} }
int r = write(fd, buf, N_BIGINTS*BIGINT_SIZE); assert(sizeof(buf) == N_BIGINTS * BIGINT_SIZE);
assert(r==N_BIGINTS*BIGINT_SIZE); {
compressed_len = sizeof(compressed_buf);
int r = compress2((Bytef*)compressed_buf, &compressed_len, (Bytef*)buf, sizeof(buf), 1);
assert(r==Z_OK);
}
{
u_int32_t v = htonl(compressed_len);
ssize_t r = write(fd, &v, sizeof(v));
assert(r==sizeof(v));
}
{
ssize_t r = write(fd, compressed_buf, compressed_len);
assert(r==(ssize_t)compressed_len);
}
{
u_int32_t v = htonl(sizeof(buf));
ssize_t r = write(fd, &v, sizeof(v));
assert(r==sizeof(v));
}
{
u_int32_t v = htonl(compressed_len);
ssize_t r = write(fd, &v, sizeof(v));
assert(r==sizeof(v));
}
} }
{ int r = close(fd); assert(r==0); } { int r = close(fd); assert(r==0); }
} }
...@@ -51,7 +77,7 @@ test (u_int64_t fsize) { ...@@ -51,7 +77,7 @@ test (u_int64_t fsize) {
// Now read it all backward // Now read it all backward
{ {
int fd = open(FNAME, O_RDONLY+O_BINARY); assert(fd>=0); int fd = open(FNAME, O_RDONLY+O_BINARY); assert(fd>=0);
BREAD br = create_bread_from_fd_initialize_at(fd, fsize, READBACK_BUFSIZE); BREAD br = create_bread_from_fd_initialize_at(fd);
while (bread_has_more(br)) { while (bread_has_more(br)) {
if (verbose>0 && (fsize/BIGINT_SIZE - i) % (1<<25) == 0) { if (verbose>0 && (fsize/BIGINT_SIZE - i) % (1<<25) == 0) {
printf(" %s:test (%"PRIu64") backwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize); printf(" %s:test (%"PRIu64") backwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment