Skip to content

M
MariaDB
Commit 4afad29c authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

Cleaned up code, 

made integer ranges be half open (at max),


git-svn-id: file:///svn/tokudb@337 c7de825b-a66e-492c-adef-691d508d4ae1
parent c17fdae9
Hide whitespace changes
Inline Side-by-side
Showing with 208 additions and 324 deletions
utils/Makefile
View file @ 4afad29c
...@@ -3,8 +3,8 @@ LFLAGS = -l CPPFLAGS = -I../include -I../newbrt ...@@ -3,8 +3,8 @@ LFLAGS = -l CPPFLAGS = -I../include -I../newbrt
#cc $(CPPFLAGS) $(DBBINS) -shared -o libdb.so $(CFLAGS) #cc $(CPPFLAGS) $(DBBINS) -shared -o libdb.so $(CFLAGS)
BDB_DUMP=/usr/local/Berkeletokudb.4.1/bin/db_dump BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
BDB_LOAD=/usr/local/Berkeletokudb.4.1/bin/db_load BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load
UTILS= \ UTILS= \
tokudb_gen \ tokudb_gen \
...@@ -22,21 +22,24 @@ test_gen: test_gen_hex ...@@ -22,21 +22,24 @@ test_gen: test_gen_hex
# SHELL=/bin/bash # SHELL=/bin/bash
BDB_LOAD=/usr/local/Berkeletokudb.4.1/bin/db_load BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load
BDB_DUMP=/usr/local/Berkeletokudb.4.1/bin/db_dump BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
TEST_GEN_HEX_FLAGS=-n 10000 -m 0 -M 1024 -r 5 TEST_GEN_HEX_NUMKEYS=10000
TEST_GEN_HEX_LENGTHMIN=0
TEST_GEN_HEX_LENGTHLIMIT=1024
TEST_GEN_HEX_FLAGS=-n $(TEST_GEN_HEX_NUMKEYS) -m $(TEST_GEN_HEX_LENGTHMIN) -M $(TEST_GEN_HEX_LENGTHLIMIT) -r 5
test_gen_hex: test_gen_hex:
#Generating 10,000 keys. 0 to 1024 bytes (not including identifier overhead) #Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text" echo "Generating text input > db > text"
rm -f $@.db.temp rm -f $@.*.temp
./tokudb_gen $(TEST_GEN_HEX_FLAGS) | $(BDB_LOAD) $@.db.temp ./tokudb_gen $(TEST_GEN_HEX_FLAGS) | $(BDB_LOAD) $@.db.temp
$(BDB_DUMP) $@.db.temp > $@.load_dump.temp $(BDB_DUMP) $@.db.temp > $@.load_dump.temp
./tokudb_gen -Hf > $@.gen_sorted.temp ./tokudb_gen -Hf > $@.gen_sorted.temp
./tokudb_gen -hf $(TEST_GEN_HEX_FLAGS) -d g -s h | tr "h" "\n" | sort -t g -k 1,1 | tr -d "\n" | tr "g" "\n" >> $@.gen_sorted.temp ./tokudb_gen -hf $(TEST_GEN_HEX_FLAGS) -d "\t" -s "\n" | sort -k 1,1 | tr -d "\n" | tr "\t" "\n" >> $@.gen_sorted.temp
./tokudb_gen -Fh >> $@.gen_sorted.temp ./tokudb_gen -Fh >> $@.gen_sorted.temp
if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo Files different!; exit 1; fi if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi
rm $@.*.temp rm $@.*.temp
......
utils/tokudb_gen.c
View file @ 4afad29c
...@@ -20,30 +20,31 @@ typedef uint8_t bool; ...@@ -20,30 +20,31 @@ typedef uint8_t bool;
int usage(void); int usage(void);
void generate_keys(void); void generate_keys(void);
int get_delimiter(char* str);
char dbt_delimiter = '\n'; char dbt_delimiter = '\n';
char* sort_delimiter = ""; char sort_delimiter[2];
char* progname; char* progname;
bool plaintext = false; bool plaintext = false;
long minsize = -1; long lengthmin = -1;
long maxsize = -1; long lengthlimit = -1;
int64_t maxnumkeys = -1; int64_t numkeys = -1;
long maxkibibytes = -1; bool header = true;
bool header = true; bool footer = true;
bool footer = true; bool justheader = false;
bool justheader = false; bool justfooter = false;
bool justfooter = false; bool outputkeys = true;
bool outputkeys = true; unsigned long seed = 1;
unsigned long seed = 1; bool printableonly = false;
bool printableonly = false; bool leadingspace = true;
bool leadingspace = true;
int main (int argc, char *argv[]) { int main (int argc, char *argv[]) {
char ch; char ch;
progname = argv[0]; progname = argv[0];
strcpy(sort_delimiter, "");
while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:N:?o:")) != EOF) { while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:o:")) != EOF) {
switch (ch) { switch (ch) {
case ('P'): { case ('P'): {
printableonly = true; printableonly = true;
...@@ -66,15 +67,15 @@ int main (int argc, char *argv[]) { ...@@ -66,15 +67,15 @@ int main (int argc, char *argv[]) {
break; break;
} }
case ('T'): { case ('T'): {
plaintext = true; plaintext = true;
leadingspace = false; leadingspace = false;
header = false; header = false;
footer = false; footer = false;
break; break;
} }
case ('p'): { case ('p'): {
plaintext = true; plaintext = true;
leadingspace = true; leadingspace = true;
break; break;
} }
case ('o'): { case ('o'): {
...@@ -87,145 +88,90 @@ int main (int argc, char *argv[]) { ...@@ -87,145 +88,90 @@ int main (int argc, char *argv[]) {
break; break;
} }
case ('d'): { case ('d'): {
if (strlen(optarg) != 1) { int temp = get_delimiter(optarg);
fprintf( if (temp == EOF) {
stderr, fprintf(stderr,
"%s: %s: (-n) Key (or value) delimiter must be one character.", "%s: %s: (-d) Key (or value) delimiter must be one character.",
progname, progname, optarg);
optarg
);
return (EXIT_FAILURE); return (EXIT_FAILURE);
} }
dbt_delimiter = optarg[0]; if (isxdigit(temp)) {
if (isxdigit(dbt_delimiter)) { fprintf(stderr,
fprintf( "%s: %c: (-d) Key (or value) delimiter cannot be a hex digit.",
stderr, progname, temp);
"%s: %c: (-n) Key (or value) delimiter cannot be a hex digit.",
progname,
dbt_delimiter
);
return (EXIT_FAILURE); return (EXIT_FAILURE);
} }
dbt_delimiter = (char)temp;
break; break;
} }
case ('s'): { case ('s'): {
sort_delimiter = optarg; int temp = get_delimiter(optarg);
if (strlen(sort_delimiter) != 1) { if (temp == EOF) {
fprintf( fprintf(stderr,
stderr, "%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.", progname, optarg);
progname,
optarg
);
return (EXIT_FAILURE); return (EXIT_FAILURE);
} }
if (isxdigit(sort_delimiter[0])) { if (isxdigit(temp)) {
fprintf( fprintf(stderr,
stderr, "%s: %c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.", progname, temp);
progname,
sort_delimiter
);
return (EXIT_FAILURE); return (EXIT_FAILURE);
} }
sort_delimiter[0] = (char)temp;
sort_delimiter[1] = '\0';
break; break;
} }
case ('r'): case ('r'): {
{
char* test; char* test;
seed = strtol(optarg, &test, 10); seed = strtol(optarg, &test, 10);
if ( if (optarg[0] == '\0' || *test != '\0') {
optarg[0] == '\0' || fprintf(stderr,
*test != '\0' "%s: %s: (-r) Random seed invalid.",
) progname, optarg);
{
fprintf(
stderr,
"%s: %s: (-r) Random seed invalid.",
progname,
optarg
);
}
break;
}
case ('m'):
{
char* test;
if (
optarg[0] == '\0' ||
(minsize = strtol(optarg, &test, 10)) < 0 ||
*test != '\0'
)
{
fprintf(
stderr,
"%s: %s: (-m) Min size of keys/values invalid.",
progname,
optarg
);
} }
break; break;
} }
case ('M'): case ('m'): {
{
char* test; char* test;
if ( if (optarg[0] == '\0' ||
optarg[0] == '\0' || (lengthmin = strtol(optarg, &test, 10)) < 0 ||
(maxsize = strtol(optarg, &test, 10)) < 0 || *test != '\0')
*test != '\0'
)
{ {
fprintf( fprintf(stderr,
stderr, "%s: %s: (-m) Min length of keys/values invalid.",
"%s: %s: (-M) Max size of keys/values invalid.", progname, optarg);
progname,
optarg
);
} }
break; break;
} }
case ('n'): case ('M'): {
{
char* test; char* test;
if ( if (optarg[0] == '\0' ||
optarg[0] == '\0' || (lengthlimit = strtol(optarg, &test, 10)) < 0 ||
(maxnumkeys = strtoll(optarg, &test, 10)) <= 0 || *test != '\0')
*test != '\0'
)
{ {
fprintf( fprintf(stderr,
stderr, "%s: %s: (-M) Limit of key/value length invalid.",
"%s: %s: (-n) Max number of keys to generate invalid.", progname, optarg);
progname,
optarg
);
} }
break; break;
} }
case ('N'): case ('n'): {
{
char* test; char* test;
if ( if (optarg[0] == '\0' ||
optarg[0] == '\0' || (numkeys = strtoll(optarg, &test, 10)) <= 0 ||
(maxkibibytes = strtol(optarg, &test, 10)) <= 0 || *test != '\0')
*test != '\0'
)
{ {
fprintf( fprintf(stderr,
stderr, "%s: %s: (-n) Number of keys to generate invalid.",
"%s: %s: (-N) Max kibibytes to generate invalid.", progname, optarg);
progname,
optarg
);
} }
break; break;
} }
case ('?'):
default: { default: {
return (usage()); return (usage());
} }
...@@ -235,193 +181,133 @@ int main (int argc, char *argv[]) { ...@@ -235,193 +181,133 @@ int main (int argc, char *argv[]) {
argv += optind; argv += optind;
if (justheader && !header) { if (justheader && !header) {
fprintf( fprintf(stderr,
stderr, "%s: The -h and -H options may not both be specified.\n",
"%s: The -h and -H options may not both be specified.\n", progname);
progname return usage();
);
usage();
return (EXIT_FAILURE);
} }
if (justfooter && !footer) { if (justfooter && !footer) {
fprintf( fprintf(stderr,
stderr, "%s: The -f and -F options may not both be specified.\n",
"%s: The -f and -F options may not both be specified.\n", progname);
progname return usage();
);
usage();
return (EXIT_FAILURE);
} }
if (justfooter && justheader) { if (justfooter && justheader) {
fprintf( fprintf(stderr,
stderr, "%s: The -H and -F options may not both be specified.\n",
"%s: The -H and -F options may not both be specified.\n", progname);
progname return usage();
);
usage();
return (EXIT_FAILURE);
} }
if (justfooter && header) { if (justfooter && header) {
fprintf( fprintf(stderr,
stderr, "%s: -F implies -h\n",
"%s: -F implies -h\n", progname);
progname
);
header = false; header = false;
} }
if (justheader && footer) { if (justheader && footer) {
fprintf( fprintf(stderr,
stderr, "%s: -H implies -f\n",
"%s: -H implies -f\n", progname);
progname
);
footer = false; footer = false;
} }
if (!leadingspace) if (!leadingspace) {
{ if (footer) {
if (footer) fprintf(stderr,
{ "%s: -p implies -f\n",
fprintf( progname);
stderr,
"%s: -p implies -f\n",
progname
);
footer = false; footer = false;
} }
if (header) if (header) {
{ fprintf(stderr,
fprintf( "%s: -p implies -h\n",
stderr, progname);
"%s: -p implies -h\n",
progname
);
header = false; header = false;
} }
} }
if (justfooter || justheader) if (justfooter || justheader) outputkeys = false;
{ else if (numkeys == -1)
outputkeys = false;
}
else if (
(maxnumkeys > 0 && maxkibibytes > 0) ||
(maxnumkeys <= 0 && maxkibibytes <= 0)
)
{ {
fprintf( fprintf(stderr,
stderr, "%s: The -n option is required.\n",
"%s: exactly one of the -n and -N options must be specified.\n", progname);
progname return usage();
);
usage();
return (EXIT_FAILURE);
} }
if (outputkeys && seed == 1) if (outputkeys && seed == 1) {
{ fprintf(stderr,
fprintf( "%s: Using default seed. (-r 1).\n",
stderr, progname);
"%s: Using default seed. (-r 1).\n",
progname
);
seed = 1; seed = 1;
} }
if (outputkeys && minsize == -1) { if (outputkeys && lengthmin == -1) {
fprintf( fprintf(stderr,
stderr, "%s: Using default lengthmin. (-m 0).\n",
"%s: Using default minsize. (-m 0).\n", progname);
progname lengthmin = 0;
);
minsize = 0;
} }
if (outputkeys && maxsize == -1) { if (outputkeys && lengthlimit == -1) {
fprintf( fprintf(stderr,
stderr, "%s: Using default lengthlimit. (-M 1024).\n",
"%s: Using default maxsize. (-M 1024).\n", progname);
progname lengthlimit = 1024;
);
maxsize = 1024;
} }
if (outputkeys && minsize > maxsize) { if (outputkeys && lengthmin >= lengthlimit) {
fprintf( fprintf(stderr,
stderr, "%s: Max key size must be greater than min key size.\n",
"%s: Max key size must be greater than min key size.\n", progname);
progname return usage();
);
usage();
return (EXIT_FAILURE);
} }
if (argc != 0) { if (argc != 0) {
return (usage()); return usage();
} }
if (header) if (header) {
{ printf("VERSION=3\n"
printf( "format=%s\n"
"VERSION=3\n" "type=btree\n"
"format=%s\n" "db_pagesize=4096\n"
"type=btree\n" "HEADER=END\n",
"db_pagesize=4096\n" plaintext ? "print" : "bytevalue");
"HEADER=END\n",
(
plaintext ?
"print" :
"bytevalue"
)
);
}
if (justheader)
{
return 0;
}
if (outputkeys)
{
/* Generate Keys! */
generate_keys();
}
if (footer)
{
printf("DATA=END\n");
} }
if (justheader) return 0;
if (outputkeys) generate_keys();
if (footer) printf("DATA=END\n");
return 0; return 0;
} }
int usage() int usage()
{ {
fprintf fprintf(stderr,
( "usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
stderr, " [-m lengthmin] [-M lengthlimit] [-r random seed]\n"
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n" " [-o filename] -n numkeys\n",
" -m minsize -M maxsize [-r random seed]\n" progname);
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n", return EXIT_FAILURE;
progname
);
return 1;
} }
unsigned char randbyte() uint8_t randbyte()
{ {
static int numsavedbits = 0; static uint32_t numsavedbits = 0;
static unsigned long long savedbits = 0; static uint64_t savedbits = 0;
unsigned char retval; uint8_t retval;
if (numsavedbits < 8) if (numsavedbits < 8) {
{ savedbits |= ((uint64_t)random()) << numsavedbits;
savedbits |= ((unsigned long long)random()) << numsavedbits;
numsavedbits += 31; /* Random generates 31 random bits. */ numsavedbits += 31; /* Random generates 31 random bits. */
} }
retval = savedbits & 0xff; retval = savedbits & 0xff;
numsavedbits -= 8; numsavedbits -= 8;
savedbits >>= 8; savedbits >>= 8;
return retval; return retval;
} }
/* Almost-uniformly random int from [0,max) */ /* Almost-uniformly random int from [0,max) */
int random_below(int max) int32_t random_below(int32_t max)
{ {
assert(max > 0); assert(max > 0);
return (random() % max); return random() % max;
} }
void outputbyte(unsigned char ch) void outputbyte(uint8_t ch)
{ {
if (plaintext) { if (plaintext) {
if (ch == '\\') printf("\\\\"); if (ch == '\\') printf("\\\\");
...@@ -435,62 +321,42 @@ void outputstring(char* str) ...@@ -435,62 +321,42 @@ void outputstring(char* str)
{ {
char* p; char* p;
for (p = str; *p != '\0'; p++) for (p = str; *p != '\0'; p++) {
{ outputbyte((uint8_t)*p);
outputbyte((unsigned char)*p);
} }
} }
void generate_keys() void generate_keys()
{ {
bool usedemptykey = false; bool usedemptykey = false;
long long numgenerated = 0; int64_t numgenerated = 0;
long long totalsize = 0; int64_t totalsize = 0;
char identifier[24]; /* 8 bytes * 2 = 16; 16+1=17; 17+null terminator = 18. Extra padding. */ char identifier[24]; /* 8 bytes * 2 = 16; 16+1=17; 17+null terminator = 18. Extra padding. */
int length; int length;
int i; int i;
uint8_t ch;
srandom(seed); srandom(seed);
while ( while (numgenerated < numkeys) {
(
maxnumkeys == -1 ||
numgenerated < maxnumkeys
) &&
(
maxkibibytes == -1 ||
totalsize >> 10 < maxkibibytes
)
)
{
numgenerated++; numgenerated++;
/* Each key is preceded by a space (unless using -T). */
if (leadingspace) printf(" ");
/* Generate a key. */ /* Generate a key. */
if (leadingspace) {
printf(" "); /* Each key is preceded by a space. */
}
{ {
/* Pick a key length. */ /* Pick a key length. */
length = random_below(maxsize - minsize + 1) + minsize; length = random_below(lengthlimit - lengthmin) + lengthmin;
/* Output 'length' random bytes. */ /* Output 'length' random bytes. */
for (i = 0; i < length; i++) for (i = 0; i < length; i++) {
{ do {ch = randbyte();}
unsigned char ch;
do {
ch = randbyte();
}
while (printableonly && !isprint(ch)); while (printableonly && !isprint(ch));
outputbyte(ch); outputbyte(ch);
} }
totalsize += length; totalsize += length;
if (length == 0 && !usedemptykey) if (length == 0 && !usedemptykey) usedemptykey = true;
{ else {
usedemptykey = true;
}
else
{
/* Append identifier to ensure uniqueness. */ /* Append identifier to ensure uniqueness. */
sprintf(identifier, "x%llx", numgenerated); sprintf(identifier, "x%llx", numgenerated);
outputstring(identifier); outputstring(identifier);
...@@ -499,24 +365,18 @@ void generate_keys() ...@@ -499,24 +365,18 @@ void generate_keys()
} }
printf("%c", dbt_delimiter); printf("%c", dbt_delimiter);
/* Each value is preceded by a space (unless using -T). */
if (leadingspace) printf(" ");
/* Generate a value. */ /* Generate a value. */
if (leadingspace) {
printf(" "); /* Each value is preceded by a space. */
}
{ {
/* Pick a key length. */ /* Pick a key length. */
length = random_range(minsize, maxsize); length = random_below(lengthlimit - lengthmin) + lengthmin;
/* Output 'length' random bytes. */ /* Output 'length' random bytes. */
for (i = 0; i < length; i++) for (i = 0; i < length; i++) {
{ do {ch = randbyte();}
unsigned char ch;
do {
ch = randbyte();
}
while (printableonly && !isprint(ch)); while (printableonly && !isprint(ch));
outputbyte(ch); outputbyte(ch);
} }
totalsize += length; totalsize += length;
...@@ -526,3 +386,24 @@ void generate_keys() ...@@ -526,3 +386,24 @@ void generate_keys()
printf("%s", sort_delimiter); printf("%s", sort_delimiter);
} }
} }
int get_delimiter(char* str)
{
if (strlen(str) == 2 && str[0] == '\\') {
switch (str[1]) {
case ('a'): return '\a';
case ('b'): return '\b';
case ('e'): return '\e';
case ('f'): return '\f';
case ('n'): return '\n';
case ('r'): return '\r';
case ('t'): return '\t';
case ('v'): return '\v';
case ('0'): return '\0';
case ('\\'): return '\\';
default: return EOF;
}
}
if (strlen(str) == 1) return str[0];
return EOF;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7