Skip to content

M
mariadb
Commit 807296a4 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

db_load/gen/dump project done until tokudb.bugs issues are worked on. 


git-svn-id: file:///svn/tokudb@479 c7de825b-a66e-492c-adef-691d508d4ae1
parent 07751626
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 459 additions and 686 deletions
utils/Makefile
View file @ 807296a4
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -I /usr/local/Berkeleydb.4.1/include/ -ldb CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -ldb
LFLAGS = -l CPPFLAGS = LFLAGS = -l CPPFLAGS =
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
...@@ -10,18 +10,17 @@ UTILS= \ ...@@ -10,18 +10,17 @@ UTILS= \
tokudb_dump \ tokudb_dump \
#End #End
.PHONY: all clean test test_gen test_gen_hex test_load .PHONY: all clean test test_gen test_gen_hex test_load test_dump
all: $(UTILS) all: $(UTILS)
test: test_gen test_load test: test_gen test_load test_dump
test_gen: test_gen_hex test_gen: test_gen_hex
# SHELL=/bin/bash BDB=/usr/local/BerkeleyDB.4.6
BDB_DUMP=$(BDB)/bin/db_dump
BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load BDB_LOAD=$(BDB)/bin/db_load
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
TEST_GEN_HEX_NUMKEYS=10000 TEST_GEN_HEX_NUMKEYS=10000
TEST_GEN_HEX_LENGTHMIN=0 TEST_GEN_HEX_LENGTHMIN=0
...@@ -41,7 +40,7 @@ test_gen_hex: ...@@ -41,7 +40,7 @@ test_gen_hex:
if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi
rm $@.*.temp rm $@.*.temp
test_load: test_gen test_load:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead #Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text" echo "Generating text input > db > text"
rm -f $@.*.temp rm -f $@.*.temp
...@@ -52,6 +51,16 @@ test_load: test_gen ...@@ -52,6 +51,16 @@ test_load: test_gen
$(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp $(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
test_dump:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
./tokudb_gen $(TEST_GEN_HEX_FLAGS) > $@.gen.temp
$(BDB_LOAD) $@.bdb.temp < $@.gen.temp
$(BDB_DUMP) $@.bdb.temp > $@.dump.bdb.temp
./tokudb_dump $@.bdb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
#if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi #if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi
clean: clean:
rm -rf *.so *.o $(UTILS) *.temp rm -rf *.so *.o $(UTILS) *.temp
......
utils/tokudb_common_funcs.h
View file @ 807296a4
...@@ -2,10 +2,27 @@ ...@@ -2,10 +2,27 @@
#define TOKUDB_COMMON_FUNCS_H #define TOKUDB_COMMON_FUNCS_H
#include "tokudb_common.h" #include "tokudb_common.h"
int strtoint32 (DB_ENV* dbenv, char* progname, char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (DB_ENV* dbenv, char* progname, char* str, uint32_t* num, uint32_t min, uint32_t max, int base); //DB_ENV->err disabled since it does not use db_strerror
int strtoint64 (DB_ENV* dbenv, char* progname, char* str, int64_t* num, int64_t min, int64_t max, int base); #define ERROR(retval, ...) \
int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint64_t min, uint64_t max, int base); if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \
fprintf(stderr, __VA_ARGS__); \
}
//DB_ENV->err disabled since it does not use db_strerror, errx does not exist.
#define ERRORX(...) \
if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: ", g.progname); \
fprintf(stderr, __VA_ARGS__); \
}
int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
/* /*
* Convert a string to an integer of type "type". * Convert a string to an integer of type "type".
...@@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint ...@@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint
* *
*/ */
#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \ #define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \
int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max, int base) \ int name(char* str, type* num, type min, type max, int base) \
{ \ { \
char* test; \ char* test; \
bigtype value; \ bigtype value; \
...@@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max ...@@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max
assert(str); \ assert(str); \
assert(num); \ assert(num); \
assert(min <= max); \ assert(min <= max); \
assert(dbenv || progname); \ assert(g.dbenv || g.progname); \
assert(base == 0 || (base >= 2 && base <= 36)); \ assert(base == 0 || (base >= 2 && base <= 36)); \
\ \
errno = 0; \ errno = 0; \
while (isspace(*str)) str++; \ while (isspace(*str)) str++; \
value = strtofunc(str, &test, base); \ value = strtofunc(str, &test, base); \
if ((*test != '\0' && *test != '\n') || test == str) { \ if ((*test != '\0' && *test != '\n') || test == str) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Invalid numeric argument\n", progname, str); \ ERRORX("%s: Invalid numeric argument\n", str); \
else dbenv->err(dbenv, 0, "%s: Invalid numeric argument", str); \
errno = EINVAL; \ errno = EINVAL; \
goto error; \ goto error; \
} \ } \
if (errno != 0) { \ if (errno != 0) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: %s\n", progname, str, strerror(errno)); \ ERROR(errno, "%s\n", str); \
else dbenv->err(dbenv, errno, "%s", str); \
goto error; \
} \ } \
if (value < min) { \ if (value < min) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Less than minimum value (%" frmt ")\n", progname, str, min); \ ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \
else dbenv->err(dbenv, 0, "%s: Less than minimum value (%" frmt ")", str, min); \
goto error; \ goto error; \
} \ } \
if (value > max) { \ if (value > max) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Greater than maximum value (%" frmt ")\n", progname, str, max); \ ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \
else dbenv->err(dbenv, 0, "%s: Greater than maximum value (%" frmt ")", str, max); \
goto error; \ goto error; \
} \ } \
*num = value; \ *num = value; \
...@@ -76,4 +88,103 @@ void outputbyte(uint8_t ch) ...@@ -76,4 +88,103 @@ void outputbyte(uint8_t ch)
else printf("%02x", ch); else printf("%02x", ch);
} }
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void outputplaintextstring(char* str)
{
bool old_plaintext = g.plaintext;
g.plaintext = true;
outputstring(str);
g.plaintext = old_plaintext;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
ERROR(errno, "printabletocstring");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
ERROR(0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
ERROR(0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
ERROR(0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int verify_library_version()
{
int major;
int minor;
db_version(&major, &minor, NULL);
if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) {
ERRORX("version %d.%d doesn't match library version %d.%d\n",
DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
#endif /* #if !defined(TOKUDB_COMMON_H) */ #endif /* #if !defined(TOKUDB_COMMON_H) */
utils/tokudb_dump.c
View file @ 807296a4
This diff is collapsed.
utils/tokudb_gen.c
View file @ 807296a4
...@@ -5,17 +5,15 @@ ...@@ -5,17 +5,15 @@
#include <string.h> #include <string.h>
#include <ctype.h> #include <ctype.h>
#include <errno.h> #include <errno.h>
#include <getopt.h>
#include <db.h>
#include "tokudb_common.h" #include "tokudb_common.h"
extern char* optarg;
extern int optind;
extern int optopt;
extern int opterr;
extern int optreset;
typedef struct { typedef struct {
DB_ENV* dbenv;
bool plaintext; bool plaintext;
char* progname;
} gen_globals; } gen_globals;
gen_globals g; gen_globals g;
...@@ -29,8 +27,6 @@ int get_delimiter(char* str); ...@@ -29,8 +27,6 @@ int get_delimiter(char* str);
char dbt_delimiter = '\n'; char dbt_delimiter = '\n';
char sort_delimiter[2]; char sort_delimiter[2];
char* progname;
uint32_t lengthmin = 0; uint32_t lengthmin = 0;
bool set_lengthmin = false; bool set_lengthmin = false;
uint32_t lengthlimit = 0; uint32_t lengthlimit = 0;
...@@ -54,10 +50,13 @@ int main (int argc, char *argv[]) { ...@@ -54,10 +50,13 @@ int main (int argc, char *argv[]) {
/* Set up the globals. */ /* Set up the globals. */
memset(&g, 0, sizeof(g)); memset(&g, 0, sizeof(g));
progname = argv[0]; g.progname = argv[0];
if (verify_library_version() != 0) goto error;
strcpy(sort_delimiter, ""); strcpy(sort_delimiter, "");
while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:o:")) != EOF) { while ((ch = getopt(argc, argv, "PfFhHTpVr:s:d:m:M:n:o:")) != EOF) {
switch (ch) { switch (ch) {
case ('P'): { case ('P'): {
printableonly = true; printableonly = true;
...@@ -80,22 +79,20 @@ int main (int argc, char *argv[]) { ...@@ -80,22 +79,20 @@ int main (int argc, char *argv[]) {
break; break;
} }
case ('T'): { case ('T'): {
g.plaintext = true; g.plaintext = true;
leadingspace = false; leadingspace = false;
header = false; header = false;
footer = false; footer = false;
break; break;
} }
case ('p'): { case ('p'): {
g.plaintext = true; g.plaintext = true;
leadingspace = true; leadingspace = true;
break; break;
} }
case ('o'): { case ('o'): {
if (freopen(optarg, "w", stdout) == NULL) { if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, ERROR(errno, "%s: reopen\n", optarg);
"%s: %s: reopen: %s\n",
progname, optarg, strerror(errno));
goto error; goto error;
} }
break; break;
...@@ -103,15 +100,13 @@ int main (int argc, char *argv[]) { ...@@ -103,15 +100,13 @@ int main (int argc, char *argv[]) {
case ('d'): { case ('d'): {
int temp = get_delimiter(optarg); int temp = get_delimiter(optarg);
if (temp == EOF) { if (temp == EOF) {
fprintf(stderr, ERRORX("%s: (-d) Key (or value) delimiter must be one character.",
"%s: %s: (-d) Key (or value) delimiter must be one character.", optarg);
progname, optarg);
goto error; goto error;
} }
if (isxdigit(temp)) { if (isxdigit(temp)) {
fprintf(stderr, ERRORX("%c: (-d) Key (or value) delimiter cannot be a hex digit.",
"%s: %c: (-d) Key (or value) delimiter cannot be a hex digit.", temp);
progname, temp);
goto error; goto error;
} }
dbt_delimiter = (char)temp; dbt_delimiter = (char)temp;
...@@ -120,15 +115,13 @@ int main (int argc, char *argv[]) { ...@@ -120,15 +115,13 @@ int main (int argc, char *argv[]) {
case ('s'): { case ('s'): {
int temp = get_delimiter(optarg); int temp = get_delimiter(optarg);
if (temp == EOF) { if (temp == EOF) {
fprintf(stderr, ERRORX("%s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.", optarg);
progname, optarg);
goto error; goto error;
} }
if (isxdigit(temp)) { if (isxdigit(temp)) {
fprintf(stderr, ERRORX("%c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
"%s: %c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.", temp);
progname, temp);
goto error; goto error;
} }
sort_delimiter[0] = (char)temp; sort_delimiter[0] = (char)temp;
...@@ -136,45 +129,41 @@ int main (int argc, char *argv[]) { ...@@ -136,45 +129,41 @@ int main (int argc, char *argv[]) {
break; break;
} }
case ('r'): { case ('r'): {
if (strtouint32(NULL, progname, optarg, &seed, 0, UINT32_MAX, 10)) { if (strtouint32(optarg, &seed, 0, UINT32_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-r) Random seed invalid.", optarg);
"%s: %s: (-r) Random seed invalid.",
progname, optarg);
goto error; goto error;
} }
set_seed = true; set_seed = true;
break; break;
} }
case ('m'): { case ('m'): {
if (strtouint32(NULL, progname, optarg, &lengthmin, 0, UINT32_MAX, 10)) { if (strtouint32(optarg, &lengthmin, 0, UINT32_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-m) Min length of keys/values invalid.", optarg);
"%s: %s: (-m) Min length of keys/values invalid.",
progname, optarg);
goto error; goto error;
} }
set_lengthmin = true; set_lengthmin = true;
break; break;
} }
case ('M'): { case ('M'): {
if (strtouint32(NULL, progname, optarg, &lengthlimit, 1, UINT32_MAX, 10)) { if (strtouint32(optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-M) Limit of key/value length invalid.", optarg);
"%s: %s: (-M) Limit of key/value length invalid.",
progname, optarg);
goto error; goto error;
} }
set_lengthlimit = true; set_lengthlimit = true;
break; break;
} }
case ('n'): { case ('n'): {
if (strtouint64(NULL, progname, optarg, &numkeys, 0, UINT64_MAX, 10)) { if (strtouint64(optarg, &numkeys, 0, UINT64_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-n) Number of keys to generate invalid.", optarg);
"%s: %s: (-n) Number of keys to generate invalid.",
progname, optarg);
goto error; goto error;
} }
set_numkeys = true; set_numkeys = true;
break; break;
} }
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
return EXIT_SUCCESS;
}
case ('?'): case ('?'):
default: { default: {
return (usage()); return (usage());
...@@ -185,61 +174,55 @@ int main (int argc, char *argv[]) { ...@@ -185,61 +174,55 @@ int main (int argc, char *argv[]) {
argv += optind; argv += optind;
if (justheader && !header) { if (justheader && !header) {
fprintf(stderr, ERRORX("The -h and -H options may not both be specified.\n");
"%s: The -h and -H options may not both be specified.\n",
progname);
goto error; goto error;
} }
if (justfooter && !footer) { if (justfooter && !footer) {
fprintf(stderr, ERRORX("The -f and -F options may not both be specified.\n");
"%s: The -f and -F options may not both be specified.\n",
progname);
goto error; goto error;
} }
if (justfooter && justheader) { if (justfooter && justheader) {
fprintf(stderr, ERRORX("The -H and -F options may not both be specified.\n");
"%s: The -H and -F options may not both be specified.\n",
progname);
goto error; goto error;
} }
if (justfooter && header) { if (justfooter && header) {
fprintf(stderr, "%s: -F implies -h\n", progname); ERRORX("-F implies -h\n");
header = false; header = false;
} }
if (justheader && footer) { if (justheader && footer) {
fprintf(stderr, "%s: -H implies -f\n", progname); ERRORX("-H implies -f\n");
footer = false; footer = false;
} }
if (!leadingspace) { if (!leadingspace) {
if (footer) { if (footer) {
fprintf(stderr, "%s: -p implies -f\n", progname); ERRORX("-p implies -f\n");
footer = false; footer = false;
} }
if (header) { if (header) {
fprintf(stderr, "%s: -p implies -h\n", progname); ERRORX("-p implies -h\n");
header = false; header = false;
} }
} }
if (justfooter || justheader) outputkeys = false; if (justfooter || justheader) outputkeys = false;
else if (!set_numkeys) else if (!set_numkeys)
{ {
fprintf(stderr, "%s: The -n option is required.\n", progname); ERRORX("The -n option is required.\n");
goto error; goto error;
} }
if (outputkeys && !set_seed) { if (outputkeys && !set_seed) {
fprintf(stderr, "%s: Using default seed. (-r 1).\n", progname); ERRORX("Using default seed. (-r 1).\n");
seed = 1; seed = 1;
} }
if (outputkeys && !set_lengthmin) { if (outputkeys && !set_lengthmin) {
fprintf(stderr, "%s: Using default lengthmin. (-m 0).\n", progname); ERRORX("Using default lengthmin. (-m 0).\n");
lengthmin = 0; lengthmin = 0;
} }
if (outputkeys && !set_lengthlimit) { if (outputkeys && !set_lengthlimit) {
fprintf(stderr, "%s: Using default lengthlimit. (-M 1024).\n", progname); ERRORX("Using default lengthlimit. (-M 1024).\n");
lengthlimit = 1024; lengthlimit = 1024;
} }
if (outputkeys && lengthmin >= lengthlimit) { if (outputkeys && lengthmin >= lengthlimit) {
fprintf(stderr, "%s: Max key size must be greater than min key size.\n", progname); ERRORX("Max key size must be greater than min key size.\n");
goto error; goto error;
} }
...@@ -256,7 +239,7 @@ int main (int argc, char *argv[]) { ...@@ -256,7 +239,7 @@ int main (int argc, char *argv[]) {
} }
if (outputkeys) generate_keys(); if (outputkeys) generate_keys();
if (footer) printf("DATA=END\n"); if (footer) printf("DATA=END\n");
return 0; return EXIT_SUCCESS;
error: error:
fprintf(stderr, "Quitting out due to errors.\n"); fprintf(stderr, "Quitting out due to errors.\n");
...@@ -266,10 +249,10 @@ error: ...@@ -266,10 +249,10 @@ error:
int usage() int usage()
{ {
fprintf(stderr, fprintf(stderr,
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n" "usage: %s [-PfFhHTpV] [-r random seed] [-s delimiter] \n"
" [-m lengthmin] [-M lengthlimit] [-r random seed]\n" " [-d delimiter] [-m lengthmin] [-M lengthlimit] \n"
" [-o filename] -n numkeys\n", " -n numkeys [-o output_file] \n",
progname); g.progname);
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -296,15 +279,6 @@ int32_t random_below(int32_t limit) ...@@ -296,15 +279,6 @@ int32_t random_below(int32_t limit)
return random() % limit; return random() % limit;
} }
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void generate_keys() void generate_keys()
{ {
bool usedemptykey = false; bool usedemptykey = false;
......
utils/tokudb_load.c
View file @ 807296a4
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7