Skip to content

M
MariaDB
Commit 807296a4 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

db_load/gen/dump project done until tokudb.bugs issues are worked on. 


git-svn-id: file:///svn/tokudb@479 c7de825b-a66e-492c-adef-691d508d4ae1
parent 07751626
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 459 additions and 686 deletions
utils/Makefile
View file @ 807296a4
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -I /usr/local/Berkeleydb.4.1/include/ -ldb
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -ldb
LFLAGS = -l CPPFLAGS =
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
......@@ -10,18 +10,17 @@ UTILS= \
tokudb_dump \
#End
.PHONY: all clean test test_gen test_gen_hex test_load
.PHONY: all clean test test_gen test_gen_hex test_load test_dump
all: $(UTILS)
test: test_gen test_load
test: test_gen test_load test_dump
test_gen: test_gen_hex
# SHELL=/bin/bash
BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
BDB=/usr/local/BerkeleyDB.4.6
BDB_DUMP=$(BDB)/bin/db_dump
BDB_LOAD=$(BDB)/bin/db_load
TEST_GEN_HEX_NUMKEYS=10000
TEST_GEN_HEX_LENGTHMIN=0
......@@ -41,7 +40,7 @@ test_gen_hex:
if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi
rm $@.*.temp
test_load: test_gen
test_load:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
......@@ -52,6 +51,16 @@ test_load: test_gen
$(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
test_dump:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
./tokudb_gen $(TEST_GEN_HEX_FLAGS) > $@.gen.temp
$(BDB_LOAD) $@.bdb.temp < $@.gen.temp
$(BDB_DUMP) $@.bdb.temp > $@.dump.bdb.temp
./tokudb_dump $@.bdb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
#if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi
clean:
rm -rf *.so *.o $(UTILS) *.temp
......
utils/tokudb_common_funcs.h
View file @ 807296a4
......@@ -2,10 +2,27 @@
#define TOKUDB_COMMON_FUNCS_H
#include "tokudb_common.h"
int strtoint32 (DB_ENV* dbenv, char* progname, char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (DB_ENV* dbenv, char* progname, char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (DB_ENV* dbenv, char* progname, char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
//DB_ENV->err disabled since it does not use db_strerror
#define ERROR(retval, ...) \
if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \
fprintf(stderr, __VA_ARGS__); \
}
//DB_ENV->err disabled since it does not use db_strerror, errx does not exist.
#define ERRORX(...) \
if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: ", g.progname); \
fprintf(stderr, __VA_ARGS__); \
}
int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
/*
* Convert a string to an integer of type "type".
......@@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint
*
*/
#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \
int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max, int base) \
int name(char* str, type* num, type min, type max, int base) \
{ \
char* test; \
bigtype value; \
......@@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max
assert(str); \
assert(num); \
assert(min <= max); \
assert(dbenv || progname); \
assert(g.dbenv || g.progname); \
assert(base == 0 || (base >= 2 && base <= 36)); \
\
errno = 0; \
while (isspace(*str)) str++; \
value = strtofunc(str, &test, base); \
if ((*test != '\0' && *test != '\n') || test == str) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Invalid numeric argument\n", progname, str); \
else dbenv->err(dbenv, 0, "%s: Invalid numeric argument", str); \
ERRORX("%s: Invalid numeric argument\n", str); \
errno = EINVAL; \
goto error; \
} \
if (errno != 0) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: %s\n", progname, str, strerror(errno)); \
else dbenv->err(dbenv, errno, "%s", str); \
goto error; \
ERROR(errno, "%s\n", str); \
} \
if (value < min) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Less than minimum value (%" frmt ")\n", progname, str, min); \
else dbenv->err(dbenv, 0, "%s: Less than minimum value (%" frmt ")", str, min); \
ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \
goto error; \
} \
if (value > max) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Greater than maximum value (%" frmt ")\n", progname, str, max); \
else dbenv->err(dbenv, 0, "%s: Greater than maximum value (%" frmt ")", str, max); \
ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \
goto error; \
} \
*num = value; \
......@@ -76,4 +88,103 @@ void outputbyte(uint8_t ch)
else printf("%02x", ch);
}
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void outputplaintextstring(char* str)
{
bool old_plaintext = g.plaintext;
g.plaintext = true;
outputstring(str);
g.plaintext = old_plaintext;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
ERROR(errno, "printabletocstring");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
ERROR(0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
ERROR(0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
ERROR(0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int verify_library_version()
{
int major;
int minor;
db_version(&major, &minor, NULL);
if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) {
ERRORX("version %d.%d doesn't match library version %d.%d\n",
DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
#endif /* #if !defined(TOKUDB_COMMON_H) */
utils/tokudb_dump.c
View file @ 807296a4
This diff is collapsed.
utils/tokudb_gen.c
View file @ 807296a4
......@@ -5,17 +5,15 @@
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <db.h>
#include "tokudb_common.h"
extern char* optarg;
extern int optind;
extern int optopt;
extern int opterr;
extern int optreset;
typedef struct {
DB_ENV* dbenv;
bool plaintext;
char* progname;
} gen_globals;
gen_globals g;
......@@ -29,8 +27,6 @@ int get_delimiter(char* str);
char dbt_delimiter = '\n';
char sort_delimiter[2];
char* progname;
uint32_t lengthmin = 0;
bool set_lengthmin = false;
uint32_t lengthlimit = 0;
......@@ -54,10 +50,13 @@ int main (int argc, char *argv[]) {
/* Set up the globals. */
memset(&g, 0, sizeof(g));
progname = argv[0];
g.progname = argv[0];
if (verify_library_version() != 0) goto error;
strcpy(sort_delimiter, "");
while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:o:")) != EOF) {
while ((ch = getopt(argc, argv, "PfFhHTpVr:s:d:m:M:n:o:")) != EOF) {
switch (ch) {
case ('P'): {
printableonly = true;
......@@ -80,22 +79,20 @@ int main (int argc, char *argv[]) {
break;
}
case ('T'): {
g.plaintext = true;
g.plaintext = true;
leadingspace = false;
header = false;
footer = false;
break;
}
case ('p'): {
g.plaintext = true;
g.plaintext = true;
leadingspace = true;
break;
}
case ('o'): {
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr,
"%s: %s: reopen: %s\n",
progname, optarg, strerror(errno));
ERROR(errno, "%s: reopen\n", optarg);
goto error;
}
break;
......@@ -103,15 +100,13 @@ int main (int argc, char *argv[]) {
case ('d'): {
int temp = get_delimiter(optarg);
if (temp == EOF) {
fprintf(stderr,
"%s: %s: (-d) Key (or value) delimiter must be one character.",
progname, optarg);
ERRORX("%s: (-d) Key (or value) delimiter must be one character.",
optarg);
goto error;
}
if (isxdigit(temp)) {
fprintf(stderr,
"%s: %c: (-d) Key (or value) delimiter cannot be a hex digit.",
progname, temp);
ERRORX("%c: (-d) Key (or value) delimiter cannot be a hex digit.",
temp);
goto error;
}
dbt_delimiter = (char)temp;
......@@ -120,15 +115,13 @@ int main (int argc, char *argv[]) {
case ('s'): {
int temp = get_delimiter(optarg);
if (temp == EOF) {
fprintf(stderr,
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
progname, optarg);
ERRORX("%s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
optarg);
goto error;
}
if (isxdigit(temp)) {
fprintf(stderr,
"%s: %c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
progname, temp);
ERRORX("%c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
temp);
goto error;
}
sort_delimiter[0] = (char)temp;
......@@ -136,45 +129,41 @@ int main (int argc, char *argv[]) {
break;
}
case ('r'): {
if (strtouint32(NULL, progname, optarg, &seed, 0, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-r) Random seed invalid.",
progname, optarg);
if (strtouint32(optarg, &seed, 0, UINT32_MAX, 10)) {
ERRORX("%s: (-r) Random seed invalid.", optarg);
goto error;
}
set_seed = true;
break;
}
case ('m'): {
if (strtouint32(NULL, progname, optarg, &lengthmin, 0, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-m) Min length of keys/values invalid.",
progname, optarg);
if (strtouint32(optarg, &lengthmin, 0, UINT32_MAX, 10)) {
ERRORX("%s: (-m) Min length of keys/values invalid.", optarg);
goto error;
}
set_lengthmin = true;
break;
}
case ('M'): {
if (strtouint32(NULL, progname, optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-M) Limit of key/value length invalid.",
progname, optarg);
if (strtouint32(optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
ERRORX("%s: (-M) Limit of key/value length invalid.", optarg);
goto error;
}
set_lengthlimit = true;
break;
}
case ('n'): {
if (strtouint64(NULL, progname, optarg, &numkeys, 0, UINT64_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-n) Number of keys to generate invalid.",
progname, optarg);
if (strtouint64(optarg, &numkeys, 0, UINT64_MAX, 10)) {
ERRORX("%s: (-n) Number of keys to generate invalid.", optarg);
goto error;
}
set_numkeys = true;
break;
}
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
return EXIT_SUCCESS;
}
case ('?'):
default: {
return (usage());
......@@ -185,61 +174,55 @@ int main (int argc, char *argv[]) {
argv += optind;
if (justheader && !header) {
fprintf(stderr,
"%s: The -h and -H options may not both be specified.\n",
progname);
ERRORX("The -h and -H options may not both be specified.\n");
goto error;
}
if (justfooter && !footer) {
fprintf(stderr,
"%s: The -f and -F options may not both be specified.\n",
progname);
ERRORX("The -f and -F options may not both be specified.\n");
goto error;
}
if (justfooter && justheader) {
fprintf(stderr,
"%s: The -H and -F options may not both be specified.\n",
progname);
ERRORX("The -H and -F options may not both be specified.\n");
goto error;
}
if (justfooter && header) {
fprintf(stderr, "%s: -F implies -h\n", progname);
ERRORX("-F implies -h\n");
header = false;
}
if (justheader && footer) {
fprintf(stderr, "%s: -H implies -f\n", progname);
ERRORX("-H implies -f\n");
footer = false;
}
if (!leadingspace) {
if (footer) {
fprintf(stderr, "%s: -p implies -f\n", progname);
ERRORX("-p implies -f\n");
footer = false;
}
if (header) {
fprintf(stderr, "%s: -p implies -h\n", progname);
ERRORX("-p implies -h\n");
header = false;
}
}
if (justfooter || justheader) outputkeys = false;
else if (!set_numkeys)
{
fprintf(stderr, "%s: The -n option is required.\n", progname);
ERRORX("The -n option is required.\n");
goto error;
}
if (outputkeys && !set_seed) {
fprintf(stderr, "%s: Using default seed. (-r 1).\n", progname);
ERRORX("Using default seed. (-r 1).\n");
seed = 1;
}
if (outputkeys && !set_lengthmin) {
fprintf(stderr, "%s: Using default lengthmin. (-m 0).\n", progname);
ERRORX("Using default lengthmin. (-m 0).\n");
lengthmin = 0;
}
if (outputkeys && !set_lengthlimit) {
fprintf(stderr, "%s: Using default lengthlimit. (-M 1024).\n", progname);
ERRORX("Using default lengthlimit. (-M 1024).\n");
lengthlimit = 1024;
}
if (outputkeys && lengthmin >= lengthlimit) {
fprintf(stderr, "%s: Max key size must be greater than min key size.\n", progname);
ERRORX("Max key size must be greater than min key size.\n");
goto error;
}
......@@ -256,7 +239,7 @@ int main (int argc, char *argv[]) {
}
if (outputkeys) generate_keys();
if (footer) printf("DATA=END\n");
return 0;
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
......@@ -266,10 +249,10 @@ int main (int argc, char *argv[]) {
int usage()
{
fprintf(stderr,
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" [-m lengthmin] [-M lengthlimit] [-r random seed]\n"
" [-o filename] -n numkeys\n",
progname);
"usage: %s [-PfFhHTpV] [-r random seed] [-s delimiter] \n"
" [-d delimiter] [-m lengthmin] [-M lengthlimit] \n"
" -n numkeys [-o output_file] \n",
g.progname);
return EXIT_FAILURE;
}
......@@ -296,15 +279,6 @@ int32_t random_below(int32_t limit)
return random() % limit;
}
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void generate_keys()
{
bool usedemptykey = false;
......
utils/tokudb_load.c
View file @ 807296a4
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7