Skip to content

M
mariadb
Commit 807296a4 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

db_load/gen/dump project done until tokudb.bugs issues are worked on. 


git-svn-id: file:///svn/tokudb@479 c7de825b-a66e-492c-adef-691d508d4ae1
parent 07751626
Hide whitespace changes
Inline Side-by-side
Showing with 459 additions and 686 deletions
utils/Makefile
View file @ 807296a4
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -I /usr/local/Berkeleydb.4.1/include/ -ldb CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -ldb
LFLAGS = -l CPPFLAGS = LFLAGS = -l CPPFLAGS =
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
...@@ -10,18 +10,17 @@ UTILS= \ ...@@ -10,18 +10,17 @@ UTILS= \
tokudb_dump \ tokudb_dump \
#End #End
.PHONY: all clean test test_gen test_gen_hex test_load .PHONY: all clean test test_gen test_gen_hex test_load test_dump
all: $(UTILS) all: $(UTILS)
test: test_gen test_load test: test_gen test_load test_dump
test_gen: test_gen_hex test_gen: test_gen_hex
# SHELL=/bin/bash BDB=/usr/local/BerkeleyDB.4.6
BDB_DUMP=$(BDB)/bin/db_dump
BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load BDB_LOAD=$(BDB)/bin/db_load
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
TEST_GEN_HEX_NUMKEYS=10000 TEST_GEN_HEX_NUMKEYS=10000
TEST_GEN_HEX_LENGTHMIN=0 TEST_GEN_HEX_LENGTHMIN=0
...@@ -41,7 +40,7 @@ test_gen_hex: ...@@ -41,7 +40,7 @@ test_gen_hex:
if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi
rm $@.*.temp rm $@.*.temp
test_load: test_gen test_load:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead #Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text" echo "Generating text input > db > text"
rm -f $@.*.temp rm -f $@.*.temp
...@@ -52,6 +51,16 @@ test_load: test_gen ...@@ -52,6 +51,16 @@ test_load: test_gen
$(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp $(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
test_dump:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
./tokudb_gen $(TEST_GEN_HEX_FLAGS) > $@.gen.temp
$(BDB_LOAD) $@.bdb.temp < $@.gen.temp
$(BDB_DUMP) $@.bdb.temp > $@.dump.bdb.temp
./tokudb_dump $@.bdb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
#if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi #if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi
clean: clean:
rm -rf *.so *.o $(UTILS) *.temp rm -rf *.so *.o $(UTILS) *.temp
......
utils/tokudb_common_funcs.h
View file @ 807296a4
...@@ -2,10 +2,27 @@ ...@@ -2,10 +2,27 @@
#define TOKUDB_COMMON_FUNCS_H #define TOKUDB_COMMON_FUNCS_H
#include "tokudb_common.h" #include "tokudb_common.h"
int strtoint32 (DB_ENV* dbenv, char* progname, char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (DB_ENV* dbenv, char* progname, char* str, uint32_t* num, uint32_t min, uint32_t max, int base); //DB_ENV->err disabled since it does not use db_strerror
int strtoint64 (DB_ENV* dbenv, char* progname, char* str, int64_t* num, int64_t min, int64_t max, int base); #define ERROR(retval, ...) \
int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint64_t min, uint64_t max, int base); if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \
fprintf(stderr, __VA_ARGS__); \
}
//DB_ENV->err disabled since it does not use db_strerror, errx does not exist.
#define ERRORX(...) \
if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: ", g.progname); \
fprintf(stderr, __VA_ARGS__); \
}
int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
/* /*
* Convert a string to an integer of type "type". * Convert a string to an integer of type "type".
...@@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint ...@@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint
* *
*/ */
#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \ #define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \
int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max, int base) \ int name(char* str, type* num, type min, type max, int base) \
{ \ { \
char* test; \ char* test; \
bigtype value; \ bigtype value; \
...@@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max ...@@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max
assert(str); \ assert(str); \
assert(num); \ assert(num); \
assert(min <= max); \ assert(min <= max); \
assert(dbenv || progname); \ assert(g.dbenv || g.progname); \
assert(base == 0 || (base >= 2 && base <= 36)); \ assert(base == 0 || (base >= 2 && base <= 36)); \
\ \
errno = 0; \ errno = 0; \
while (isspace(*str)) str++; \ while (isspace(*str)) str++; \
value = strtofunc(str, &test, base); \ value = strtofunc(str, &test, base); \
if ((*test != '\0' && *test != '\n') || test == str) { \ if ((*test != '\0' && *test != '\n') || test == str) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Invalid numeric argument\n", progname, str); \ ERRORX("%s: Invalid numeric argument\n", str); \
else dbenv->err(dbenv, 0, "%s: Invalid numeric argument", str); \
errno = EINVAL; \ errno = EINVAL; \
goto error; \ goto error; \
} \ } \
if (errno != 0) { \ if (errno != 0) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: %s\n", progname, str, strerror(errno)); \ ERROR(errno, "%s\n", str); \
else dbenv->err(dbenv, errno, "%s", str); \
goto error; \
} \ } \
if (value < min) { \ if (value < min) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Less than minimum value (%" frmt ")\n", progname, str, min); \ ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \
else dbenv->err(dbenv, 0, "%s: Less than minimum value (%" frmt ")", str, min); \
goto error; \ goto error; \
} \ } \
if (value > max) { \ if (value > max) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Greater than maximum value (%" frmt ")\n", progname, str, max); \ ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \
else dbenv->err(dbenv, 0, "%s: Greater than maximum value (%" frmt ")", str, max); \
goto error; \ goto error; \
} \ } \
*num = value; \ *num = value; \
...@@ -76,4 +88,103 @@ void outputbyte(uint8_t ch) ...@@ -76,4 +88,103 @@ void outputbyte(uint8_t ch)
else printf("%02x", ch); else printf("%02x", ch);
} }
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void outputplaintextstring(char* str)
{
bool old_plaintext = g.plaintext;
g.plaintext = true;
outputstring(str);
g.plaintext = old_plaintext;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
ERROR(errno, "printabletocstring");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
ERROR(0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
ERROR(0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
ERROR(0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int verify_library_version()
{
int major;
int minor;
db_version(&major, &minor, NULL);
if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) {
ERRORX("version %d.%d doesn't match library version %d.%d\n",
DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
#endif /* #if !defined(TOKUDB_COMMON_H) */ #endif /* #if !defined(TOKUDB_COMMON_H) */
utils/tokudb_dump.c
View file @ 807296a4
...@@ -9,69 +9,39 @@ ...@@ -9,69 +9,39 @@
#include <db.h> #include <db.h>
#include "tokudb_common.h" #include "tokudb_common.h"
typedef struct {
char* data[2];
} gdbt;
typedef struct {
char* data;
} rhdr;
typedef struct { typedef struct {
bool leadingspace; bool leadingspace;
bool plaintext; bool plaintext;
bool header; bool header;
bool keys; bool footer;
bool is_private; bool is_private;
bool file_error;
char* progname; char* progname;
char* homedir; char* homedir;
char* database; char* database;
char* subdatabase; char* subdatabase;
char** config_options;
int32_t version;
int exitcode; int exitcode;
int recover_flags;
DBTYPE dbtype; DBTYPE dbtype;
DBTYPE opened_dbtype;
DB* db; DB* db;
DB_ENV* dbenv; DB_ENV* dbenv;
struct {
char* data[2];
} dump_dbt;
struct {
char* data;
} dump_header;
} dump_globals; } dump_globals;
dump_globals g; dump_globals g;
#include "tokudb_common_funcs.h" #include "tokudb_common_funcs.h"
int usage (); int usage ();
int longusage ();
int dump_database ();
int create_init_env(); int create_init_env();
int dump_header (); int dump_database ();
int dump_footer ();
int open_database (); int open_database ();
int dump_pairs (); int dump_pairs ();
int apply_commandline_options(); int dump_footer ();
int dump_header ();
int close_database (); int close_database ();
int hextoint (int ch);
int doublechararray(char** pmem, uint64_t* size);
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
int ch; int ch;
int retval; int retval;
struct option options[] = {
{ "help", no_argument, NULL, 'H' },
{ "plain_text", no_argument, NULL, 'T' },
{ "Version", no_argument, NULL, 'V' },
{ "config", required_argument, NULL, 'c' },
{ "input_file", required_argument, NULL, 'f' },
{ "home", required_argument, NULL, 'h' },
{ "password", required_argument, NULL, 'p' },
{ NULL, 0, NULL, 0 }
};
char** next_config_option;
/* Set up the globals. */ /* Set up the globals. */
memset(&g, 0, sizeof(g)); memset(&g, 0, sizeof(g));
...@@ -79,32 +49,57 @@ int main(int argc, char *argv[]) { ...@@ -79,32 +49,57 @@ int main(int argc, char *argv[]) {
g.dbtype = DB_UNKNOWN; g.dbtype = DB_UNKNOWN;
g.progname = argv[0]; g.progname = argv[0];
g.header = true; g.header = true;
g.footer = true;
next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*)); if (verify_library_version() != 0) goto error;
if (next_config_option == NULL) {
fprintf(stderr, "%s: %s\n", g.progname, strerror(errno));
goto error;
}
while ((ch = getopt_long_only(argc, argv, "HTVc:f:h:p:t:", options, NULL)) != EOF) { while ((ch = getopt(argc, argv, "klNpRrVd:f:h:P:s:")) != EOF) {
switch (ch) { switch (ch) {
case ('H'): { case ('k'): {
g.exitcode = longusage(); ERRORX("-%c option not supported.\n", ch);
goto cleanup; goto error;
}
case ('l'): {
//TODO: Implement (Requires master database support)
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('N'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
} }
case ('T'): { case ('T'): {
g.plaintext = true; g.plaintext = true;
g.leadingspace = false; g.leadingspace = false;
g.header = false; g.header = false;
g.footer = false;
break; break;
} }
case ('V'): { case ('p'): {
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch); g.plaintext = true;
break;
}
case ('R'): {
g.recover_flags |= DB_SALVAGE | DB_AGGRESSIVE;
//TODO: Implement aggressive recovery (requires db->verify())
ERRORX("-%c option not supported.\n", ch);
goto error; goto error;
} }
case ('c'): { case ('r'): {
*next_config_option++ = optarg; g.recover_flags |= DB_SALVAGE;
break;
//TODO: Implement recovery (requires db->verify())
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
goto cleanup;
}
case ('d'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
} }
case ('f'): { case ('f'): {
if (freopen(optarg, "w", stdout) == NULL) { if (freopen(optarg, "w", stdout) == NULL) {
...@@ -119,10 +114,14 @@ int main(int argc, char *argv[]) { ...@@ -119,10 +114,14 @@ int main(int argc, char *argv[]) {
g.homedir = optarg; g.homedir = optarg;
break; break;
} }
case ('p'): { case ('P'): {
/* Clear password. */ /* Clear password. */
memset(optarg, 0, strlen(optarg)); memset(optarg, 0, strlen(optarg));
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch); ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('s'): {
g.subdatabase = optarg;
goto error; goto error;
} }
case ('?'): case ('?'):
...@@ -134,6 +133,18 @@ int main(int argc, char *argv[]) { ...@@ -134,6 +133,18 @@ int main(int argc, char *argv[]) {
} }
argc -= optind; argc -= optind;
argv += optind; argv += optind;
if (g.plaintext) g.recover_flags |= DB_PRINTABLE;
if (g.subdatabase != NULL && IS_SET_ALL(g.recover_flags, DB_SALVAGE)) {
if (IS_SET_ALL(g.recover_flags, DB_AGGRESSIVE)) {
ERRORX("The -s and -R options may not both be specified.\n");
goto error;
}
ERRORX("The -s and -r options may not both be specified.\n");
goto error;
}
if (argc != 1) { if (argc != 1) {
g.exitcode = usage(); g.exitcode = usage();
...@@ -143,7 +154,6 @@ int main(int argc, char *argv[]) { ...@@ -143,7 +154,6 @@ int main(int argc, char *argv[]) {
g.database = argv[0]; g.database = argv[0];
if (create_init_env() != 0) goto error; if (create_init_env() != 0) goto error;
//TODO: Figure out how to dump multiple databases at once.
if (dump_database() != 0) goto error; if (dump_database() != 0) goto error;
if (false) { if (false) {
error: error:
...@@ -153,52 +163,36 @@ error: ...@@ -153,52 +163,36 @@ error:
cleanup: cleanup:
if (g.dbenv && (retval = g.dbenv->close(g.dbenv, 0)) != 0) { if (g.dbenv && (retval = g.dbenv->close(g.dbenv, 0)) != 0) {
g.exitcode = EXIT_FAILURE; g.exitcode = EXIT_FAILURE;
fprintf(stderr, "%s: dbenv->close: %s\n", g.progname, db_strerror(retval)); fprintf(stderr, "%s: %s: dbenv->close\n", g.progname, db_strerror(retval));
} }
//TODO: /* Resend any caught signal. */ //TODO: /* Resend any caught signal. */
if (g.config_options) free(g.config_options);
if (g.subdatabase) free(g.subdatabase); if (g.subdatabase) free(g.subdatabase);
// if (g.read_header.data) free(g.read_header.data);
// if (g.get_dbt.data[0]) free(g.get_dbt.data[0]);
// if (g.get_dbt.data[1]) free(g.get_dbt.data[1]);
return g.exitcode; return g.exitcode;
} }
int dump_database() int dump_database()
{ {
DB_ENV* dbenv = g.dbenv;
int retval; int retval;
/* Create a database handle. */ /* Create a database handle. */
retval = db_create(&g.db, g.dbenv, 0); retval = db_create(&g.db, g.dbenv, 0);
if (retval != 0) { if (retval != 0) {
dbenv->err(dbenv, retval, "db_create"); ERROR(retval, "db_create");
return EXIT_FAILURE; return EXIT_FAILURE;
} }
if (apply_commandline_options() != 0) goto error;
if (g.file_error) goto cleanup;
/*
//TODO: Uncomment.. implement get_type, and verify the internal type is btree.
if (g.dbtype == DB_UNKNOWN) {
dbenv->err(dbenv, 0, "no database type specified");
goto error;
}
*/
/* /*
TODO: If/when supporting encryption TODO: If/when supporting encryption
if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) { if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
dbenv->err(dbenv, ret, "DB->set_flags: DB_ENCRYPT"); ERROR(ret, "DB->set_flags: DB_ENCRYPT");
goto error; goto error;
} }
*/ */
//TODO: dump_header()
if (open_database() != 0) goto error; if (open_database() != 0) goto error;
if (g.header && dump_header() != 0) goto error;
if (dump_pairs() != 0) goto error; if (dump_pairs() != 0) goto error;
//TODO: dump_footer() if (g.footer && dump_footer() != 0) goto error;
if (g.file_error) goto cleanup;
if (false) { if (false) {
error: error:
...@@ -213,20 +207,10 @@ cleanup: ...@@ -213,20 +207,10 @@ cleanup:
int usage() int usage()
{ {
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__); fprintf(stderr,
printf "usage: %s [-klNprRV] [-f output_file] [-h home]\n"
( " [-s subdatabase] db_file\n",
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n" g.progname);
" -m minsize -M maxsize [-r random seed]\n"
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n",
g.progname
);
return EXIT_FAILURE;
}
int longusage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -264,20 +248,20 @@ int create_init_env() ...@@ -264,20 +248,20 @@ int create_init_env()
///TODO: UNCOMMENT/IMPLEMENT ///TODO: UNCOMMENT/IMPLEMENT
retval = dbenv->set_cachesize(dbenv, 0, cache, 1); retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
if (retval) { if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->set_cachesize"); ERROR(retval, "DB_ENV->set_cachesize");
goto error; goto error;
} }
*/ */
g.is_private = true; g.is_private = true;
//TODO: Do we want to support transactions/logging even in single-process mode? //TODO: Do we want to support transactions even in single-process mode?
//Maybe if the db already exists. //Logging is not necessary.. this is read-only.
//If db does not exist.. makes sense not to log or have transactions //However, do we need to use DB_INIT_LOG to join a logging environment?
REMOVE_BITS(flags, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN); REMOVE_BITS(flags, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN);
SET_BITS(flags, DB_CREATE | DB_PRIVATE); SET_BITS(flags, DB_CREATE | DB_PRIVATE);
retval = dbenv->open(dbenv, g.homedir, flags, 0); retval = dbenv->open(dbenv, g.homedir, flags, 0);
if (retval) { if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->open"); ERROR(retval, "DB_ENV->open");
goto error; goto error;
} }
success: success:
...@@ -288,219 +272,52 @@ error: ...@@ -288,219 +272,52 @@ error:
return EXIT_FAILURE; return EXIT_FAILURE;
} }
int printabletocstring(char* inputstr, char** poutputstr) #define DUMP_FLAG(bit, dump) if (IS_SET_ALL(flags, bit)) printf(dump);
{
char highch;
char lowch;
char nextch;
char* cstring;
DB_ENV* dbenv = g.dbenv;
#define DUMP_IGNORED_FLAG(bit, dump)
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char)); int dump_header()
if (cstring == NULL) { {
dbenv->err(dbenv, ENOMEM, ""); u_int32_t flags;
int retval;
DB* db = g.db;
assert(g.header);
printf("VERSION=3\n");
printf("format=%s\n", g.plaintext ? "print" : "bytevalue");
assert(g.dbtype == DB_BTREE || (g.dbtype == DB_UNKNOWN && g.opened_dbtype == DB_BTREE));
printf("type=btree\n");
//TODO: Get page size from db. Currently tokudb does not support db->get_pagesize.
printf("db_pagesize=4096\n");
if (g.subdatabase) {
printf("subdatabase=");
outputplaintextstring(g.subdatabase);
printf("\n");
}
if ((retval = db->get_flags(db, &flags)) != 0) {
ERROR(retval, "DB->get_flags");
goto error; goto error;
} }
DUMP_IGNORED_FLAG(DB_CHKSUM, "chksum=1");
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) { DUMP_FLAG( DB_DUP, "duplicates=1");
if (*inputstr == '\\') { DUMP_IGNORED_FLAG(DB_DUPSORT, "dupsort=1");
if ((highch = *++inputstr) == '\\') { DUMP_IGNORED_FLAG(DB_RECNUM, "recnum=1");
*cstring++ = '\\'; printf("HEADER=END\n");
continue;
} if (ferror(stdout)) goto error;
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
dbenv->err(dbenv, 0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
dbenv->err(dbenv, 0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS; return EXIT_SUCCESS;
error: error:
dbenv->err(dbenv, 0, "Quitting out due to errors.\n");
return EXIT_FAILURE; return EXIT_FAILURE;
} }
///TODO: IMPLEMENT/Replace original line. int dump_footer()
#define PARSE_NUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
/*if ((retval = dbfunction(db, num)) != 0) goto printerror;*/ \
continue; \
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
goto error; \
}
#define PARSE_FLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
if ((retval = db->set_flags(db, flag)) != 0) { \
dbenv->err(dbenv, retval, \
"set_flags: %s", \
field); \
goto error; \
} \
continue; \
}
#define PARSE_UNSUPPORTEDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
goto error; \
}
#define PARSE_CHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
dbenv->err(dbenv, 0, \
"%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
if ((retval = dbfunction(db, value[0])) != 0) { \
goto printerror; \
} \
continue; \
}
int dump_header()
{
//TODO: All of this.. have not started.
return 0;
}
int apply_commandline_options()
{ {
char** next_config_option = g.config_options; printf("DATA=END\n");
unsigned index; if (ferror(stdout)) goto error;
char* field;
char* value = NULL;
bool first;
int ch;
int32_t num;
int retval;
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
assert(g.header);
for (index = 0; g.config_options[index]; index++) {
if (value) {
/* Restore the field=value format. */
value[-1] = '=';
value = NULL;
}
field = g.config_options[index];
if ((value = strchr(field, '=')) == NULL) {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
goto error;
}
value[0] = '\0';
value++;
if (field[0] == '\0' || value[0] == '\0') {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
goto error;
}
if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) {
if (g.subdatabase != NULL) {
free(g.subdatabase);
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
goto error;
}
continue;
}
/*
///TODO: UNCOMMENT/IMPLEMENT
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
*/
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
if (value) {
/* Restore the field=value format. */
value[-1] = '=';
value = NULL;
}
return EXIT_SUCCESS; return EXIT_SUCCESS;
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
}
error: error:
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -508,77 +325,33 @@ error: ...@@ -508,77 +325,33 @@ error:
int open_database() int open_database()
{ {
DB* db = g.db; DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval; int retval;
int open_flags = DB_RDONLY; int open_flags = DB_RDONLY;
//TODO: Transaction auto commit stuff //TODO: Transaction auto commit stuff
//if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT); //if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT);
//TODO: First see if it exists.. THEN create it?
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666); retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) { if (retval != 0) {
dbenv->err(dbenv, retval, "DB->open: %s", g.database); ERROR(retval, "DB->open: %s", g.database);
goto error; goto error;
} }
//TODO: Ensure we have enough cache to store some min number of btree pages. retval = db->get_type(db, &g.opened_dbtype);
//NOTE: This may require closing db, environment, and creating new ones.
/*
///TODO: UNCOMMENT/IMPLEMENT
DBTYPE existingtype;
retval = db->get_type(db, &existingtype);
if (retval != 0) { if (retval != 0) {
dbenv->err(dbenv, retval, "DB->get_type: %s", g.database); ERROR(retval, "DB->get_type");
goto error; goto error;
} }
assert(g.dbtype == DB_BTREE); if (g.opened_dbtype != DB_BTREE) {
if (existingtype != g.dbtype) { ERRORX("Unsupported db type %d\n", g.opened_dbtype);
fprintf(stderr, "Existing database is not a dictionary (DB_BTREE).\n");
goto error; goto error;
} }
*/ if (g.dbtype != DB_UNKNOWN && g.opened_dbtype != g.dbtype) {
return EXIT_SUCCESS; ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, g.opened_dbtype);
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int doublechararray(char** pmem, uint64_t* size)
{
DB_ENV* dbenv = g.dbenv;
assert(pmem);
assert(size);
assert(IS_POWER_OF_2(*size));
*size <<= 1;
if (*size == 0) {
/* Overflowed uint64_t. */
dbenv->err(dbenv, 0, "doublechararray\n");
goto error;
}
if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) {
dbenv->err(dbenv, errno, "");
goto error; goto error;
} }
return EXIT_SUCCESS; return EXIT_SUCCESS;
error: error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -615,15 +388,14 @@ int dump_pairs() ...@@ -615,15 +388,14 @@ int dump_pairs()
DBT key; DBT key;
DBT data; DBT data;
int spacech; int spacech;
DB* db = g.db;/////////////// DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
DBC* dbc = NULL; DBC* dbc = NULL;
memset(&key, 0, sizeof(key)); memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data)); memset(&data, 0, sizeof(data));
if ((retval = db->cursor(db, (DB_TXN*)NULL, &dbc, 0)) != 0) { if ((retval = db->cursor(db, (DB_TXN*)NULL, &dbc, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->cursor"); ERROR(retval, "DB->cursor");
goto error; goto error;
} }
while ((retval = dbc->c_get(dbc, &key, &data, DB_NEXT)) == 0) { while ((retval = dbc->c_get(dbc, &key, &data, DB_NEXT)) == 0) {
...@@ -631,7 +403,7 @@ int dump_pairs() ...@@ -631,7 +403,7 @@ int dump_pairs()
if (dump_dbt(&data) != 0) goto error; if (dump_dbt(&data) != 0) goto error;
} }
if (retval != DB_NOTFOUND) { if (retval != DB_NOTFOUND) {
dbenv->err(dbenv, retval, "DBC->c_get"); ERROR(retval, "DBC->c_get");
goto error; goto error;
} }
...@@ -642,7 +414,7 @@ error: ...@@ -642,7 +414,7 @@ error:
} }
cleanup: cleanup:
if (dbc && (retval = dbc->c_close(dbc)) != 0) { if (dbc && (retval = dbc->c_close(dbc)) != 0) {
dbenv->err(dbenv, retval, "DBC->c_close"); ERROR(retval, "DBC->c_close");
g.exitcode = EXIT_FAILURE; g.exitcode = EXIT_FAILURE;
} }
success: success:
...@@ -652,12 +424,11 @@ success: ...@@ -652,12 +424,11 @@ success:
int close_database() int close_database()
{ {
DB* db = g.db; DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval; int retval;
assert(db); assert(db);
if ((retval = db->close(db, 0)) != 0) { if ((retval = db->close(db, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->close"); ERROR(retval, "DB->close");
goto error; goto error;
} }
return EXIT_SUCCESS; return EXIT_SUCCESS;
......
utils/tokudb_gen.c
View file @ 807296a4
...@@ -5,17 +5,15 @@ ...@@ -5,17 +5,15 @@
#include <string.h> #include <string.h>
#include <ctype.h> #include <ctype.h>
#include <errno.h> #include <errno.h>
#include <getopt.h>
#include <db.h>
#include "tokudb_common.h" #include "tokudb_common.h"
extern char* optarg;
extern int optind;
extern int optopt;
extern int opterr;
extern int optreset;
typedef struct { typedef struct {
DB_ENV* dbenv;
bool plaintext; bool plaintext;
char* progname;
} gen_globals; } gen_globals;
gen_globals g; gen_globals g;
...@@ -29,8 +27,6 @@ int get_delimiter(char* str); ...@@ -29,8 +27,6 @@ int get_delimiter(char* str);
char dbt_delimiter = '\n'; char dbt_delimiter = '\n';
char sort_delimiter[2]; char sort_delimiter[2];
char* progname;
uint32_t lengthmin = 0; uint32_t lengthmin = 0;
bool set_lengthmin = false; bool set_lengthmin = false;
uint32_t lengthlimit = 0; uint32_t lengthlimit = 0;
...@@ -54,10 +50,13 @@ int main (int argc, char *argv[]) { ...@@ -54,10 +50,13 @@ int main (int argc, char *argv[]) {
/* Set up the globals. */ /* Set up the globals. */
memset(&g, 0, sizeof(g)); memset(&g, 0, sizeof(g));
progname = argv[0]; g.progname = argv[0];
if (verify_library_version() != 0) goto error;
strcpy(sort_delimiter, ""); strcpy(sort_delimiter, "");
while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:o:")) != EOF) { while ((ch = getopt(argc, argv, "PfFhHTpVr:s:d:m:M:n:o:")) != EOF) {
switch (ch) { switch (ch) {
case ('P'): { case ('P'): {
printableonly = true; printableonly = true;
...@@ -80,22 +79,20 @@ int main (int argc, char *argv[]) { ...@@ -80,22 +79,20 @@ int main (int argc, char *argv[]) {
break; break;
} }
case ('T'): { case ('T'): {
g.plaintext = true; g.plaintext = true;
leadingspace = false; leadingspace = false;
header = false; header = false;
footer = false; footer = false;
break; break;
} }
case ('p'): { case ('p'): {
g.plaintext = true; g.plaintext = true;
leadingspace = true; leadingspace = true;
break; break;
} }
case ('o'): { case ('o'): {
if (freopen(optarg, "w", stdout) == NULL) { if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, ERROR(errno, "%s: reopen\n", optarg);
"%s: %s: reopen: %s\n",
progname, optarg, strerror(errno));
goto error; goto error;
} }
break; break;
...@@ -103,15 +100,13 @@ int main (int argc, char *argv[]) { ...@@ -103,15 +100,13 @@ int main (int argc, char *argv[]) {
case ('d'): { case ('d'): {
int temp = get_delimiter(optarg); int temp = get_delimiter(optarg);
if (temp == EOF) { if (temp == EOF) {
fprintf(stderr, ERRORX("%s: (-d) Key (or value) delimiter must be one character.",
"%s: %s: (-d) Key (or value) delimiter must be one character.", optarg);
progname, optarg);
goto error; goto error;
} }
if (isxdigit(temp)) { if (isxdigit(temp)) {
fprintf(stderr, ERRORX("%c: (-d) Key (or value) delimiter cannot be a hex digit.",
"%s: %c: (-d) Key (or value) delimiter cannot be a hex digit.", temp);
progname, temp);
goto error; goto error;
} }
dbt_delimiter = (char)temp; dbt_delimiter = (char)temp;
...@@ -120,15 +115,13 @@ int main (int argc, char *argv[]) { ...@@ -120,15 +115,13 @@ int main (int argc, char *argv[]) {
case ('s'): { case ('s'): {
int temp = get_delimiter(optarg); int temp = get_delimiter(optarg);
if (temp == EOF) { if (temp == EOF) {
fprintf(stderr, ERRORX("%s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.", optarg);
progname, optarg);
goto error; goto error;
} }
if (isxdigit(temp)) { if (isxdigit(temp)) {
fprintf(stderr, ERRORX("%c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
"%s: %c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.", temp);
progname, temp);
goto error; goto error;
} }
sort_delimiter[0] = (char)temp; sort_delimiter[0] = (char)temp;
...@@ -136,45 +129,41 @@ int main (int argc, char *argv[]) { ...@@ -136,45 +129,41 @@ int main (int argc, char *argv[]) {
break; break;
} }
case ('r'): { case ('r'): {
if (strtouint32(NULL, progname, optarg, &seed, 0, UINT32_MAX, 10)) { if (strtouint32(optarg, &seed, 0, UINT32_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-r) Random seed invalid.", optarg);
"%s: %s: (-r) Random seed invalid.",
progname, optarg);
goto error; goto error;
} }
set_seed = true; set_seed = true;
break; break;
} }
case ('m'): { case ('m'): {
if (strtouint32(NULL, progname, optarg, &lengthmin, 0, UINT32_MAX, 10)) { if (strtouint32(optarg, &lengthmin, 0, UINT32_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-m) Min length of keys/values invalid.", optarg);
"%s: %s: (-m) Min length of keys/values invalid.",
progname, optarg);
goto error; goto error;
} }
set_lengthmin = true; set_lengthmin = true;
break; break;
} }
case ('M'): { case ('M'): {
if (strtouint32(NULL, progname, optarg, &lengthlimit, 1, UINT32_MAX, 10)) { if (strtouint32(optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-M) Limit of key/value length invalid.", optarg);
"%s: %s: (-M) Limit of key/value length invalid.",
progname, optarg);
goto error; goto error;
} }
set_lengthlimit = true; set_lengthlimit = true;
break; break;
} }
case ('n'): { case ('n'): {
if (strtouint64(NULL, progname, optarg, &numkeys, 0, UINT64_MAX, 10)) { if (strtouint64(optarg, &numkeys, 0, UINT64_MAX, 10)) {
fprintf(stderr, ERRORX("%s: (-n) Number of keys to generate invalid.", optarg);
"%s: %s: (-n) Number of keys to generate invalid.",
progname, optarg);
goto error; goto error;
} }
set_numkeys = true; set_numkeys = true;
break; break;
} }
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
return EXIT_SUCCESS;
}
case ('?'): case ('?'):
default: { default: {
return (usage()); return (usage());
...@@ -185,61 +174,55 @@ int main (int argc, char *argv[]) { ...@@ -185,61 +174,55 @@ int main (int argc, char *argv[]) {
argv += optind; argv += optind;
if (justheader && !header) { if (justheader && !header) {
fprintf(stderr, ERRORX("The -h and -H options may not both be specified.\n");
"%s: The -h and -H options may not both be specified.\n",
progname);
goto error; goto error;
} }
if (justfooter && !footer) { if (justfooter && !footer) {
fprintf(stderr, ERRORX("The -f and -F options may not both be specified.\n");
"%s: The -f and -F options may not both be specified.\n",
progname);
goto error; goto error;
} }
if (justfooter && justheader) { if (justfooter && justheader) {
fprintf(stderr, ERRORX("The -H and -F options may not both be specified.\n");
"%s: The -H and -F options may not both be specified.\n",
progname);
goto error; goto error;
} }
if (justfooter && header) { if (justfooter && header) {
fprintf(stderr, "%s: -F implies -h\n", progname); ERRORX("-F implies -h\n");
header = false; header = false;
} }
if (justheader && footer) { if (justheader && footer) {
fprintf(stderr, "%s: -H implies -f\n", progname); ERRORX("-H implies -f\n");
footer = false; footer = false;
} }
if (!leadingspace) { if (!leadingspace) {
if (footer) { if (footer) {
fprintf(stderr, "%s: -p implies -f\n", progname); ERRORX("-p implies -f\n");
footer = false; footer = false;
} }
if (header) { if (header) {
fprintf(stderr, "%s: -p implies -h\n", progname); ERRORX("-p implies -h\n");
header = false; header = false;
} }
} }
if (justfooter || justheader) outputkeys = false; if (justfooter || justheader) outputkeys = false;
else if (!set_numkeys) else if (!set_numkeys)
{ {
fprintf(stderr, "%s: The -n option is required.\n", progname); ERRORX("The -n option is required.\n");
goto error; goto error;
} }
if (outputkeys && !set_seed) { if (outputkeys && !set_seed) {
fprintf(stderr, "%s: Using default seed. (-r 1).\n", progname); ERRORX("Using default seed. (-r 1).\n");
seed = 1; seed = 1;
} }
if (outputkeys && !set_lengthmin) { if (outputkeys && !set_lengthmin) {
fprintf(stderr, "%s: Using default lengthmin. (-m 0).\n", progname); ERRORX("Using default lengthmin. (-m 0).\n");
lengthmin = 0; lengthmin = 0;
} }
if (outputkeys && !set_lengthlimit) { if (outputkeys && !set_lengthlimit) {
fprintf(stderr, "%s: Using default lengthlimit. (-M 1024).\n", progname); ERRORX("Using default lengthlimit. (-M 1024).\n");
lengthlimit = 1024; lengthlimit = 1024;
} }
if (outputkeys && lengthmin >= lengthlimit) { if (outputkeys && lengthmin >= lengthlimit) {
fprintf(stderr, "%s: Max key size must be greater than min key size.\n", progname); ERRORX("Max key size must be greater than min key size.\n");
goto error; goto error;
} }
...@@ -256,7 +239,7 @@ int main (int argc, char *argv[]) { ...@@ -256,7 +239,7 @@ int main (int argc, char *argv[]) {
} }
if (outputkeys) generate_keys(); if (outputkeys) generate_keys();
if (footer) printf("DATA=END\n"); if (footer) printf("DATA=END\n");
return 0; return EXIT_SUCCESS;
error: error:
fprintf(stderr, "Quitting out due to errors.\n"); fprintf(stderr, "Quitting out due to errors.\n");
...@@ -266,10 +249,10 @@ error: ...@@ -266,10 +249,10 @@ error:
int usage() int usage()
{ {
fprintf(stderr, fprintf(stderr,
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n" "usage: %s [-PfFhHTpV] [-r random seed] [-s delimiter] \n"
" [-m lengthmin] [-M lengthlimit] [-r random seed]\n" " [-d delimiter] [-m lengthmin] [-M lengthlimit] \n"
" [-o filename] -n numkeys\n", " -n numkeys [-o output_file] \n",
progname); g.progname);
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -296,15 +279,6 @@ int32_t random_below(int32_t limit) ...@@ -296,15 +279,6 @@ int32_t random_below(int32_t limit)
return random() % limit; return random() % limit;
} }
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void generate_keys() void generate_keys()
{ {
bool usedemptykey = false; bool usedemptykey = false;
......
utils/tokudb_load.c
View file @ 807296a4
...@@ -9,14 +9,6 @@ ...@@ -9,14 +9,6 @@
#include <db.h> #include <db.h>
#include "tokudb_common.h" #include "tokudb_common.h"
typedef struct {
char* data[2];
} gdbt;
typedef struct {
char* data;
} rhdr;
typedef struct { typedef struct {
bool leadingspace; bool leadingspace;
bool plaintext; bool plaintext;
...@@ -56,24 +48,11 @@ int open_database (); ...@@ -56,24 +48,11 @@ int open_database ();
int read_keys (); int read_keys ();
int apply_commandline_options(); int apply_commandline_options();
int close_database (); int close_database ();
int hextoint (int ch);
int doublechararray(char** pmem, uint64_t* size); int doublechararray(char** pmem, uint64_t* size);
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
int ch; int ch;
int retval; int retval;
struct option options[] = {
{ "no_overwrite", no_argument, NULL, 'n' },
{ "help", no_argument, NULL, 'H' },
{ "plain_text", no_argument, NULL, 'T' },
{ "Version", no_argument, NULL, 'V' },
{ "config", required_argument, NULL, 'c' },
{ "input_file", required_argument, NULL, 'f' },
{ "home", required_argument, NULL, 'h' },
{ "password", required_argument, NULL, 'p' },
{ "type", required_argument, NULL, 't' },
{ NULL, 0, NULL, 0 }
};
char** next_config_option; char** next_config_option;
/* Set up the globals. */ /* Set up the globals. */
...@@ -83,24 +62,21 @@ int main(int argc, char *argv[]) { ...@@ -83,24 +62,21 @@ int main(int argc, char *argv[]) {
g.dbtype = DB_UNKNOWN; g.dbtype = DB_UNKNOWN;
g.progname = argv[0]; g.progname = argv[0];
g.header = true; g.header = true;
if (verify_library_version() != 0) goto error;
next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*)); next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*));
if (next_config_option == NULL) { if (next_config_option == NULL) {
fprintf(stderr, "%s: %s\n", g.progname, strerror(errno)); ERROR(errno, "main: calloc\n");
goto error; goto error;
} }
while ((ch = getopt(argc, argv, "nTVc:f:h:p:t:r:")) != EOF) {
while ((ch = getopt_long_only(argc, argv, "nHTVc:f:h:p:t:", options, NULL)) != EOF) {
switch (ch) { switch (ch) {
case ('n'): { case ('n'): {
/* g.overwritekeys = false; */ /* g.overwritekeys = false; */
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch); ERRORX("-%c option not supported.\n", ch);
goto error; goto error;
} }
case ('H'): {
g.exitcode = longusage();
goto cleanup;
}
case ('T'): { case ('T'): {
g.plaintext = true; g.plaintext = true;
g.leadingspace = false; g.leadingspace = false;
...@@ -108,8 +84,8 @@ int main(int argc, char *argv[]) { ...@@ -108,8 +84,8 @@ int main(int argc, char *argv[]) {
break; break;
} }
case ('V'): { case ('V'): {
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch); printf("%s\n", db_version(NULL, NULL, NULL));
goto error; goto cleanup;
} }
case ('c'): { case ('c'): {
*next_config_option++ = optarg; *next_config_option++ = optarg;
...@@ -131,7 +107,7 @@ int main(int argc, char *argv[]) { ...@@ -131,7 +107,7 @@ int main(int argc, char *argv[]) {
case ('p'): { case ('p'): {
/* Clear password. */ /* Clear password. */
memset(optarg, 0, strlen(optarg)); memset(optarg, 0, strlen(optarg));
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch); ERRORX("-%c option not supported.\n", ch);
goto error; goto error;
} }
case ('t'): { case ('t'): {
...@@ -146,6 +122,10 @@ int main(int argc, char *argv[]) { ...@@ -146,6 +122,10 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "%s: Unrecognized db type %s.\n", g.progname, optarg); fprintf(stderr, "%s: Unrecognized db type %s.\n", g.progname, optarg);
goto error; goto error;
} }
case ('r'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('?'): case ('?'):
default: { default: {
g.exitcode = usage(); g.exitcode = usage();
...@@ -195,7 +175,7 @@ int load_database() ...@@ -195,7 +175,7 @@ int load_database()
/* Create a database handle. */ /* Create a database handle. */
retval = db_create(&g.db, g.dbenv, 0); retval = db_create(&g.db, g.dbenv, 0);
if (retval != 0) { if (retval != 0) {
dbenv->err(dbenv, retval, "db_create"); ERROR(retval, "db_create");
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -204,15 +184,10 @@ int load_database() ...@@ -204,15 +184,10 @@ int load_database()
if (apply_commandline_options() != 0) goto error; if (apply_commandline_options() != 0) goto error;
if (g.eof) goto cleanup; if (g.eof) goto cleanup;
//TODO: Only quit out if DB does NOT EXIST.
if (g.dbtype == DB_UNKNOWN) {
dbenv->err(dbenv, 0, "no database type specified");
goto error;
}
/* /*
TODO: If/when supporting encryption TODO: If/when supporting encryption
if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) { if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
dbenv->err(dbenv, ret, "DB->set_flags: DB_ENCRYPT"); ERROR(ret, "DB->set_flags: DB_ENCRYPT");
goto error; goto error;
} }
*/ */
...@@ -234,20 +209,10 @@ cleanup: ...@@ -234,20 +209,10 @@ cleanup:
int usage() int usage()
{ {
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__); fprintf(stderr,
printf "usage: %s [-nTV] [-c name=value] [-f input_file]\n"
( " [-h home] [-t btree] db_file\n",
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n" g.progname);
" -m minsize -M maxsize [-r random seed]\n"
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n",
g.progname
);
return EXIT_FAILURE;
}
int longusage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
return EXIT_FAILURE; return EXIT_FAILURE;
} }
...@@ -269,14 +234,13 @@ int create_init_env() ...@@ -269,14 +234,13 @@ int create_init_env()
/* /*
TODO: If/when supporting encryption TODO: If/when supporting encryption
if (g.password && (retval = dbenv->set_encrypt(dbenv, g.password, DB_ENCRYPT_AES))) { if (g.password && (retval = dbenv->set_encrypt(dbenv, g.password, DB_ENCRYPT_AES))) {
dbenv->err(dbenv, retval, "set_passwd"); ERROR(retval, "set_passwd");
goto error; goto error;
} }
*/ */
/* Open the dbenvironment. */ /* Open the dbenvironment. */
g.is_private = false; g.is_private = false;
// flags = DB_JOINENV | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_USE_ENVIRON;
flags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL; ///TODO: UNCOMMENT/IMPLEMENT | DB_USE_ENVIRON; flags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL; ///TODO: UNCOMMENT/IMPLEMENT | DB_USE_ENVIRON;
//TODO: Transactions.. SET_BITS(flags, DB_INIT_TXN); //TODO: Transactions.. SET_BITS(flags, DB_INIT_TXN);
...@@ -289,7 +253,7 @@ int create_init_env() ...@@ -289,7 +253,7 @@ int create_init_env()
///TODO: UNCOMMENT/IMPLEMENT ///TODO: UNCOMMENT/IMPLEMENT
retval = dbenv->set_cachesize(dbenv, 0, cache, 1); retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
if (retval) { if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->set_cachesize"); ERROR(retval, "DB_ENV->set_cachesize");
goto error; goto error;
} }
*/ */
...@@ -302,7 +266,7 @@ int create_init_env() ...@@ -302,7 +266,7 @@ int create_init_env()
retval = dbenv->open(dbenv, g.homedir ? g.homedir : ".", flags, 0); retval = dbenv->open(dbenv, g.homedir ? g.homedir : ".", flags, 0);
if (retval) { if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->open"); ERROR(retval, "DB_ENV->open");
goto error; goto error;
} }
success: success:
...@@ -313,90 +277,34 @@ error: ...@@ -313,90 +277,34 @@ error:
return EXIT_FAILURE; return EXIT_FAILURE;
} }
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
DB_ENV* dbenv = g.dbenv;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
dbenv->err(dbenv, ENOMEM, "");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
dbenv->err(dbenv, 0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
dbenv->err(dbenv, 0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
dbenv->err(dbenv, 0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_NUMBER(match, dbfunction) \ #define PARSE_NUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \ if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \ if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
/*if ((retval = dbfunction(db, num)) != 0) goto printerror;*/ \ if ((retval = dbfunction(db, num)) != 0) goto printerror; \
continue; \ continue; \
} }
///TODO: IMPLEMENT/Replace original line.
#define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \ #define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \ if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \ if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \ ERRORX("%s option not supported.\n", field); \
goto error; \ goto error; \
} }
#define PARSE_IGNOREDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
ERRORX("%s option not supported yet (ignored).\n", field); \
continue; \
}
#define PARSE_FLAG(match, flag) \ #define PARSE_FLAG(match, flag) \
if (!strcmp(field, match)) { \ if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \ if (strtoint32(value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \ ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
"%s: boolean name=value pairs require a value of 0 or 1", \ field); \
field); \
goto error; \ goto error; \
} \ } \
if ((retval = db->set_flags(db, flag)) != 0) { \ if ((retval = db->set_flags(db, flag)) != 0) { \
dbenv->err(dbenv, retval, \ ERROR(retval, "set_flags: %s", field); \
"set_flags: %s", \
field); \
goto error; \ goto error; \
} \ } \
continue; \ continue; \
...@@ -404,22 +312,31 @@ if (!strcmp(field, match)) { \ ...@@ -404,22 +312,31 @@ if (!strcmp(field, match)) { \
#define PARSE_UNSUPPORTEDFLAG(match, flag) \ #define PARSE_UNSUPPORTEDFLAG(match, flag) \
if (!strcmp(field, match)) { \ if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \ if (strtoint32(value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \ ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
"%s: boolean name=value pairs require a value of 0 or 1", \ field); \
field); \
goto error; \ goto error; \
} \ } \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \ ERRORX("%s option not supported.\n", field); \
goto error; \ goto error; \
} }
#define PARSE_IGNOREDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
ERRORX("%s option not supported yet (ignored).\n", field); \
continue; \
}
#define PARSE_CHAR(match, dbfunction) \ #define PARSE_CHAR(match, dbfunction) \
if (!strcmp(field, match)) { \ if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \ if (strlen(value) != 1) { \
dbenv->err(dbenv, 0, \ ERRORX("%s=%s: Expected 1-byte value", \
"%s=%s: Expected 1-byte value", \ field, value); \
field, value); \
goto error; \ goto error; \
} \ } \
if ((retval = dbfunction(db, value[0])) != 0) { \ if ((retval = dbfunction(db, value[0])) != 0) { \
...@@ -428,6 +345,34 @@ if (!strcmp(field, match)) { \ ...@@ -428,6 +345,34 @@ if (!strcmp(field, match)) { \
continue; \ continue; \
} }
#define PARSE_UNSUPPORTEDCHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
ERRORX("%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_COMMON_CONFIGURATIONS() \
PARSE_IGNOREDNUMBER( "bt_minkey", db->set_bt_minkey); \
PARSE_IGNOREDFLAG( "chksum", DB_CHKSUM); \
PARSE_IGNOREDNUMBER( "db_lorder", db->set_lorder); \
PARSE_IGNOREDNUMBER( "db_pagesize", db->set_pagesize); \
PARSE_FLAG( "duplicates", DB_DUP); \
PARSE_IGNOREDFLAG( "dupsort", DB_DUPSORT); \
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize); \
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor); \
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem); \
PARSE_UNSUPPORTEDNUMBER("re_len", db->set_re_len); \
PARSE_UNSUPPORTEDCHAR( "re_pad", db->set_re_pad); \
PARSE_IGNOREDFLAG( "recnum", DB_RECNUM); \
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
int read_header() int read_header()
{ {
static uint64_t datasize = 1 << 10; static uint64_t datasize = 1 << 10;
...@@ -443,7 +388,7 @@ int read_header() ...@@ -443,7 +388,7 @@ int read_header()
assert(g.header); assert(g.header);
if (g.read_header.data == NULL && (g.read_header.data = (char*)malloc(datasize * sizeof(char))) == NULL) { if (g.read_header.data == NULL && (g.read_header.data = (char*)malloc(datasize * sizeof(char))) == NULL) {
dbenv->err(dbenv, errno, ""); ERROR(errno, "read_header: malloc");
goto error; goto error;
} }
while (!g.eof) { while (!g.eof) {
...@@ -453,7 +398,6 @@ int read_header() ...@@ -453,7 +398,6 @@ int read_header()
while (true) { while (true) {
if ((ch = getchar()) == EOF) { if ((ch = getchar()) == EOF) {
g.eof = true; g.eof = true;
//TODO: Check ferror for every EOF test.
if (ferror(stdin)) goto formaterror; if (ferror(stdin)) goto formaterror;
break; break;
} }
...@@ -477,9 +421,9 @@ int read_header() ...@@ -477,9 +421,9 @@ int read_header()
if (!strcmp(field, "HEADER")) break; if (!strcmp(field, "HEADER")) break;
if (!strcmp(field, "VERSION")) { if (!strcmp(field, "VERSION")) {
if (strtoint32(dbenv, NULL, value, &g.version, 1, INT32_MAX, 10)) goto error; if (strtoint32(value, &g.version, 1, INT32_MAX, 10)) goto error;
if (g.version != 3) { if (g.version != 3) {
dbenv->err(dbenv, 0, "line %lu: VERSION %d is unsupported", g.linenumber, g.version); ERRORX("line %llu: VERSION %d is unsupported", g.linenumber, g.version);
goto error; goto error;
} }
continue; continue;
...@@ -501,54 +445,41 @@ int read_header() ...@@ -501,54 +445,41 @@ int read_header()
continue; continue;
} }
if (!strcmp(value, "hash") || strcmp(value, "recno") || strcmp(value, "queue")) { if (!strcmp(value, "hash") || strcmp(value, "recno") || strcmp(value, "queue")) {
dbenv->err(dbenv, 0, "db type %s not supported.\n", value); ERRORX("db type %s not supported.\n", value);
goto error; goto error;
} }
dbenv->err(dbenv, 0, "line %lu: unknown type %s", g.linenumber, value); ERRORX("line %llu: unknown type %s", g.linenumber, value);
goto error; goto error;
} }
if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) { if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) {
if (g.subdatabase != NULL) { if (g.subdatabase != NULL) {
//TODO: Free before quitting main., clear at start too.. could be a new db without a name?
free(g.subdatabase); free(g.subdatabase);
g.subdatabase = NULL; g.subdatabase = NULL;
} }
if ((retval = printabletocstring(value, &g.subdatabase))) { if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name"); ERROR(retval, "error reading db name");
goto error; goto error;
} }
continue; continue;
} }
if (!strcmp(field, "keys")) { if (!strcmp(field, "keys")) {
int32_t temp; int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) { if (strtoint32(value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0, ERROR(0,
"%s: boolean name=value pairs require a value of 0 or 1", "%s: boolean name=value pairs require a value of 0 or 1",
field); field);
goto error; goto error;
} }
g.keys = temp; g.keys = temp;
if (!g.keys) { if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field); ERRORX("keys=0 not supported");
goto error; goto error;
} }
continue; continue;
} }
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey); PARSE_COMMON_CONFIGURATIONS();
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field); ERRORX("unknown input-file header configuration keyword \"%s\"", field);
goto error; goto error;
} }
success: success:
...@@ -556,11 +487,11 @@ success: ...@@ -556,11 +487,11 @@ success:
if (false) { if (false) {
printerror: printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value); ERROR(retval, "%s=%s", field, value);
} }
if (false) { if (false) {
formaterror: formaterror:
dbenv->err(dbenv, 0, "line %lu: unexpected format", g.linenumber); ERRORX("line %llu: unexpected format", g.linenumber);
} }
error: error:
return EXIT_FAILURE; return EXIT_FAILURE;
...@@ -590,14 +521,14 @@ int apply_commandline_options() ...@@ -590,14 +521,14 @@ int apply_commandline_options()
field = g.config_options[index]; field = g.config_options[index];
if ((value = strchr(field, '=')) == NULL) { if ((value = strchr(field, '=')) == NULL) {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format"); ERRORX("command-line configuration uses name=value format");
goto error; goto error;
} }
value[0] = '\0'; value[0] = '\0';
value++; value++;
if (field[0] == '\0' || value[0] == '\0') { if (field[0] == '\0' || value[0] == '\0') {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format"); ERRORX("command-line configuration uses name=value format");
goto error; goto error;
} }
...@@ -607,44 +538,29 @@ int apply_commandline_options() ...@@ -607,44 +538,29 @@ int apply_commandline_options()
g.subdatabase = NULL; g.subdatabase = NULL;
} }
if ((retval = printabletocstring(value, &g.subdatabase))) { if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name"); ERROR(retval, "error reading db name");
goto error; goto error;
} }
continue; continue;
} }
if (!strcmp(field, "keys")) { if (!strcmp(field, "keys")) {
int32_t temp; int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) { if (strtoint32(value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0, ERROR(0,
"%s: boolean name=value pairs require a value of 0 or 1", "%s: boolean name=value pairs require a value of 0 or 1",
field); field);
goto error; goto error;
} }
g.keys = temp; g.keys = temp;
if (!g.keys) { if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field); ERRORX("keys=0 not supported");
goto error; goto error;
} }
continue; continue;
} }
/* PARSE_COMMON_CONFIGURATIONS();
///TODO: UNCOMMENT/IMPLEMENT
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
*/
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field); ERRORX("unknown input-file header configuration keyword \"%s\"", field);
goto error; goto error;
} }
if (value) { if (value) {
...@@ -656,7 +572,7 @@ int apply_commandline_options() ...@@ -656,7 +572,7 @@ int apply_commandline_options()
if (false) { if (false) {
printerror: printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value); ERROR(retval, "%s=%s", field, value);
} }
error: error:
return EXIT_FAILURE; return EXIT_FAILURE;
...@@ -667,54 +583,46 @@ int open_database() ...@@ -667,54 +583,46 @@ int open_database()
DB* db = g.db; DB* db = g.db;
DB_ENV* dbenv = g.dbenv; DB_ENV* dbenv = g.dbenv;
int retval; int retval;
DBTYPE opened_type;
int open_flags = DB_CREATE; int open_flags = 0;
//TODO: Transaction auto commit stuff //TODO: Transaction auto commit stuff
//if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT); //if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT);
//TODO: First see if it exists.. THEN create it?
//Try to see if it exists first.
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666); retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) { if (retval != 0) {
dbenv->err(dbenv, retval, "DB->open: %s", g.database); //Does not exist and we did not specify a type.
if (g.dbtype == DB_UNKNOWN) {
ERRORX("no database type specified");
goto error;
}
SET_BITS(open_flags, DB_CREATE);
//Try creating it.
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
ERROR(retval, "DB->open: %s", g.database);
goto error;
}
}
if ((retval = db->get_type(db, &opened_type)) != 0) {
ERROR(retval, "DB->get_type");
goto error; goto error;
} }
//TODO: Ensure we have enough cache to store some min number of btree pages. if (opened_type != DB_BTREE) {
//NOTE: This may require closing db, environment, and creating new ones. ERRORX("Unsupported db type %d\n", opened_type);
/*
///TODO: UNCOMMENT/IMPLEMENT
DBTYPE existingtype;
retval = db->get_type(db, &existingtype);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->get_type: %s", g.database);
goto error; goto error;
} }
assert(g.dbtype == DB_BTREE); if (g.dbtype != DB_UNKNOWN && opened_type != g.dbtype) {
if (existingtype != g.dbtype) { ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, opened_type);
fprintf(stderr, "Existing database is not a dictionary (DB_BTREE).\n");
goto error; goto error;
} }
*/
return EXIT_SUCCESS; return EXIT_SUCCESS;
error: error:
fprintf(stderr, "Quitting out due to errors.\n"); fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE; return EXIT_FAILURE;
} }
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int doublechararray(char** pmem, uint64_t* size) int doublechararray(char** pmem, uint64_t* size)
{ {
DB_ENV* dbenv = g.dbenv; DB_ENV* dbenv = g.dbenv;
...@@ -726,11 +634,11 @@ int doublechararray(char** pmem, uint64_t* size) ...@@ -726,11 +634,11 @@ int doublechararray(char** pmem, uint64_t* size)
*size <<= 1; *size <<= 1;
if (*size == 0) { if (*size == 0) {
/* Overflowed uint64_t. */ /* Overflowed uint64_t. */
dbenv->err(dbenv, 0, "Line %llu: Line too long.\n", g.linenumber); ERRORX("Line %llu: Line too long.\n", g.linenumber);
goto error; goto error;
} }
if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) { if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) {
dbenv->err(dbenv, errno, ""); ERROR(errno, "doublechararray: realloc");
goto error; goto error;
} }
return EXIT_SUCCESS; return EXIT_SUCCESS;
...@@ -755,7 +663,7 @@ int get_dbt(DBT* pdbt) ...@@ -755,7 +663,7 @@ int get_dbt(DBT* pdbt)
which = 1 - which; which = 1 - which;
if (g.get_dbt.data[which] == NULL && if (g.get_dbt.data[which] == NULL &&
(g.get_dbt.data[which] = (char*)malloc(datasize[which] * sizeof(char))) == NULL) { (g.get_dbt.data[which] = (char*)malloc(datasize[which] * sizeof(char))) == NULL) {
dbenv->err(dbenv, errno, ""); ERROR(errno, "get_dbt: malloc");
goto error; goto error;
} }
...@@ -781,22 +689,22 @@ int get_dbt(DBT* pdbt) ...@@ -781,22 +689,22 @@ int get_dbt(DBT* pdbt)
} }
else if (highch == EOF) { else if (highch == EOF) {
g.eof = true; g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber); ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error; goto error;
} }
else if (!isxdigit(highch)) { else if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch); ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
goto error; goto error;
} }
lowch = getchar(); lowch = getchar();
if (lowch == EOF) { if (lowch == EOF) {
g.eof = true; g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber); ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error; goto error;
} }
else if (!isxdigit(lowch)) { else if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch); ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
goto error; goto error;
} }
...@@ -808,7 +716,7 @@ int get_dbt(DBT* pdbt) ...@@ -808,7 +716,7 @@ int get_dbt(DBT* pdbt)
nextch = firstch; nextch = firstch;
break; break;
} }
dbenv->err(dbenv, 0, "Line %llu: Nonprintable character found.", g.linenumber); ERRORX("Line %llu: Nonprintable character found.", g.linenumber);
goto error; goto error;
} }
} }
...@@ -835,15 +743,15 @@ int get_dbt(DBT* pdbt) ...@@ -835,15 +743,15 @@ int get_dbt(DBT* pdbt)
lowch = getchar(); lowch = getchar();
if (lowch == EOF) { if (lowch == EOF) {
g.eof = true; g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber); ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error; goto error;
} }
if (!isxdigit(highch)) { if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch); ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
goto error; goto error;
} }
if (!isxdigit(lowch)) { if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch); ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
goto error; goto error;
} }
if (index == datasize[which]) { if (index == datasize[which]) {
...@@ -873,7 +781,7 @@ int insert_pair(DBT* key, DBT* data) ...@@ -873,7 +781,7 @@ int insert_pair(DBT* key, DBT* data)
int retval = db->put(db, NULL, key, data, g.overwritekeys ? 0 : DB_NOOVERWRITE); int retval = db->put(db, NULL, key, data, g.overwritekeys ? 0 : DB_NOOVERWRITE);
if (retval != 0) { if (retval != 0) {
//TODO: Check for transaction failures/etc.. retry if necessary. //TODO: Check for transaction failures/etc.. retry if necessary.
dbenv->err(dbenv, retval, "DB->put"); ERROR(retval, "DB->put");
goto error; goto error;
} }
return EXIT_SUCCESS; return EXIT_SUCCESS;
...@@ -905,7 +813,7 @@ int read_keys() ...@@ -905,7 +813,7 @@ int read_keys()
g.linenumber++; g.linenumber++;
if (get_dbt(&key) != 0) goto error; if (get_dbt(&key) != 0) goto error;
if (g.eof) { if (g.eof) {
dbenv->err(dbenv, 0, "Line %llu: Key exists but value missing.", g.linenumber); ERRORX("Line %llu: Key exists but value missing.", g.linenumber);
goto error; goto error;
} }
g.linenumber++; g.linenumber++;
...@@ -937,13 +845,13 @@ int read_keys() ...@@ -937,13 +845,13 @@ int read_keys()
} }
default: { default: {
unexpectedinput: unexpectedinput:
dbenv->err(dbenv, 0, "Line %llu: Unexpected input while reading key.\n", g.linenumber); ERRORX("Line %llu: Unexpected input while reading key.\n", g.linenumber);
goto error; goto error;
} }
} }
if (g.eof) { if (g.eof) {
dbenv->err(dbenv, 0, "Line %llu: Key exists but value missing.", g.linenumber); ERRORX("Line %llu: Key exists but value missing.", g.linenumber);
goto error; goto error;
} }
g.linenumber++; g.linenumber++;
...@@ -951,7 +859,7 @@ unexpectedinput: ...@@ -951,7 +859,7 @@ unexpectedinput:
switch (spacech) { switch (spacech) {
case (EOF): { case (EOF): {
g.eof = true; g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file while reading value.\n", g.linenumber); ERRORX("Line %llu: Unexpected end of file while reading value.\n", g.linenumber);
goto error; goto error;
} }
case (' '): { case (' '): {
...@@ -960,7 +868,7 @@ unexpectedinput: ...@@ -960,7 +868,7 @@ unexpectedinput:
break; break;
} }
default: { default: {
dbenv->err(dbenv, 0, "Line %llu: Unexpected input while reading value.\n", g.linenumber); ERRORX("Line %llu: Unexpected input while reading value.\n", g.linenumber);
goto error; goto error;
} }
} }
...@@ -980,7 +888,7 @@ int close_database() ...@@ -980,7 +888,7 @@ int close_database()
assert(db); assert(db);
if ((retval = db->close(db, 0)) != 0) { if ((retval = db->close(db, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->close"); ERROR(retval, "DB->close");
goto error; goto error;
} }
return EXIT_SUCCESS; return EXIT_SUCCESS;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7