Skip to content

M
MariaDB
Commit 807296a4 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

db_load/gen/dump project done until tokudb.bugs issues are worked on. 


git-svn-id: file:///svn/tokudb@479 c7de825b-a66e-492c-adef-691d508d4ae1
parent 07751626
Hide whitespace changes
Inline Side-by-side
Showing with 459 additions and 686 deletions
utils/Makefile
View file @ 807296a4
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -I /usr/local/Berkeleydb.4.1/include/ -ldb
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -ldb
LFLAGS = -l CPPFLAGS =
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
......@@ -10,18 +10,17 @@ UTILS= \
tokudb_dump \
#End
.PHONY: all clean test test_gen test_gen_hex test_load
.PHONY: all clean test test_gen test_gen_hex test_load test_dump
all: $(UTILS)
test: test_gen test_load
test: test_gen test_load test_dump
test_gen: test_gen_hex
# SHELL=/bin/bash
BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
BDB=/usr/local/BerkeleyDB.4.6
BDB_DUMP=$(BDB)/bin/db_dump
BDB_LOAD=$(BDB)/bin/db_load
TEST_GEN_HEX_NUMKEYS=10000
TEST_GEN_HEX_LENGTHMIN=0
......@@ -41,7 +40,7 @@ test_gen_hex:
if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi
rm $@.*.temp
test_load: test_gen
test_load:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
......@@ -52,6 +51,16 @@ test_load: test_gen
$(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
test_dump:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
./tokudb_gen $(TEST_GEN_HEX_FLAGS) > $@.gen.temp
$(BDB_LOAD) $@.bdb.temp < $@.gen.temp
$(BDB_DUMP) $@.bdb.temp > $@.dump.bdb.temp
./tokudb_dump $@.bdb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
#if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi
clean:
rm -rf *.so *.o $(UTILS) *.temp
......
utils/tokudb_common_funcs.h
View file @ 807296a4
......@@ -2,10 +2,27 @@
#define TOKUDB_COMMON_FUNCS_H
#include "tokudb_common.h"
int strtoint32 (DB_ENV* dbenv, char* progname, char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (DB_ENV* dbenv, char* progname, char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (DB_ENV* dbenv, char* progname, char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
//DB_ENV->err disabled since it does not use db_strerror
#define ERROR(retval, ...) \
if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \
fprintf(stderr, __VA_ARGS__); \
}
//DB_ENV->err disabled since it does not use db_strerror, errx does not exist.
#define ERRORX(...) \
if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: ", g.progname); \
fprintf(stderr, __VA_ARGS__); \
}
int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
/*
* Convert a string to an integer of type "type".
......@@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint
*
*/
#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \
int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max, int base) \
int name(char* str, type* num, type min, type max, int base) \
{ \
char* test; \
bigtype value; \
......@@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max
assert(str); \
assert(num); \
assert(min <= max); \
assert(dbenv || progname); \
assert(g.dbenv || g.progname); \
assert(base == 0 || (base >= 2 && base <= 36)); \
\
errno = 0; \
while (isspace(*str)) str++; \
value = strtofunc(str, &test, base); \
if ((*test != '\0' && *test != '\n') || test == str) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Invalid numeric argument\n", progname, str); \
else dbenv->err(dbenv, 0, "%s: Invalid numeric argument", str); \
ERRORX("%s: Invalid numeric argument\n", str); \
errno = EINVAL; \
goto error; \
} \
if (errno != 0) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: %s\n", progname, str, strerror(errno)); \
else dbenv->err(dbenv, errno, "%s", str); \
goto error; \
ERROR(errno, "%s\n", str); \
} \
if (value < min) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Less than minimum value (%" frmt ")\n", progname, str, min); \
else dbenv->err(dbenv, 0, "%s: Less than minimum value (%" frmt ")", str, min); \
ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \
goto error; \
} \
if (value > max) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Greater than maximum value (%" frmt ")\n", progname, str, max); \
else dbenv->err(dbenv, 0, "%s: Greater than maximum value (%" frmt ")", str, max); \
ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \
goto error; \
} \
*num = value; \
......@@ -76,4 +88,103 @@ void outputbyte(uint8_t ch)
else printf("%02x", ch);
}
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void outputplaintextstring(char* str)
{
bool old_plaintext = g.plaintext;
g.plaintext = true;
outputstring(str);
g.plaintext = old_plaintext;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
ERROR(errno, "printabletocstring");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
ERROR(0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
ERROR(0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
ERROR(0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int verify_library_version()
{
int major;
int minor;
db_version(&major, &minor, NULL);
if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) {
ERRORX("version %d.%d doesn't match library version %d.%d\n",
DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
#endif /* #if !defined(TOKUDB_COMMON_H) */
utils/tokudb_dump.c
View file @ 807296a4
......@@ -9,69 +9,39 @@
#include <db.h>
#include "tokudb_common.h"
typedef struct {
char* data[2];
} gdbt;
typedef struct {
char* data;
} rhdr;
typedef struct {
bool leadingspace;
bool plaintext;
bool header;
bool keys;
bool footer;
bool is_private;
bool file_error;
char* progname;
char* homedir;
char* database;
char* subdatabase;
char** config_options;
int32_t version;
int exitcode;
int recover_flags;
DBTYPE dbtype;
DBTYPE opened_dbtype;
DB* db;
DB_ENV* dbenv;
struct {
char* data[2];
} dump_dbt;
struct {
char* data;
} dump_header;
} dump_globals;
dump_globals g;
#include "tokudb_common_funcs.h"
int usage ();
int longusage ();
int dump_database ();
int create_init_env();
int dump_header ();
int dump_footer ();
int dump_database ();
int open_database ();
int dump_pairs ();
int apply_commandline_options();
int dump_pairs ();
int dump_footer ();
int dump_header ();
int close_database ();
int hextoint (int ch);
int doublechararray(char** pmem, uint64_t* size);
int main(int argc, char *argv[]) {
int ch;
int retval;
struct option options[] = {
{ "help", no_argument, NULL, 'H' },
{ "plain_text", no_argument, NULL, 'T' },
{ "Version", no_argument, NULL, 'V' },
{ "config", required_argument, NULL, 'c' },
{ "input_file", required_argument, NULL, 'f' },
{ "home", required_argument, NULL, 'h' },
{ "password", required_argument, NULL, 'p' },
{ NULL, 0, NULL, 0 }
};
char** next_config_option;
/* Set up the globals. */
memset(&g, 0, sizeof(g));
......@@ -79,32 +49,57 @@ int main(int argc, char *argv[]) {
g.dbtype = DB_UNKNOWN;
g.progname = argv[0];
g.header = true;
g.footer = true;
next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*));
if (next_config_option == NULL) {
fprintf(stderr, "%s: %s\n", g.progname, strerror(errno));
goto error;
}
if (verify_library_version() != 0) goto error;
while ((ch = getopt_long_only(argc, argv, "HTVc:f:h:p:t:", options, NULL)) != EOF) {
while ((ch = getopt(argc, argv, "klNpRrVd:f:h:P:s:")) != EOF) {
switch (ch) {
case ('H'): {
g.exitcode = longusage();
goto cleanup;
case ('k'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('l'): {
//TODO: Implement (Requires master database support)
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('N'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('T'): {
g.plaintext = true;
g.leadingspace = false;
g.header = false;
g.footer = false;
break;
}
case ('V'): {
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
case ('p'): {
g.plaintext = true;
break;
}
case ('R'): {
g.recover_flags |= DB_SALVAGE | DB_AGGRESSIVE;
//TODO: Implement aggressive recovery (requires db->verify())
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('c'): {
*next_config_option++ = optarg;
break;
case ('r'): {
g.recover_flags |= DB_SALVAGE;
//TODO: Implement recovery (requires db->verify())
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
goto cleanup;
}
case ('d'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('f'): {
if (freopen(optarg, "w", stdout) == NULL) {
......@@ -119,10 +114,14 @@ int main(int argc, char *argv[]) {
g.homedir = optarg;
break;
}
case ('p'): {
case ('P'): {
/* Clear password. */
memset(optarg, 0, strlen(optarg));
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('s'): {
g.subdatabase = optarg;
goto error;
}
case ('?'):
......@@ -134,6 +133,18 @@ int main(int argc, char *argv[]) {
}
argc -= optind;
argv += optind;
if (g.plaintext) g.recover_flags |= DB_PRINTABLE;
if (g.subdatabase != NULL && IS_SET_ALL(g.recover_flags, DB_SALVAGE)) {
if (IS_SET_ALL(g.recover_flags, DB_AGGRESSIVE)) {
ERRORX("The -s and -R options may not both be specified.\n");
goto error;
}
ERRORX("The -s and -r options may not both be specified.\n");
goto error;
}
if (argc != 1) {
g.exitcode = usage();
......@@ -143,7 +154,6 @@ int main(int argc, char *argv[]) {
g.database = argv[0];
if (create_init_env() != 0) goto error;
//TODO: Figure out how to dump multiple databases at once.
if (dump_database() != 0) goto error;
if (false) {
error:
......@@ -153,52 +163,36 @@ int main(int argc, char *argv[]) {
cleanup:
if (g.dbenv && (retval = g.dbenv->close(g.dbenv, 0)) != 0) {
g.exitcode = EXIT_FAILURE;
fprintf(stderr, "%s: dbenv->close: %s\n", g.progname, db_strerror(retval));
fprintf(stderr, "%s: %s: dbenv->close\n", g.progname, db_strerror(retval));
}
//TODO: /* Resend any caught signal. */
if (g.config_options) free(g.config_options);
if (g.subdatabase) free(g.subdatabase);
// if (g.read_header.data) free(g.read_header.data);
// if (g.get_dbt.data[0]) free(g.get_dbt.data[0]);
// if (g.get_dbt.data[1]) free(g.get_dbt.data[1]);
return g.exitcode;
}
int dump_database()
{
DB_ENV* dbenv = g.dbenv;
int retval;
/* Create a database handle. */
retval = db_create(&g.db, g.dbenv, 0);
if (retval != 0) {
dbenv->err(dbenv, retval, "db_create");
ERROR(retval, "db_create");
return EXIT_FAILURE;
}
if (apply_commandline_options() != 0) goto error;
if (g.file_error) goto cleanup;
/*
//TODO: Uncomment.. implement get_type, and verify the internal type is btree.
if (g.dbtype == DB_UNKNOWN) {
dbenv->err(dbenv, 0, "no database type specified");
goto error;
}
*/
/*
TODO: If/when supporting encryption
if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
dbenv->err(dbenv, ret, "DB->set_flags: DB_ENCRYPT");
ERROR(ret, "DB->set_flags: DB_ENCRYPT");
goto error;
}
*/
//TODO: dump_header()
if (open_database() != 0) goto error;
if (g.header && dump_header() != 0) goto error;
if (dump_pairs() != 0) goto error;
//TODO: dump_footer()
if (g.file_error) goto cleanup;
if (g.footer && dump_footer() != 0) goto error;
if (false) {
error:
......@@ -213,20 +207,10 @@ int dump_database()
int usage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
printf
(
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" -m minsize -M maxsize [-r random seed]\n"
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n",
g.progname
);
return EXIT_FAILURE;
}
int longusage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
fprintf(stderr,
"usage: %s [-klNprRV] [-f output_file] [-h home]\n"
" [-s subdatabase] db_file\n",
g.progname);
return EXIT_FAILURE;
}
......@@ -264,20 +248,20 @@ int create_init_env()
///TODO: UNCOMMENT/IMPLEMENT
retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->set_cachesize");
ERROR(retval, "DB_ENV->set_cachesize");
goto error;
}
*/
g.is_private = true;
//TODO: Do we want to support transactions/logging even in single-process mode?
//Maybe if the db already exists.
//If db does not exist.. makes sense not to log or have transactions
//TODO: Do we want to support transactions even in single-process mode?
//Logging is not necessary.. this is read-only.
//However, do we need to use DB_INIT_LOG to join a logging environment?
REMOVE_BITS(flags, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN);
SET_BITS(flags, DB_CREATE | DB_PRIVATE);
retval = dbenv->open(dbenv, g.homedir, flags, 0);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->open");
ERROR(retval, "DB_ENV->open");
goto error;
}
success:
......@@ -288,219 +272,52 @@ int create_init_env()
return EXIT_FAILURE;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
DB_ENV* dbenv = g.dbenv;
#define DUMP_FLAG(bit, dump) if (IS_SET_ALL(flags, bit)) printf(dump);
#define DUMP_IGNORED_FLAG(bit, dump)
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
dbenv->err(dbenv, ENOMEM, "");
int dump_header()
{
u_int32_t flags;
int retval;
DB* db = g.db;
assert(g.header);
printf("VERSION=3\n");
printf("format=%s\n", g.plaintext ? "print" : "bytevalue");
assert(g.dbtype == DB_BTREE || (g.dbtype == DB_UNKNOWN && g.opened_dbtype == DB_BTREE));
printf("type=btree\n");
//TODO: Get page size from db. Currently tokudb does not support db->get_pagesize.
printf("db_pagesize=4096\n");
if (g.subdatabase) {
printf("subdatabase=");
outputplaintextstring(g.subdatabase);
printf("\n");
}
if ((retval = db->get_flags(db, &flags)) != 0) {
ERROR(retval, "DB->get_flags");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
dbenv->err(dbenv, 0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
dbenv->err(dbenv, 0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
DUMP_IGNORED_FLAG(DB_CHKSUM, "chksum=1");
DUMP_FLAG( DB_DUP, "duplicates=1");
DUMP_IGNORED_FLAG(DB_DUPSORT, "dupsort=1");
DUMP_IGNORED_FLAG(DB_RECNUM, "recnum=1");
printf("HEADER=END\n");
if (ferror(stdout)) goto error;
return EXIT_SUCCESS;
error:
dbenv->err(dbenv, 0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_NUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
/*if ((retval = dbfunction(db, num)) != 0) goto printerror;*/ \
continue; \
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
goto error; \
}
#define PARSE_FLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
if ((retval = db->set_flags(db, flag)) != 0) { \
dbenv->err(dbenv, retval, \
"set_flags: %s", \
field); \
goto error; \
} \
continue; \
}
#define PARSE_UNSUPPORTEDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
goto error; \
}
#define PARSE_CHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
dbenv->err(dbenv, 0, \
"%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
if ((retval = dbfunction(db, value[0])) != 0) { \
goto printerror; \
} \
continue; \
}
int dump_header()
{
//TODO: All of this.. have not started.
return 0;
}
int apply_commandline_options()
int dump_footer()
{
char** next_config_option = g.config_options;
unsigned index;
char* field;
char* value = NULL;
bool first;
int ch;
int32_t num;
int retval;
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
assert(g.header);
for (index = 0; g.config_options[index]; index++) {
if (value) {
/* Restore the field=value format. */
value[-1] = '=';
value = NULL;
}
field = g.config_options[index];
printf("DATA=END\n");
if (ferror(stdout)) goto error;
if ((value = strchr(field, '=')) == NULL) {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
goto error;
}
value[0] = '\0';
value++;
if (field[0] == '\0' || value[0] == '\0') {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
goto error;
}
if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) {
if (g.subdatabase != NULL) {
free(g.subdatabase);
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
goto error;
}
continue;
}
/*
///TODO: UNCOMMENT/IMPLEMENT
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
*/
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
if (value) {
/* Restore the field=value format. */
value[-1] = '=';
value = NULL;
}
return EXIT_SUCCESS;
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
}
error:
return EXIT_FAILURE;
}
......@@ -508,77 +325,33 @@ int apply_commandline_options()
int open_database()
{
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval;
int open_flags = DB_RDONLY;
//TODO: Transaction auto commit stuff
//if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT);
//TODO: First see if it exists.. THEN create it?
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->open: %s", g.database);
ERROR(retval, "DB->open: %s", g.database);
goto error;
}
//TODO: Ensure we have enough cache to store some min number of btree pages.
//NOTE: This may require closing db, environment, and creating new ones.
/*
///TODO: UNCOMMENT/IMPLEMENT
DBTYPE existingtype;
retval = db->get_type(db, &existingtype);
retval = db->get_type(db, &g.opened_dbtype);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->get_type: %s", g.database);
ERROR(retval, "DB->get_type");
goto error;
}
assert(g.dbtype == DB_BTREE);
if (existingtype != g.dbtype) {
fprintf(stderr, "Existing database is not a dictionary (DB_BTREE).\n");
if (g.opened_dbtype != DB_BTREE) {
ERRORX("Unsupported db type %d\n", g.opened_dbtype);
goto error;
}
*/
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int doublechararray(char** pmem, uint64_t* size)
{
DB_ENV* dbenv = g.dbenv;
assert(pmem);
assert(size);
assert(IS_POWER_OF_2(*size));
*size <<= 1;
if (*size == 0) {
/* Overflowed uint64_t. */
dbenv->err(dbenv, 0, "doublechararray\n");
goto error;
}
if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) {
dbenv->err(dbenv, errno, "");
if (g.dbtype != DB_UNKNOWN && g.opened_dbtype != g.dbtype) {
ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, g.opened_dbtype);
goto error;
}
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
......@@ -615,15 +388,14 @@ int dump_pairs()
DBT key;
DBT data;
int spacech;
DB* db = g.db;///////////////
DB_ENV* dbenv = g.dbenv;
DB* db = g.db;
DBC* dbc = NULL;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
if ((retval = db->cursor(db, (DB_TXN*)NULL, &dbc, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->cursor");
ERROR(retval, "DB->cursor");
goto error;
}
while ((retval = dbc->c_get(dbc, &key, &data, DB_NEXT)) == 0) {
......@@ -631,7 +403,7 @@ int dump_pairs()
if (dump_dbt(&data) != 0) goto error;
}
if (retval != DB_NOTFOUND) {
dbenv->err(dbenv, retval, "DBC->c_get");
ERROR(retval, "DBC->c_get");
goto error;
}
......@@ -642,7 +414,7 @@ int dump_pairs()
}
cleanup:
if (dbc && (retval = dbc->c_close(dbc)) != 0) {
dbenv->err(dbenv, retval, "DBC->c_close");
ERROR(retval, "DBC->c_close");
g.exitcode = EXIT_FAILURE;
}
success:
......@@ -652,12 +424,11 @@ int dump_pairs()
int close_database()
{
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval;
assert(db);
if ((retval = db->close(db, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->close");
ERROR(retval, "DB->close");
goto error;
}
return EXIT_SUCCESS;
......
utils/tokudb_gen.c
View file @ 807296a4
......@@ -5,17 +5,15 @@
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <db.h>
#include "tokudb_common.h"
extern char* optarg;
extern int optind;
extern int optopt;
extern int opterr;
extern int optreset;
typedef struct {
DB_ENV* dbenv;
bool plaintext;
char* progname;
} gen_globals;
gen_globals g;
......@@ -29,8 +27,6 @@ int get_delimiter(char* str);
char dbt_delimiter = '\n';
char sort_delimiter[2];
char* progname;
uint32_t lengthmin = 0;
bool set_lengthmin = false;
uint32_t lengthlimit = 0;
......@@ -54,10 +50,13 @@ int main (int argc, char *argv[]) {
/* Set up the globals. */
memset(&g, 0, sizeof(g));
progname = argv[0];
g.progname = argv[0];
if (verify_library_version() != 0) goto error;
strcpy(sort_delimiter, "");
while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:o:")) != EOF) {
while ((ch = getopt(argc, argv, "PfFhHTpVr:s:d:m:M:n:o:")) != EOF) {
switch (ch) {
case ('P'): {
printableonly = true;
......@@ -80,22 +79,20 @@ int main (int argc, char *argv[]) {
break;
}
case ('T'): {
g.plaintext = true;
g.plaintext = true;
leadingspace = false;
header = false;
footer = false;
break;
}
case ('p'): {
g.plaintext = true;
g.plaintext = true;
leadingspace = true;
break;
}
case ('o'): {
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr,
"%s: %s: reopen: %s\n",
progname, optarg, strerror(errno));
ERROR(errno, "%s: reopen\n", optarg);
goto error;
}
break;
......@@ -103,15 +100,13 @@ int main (int argc, char *argv[]) {
case ('d'): {
int temp = get_delimiter(optarg);
if (temp == EOF) {
fprintf(stderr,
"%s: %s: (-d) Key (or value) delimiter must be one character.",
progname, optarg);
ERRORX("%s: (-d) Key (or value) delimiter must be one character.",
optarg);
goto error;
}
if (isxdigit(temp)) {
fprintf(stderr,
"%s: %c: (-d) Key (or value) delimiter cannot be a hex digit.",
progname, temp);
ERRORX("%c: (-d) Key (or value) delimiter cannot be a hex digit.",
temp);
goto error;
}
dbt_delimiter = (char)temp;
......@@ -120,15 +115,13 @@ int main (int argc, char *argv[]) {
case ('s'): {
int temp = get_delimiter(optarg);
if (temp == EOF) {
fprintf(stderr,
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
progname, optarg);
ERRORX("%s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
optarg);
goto error;
}
if (isxdigit(temp)) {
fprintf(stderr,
"%s: %c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
progname, temp);
ERRORX("%c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
temp);
goto error;
}
sort_delimiter[0] = (char)temp;
......@@ -136,45 +129,41 @@ int main (int argc, char *argv[]) {
break;
}
case ('r'): {
if (strtouint32(NULL, progname, optarg, &seed, 0, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-r) Random seed invalid.",
progname, optarg);
if (strtouint32(optarg, &seed, 0, UINT32_MAX, 10)) {
ERRORX("%s: (-r) Random seed invalid.", optarg);
goto error;
}
set_seed = true;
break;
}
case ('m'): {
if (strtouint32(NULL, progname, optarg, &lengthmin, 0, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-m) Min length of keys/values invalid.",
progname, optarg);
if (strtouint32(optarg, &lengthmin, 0, UINT32_MAX, 10)) {
ERRORX("%s: (-m) Min length of keys/values invalid.", optarg);
goto error;
}
set_lengthmin = true;
break;
}
case ('M'): {
if (strtouint32(NULL, progname, optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-M) Limit of key/value length invalid.",
progname, optarg);
if (strtouint32(optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
ERRORX("%s: (-M) Limit of key/value length invalid.", optarg);
goto error;
}
set_lengthlimit = true;
break;
}
case ('n'): {
if (strtouint64(NULL, progname, optarg, &numkeys, 0, UINT64_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-n) Number of keys to generate invalid.",
progname, optarg);
if (strtouint64(optarg, &numkeys, 0, UINT64_MAX, 10)) {
ERRORX("%s: (-n) Number of keys to generate invalid.", optarg);
goto error;
}
set_numkeys = true;
break;
}
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
return EXIT_SUCCESS;
}
case ('?'):
default: {
return (usage());
......@@ -185,61 +174,55 @@ int main (int argc, char *argv[]) {
argv += optind;
if (justheader && !header) {
fprintf(stderr,
"%s: The -h and -H options may not both be specified.\n",
progname);
ERRORX("The -h and -H options may not both be specified.\n");
goto error;
}
if (justfooter && !footer) {
fprintf(stderr,
"%s: The -f and -F options may not both be specified.\n",
progname);
ERRORX("The -f and -F options may not both be specified.\n");
goto error;
}
if (justfooter && justheader) {
fprintf(stderr,
"%s: The -H and -F options may not both be specified.\n",
progname);
ERRORX("The -H and -F options may not both be specified.\n");
goto error;
}
if (justfooter && header) {
fprintf(stderr, "%s: -F implies -h\n", progname);
ERRORX("-F implies -h\n");
header = false;
}
if (justheader && footer) {
fprintf(stderr, "%s: -H implies -f\n", progname);
ERRORX("-H implies -f\n");
footer = false;
}
if (!leadingspace) {
if (footer) {
fprintf(stderr, "%s: -p implies -f\n", progname);
ERRORX("-p implies -f\n");
footer = false;
}
if (header) {
fprintf(stderr, "%s: -p implies -h\n", progname);
ERRORX("-p implies -h\n");
header = false;
}
}
if (justfooter || justheader) outputkeys = false;
else if (!set_numkeys)
{
fprintf(stderr, "%s: The -n option is required.\n", progname);
ERRORX("The -n option is required.\n");
goto error;
}
if (outputkeys && !set_seed) {
fprintf(stderr, "%s: Using default seed. (-r 1).\n", progname);
ERRORX("Using default seed. (-r 1).\n");
seed = 1;
}
if (outputkeys && !set_lengthmin) {
fprintf(stderr, "%s: Using default lengthmin. (-m 0).\n", progname);
ERRORX("Using default lengthmin. (-m 0).\n");
lengthmin = 0;
}
if (outputkeys && !set_lengthlimit) {
fprintf(stderr, "%s: Using default lengthlimit. (-M 1024).\n", progname);
ERRORX("Using default lengthlimit. (-M 1024).\n");
lengthlimit = 1024;
}
if (outputkeys && lengthmin >= lengthlimit) {
fprintf(stderr, "%s: Max key size must be greater than min key size.\n", progname);
ERRORX("Max key size must be greater than min key size.\n");
goto error;
}
......@@ -256,7 +239,7 @@ int main (int argc, char *argv[]) {
}
if (outputkeys) generate_keys();
if (footer) printf("DATA=END\n");
return 0;
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
......@@ -266,10 +249,10 @@ int main (int argc, char *argv[]) {
int usage()
{
fprintf(stderr,
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" [-m lengthmin] [-M lengthlimit] [-r random seed]\n"
" [-o filename] -n numkeys\n",
progname);
"usage: %s [-PfFhHTpV] [-r random seed] [-s delimiter] \n"
" [-d delimiter] [-m lengthmin] [-M lengthlimit] \n"
" -n numkeys [-o output_file] \n",
g.progname);
return EXIT_FAILURE;
}
......@@ -296,15 +279,6 @@ int32_t random_below(int32_t limit)
return random() % limit;
}
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void generate_keys()
{
bool usedemptykey = false;
......
utils/tokudb_load.c
View file @ 807296a4
......@@ -9,14 +9,6 @@
#include <db.h>
#include "tokudb_common.h"
typedef struct {
char* data[2];
} gdbt;
typedef struct {
char* data;
} rhdr;
typedef struct {
bool leadingspace;
bool plaintext;
......@@ -56,24 +48,11 @@ int open_database ();
int read_keys ();
int apply_commandline_options();
int close_database ();
int hextoint (int ch);
int doublechararray(char** pmem, uint64_t* size);
int main(int argc, char *argv[]) {
int ch;
int retval;
struct option options[] = {
{ "no_overwrite", no_argument, NULL, 'n' },
{ "help", no_argument, NULL, 'H' },
{ "plain_text", no_argument, NULL, 'T' },
{ "Version", no_argument, NULL, 'V' },
{ "config", required_argument, NULL, 'c' },
{ "input_file", required_argument, NULL, 'f' },
{ "home", required_argument, NULL, 'h' },
{ "password", required_argument, NULL, 'p' },
{ "type", required_argument, NULL, 't' },
{ NULL, 0, NULL, 0 }
};
char** next_config_option;
/* Set up the globals. */
......@@ -83,24 +62,21 @@ int main(int argc, char *argv[]) {
g.dbtype = DB_UNKNOWN;
g.progname = argv[0];
g.header = true;
if (verify_library_version() != 0) goto error;
next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*));
if (next_config_option == NULL) {
fprintf(stderr, "%s: %s\n", g.progname, strerror(errno));
ERROR(errno, "main: calloc\n");
goto error;
}
while ((ch = getopt_long_only(argc, argv, "nHTVc:f:h:p:t:", options, NULL)) != EOF) {
while ((ch = getopt(argc, argv, "nTVc:f:h:p:t:r:")) != EOF) {
switch (ch) {
case ('n'): {
/* g.overwritekeys = false; */
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('H'): {
g.exitcode = longusage();
goto cleanup;
}
case ('T'): {
g.plaintext = true;
g.leadingspace = false;
......@@ -108,8 +84,8 @@ int main(int argc, char *argv[]) {
break;
}
case ('V'): {
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
goto error;
printf("%s\n", db_version(NULL, NULL, NULL));
goto cleanup;
}
case ('c'): {
*next_config_option++ = optarg;
......@@ -131,7 +107,7 @@ int main(int argc, char *argv[]) {
case ('p'): {
/* Clear password. */
memset(optarg, 0, strlen(optarg));
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('t'): {
......@@ -146,6 +122,10 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "%s: Unrecognized db type %s.\n", g.progname, optarg);
goto error;
}
case ('r'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('?'):
default: {
g.exitcode = usage();
......@@ -195,7 +175,7 @@ int load_database()
/* Create a database handle. */
retval = db_create(&g.db, g.dbenv, 0);
if (retval != 0) {
dbenv->err(dbenv, retval, "db_create");
ERROR(retval, "db_create");
return EXIT_FAILURE;
}
......@@ -204,15 +184,10 @@ int load_database()
if (apply_commandline_options() != 0) goto error;
if (g.eof) goto cleanup;
//TODO: Only quit out if DB does NOT EXIST.
if (g.dbtype == DB_UNKNOWN) {
dbenv->err(dbenv, 0, "no database type specified");
goto error;
}
/*
TODO: If/when supporting encryption
if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
dbenv->err(dbenv, ret, "DB->set_flags: DB_ENCRYPT");
ERROR(ret, "DB->set_flags: DB_ENCRYPT");
goto error;
}
*/
......@@ -234,20 +209,10 @@ int load_database()
int usage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
printf
(
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" -m minsize -M maxsize [-r random seed]\n"
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n",
g.progname
);
return EXIT_FAILURE;
}
int longusage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
fprintf(stderr,
"usage: %s [-nTV] [-c name=value] [-f input_file]\n"
" [-h home] [-t btree] db_file\n",
g.progname);
return EXIT_FAILURE;
}
......@@ -269,14 +234,13 @@ int create_init_env()
/*
TODO: If/when supporting encryption
if (g.password && (retval = dbenv->set_encrypt(dbenv, g.password, DB_ENCRYPT_AES))) {
dbenv->err(dbenv, retval, "set_passwd");
ERROR(retval, "set_passwd");
goto error;
}
*/
/* Open the dbenvironment. */
g.is_private = false;
// flags = DB_JOINENV | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_USE_ENVIRON;
flags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL; ///TODO: UNCOMMENT/IMPLEMENT | DB_USE_ENVIRON;
//TODO: Transactions.. SET_BITS(flags, DB_INIT_TXN);
......@@ -289,7 +253,7 @@ int create_init_env()
///TODO: UNCOMMENT/IMPLEMENT
retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->set_cachesize");
ERROR(retval, "DB_ENV->set_cachesize");
goto error;
}
*/
......@@ -302,7 +266,7 @@ int create_init_env()
retval = dbenv->open(dbenv, g.homedir ? g.homedir : ".", flags, 0);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->open");
ERROR(retval, "DB_ENV->open");
goto error;
}
success:
......@@ -313,90 +277,34 @@ int create_init_env()
return EXIT_FAILURE;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
DB_ENV* dbenv = g.dbenv;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
dbenv->err(dbenv, ENOMEM, "");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
dbenv->err(dbenv, 0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
dbenv->err(dbenv, 0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
dbenv->err(dbenv, 0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_NUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
/*if ((retval = dbfunction(db, num)) != 0) goto printerror;*/ \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
if ((retval = dbfunction(db, num)) != 0) goto printerror; \
continue; \
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_IGNOREDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
ERRORX("%s option not supported yet (ignored).\n", field); \
continue; \
}
#define PARSE_FLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
if ((retval = db->set_flags(db, flag)) != 0) { \
dbenv->err(dbenv, retval, \
"set_flags: %s", \
field); \
ERROR(retval, "set_flags: %s", field); \
goto error; \
} \
continue; \
......@@ -404,22 +312,31 @@ if (!strcmp(field, match)) { \
#define PARSE_UNSUPPORTEDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_IGNOREDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
ERRORX("%s option not supported yet (ignored).\n", field); \
continue; \
}
#define PARSE_CHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
dbenv->err(dbenv, 0, \
"%s=%s: Expected 1-byte value", \
field, value); \
ERRORX("%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
if ((retval = dbfunction(db, value[0])) != 0) { \
......@@ -428,6 +345,34 @@ if (!strcmp(field, match)) { \
continue; \
}
#define PARSE_UNSUPPORTEDCHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
ERRORX("%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_COMMON_CONFIGURATIONS() \
PARSE_IGNOREDNUMBER( "bt_minkey", db->set_bt_minkey); \
PARSE_IGNOREDFLAG( "chksum", DB_CHKSUM); \
PARSE_IGNOREDNUMBER( "db_lorder", db->set_lorder); \
PARSE_IGNOREDNUMBER( "db_pagesize", db->set_pagesize); \
PARSE_FLAG( "duplicates", DB_DUP); \
PARSE_IGNOREDFLAG( "dupsort", DB_DUPSORT); \
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize); \
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor); \
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem); \
PARSE_UNSUPPORTEDNUMBER("re_len", db->set_re_len); \
PARSE_UNSUPPORTEDCHAR( "re_pad", db->set_re_pad); \
PARSE_IGNOREDFLAG( "recnum", DB_RECNUM); \
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
int read_header()
{
static uint64_t datasize = 1 << 10;
......@@ -443,7 +388,7 @@ int read_header()
assert(g.header);
if (g.read_header.data == NULL && (g.read_header.data = (char*)malloc(datasize * sizeof(char))) == NULL) {
dbenv->err(dbenv, errno, "");
ERROR(errno, "read_header: malloc");
goto error;
}
while (!g.eof) {
......@@ -453,7 +398,6 @@ int read_header()
while (true) {
if ((ch = getchar()) == EOF) {
g.eof = true;
//TODO: Check ferror for every EOF test.
if (ferror(stdin)) goto formaterror;
break;
}
......@@ -477,9 +421,9 @@ int read_header()
if (!strcmp(field, "HEADER")) break;
if (!strcmp(field, "VERSION")) {
if (strtoint32(dbenv, NULL, value, &g.version, 1, INT32_MAX, 10)) goto error;
if (strtoint32(value, &g.version, 1, INT32_MAX, 10)) goto error;
if (g.version != 3) {
dbenv->err(dbenv, 0, "line %lu: VERSION %d is unsupported", g.linenumber, g.version);
ERRORX("line %llu: VERSION %d is unsupported", g.linenumber, g.version);
goto error;
}
continue;
......@@ -501,54 +445,41 @@ int read_header()
continue;
}
if (!strcmp(value, "hash") || strcmp(value, "recno") || strcmp(value, "queue")) {
dbenv->err(dbenv, 0, "db type %s not supported.\n", value);
ERRORX("db type %s not supported.\n", value);
goto error;
}
dbenv->err(dbenv, 0, "line %lu: unknown type %s", g.linenumber, value);
ERRORX("line %llu: unknown type %s", g.linenumber, value);
goto error;
}
if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) {
if (g.subdatabase != NULL) {
//TODO: Free before quitting main., clear at start too.. could be a new db without a name?
free(g.subdatabase);
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
ERROR(retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
if (strtoint32(value, &temp, 0, 1, 10)) {
ERROR(0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
ERRORX("keys=0 not supported");
goto error;
}
continue;
}
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
PARSE_COMMON_CONFIGURATIONS();
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
ERRORX("unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
success:
......@@ -556,11 +487,11 @@ int read_header()
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
ERROR(retval, "%s=%s", field, value);
}
if (false) {
formaterror:
dbenv->err(dbenv, 0, "line %lu: unexpected format", g.linenumber);
ERRORX("line %llu: unexpected format", g.linenumber);
}
error:
return EXIT_FAILURE;
......@@ -590,14 +521,14 @@ int apply_commandline_options()
field = g.config_options[index];
if ((value = strchr(field, '=')) == NULL) {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
ERRORX("command-line configuration uses name=value format");
goto error;
}
value[0] = '\0';
value++;
if (field[0] == '\0' || value[0] == '\0') {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
ERRORX("command-line configuration uses name=value format");
goto error;
}
......@@ -607,44 +538,29 @@ int apply_commandline_options()
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
ERROR(retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
if (strtoint32(value, &temp, 0, 1, 10)) {
ERROR(0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
ERRORX("keys=0 not supported");
goto error;
}
continue;
}
/*
///TODO: UNCOMMENT/IMPLEMENT
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
*/
PARSE_COMMON_CONFIGURATIONS();
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
ERRORX("unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
if (value) {
......@@ -656,7 +572,7 @@ int apply_commandline_options()
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
ERROR(retval, "%s=%s", field, value);
}
error:
return EXIT_FAILURE;
......@@ -667,54 +583,46 @@ int open_database()
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval;
DBTYPE opened_type;
int open_flags = DB_CREATE;
int open_flags = 0;
//TODO: Transaction auto commit stuff
//if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT);
//TODO: First see if it exists.. THEN create it?
//Try to see if it exists first.
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->open: %s", g.database);
//Does not exist and we did not specify a type.
if (g.dbtype == DB_UNKNOWN) {
ERRORX("no database type specified");
goto error;
}
SET_BITS(open_flags, DB_CREATE);
//Try creating it.
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
ERROR(retval, "DB->open: %s", g.database);
goto error;
}
}
if ((retval = db->get_type(db, &opened_type)) != 0) {
ERROR(retval, "DB->get_type");
goto error;
}
//TODO: Ensure we have enough cache to store some min number of btree pages.
//NOTE: This may require closing db, environment, and creating new ones.
/*
///TODO: UNCOMMENT/IMPLEMENT
DBTYPE existingtype;
retval = db->get_type(db, &existingtype);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->get_type: %s", g.database);
if (opened_type != DB_BTREE) {
ERRORX("Unsupported db type %d\n", opened_type);
goto error;
}
assert(g.dbtype == DB_BTREE);
if (existingtype != g.dbtype) {
fprintf(stderr, "Existing database is not a dictionary (DB_BTREE).\n");
if (g.dbtype != DB_UNKNOWN && opened_type != g.dbtype) {
ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, opened_type);
goto error;
}
*/
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int doublechararray(char** pmem, uint64_t* size)
{
DB_ENV* dbenv = g.dbenv;
......@@ -726,11 +634,11 @@ int doublechararray(char** pmem, uint64_t* size)
*size <<= 1;
if (*size == 0) {
/* Overflowed uint64_t. */
dbenv->err(dbenv, 0, "Line %llu: Line too long.\n", g.linenumber);
ERRORX("Line %llu: Line too long.\n", g.linenumber);
goto error;
}
if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) {
dbenv->err(dbenv, errno, "");
ERROR(errno, "doublechararray: realloc");
goto error;
}
return EXIT_SUCCESS;
......@@ -755,7 +663,7 @@ int get_dbt(DBT* pdbt)
which = 1 - which;
if (g.get_dbt.data[which] == NULL &&
(g.get_dbt.data[which] = (char*)malloc(datasize[which] * sizeof(char))) == NULL) {
dbenv->err(dbenv, errno, "");
ERROR(errno, "get_dbt: malloc");
goto error;
}
......@@ -781,22 +689,22 @@ int get_dbt(DBT* pdbt)
}
else if (highch == EOF) {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error;
}
else if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
goto error;
}
lowch = getchar();
if (lowch == EOF) {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error;
}
else if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
goto error;
}
......@@ -808,7 +716,7 @@ int get_dbt(DBT* pdbt)
nextch = firstch;
break;
}
dbenv->err(dbenv, 0, "Line %llu: Nonprintable character found.", g.linenumber);
ERRORX("Line %llu: Nonprintable character found.", g.linenumber);
goto error;
}
}
......@@ -835,15 +743,15 @@ int get_dbt(DBT* pdbt)
lowch = getchar();
if (lowch == EOF) {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
goto error;
}
if (index == datasize[which]) {
......@@ -873,7 +781,7 @@ int insert_pair(DBT* key, DBT* data)
int retval = db->put(db, NULL, key, data, g.overwritekeys ? 0 : DB_NOOVERWRITE);
if (retval != 0) {
//TODO: Check for transaction failures/etc.. retry if necessary.
dbenv->err(dbenv, retval, "DB->put");
ERROR(retval, "DB->put");
goto error;
}
return EXIT_SUCCESS;
......@@ -905,7 +813,7 @@ int read_keys()
g.linenumber++;
if (get_dbt(&key) != 0) goto error;
if (g.eof) {
dbenv->err(dbenv, 0, "Line %llu: Key exists but value missing.", g.linenumber);
ERRORX("Line %llu: Key exists but value missing.", g.linenumber);
goto error;
}
g.linenumber++;
......@@ -937,13 +845,13 @@ int read_keys()
}
default: {
unexpectedinput:
dbenv->err(dbenv, 0, "Line %llu: Unexpected input while reading key.\n", g.linenumber);
ERRORX("Line %llu: Unexpected input while reading key.\n", g.linenumber);
goto error;
}
}
if (g.eof) {
dbenv->err(dbenv, 0, "Line %llu: Key exists but value missing.", g.linenumber);
ERRORX("Line %llu: Key exists but value missing.", g.linenumber);
goto error;
}
g.linenumber++;
......@@ -951,7 +859,7 @@ int read_keys()
switch (spacech) {
case (EOF): {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file while reading value.\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file while reading value.\n", g.linenumber);
goto error;
}
case (' '): {
......@@ -960,7 +868,7 @@ int read_keys()
break;
}
default: {
dbenv->err(dbenv, 0, "Line %llu: Unexpected input while reading value.\n", g.linenumber);
ERRORX("Line %llu: Unexpected input while reading value.\n", g.linenumber);
goto error;
}
}
......@@ -980,7 +888,7 @@ int close_database()
assert(db);
if ((retval = db->close(db, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->close");
ERROR(retval, "DB->close");
goto error;
}
return EXIT_SUCCESS;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7