logformat.cc 46.1 KB
Newer Older
1 2
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
3
#ident "$Id$"
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
/*
COPYING CONDITIONS NOTICE:

  This program is free software; you can redistribute it and/or modify
  it under the terms of version 2 of the GNU General Public License as
  published by the Free Software Foundation, and provided that the
  following conditions are met:

      * Redistributions of source code must retain this COPYING
        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
        GRANT (below).

      * Redistributions in binary form must reproduce this COPYING
        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
        GRANT (below) in the documentation and/or other materials
        provided with the distribution.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  02110-1301, USA.

COPYRIGHT NOTICE:

  TokuDB, Tokutek Fractal Tree Indexing Library.
  Copyright (C) 2007-2013 Tokutek, Inc.

DISCLAIMER:

  This program is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  General Public License for more details.

UNIVERSITY PATENT NOTICE:

  The technology is licensed by the Massachusetts Institute of
  Technology, Rutgers State University of New Jersey, and the Research
  Foundation of State University of New York at Stony Brook under
  United States of America Serial No. 11/760379 and to the patents
  and/or patent applications resulting from it.

PATENT MARKING NOTICE:

  This software is covered by US Patent No. 8,185,551.
53
  This software is covered by US Patent No. 8,489,638.
54 55 56

PATENT RIGHTS GRANT:

57
  "THIS IMPLEMENTATION" means the copyrightable works distributed by
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
  Tokutek as part of the Fractal Tree project.

  "PATENT CLAIMS" means the claims of patents that are owned or
  licensable by Tokutek, both currently or in the future; and that in
  the absence of this license would be infringed by THIS
  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.

  "PATENT CHALLENGE" shall mean a challenge to the validity,
  patentability, enforceability and/or non-infringement of any of the
  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.

  Tokutek hereby grants to you, for the term and geographical scope of
  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
  irrevocable (except as stated in this section) patent license to
  make, have made, use, offer to sell, sell, import, transfer, and
  otherwise run, modify, and propagate the contents of THIS
  IMPLEMENTATION, where such license applies only to the PATENT
  CLAIMS.  This grant does not include claims that would be infringed
  only as a consequence of further modifications of THIS
  IMPLEMENTATION.  If you or your agent or licensee institute or order
  or agree to the institution of patent litigation against any entity
  (including a cross-claim or counterclaim in a lawsuit) alleging that
  THIS IMPLEMENTATION constitutes direct or contributory patent
  infringement, or inducement of patent infringement, then any rights
  granted to you under this License shall terminate as of the date
  such litigation is filed.  If you or your agent or exclusive
  licensee institute or order or agree to the institution of a PATENT
  CHALLENGE, then Tokutek may terminate any rights granted to you
  under this License.
*/

89
#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
90
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
91

Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
92 93 94 95 96 97 98 99
/* This file defines the logformat in an executable fashion.
 * This code is used to generate
 *   The code that writes into the log.
 *   The code that reads the log and prints it to stdout (the log_print utility)
 *   The code that reads the log for recovery.
 *   The struct definitions.
 *   The Latex documentation.
 */
100
#include <ctype.h>
101
#include <errno.h>
102
#include <stdarg.h>
103
#include <stdio.h>
104
#include <stdlib.h>
105
#include <string.h>
106 107 108
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
109 110
#include <toku_portability.h>
#include <toku_assert.h>
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
111

112

Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
113
typedef struct field {
114 115 116
    const char *type;
    const char *name;
    const char *format; // optional format string
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
117 118
} F;

119
#define NULLFIELD {0,0,0}
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
120 121
#define FA (F[])

Leif Walsh's avatar
Leif Walsh committed
122 123 124
enum log_begin_action {
    IGNORE_LOG_BEGIN,
    SHOULD_LOG_BEGIN,
125 126
    ASSERT_BEGIN_WAS_LOGGED,
    LOG_BEGIN_ACTION_NA = IGNORE_LOG_BEGIN
Leif Walsh's avatar
Leif Walsh committed
127 128
};

Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
129
struct logtype {
130
    const char *name;
131
    unsigned int command_and_flags;
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
132
    struct field *fields;
Leif Walsh's avatar
Leif Walsh committed
133
    enum log_begin_action log_begin_action;
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
134 135 136 137
};

// In the fields, don't mention the command, the LSN, the CRC or the trailing LEN.

138
const struct logtype rollbacks[] = {
139
    //TODO: #2037 Add dname
140
    {"fdelete", 'U', FA{{"FILENUM",    "filenum", 0},
141
                        NULLFIELD}, LOG_BEGIN_ACTION_NA},
142 143
    //TODO: #2037 Add dname
    {"fcreate", 'F', FA{{"FILENUM", "filenum", 0},
Leif Walsh's avatar
Leif Walsh committed
144
                        {"BYTESTRING", "iname", 0},
145
                        NULLFIELD}, LOG_BEGIN_ACTION_NA},
146
    // cmdinsert is used to insert a key-value pair into a DB.  For rollback we don't need the data.
147
    {"cmdinsert", 'i', FA{
Leif Walsh's avatar
Leif Walsh committed
148 149
                          {"FILENUM", "filenum", 0},
                          {"BYTESTRING", "key", 0},
150
                          NULLFIELD}, LOG_BEGIN_ACTION_NA},
151
    {"cmddelete", 'd', FA{
Leif Walsh's avatar
Leif Walsh committed
152 153
                          {"FILENUM", "filenum", 0},
                          {"BYTESTRING", "key", 0},
154
                          NULLFIELD}, LOG_BEGIN_ACTION_NA},
155
    {"rollinclude", 'r', FA{{"TXNID_PAIR",     "xid", 0},
Yoni Fogel's avatar
Yoni Fogel committed
156
                            {"uint64_t", "num_nodes", 0},
157 158
                            {"BLOCKNUM",  "spilled_head", 0},
                            {"BLOCKNUM",  "spilled_tail", 0},
159
                            NULLFIELD}, LOG_BEGIN_ACTION_NA},
160
    {"load", 'l', FA{{"FILENUM",    "old_filenum", 0},
161
                     {"BYTESTRING", "new_iname", 0},
162
                     NULLFIELD}, LOG_BEGIN_ACTION_NA},
163 164
    // #2954
    {"hot_index", 'h', FA{{"FILENUMS",  "hot_index_filenums", 0},
165
                          NULLFIELD}, LOG_BEGIN_ACTION_NA},
166 167
    {"dictionary_redirect", 'R', FA{{"FILENUM", "old_filenum", 0},
                                    {"FILENUM", "new_filenum", 0},
168
                                    NULLFIELD}, LOG_BEGIN_ACTION_NA},
169 170
    {"cmdupdate", 'u', FA{{"FILENUM", "filenum", 0},
                          {"BYTESTRING", "key", 0},
171
                          NULLFIELD}, LOG_BEGIN_ACTION_NA},
172
    {"cmdupdatebroadcast", 'B', FA{{"FILENUM", "filenum", 0},
Yoni Fogel's avatar
Yoni Fogel committed
173
                                   {"bool",    "is_resetting_op", 0},
174
                                   NULLFIELD}, LOG_BEGIN_ACTION_NA},
175 176
    {"change_fdescriptor", 'D', FA{{"FILENUM",    "filenum", 0},
                            {"BYTESTRING", "old_descriptor", 0},
177 178
                            NULLFIELD}, LOG_BEGIN_ACTION_NA},
    {0,0,FA{NULLFIELD}, LOG_BEGIN_ACTION_NA}
179 180
};

Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
181
const struct logtype logtypes[] = {
182
    // Records produced by checkpoints
183
#if 0 // no longer used, but reserve the type
184
    {"local_txn_checkpoint", 'c', FA{{"TXNID",      "xid", 0}, NULLFIELD}},
185
#endif
Yoni Fogel's avatar
Yoni Fogel committed
186
    {"begin_checkpoint", 'x', FA{{"uint64_t", "timestamp", 0}, {"TXNID", "last_xid", 0}, NULLFIELD}, IGNORE_LOG_BEGIN},
187
    {"end_checkpoint",   'X', FA{{"LSN", "lsn_begin_checkpoint", 0},
Yoni Fogel's avatar
Yoni Fogel committed
188 189 190
                                 {"uint64_t", "timestamp", 0},
                                 {"uint32_t", "num_fassociate_entries", 0}, // how many files were checkpointed
                                 {"uint32_t", "num_xstillopen_entries", 0},  // how many txns  were checkpointed
Leif Walsh's avatar
Leif Walsh committed
191
                                 NULLFIELD}, IGNORE_LOG_BEGIN},
192
    //TODO: #2037 Add dname
193
    {"fassociate",  'f', FA{{"FILENUM", "filenum", 0},
Yoni Fogel's avatar
Yoni Fogel committed
194
                            {"uint32_t",  "treeflags", 0},
Leif Walsh's avatar
Leif Walsh committed
195
                            {"BYTESTRING", "iname", 0},   // pathname of file
Yoni Fogel's avatar
Yoni Fogel committed
196
                            {"uint8_t", "unlink_on_close", 0},
Leif Walsh's avatar
Leif Walsh committed
197
                            NULLFIELD}, IGNORE_LOG_BEGIN},
198
    //We do not use a TXNINFO struct since recovery log has
199
    //FILENUMS and TOKUTXN has FTs (for open_fts)
200 201
    {"xstillopen", 's', FA{{"TXNID_PAIR", "xid", 0}, 
                           {"TXNID_PAIR", "parentxid", 0}, 
Yoni Fogel's avatar
Yoni Fogel committed
202
                           {"uint64_t", "rollentry_raw_count", 0}, 
203
                           {"FILENUMS",  "open_filenums", 0},
Yoni Fogel's avatar
Yoni Fogel committed
204 205 206
                           {"uint8_t",  "force_fsync_on_commit", 0}, 
                           {"uint64_t", "num_rollback_nodes", 0}, 
                           {"uint64_t", "num_rollentries", 0}, 
207 208 209
                           {"BLOCKNUM",  "spilled_rollback_head", 0}, 
                           {"BLOCKNUM",  "spilled_rollback_tail", 0}, 
                           {"BLOCKNUM",  "current_rollback", 0}, 
210
                           NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED}, // record all transactions
211
    // prepared txns need a gid
212
    {"xstillopenprepared", 'p', FA{{"TXNID_PAIR", "xid", 0},
Leif Walsh's avatar
Leif Walsh committed
213
                                   {"XIDP",  "xa_xid", 0}, // prepared transactions need a gid, and have no parentxid.
Yoni Fogel's avatar
Yoni Fogel committed
214
                                   {"uint64_t", "rollentry_raw_count", 0}, 
Leif Walsh's avatar
Leif Walsh committed
215
                                   {"FILENUMS",  "open_filenums", 0},
Yoni Fogel's avatar
Yoni Fogel committed
216
                                   {"uint8_t",  "force_fsync_on_commit", 0}, 
217 218 219 220 221
                                   {"uint64_t", "num_rollback_nodes", 0},
                                   {"uint64_t", "num_rollentries", 0},
                                   {"BLOCKNUM",  "spilled_rollback_head", 0},
                                   {"BLOCKNUM",  "spilled_rollback_tail", 0},
                                   {"BLOCKNUM",  "current_rollback", 0},
222
                                   NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED}, // record all transactions
223
    // Records produced by transactions
224 225 226 227
    {"xbegin", 'b', FA{{"TXNID_PAIR", "xid", 0},{"TXNID_PAIR", "parentxid", 0},NULLFIELD}, IGNORE_LOG_BEGIN},
    {"xcommit",'C', FA{{"TXNID_PAIR", "xid", 0},NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED},
    {"xprepare",'P', FA{{"TXNID_PAIR", "xid", 0}, {"XIDP", "xa_xid", 0}, NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED},
    {"xabort", 'q', FA{{"TXNID_PAIR", "xid", 0},NULLFIELD}, ASSERT_BEGIN_WAS_LOGGED},
228
    //TODO: #2037 Add dname
229
    {"fcreate", 'F', FA{{"TXNID_PAIR",      "xid", 0},
Yoni Fogel's avatar
Yoni Fogel committed
230
                        {"FILENUM",    "filenum", 0},
Leif Walsh's avatar
Leif Walsh committed
231
                        {"BYTESTRING", "iname", 0},
Yoni Fogel's avatar
Yoni Fogel committed
232 233 234 235 236
                        {"uint32_t",  "mode",  "0%o"},
                        {"uint32_t",  "treeflags", 0},
                        {"uint32_t", "nodesize", 0},
                        {"uint32_t", "basementnodesize", 0},
                        {"uint32_t", "compression_method", 0},
Leif Walsh's avatar
Leif Walsh committed
237
                        NULLFIELD}, SHOULD_LOG_BEGIN},
238
    //TODO: #2037 Add dname
239
    {"fopen",   'O', FA{{"BYTESTRING", "iname", 0},
Leif Walsh's avatar
Leif Walsh committed
240
                        {"FILENUM",    "filenum", 0},
Yoni Fogel's avatar
Yoni Fogel committed
241
                        {"uint32_t",  "treeflags", 0},
Leif Walsh's avatar
Leif Walsh committed
242
                        NULLFIELD}, IGNORE_LOG_BEGIN},
243 244
    //TODO: #2037 Add dname
    {"fclose",   'e', FA{{"BYTESTRING", "iname", 0},
245
                         {"FILENUM",    "filenum", 0},
Leif Walsh's avatar
Leif Walsh committed
246
                         NULLFIELD}, IGNORE_LOG_BEGIN},
247
    //TODO: #2037 Add dname
248
    {"fdelete", 'U', FA{{"TXNID_PAIR",      "xid", 0},
Leif Walsh's avatar
Leif Walsh committed
249 250
                        {"FILENUM", "filenum", 0},
                        NULLFIELD}, SHOULD_LOG_BEGIN},
251
    {"enq_insert", 'I', FA{{"FILENUM",    "filenum", 0},
252
                           {"TXNID_PAIR",      "xid", 0},
253 254
                           {"BYTESTRING", "key", 0},
                           {"BYTESTRING", "value", 0},
Leif Walsh's avatar
Leif Walsh committed
255
                           NULLFIELD}, SHOULD_LOG_BEGIN},
256
    {"enq_insert_no_overwrite", 'i', FA{{"FILENUM",    "filenum", 0},
257
                                        {"TXNID_PAIR",      "xid", 0},
258 259
                                        {"BYTESTRING", "key", 0},
                                        {"BYTESTRING", "value", 0},
Leif Walsh's avatar
Leif Walsh committed
260
                                        NULLFIELD}, SHOULD_LOG_BEGIN},
261
    {"enq_delete_any", 'E', FA{{"FILENUM",    "filenum", 0},
262
                               {"TXNID_PAIR",      "xid", 0},
263
                               {"BYTESTRING", "key", 0},
Leif Walsh's avatar
Leif Walsh committed
264
                               NULLFIELD}, SHOULD_LOG_BEGIN},
265 266
    {"enq_insert_multiple", 'm', FA{{"FILENUM",    "src_filenum", 0},
                                    {"FILENUMS",   "dest_filenums", 0},
267
                                    {"TXNID_PAIR",      "xid", 0},
268 269
                                    {"BYTESTRING", "src_key", 0},
                                    {"BYTESTRING", "src_val", 0},
Leif Walsh's avatar
Leif Walsh committed
270
                                    NULLFIELD}, SHOULD_LOG_BEGIN},
271 272
    {"enq_delete_multiple", 'M', FA{{"FILENUM",    "src_filenum", 0},
                                    {"FILENUMS",   "dest_filenums", 0},
273
                                    {"TXNID_PAIR",      "xid", 0},
274 275
                                    {"BYTESTRING", "src_key", 0},
                                    {"BYTESTRING", "src_val", 0},
Leif Walsh's avatar
Leif Walsh committed
276
                                    NULLFIELD}, SHOULD_LOG_BEGIN},
Yoni Fogel's avatar
Yoni Fogel committed
277
    {"comment", 'T', FA{{"uint64_t", "timestamp", 0},
278
                        {"BYTESTRING", "comment", 0},
Leif Walsh's avatar
Leif Walsh committed
279
                        NULLFIELD}, IGNORE_LOG_BEGIN},
280
    // Note: shutdown_up_to_19 log entry is NOT ALLOWED TO BE CHANGED.
281 282
    // Do not change the letter ('Q'), do not add fields,
    // do not remove fields.
283
    // TODO: Kill this logentry entirely once we no longer support version 19.
Yoni Fogel's avatar
Yoni Fogel committed
284
    {"shutdown_up_to_19", 'Q', FA{{"uint64_t", "timestamp", 0},
285 286 287 288 289 290
                         NULLFIELD}, IGNORE_LOG_BEGIN},
    // Note: Shutdown log entry is NOT ALLOWED TO BE CHANGED.
    // Do not change the letter ('0'), do not add fields,
    // do not remove fields.
    // You CAN leave this alone and add a new one, but then you have
    // to deal with the upgrade mechanism again.
291 292 293
    // This is how we detect clean shutdowns from OLDER VERSIONS.
    // This log entry must always be readable for future versions.
    // If you DO change it, you need to write a separate log upgrade mechanism.
Yoni Fogel's avatar
Yoni Fogel committed
294
    {"shutdown", '0', FA{{"uint64_t", "timestamp", 0},
295
                         {"TXNID", "last_xid", 0},
Leif Walsh's avatar
Leif Walsh committed
296
                         NULLFIELD}, IGNORE_LOG_BEGIN},
297
    {"load", 'l', FA{{"TXNID_PAIR",      "xid", 0},
298
                     {"FILENUM",    "old_filenum", 0},
299
                     {"BYTESTRING", "new_iname", 0},
Leif Walsh's avatar
Leif Walsh committed
300
                     NULLFIELD}, SHOULD_LOG_BEGIN},
301
    // #2954
302
    {"hot_index", 'h', FA{{"TXNID_PAIR",     "xid", 0},
303
                          {"FILENUMS",  "hot_index_filenums", 0},
Leif Walsh's avatar
Leif Walsh committed
304
                          NULLFIELD}, SHOULD_LOG_BEGIN},
305
    {"enq_update", 'u', FA{{"FILENUM",    "filenum", 0},
306
                           {"TXNID_PAIR",      "xid", 0},
307 308
                           {"BYTESTRING", "key", 0},
                           {"BYTESTRING", "extra", 0},
Leif Walsh's avatar
Leif Walsh committed
309
                           NULLFIELD}, SHOULD_LOG_BEGIN},
310
    {"enq_updatebroadcast", 'B', FA{{"FILENUM",    "filenum", 0},
311
                                    {"TXNID_PAIR",      "xid", 0},
312
                                    {"BYTESTRING", "extra", 0},
Yoni Fogel's avatar
Yoni Fogel committed
313
                                    {"bool",       "is_resetting_op", 0},
Leif Walsh's avatar
Leif Walsh committed
314
                                    NULLFIELD}, SHOULD_LOG_BEGIN},
315
    {"change_fdescriptor", 'D', FA{{"FILENUM",    "filenum", 0},
316
                            {"TXNID_PAIR",      "xid", 0},
317 318
                            {"BYTESTRING", "old_descriptor", 0},
                            {"BYTESTRING", "new_descriptor", 0},
Yoni Fogel's avatar
Yoni Fogel committed
319
                            {"bool",       "update_cmp_descriptor", 0},
Leif Walsh's avatar
Leif Walsh committed
320 321
                            NULLFIELD}, SHOULD_LOG_BEGIN},
    {0,0,FA{NULLFIELD}, (enum log_begin_action) 0}
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
322
};
323

324

Leif Walsh's avatar
Leif Walsh committed
325
#define DO_STRUCTS(lt, array, body) do {        \
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
326
    const struct logtype *lt;    \
Leif Walsh's avatar
Leif Walsh committed
327 328
    for (lt=&array[0]; lt->name; lt++) {        \
        body; \
329
    } } while (0)
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
330

331 332 333 334 335 336
#define DO_ROLLBACKS(lt, body) DO_STRUCTS(lt, rollbacks, body)

#define DO_LOGTYPES(lt, body) DO_STRUCTS(lt, logtypes, body)

#define DO_LOGTYPES_AND_ROLLBACKS(lt, body) (DO_ROLLBACKS(lt,body), DO_LOGTYPES(lt, body))

337
#define DO_FIELDS(fld, lt, body) do { \
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
338 339 340
    struct field *fld; \
    for (fld=lt->fields; fld->type; fld++) { \
        body; \
341
    } } while (0)
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
342

343

344
static void __attribute__((format (printf, 3, 4))) fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
345 346 347 348
    va_list ap;
    int r;
    va_start(ap, format);
    r=vfprintf(f1, format, ap); assert(r>=0);
349 350
    va_end(ap);
    va_start(ap, format);
351 352 353 354
    r=vfprintf(f2, format, ap); assert(r>=0);
    va_end(ap);
}

355
FILE *hf=0, *cf=0, *pf=0;
356

357
static void
358
generate_enum_internal (const char *enum_name, const char *enum_prefix, const struct logtype *lts) {
359
    char used_cmds[256];
360
    int count=0;
361
    memset(used_cmds, 0, 256);
362 363
    fprintf(hf, "enum %s {", enum_name);
    DO_STRUCTS(lt, lts,
Leif Walsh's avatar
Leif Walsh committed
364 365 366 367 368 369 370 371 372
                {
                    unsigned char cmd = (unsigned char)(lt->command_and_flags&0xff);
                    if (count!=0) fprintf(hf, ",");
                    count++;
                    fprintf(hf, "\n");
                    fprintf(hf,"    %s_%-16s = '%c'", enum_prefix, lt->name, cmd);
                    if (used_cmds[cmd]!=0) { fprintf(stderr, "%s:%d: error: Command %d (%c) was used twice (second time for %s)\n", __FILE__, __LINE__, cmd, cmd, lt->name); abort(); }
                    used_cmds[cmd]=1;
                });
373
    fprintf(hf, "\n};\n\n");
374

375 376
}

377 378
static void
generate_enum (void) {
379 380
    generate_enum_internal("lt_cmd", "LT", logtypes);
    generate_enum_internal("rt_cmd", "RT", rollbacks);
381 382
}

383 384
static void
generate_log_struct (void) {
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
385
    DO_LOGTYPES(lt,
Leif Walsh's avatar
Leif Walsh committed
386 387 388 389
                {  fprintf(hf, "struct logtype_%s {\n", lt->name);
                    fprintf(hf, "  %-16s lsn;\n", "LSN");
                    DO_FIELDS(field_type, lt,
                              fprintf(hf, "  %-16s %s;\n", field_type->type, field_type->name));
Yoni Fogel's avatar
Yoni Fogel committed
390 391
                    fprintf(hf, "  %-16s crc;\n", "uint32_t");
                    fprintf(hf, "  %-16s len;\n", "uint32_t");
Leif Walsh's avatar
Leif Walsh committed
392 393 394 395 396
                    fprintf(hf, "};\n");
                    //fprintf(hf, "void toku_recover_%s (LSN lsn", lt->name);
                    //DO_FIELDS(field_type, lt, fprintf(hf, ", %s %s", field_type->type, field_type->name));
                    //fprintf(hf, ");\n");
                });
397
    DO_ROLLBACKS(lt,
Leif Walsh's avatar
Leif Walsh committed
398 399 400 401 402 403 404 405 406 407 408
                 {  fprintf(hf, "struct rolltype_%s {\n", lt->name);
                     DO_FIELDS(field_type, lt,
                               fprintf(hf, "  %-16s %s;\n", field_type->type, field_type->name));
                     fprintf(hf, "};\n");
                     fprintf(hf, "int toku_rollback_%s (", lt->name);
                     DO_FIELDS(field_type, lt, fprintf(hf, "%s %s,", field_type->type, field_type->name));
                     fprintf(hf, "TOKUTXN txn, LSN oplsn);\n");
                     fprintf(hf, "int toku_commit_%s (", lt->name);
                     DO_FIELDS(field_type, lt, fprintf(hf, "%s %s,", field_type->type, field_type->name));
                     fprintf(hf, "TOKUTXN txn, LSN oplsn);\n");
                 });
409 410 411 412 413 414
    fprintf(hf, "struct log_entry {\n");
    fprintf(hf, "  enum lt_cmd cmd;\n");
    fprintf(hf, "  union {\n");
    DO_LOGTYPES(lt, fprintf(hf,"    struct logtype_%s %s;\n", lt->name, lt->name));
    fprintf(hf, "  } u;\n");
    fprintf(hf, "};\n");
415

416 417
    fprintf(hf, "struct roll_entry {\n");
    fprintf(hf, "  enum rt_cmd cmd;\n");
418
    fprintf(hf, "  struct roll_entry *prev; /* for in-memory list of log entries.  Threads from newest to oldest. */\n");
419 420 421 422 423
    fprintf(hf, "  union {\n");
    DO_ROLLBACKS(lt, fprintf(hf,"    struct rolltype_%s %s;\n", lt->name, lt->name));
    fprintf(hf, "  } u;\n");
    fprintf(hf, "};\n");

Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
424 425
}

426 427
static void
generate_dispatch (void) {
428 429
    fprintf(hf, "#define rolltype_dispatch(s, funprefix) ({ switch((s)->cmd) {\\\n");
    DO_ROLLBACKS(lt, fprintf(hf, "  case RT_%s: funprefix ## %s (&(s)->u.%s); break;\\\n", lt->name, lt->name, lt->name));
430
    fprintf(hf, " }})\n");
431

Yoni Fogel's avatar
Yoni Fogel committed
432 433
    fprintf(hf, "#define logtype_dispatch_assign(s, funprefix, var, ...) do { switch((s)->cmd) {\\\n");
    DO_LOGTYPES(lt, fprintf(hf, "  case LT_%s: var = funprefix ## %s (&(s)->u.%s, __VA_ARGS__); break;\\\n", lt->name, lt->name, lt->name));
434
    fprintf(hf, " }} while (0)\n");
435

Yoni Fogel's avatar
Yoni Fogel committed
436
    fprintf(hf, "#define rolltype_dispatch_assign(s, funprefix, var, ...) do { \\\n");
437
    fprintf(hf, "  switch((s)->cmd) {\\\n");
438
    DO_ROLLBACKS(lt, {
Leif Walsh's avatar
Leif Walsh committed
439 440 441 442 443 444 445 446 447
                    fprintf(hf, "  case RT_%s: var = funprefix ## %s (", lt->name, lt->name);
                    int fieldcount=0;
                    DO_FIELDS(field_type, lt, {
                                if (fieldcount>0) fprintf(hf, ",");
                                fprintf(hf, "(s)->u.%s.%s", lt->name, field_type->name);
                                fieldcount++;
                            });
                    fprintf(hf, ", __VA_ARGS__); break;\\\n");
                });
448
    fprintf(hf, "  default: assert(0);} } while (0)\n");
449

450
    fprintf(hf, "#define logtype_dispatch_args(s, funprefix, ...) do { switch((s)->cmd) {\\\n");
451
    DO_LOGTYPES(lt,
Leif Walsh's avatar
Leif Walsh committed
452 453 454 455 456
                {
                    fprintf(hf, "  case LT_%s: funprefix ## %s ((s)->u.%s.lsn", lt->name, lt->name, lt->name);
                    DO_FIELDS(field_type, lt, fprintf(hf, ",(s)->u.%s.%s", lt->name, field_type->name));
                    fprintf(hf, ", __VA_ARGS__); break;\\\n");
                });
457
    fprintf(hf, " }} while (0)\n");
458
}
459 460 461

static void
generate_get_timestamp(void) {
Yoni Fogel's avatar
Yoni Fogel committed
462
    fprintf(cf, "static uint64_t toku_get_timestamp(void) {\n");
463 464 465 466 467
    fprintf(cf, "  struct timeval tv; int r = gettimeofday(&tv, NULL);\n");
    fprintf(cf, "  assert(r==0);\n");
    fprintf(cf, "  return tv.tv_sec * 1000000ULL + tv.tv_usec;\n");
    fprintf(cf, "}\n");
}
468

469 470
static void
generate_log_writer (void) {
471
    generate_get_timestamp();
472
    DO_LOGTYPES(lt, {
473 474
            //TODO(yoni): The overhead variables are NOT correct for BYTESTRING, FILENUMS (or any other variable length type)
            //            We should switch to something like using toku_logsizeof_*.
Rich Prohaska's avatar
Rich Prohaska committed
475
            fprintf(hf, "static const size_t toku_log_%s_overhead = (+4+1+8", lt->name);
476
            DO_FIELDS(field_type, lt, fprintf(hf, "+sizeof(%s)", field_type->type));
Rich Prohaska's avatar
Rich Prohaska committed
477
            fprintf(hf, "+8);\n");
478
                        fprintf2(cf, hf, "void toku_log_%s (TOKULOGGER logger, LSN *lsnp, int do_fsync", lt->name);
Leif Walsh's avatar
Leif Walsh committed
479 480 481
                        switch (lt->log_begin_action) {
                        case SHOULD_LOG_BEGIN:
                        case ASSERT_BEGIN_WAS_LOGGED: {
482
                            fprintf2(cf, hf, ", TOKUTXN txn");
Leif Walsh's avatar
Leif Walsh committed
483 484 485
                            break;
                        }
                        case IGNORE_LOG_BEGIN: break;
486
                        }
Leif Walsh's avatar
Leif Walsh committed
487 488 489
                        DO_FIELDS(field_type, lt, fprintf2(cf, hf, ", %s %s", field_type->type, field_type->name));
                        fprintf(hf, ");\n");
                        fprintf(cf, ") {\n");
490 491 492
                        fprintf(cf, "  if (logger == NULL) {\n");
                        fprintf(cf, "     return;\n");
                        fprintf(cf, "  }\n");
Leif Walsh's avatar
Leif Walsh committed
493 494
                        switch (lt->log_begin_action) {
                        case SHOULD_LOG_BEGIN: {
495 496
                            fprintf(cf, "  //txn can be NULL during tests\n");
                            fprintf(cf, "  //never null when not checkpoint.\n");
497
                            fprintf(cf, "  if (txn && !txn->begin_was_logged) {\n");
498
                            fprintf(cf, "    invariant(!txn_declared_read_only(txn));\n");
499 500
                            fprintf(cf, "    toku_maybe_log_begin_txn_for_write_operation(txn);\n");
                            fprintf(cf, "  }\n");
Leif Walsh's avatar
Leif Walsh committed
501 502 503
                            break;
                        }
                        case ASSERT_BEGIN_WAS_LOGGED: {
504
                            fprintf(cf, "  //txn can be NULL during tests\n");
505
                            fprintf(cf, "  invariant(!txn || txn->begin_was_logged);\n");
506
                            fprintf(cf, "  invariant(!txn || !txn_declared_read_only(txn));\n");
Leif Walsh's avatar
Leif Walsh committed
507 508 509
                            break;
                        }
                        case IGNORE_LOG_BEGIN: break;
510
                        }
Leif Walsh's avatar
Leif Walsh committed
511 512 513 514 515
                        fprintf(cf, "  if (!logger->write_log_files) {\n");
                        fprintf(cf, "    ml_lock(&logger->input_lock);\n");
                        fprintf(cf, "    logger->lsn.lsn++;\n");
                        fprintf(cf, "    if (lsnp) *lsnp=logger->lsn;\n");
                        fprintf(cf, "    ml_unlock(&logger->input_lock);\n");
516
                        fprintf(cf, "    return;\n");
Leif Walsh's avatar
Leif Walsh committed
517 518 519 520 521 522 523 524 525 526
                        fprintf(cf, "  }\n");
                        fprintf(cf, "  const unsigned int buflen= (+4 // len at the beginning\n");
                        fprintf(cf, "                              +1 // log command\n");
                        fprintf(cf, "                              +8 // lsn\n");
                        DO_FIELDS(field_type, lt,
                                  fprintf(cf, "                              +toku_logsizeof_%s(%s)\n", field_type->type, field_type->name));
                        fprintf(cf, "                              +8 // crc + len\n");
                        fprintf(cf, "                     );\n");
                        fprintf(cf, "  struct wbuf wbuf;\n");
                        fprintf(cf, "  ml_lock(&logger->input_lock);\n");
527
                        fprintf(cf, "  toku_logger_make_space_in_inbuf(logger, buflen);\n");
Leif Walsh's avatar
Leif Walsh committed
528 529 530 531
                        fprintf(cf, "  wbuf_nocrc_init(&wbuf, logger->inbuf.buf+logger->inbuf.n_in_buf, buflen);\n");
                        fprintf(cf, "  wbuf_nocrc_int(&wbuf, buflen);\n");
                        fprintf(cf, "  wbuf_nocrc_char(&wbuf, '%c');\n", (char)(0xff&lt->command_and_flags));
                        fprintf(cf, "  logger->lsn.lsn++;\n");
532 533 534
                        fprintf(cf, "  logger->inbuf.max_lsn_in_buf = logger->lsn;\n");
                        fprintf(cf, "  wbuf_nocrc_LSN(&wbuf, logger->lsn);\n");
                        fprintf(cf, "  if (lsnp) *lsnp=logger->lsn;\n");
Leif Walsh's avatar
Leif Walsh committed
535
                        DO_FIELDS(field_type, lt,
536
                                  if (strcmp(field_type->name, "timestamp") == 0)
537
                                      fprintf(cf, "  if (timestamp == 0) timestamp = toku_get_timestamp();\n");
Leif Walsh's avatar
Leif Walsh committed
538
                                  fprintf(cf, "  wbuf_nocrc_%s(&wbuf, %s);\n", field_type->type, field_type->name));
539
                        fprintf(cf, "  wbuf_nocrc_int(&wbuf, toku_x1764_memory(wbuf.buf, wbuf.ndone));\n");
Leif Walsh's avatar
Leif Walsh committed
540 541 542
                        fprintf(cf, "  wbuf_nocrc_int(&wbuf, buflen);\n");
                        fprintf(cf, "  assert(wbuf.ndone==buflen);\n");
                        fprintf(cf, "  logger->inbuf.n_in_buf += buflen;\n");
543
                        fprintf(cf, "  toku_logger_maybe_fsync(logger, logger->lsn, do_fsync, true);\n");
Leif Walsh's avatar
Leif Walsh committed
544 545
                        fprintf(cf, "}\n\n");
                    });
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
546 547
}

548 549
static void
generate_log_reader (void) {
550
    DO_LOGTYPES(lt, {
Yoni Fogel's avatar
Yoni Fogel committed
551
                        fprintf(cf, "static int toku_log_fread_%s (FILE *infile, uint32_t len1, struct logtype_%s *data, struct x1764 *checksum)", lt->name, lt->name);
Leif Walsh's avatar
Leif Walsh committed
552 553
                        fprintf(cf, " {\n");
                        fprintf(cf, "  int r=0;\n");
Yoni Fogel's avatar
Yoni Fogel committed
554
                        fprintf(cf, "  uint32_t actual_len=5; // 1 for the command, 4 for the first len.\n");
Leif Walsh's avatar
Leif Walsh committed
555 556 557
                        fprintf(cf, "  r=toku_fread_%-16s(infile, &data->%-16s, checksum, &actual_len); if (r!=0) return r;\n", "LSN", "lsn");
                        DO_FIELDS(field_type, lt,
                                  fprintf(cf, "  r=toku_fread_%-16s(infile, &data->%-16s, checksum, &actual_len); if (r!=0) return r;\n", field_type->type, field_type->name));
Yoni Fogel's avatar
Yoni Fogel committed
558 559 560
                        fprintf(cf, "  uint32_t checksum_in_file, len_in_file;\n");
                        fprintf(cf, "  r=toku_fread_uint32_t_nocrclen(infile, &checksum_in_file); actual_len+=4;   if (r!=0) return r;\n");
                        fprintf(cf, "  r=toku_fread_uint32_t_nocrclen(infile, &len_in_file);    actual_len+=4;   if (r!=0) return r;\n");
561
                        fprintf(cf, "  if (checksum_in_file!=toku_x1764_finish(checksum) || len_in_file!=actual_len || len1 != len_in_file) return DB_BADFORMAT;\n");
Leif Walsh's avatar
Leif Walsh committed
562 563 564
                        fprintf(cf, "  return 0;\n");
                        fprintf(cf, "}\n\n");
                    });
565
    fprintf2(cf, hf, "int toku_log_fread (FILE *infile, struct log_entry *le)");
566 567
    fprintf(hf, ";\n");
    fprintf(cf, " {\n");
Yoni Fogel's avatar
Yoni Fogel committed
568 569
    fprintf(cf, "  uint32_t len1; int r;\n");
    fprintf(cf, "  uint32_t ignorelen=0;\n");
570
    fprintf(cf, "  struct x1764 checksum;\n");
571
    fprintf(cf, "  toku_x1764_init(&checksum);\n");
Yoni Fogel's avatar
Yoni Fogel committed
572
    fprintf(cf, "  r = toku_fread_uint32_t(infile, &len1, &checksum, &ignorelen); if (r!=0) return r;\n");
573 574
    fprintf(cf, "  int cmd=fgetc(infile);\n");
    fprintf(cf, "  if (cmd==EOF) return EOF;\n");
575
    fprintf(cf, "  char cmdchar = (char)cmd;\n");
576
    fprintf(cf, "  toku_x1764_add(&checksum, &cmdchar, 1);\n");
577
    fprintf(cf, "  le->cmd=(enum lt_cmd)cmd;\n");
578
    fprintf(cf, "  switch ((enum lt_cmd)cmd) {\n");
579
    DO_LOGTYPES(lt, {
Leif Walsh's avatar
Leif Walsh committed
580 581 582
                        fprintf(cf, "  case LT_%s:\n", lt->name);
                        fprintf(cf, "    return toku_log_fread_%s (infile, len1, &le->u.%s, &checksum);\n", lt->name, lt->name);
                    });
583 584 585
    fprintf(cf, "  };\n");
    fprintf(cf, "  return DB_BADFORMAT;\n"); // Should read past the record using the len field.
    fprintf(cf, "}\n\n");
Dave Wells's avatar
Dave Wells committed
586 587
    //fprintf2(cf, hf, "// Return 0 if there is something to read, return -1 if nothing to read, abort if an error.\n");
    fprintf2(cf, hf, "// Return 0 if there is something to read, -1 if nothing to read, >0 on error\n");
588 589 590
    fprintf2(cf, hf, "int toku_log_fread_backward (FILE *infile, struct log_entry *le)");
    fprintf(hf, ";\n");
    fprintf(cf, "{\n");
591
    fprintf(cf, "  memset(le, 0, sizeof(*le));\n");
592 593
    fprintf(cf, "  long pos = ftell(infile);\n");
    fprintf(cf, "  if (pos<=12) return -1;\n");
Dave Wells's avatar
Dave Wells committed
594
    fprintf(cf, "  int r = fseek(infile, -4, SEEK_CUR); \n");//              assert(r==0);\n");
595
    fprintf(cf, "  if (r!=0) return get_error_errno();\n");
Yoni Fogel's avatar
Yoni Fogel committed
596 597
    fprintf(cf, "  uint32_t len;\n");
    fprintf(cf, "  r = toku_fread_uint32_t_nocrclen(infile, &len); \n");//  assert(r==0);\n");
Dave Wells's avatar
Dave Wells committed
598 599
    fprintf(cf, "  if (r!=0) return 1;\n");
    fprintf(cf, "  r = fseek(infile, -(int)len, SEEK_CUR) ;  \n");//         assert(r==0);\n");
600
    fprintf(cf, "  if (r!=0) return get_error_errno();\n");
Dave Wells's avatar
Dave Wells committed
601 602
    fprintf(cf, "  r = toku_log_fread(infile, le); \n");//                   assert(r==0);\n");
    fprintf(cf, "  if (r!=0) return 1;\n");
603 604
    fprintf(cf, "  long afterpos = ftell(infile);\n");
    fprintf(cf, "  if (afterpos != pos) return 1;\n");
Dave Wells's avatar
Dave Wells committed
605
    fprintf(cf, "  r = fseek(infile, -(int)len, SEEK_CUR); \n");//           assert(r==0);\n");
606
    fprintf(cf, "  if (r!=0) return get_error_errno();\n");
607
    fprintf(cf, "  return 0;\n");
608 609
    fprintf(cf, "}\n\n");

610 611
    DO_LOGTYPES(lt, ({
            fprintf(cf, "static void toku_log_free_log_entry_%s_resources (struct logtype_%s *data", lt->name, lt->name);
Leif Walsh's avatar
Leif Walsh committed
612
            if (!lt->fields->type) fprintf(cf, " __attribute__((__unused__))");
613
            fprintf(cf, ") {\n");
614
            DO_FIELDS(field_type, lt,
Leif Walsh's avatar
Leif Walsh committed
615 616 617 618
                      fprintf(cf, "    toku_free_%s(data->%s);\n", field_type->type, field_type->name);
                      );
            fprintf(cf, "}\n\n");
            }));
619 620 621 622 623 624 625 626 627 628 629
    fprintf2(cf, hf, "void toku_log_free_log_entry_resources (struct log_entry *le)");
    fprintf(hf, ";\n");
    fprintf(cf, " {\n");
    fprintf(cf, "    switch ((enum lt_cmd)le->cmd) {\n");
    DO_LOGTYPES(lt, {
            fprintf(cf, "    case LT_%s:\n", lt->name);
            fprintf(cf, "        return toku_log_free_log_entry_%s_resources (&(le->u.%s));\n", lt->name, lt->name);
        });
    fprintf(cf, "    };\n");
    fprintf(cf, "    return;\n");
    fprintf(cf, "}\n\n");
630 631
}

632 633
static void
generate_logprint (void) {
634
    unsigned maxnamelen=0;
635
    fprintf2(pf, hf, "int toku_logprint_one_record(FILE *outf, FILE *f)");
636
    fprintf(hf, ";\n");
637 638
    fprintf(pf, " {\n");
    fprintf(pf, "    int cmd, r;\n");
Yoni Fogel's avatar
Yoni Fogel committed
639 640
    fprintf(pf, "    uint32_t len1, crc_in_file;\n");
    fprintf(pf, "    uint32_t ignorelen=0;\n");
641
    fprintf(pf, "    struct x1764 checksum;\n");
642
    fprintf(pf, "    toku_x1764_init(&checksum);\n");
Yoni Fogel's avatar
Yoni Fogel committed
643
    fprintf(pf, "    r=toku_fread_uint32_t(f, &len1, &checksum, &ignorelen);\n");
644 645 646
    fprintf(pf, "    if (r==EOF) return EOF;\n");
    fprintf(pf, "    cmd=fgetc(f);\n");
    fprintf(pf, "    if (cmd==EOF) return DB_BADFORMAT;\n");
Yoni Fogel's avatar
Yoni Fogel committed
647
    fprintf(pf, "    uint32_t len_in_file, len=1+4; // cmd + len1\n");
648
    fprintf(pf, "    char charcmd = (char)cmd;\n");
649
    fprintf(pf, "    toku_x1764_add(&checksum, &charcmd, 1);\n");
650
    fprintf(pf, "    switch ((enum lt_cmd)cmd) {\n");
651 652
    DO_LOGTYPES(lt, { if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); });
    DO_LOGTYPES(lt, {
Leif Walsh's avatar
Leif Walsh committed
653 654 655 656 657 658 659 660 661 662 663 664 665 666
                unsigned char cmd = (unsigned char)(0xff&lt->command_and_flags);
                fprintf(pf, "    case LT_%s: \n", lt->name);
                // We aren't using the log reader here because we want better diagnostics as soon as things go wrong.
                fprintf(pf, "        fprintf(outf, \"%%-%us \", \"%s\");\n", maxnamelen, lt->name);
                if (isprint(cmd)) fprintf(pf,"        fprintf(outf, \" '%c':\");\n", cmd);
                else                      fprintf(pf,"        fprintf(outf, \"0%03o:\");\n", cmd);
                fprintf(pf, "        r = toku_logprint_%-16s(outf, f, \"lsn\", &checksum, &len, 0);     if (r!=0) return r;\n", "LSN");
                DO_FIELDS(field_type, lt, {
                            fprintf(pf, "        r = toku_logprint_%-16s(outf, f, \"%s\", &checksum, &len,", field_type->type, field_type->name);
                            if (field_type->format) fprintf(pf, "\"%s\"", field_type->format);
                            else            fprintf(pf, "0");
                            fprintf(pf, "); if (r!=0) return r;\n");
                        });
                fprintf(pf, "        {\n");
667
                fprintf(pf, "          uint32_t actual_murmur = toku_x1764_finish(&checksum);\n");
Yoni Fogel's avatar
Yoni Fogel committed
668
                fprintf(pf, "          r = toku_fread_uint32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
Leif Walsh's avatar
Leif Walsh committed
669 670
                fprintf(pf, "          fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
                fprintf(pf, "          if (crc_in_file!=actual_murmur) fprintf(outf, \" checksum=%%08x\", actual_murmur);\n");
Yoni Fogel's avatar
Yoni Fogel committed
671
                fprintf(pf, "          r = toku_fread_uint32_t_nocrclen (f, &len_in_file); len+=4; if (r!=0) return r;\n");
Leif Walsh's avatar
Leif Walsh committed
672 673 674 675 676
                fprintf(pf, "          fprintf(outf, \" len=%%u\", len_in_file);\n");
                fprintf(pf, "          if (len_in_file!=len) fprintf(outf, \" actual_len=%%u\", len);\n");
                fprintf(pf, "          if (len_in_file!=len || crc_in_file!=actual_murmur) return DB_BADFORMAT;\n");
                fprintf(pf, "        };\n");
                fprintf(pf, "        fprintf(outf, \"\\n\");\n");
677
                fprintf(pf, "        return 0;\n\n");
Leif Walsh's avatar
Leif Walsh committed
678
            });
679 680 681 682
    fprintf(pf, "    }\n");
    fprintf(pf, "    fprintf(outf, \"Unknown command %%d ('%%c')\", cmd, cmd);\n");
    fprintf(pf, "    return DB_BADFORMAT;\n");
    fprintf(pf, "}\n\n");
683
}
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
684

685 686
static void
generate_rollbacks (void) {
687
    DO_ROLLBACKS(lt, {
688
                    fprintf2(cf, hf, "void toku_logger_save_rollback_%s (TOKUTXN txn", lt->name);
Leif Walsh's avatar
Leif Walsh committed
689
                    DO_FIELDS(field_type, lt, {
690 691
                        if ( strcmp(field_type->type, "BYTESTRING") == 0 ) {
                            fprintf2(cf, hf, ", BYTESTRING *%s_ptr", field_type->name);
692
                        } 
693 694
                        else if ( strcmp(field_type->type, "FILENUMS") == 0 ) {
                            fprintf2(cf, hf, ", FILENUMS *%s_ptr", field_type->name);
695 696
                        }
                        else {
697
                            fprintf2(cf, hf, ", %s %s", field_type->type, field_type->name);
698 699 700
                        }
                    });

Leif Walsh's avatar
Leif Walsh committed
701 702
                    fprintf(hf, ");\n");
                    fprintf(cf, ") {\n");
703
                    fprintf(cf, "  toku_txn_lock(txn);\n");
704
                    fprintf(cf, "  ROLLBACK_LOG_NODE log;\n");
705
                    fprintf(cf, "  toku_get_and_pin_rollback_log_for_new_entry(txn, &log);\n");
Leif Walsh's avatar
Leif Walsh committed
706 707
                    // 'memdup' all BYTESTRINGS here
                    DO_FIELDS(field_type, lt, {
708
                        if ( strcmp(field_type->type, "BYTESTRING") == 0 ) {
709
                            fprintf(cf, "  BYTESTRING %s   = {\n"
710
                                    "    .len  = %s_ptr->len,\n"
711
                                    "    .data = cast_to_typeof(%s.data) toku_memdup_in_rollback(log, %s_ptr->data, %s_ptr->len)\n"
712
                                    "  };\n",
713
                                    field_type->name, field_type->name, field_type->name, field_type->name, field_type->name);
714
                        }
715
                        if ( strcmp(field_type->type, "FILENUMS") == 0 ) {
716 717
                            fprintf(cf, "  FILENUMS %s   = {\n"
                                    "    .num  = %s_ptr->num,\n"
718
                                    "    .filenums = cast_to_typeof(%s.filenums) toku_memdup_in_rollback(log, %s_ptr->filenums, %s_ptr->num * (sizeof (FILENUM)))\n"
719
                                    "  };\n",
720
                                    field_type->name, field_type->name, field_type->name, field_type->name, field_type->name);
721
                        }
722
                    });
Leif Walsh's avatar
Leif Walsh committed
723 724
                    {
                        int count=0;
Yoni Fogel's avatar
Yoni Fogel committed
725
                        fprintf(cf, "  uint32_t rollback_fsize = toku_logger_rollback_fsize_%s(", lt->name);
Leif Walsh's avatar
Leif Walsh committed
726 727 728 729 730
                        DO_FIELDS(field_type, lt, fprintf(cf, "%s%s", (count++>0)?", ":"", field_type->name));
                        fprintf(cf, ");\n");
                    }
                    fprintf(cf, "  struct roll_entry *v;\n");
                    fprintf(cf, "  size_t mem_needed = sizeof(v->u.%s) + __builtin_offsetof(struct roll_entry, u.%s);\n", lt->name, lt->name);
731
                    fprintf(cf, "  CAST_FROM_VOIDP(v, toku_malloc_in_rollback(log, mem_needed));\n");
Leif Walsh's avatar
Leif Walsh committed
732 733 734 735 736 737 738 739
                    fprintf(cf, "  assert(v);\n");
                    fprintf(cf, "  v->cmd = (enum rt_cmd)%u;\n", lt->command_and_flags&0xff);
                    DO_FIELDS(field_type, lt, fprintf(cf, "  v->u.%s.%s = %s;\n", lt->name, field_type->name, field_type->name));
                    fprintf(cf, "  v->prev = log->newest_logentry;\n");
                    fprintf(cf, "  if (log->oldest_logentry==NULL) log->oldest_logentry=v;\n");
                    fprintf(cf, "  log->newest_logentry = v;\n");
                    fprintf(cf, "  log->rollentry_resident_bytecount += rollback_fsize;\n");
                    fprintf(cf, "  txn->roll_info.rollentry_raw_count          += rollback_fsize;\n");
740
                    fprintf(cf, "  txn->roll_info.num_rollentries++;\n");
Yoni Fogel's avatar
Yoni Fogel committed
741
                    fprintf(cf, "  log->dirty = true;\n");
Leif Walsh's avatar
Leif Walsh committed
742 743 744
                    fprintf(cf, "  // spill and unpin assert success internally\n");
                    fprintf(cf, "  toku_maybe_spill_rollbacks(txn, log);\n");
                    fprintf(cf, "  toku_rollback_log_unpin(txn, log);\n");
745
                    fprintf(cf, "  toku_txn_unlock(txn);\n");
746
                    fprintf(cf, "}\n");
Leif Walsh's avatar
Leif Walsh committed
747
            });
748

749
    DO_ROLLBACKS(lt, {
Leif Walsh's avatar
Leif Walsh committed
750 751 752 753 754
                fprintf2(cf, hf, "void toku_logger_rollback_wbuf_nocrc_write_%s (struct wbuf *wbuf", lt->name);
                DO_FIELDS(field_type, lt, fprintf2(cf, hf, ", %s %s", field_type->type, field_type->name));
                fprintf2(cf, hf, ")");
                fprintf(hf, ";\n");
                fprintf(cf, " {\n");
755 756 757

                {
                    int count=0;
Yoni Fogel's avatar
Yoni Fogel committed
758
                    fprintf(cf, "  uint32_t rollback_fsize = toku_logger_rollback_fsize_%s(", lt->name);
759
                    DO_FIELDS(field_type, lt, fprintf(cf, "%s%s", (count++>0)?", ":"", field_type->name));
760 761 762
                    fprintf(cf, ");\n");
                    fprintf(cf, "  wbuf_nocrc_int(wbuf, rollback_fsize);\n");
                }
Leif Walsh's avatar
Leif Walsh committed
763 764 765 766
                fprintf(cf, "  wbuf_nocrc_char(wbuf, '%c');\n", (char)(0xff&lt->command_and_flags));
                DO_FIELDS(field_type, lt, fprintf(cf, "  wbuf_nocrc_%s(wbuf, %s);\n", field_type->type, field_type->name));
                fprintf(cf, "}\n");
            });
767
    fprintf2(cf, hf, "void toku_logger_rollback_wbuf_nocrc_write (struct wbuf *wbuf, struct roll_entry *r)");
768 769
    fprintf(hf, ";\n");
    fprintf(cf, " {\n  switch (r->cmd) {\n");
770
    DO_ROLLBACKS(lt, {
Leif Walsh's avatar
Leif Walsh committed
771 772 773 774
                fprintf(cf, "    case RT_%s: toku_logger_rollback_wbuf_nocrc_write_%s(wbuf", lt->name, lt->name);
                DO_FIELDS(field_type, lt, fprintf(cf, ", r->u.%s.%s", lt->name, field_type->name));
                fprintf(cf, "); return;\n");
            });
775 776
    fprintf(cf, "  }\n  assert(0);\n");
    fprintf(cf, "}\n");
777
    DO_ROLLBACKS(lt, {
Yoni Fogel's avatar
Yoni Fogel committed
778
                fprintf2(cf, hf, "uint32_t toku_logger_rollback_fsize_%s (", lt->name);
Leif Walsh's avatar
Leif Walsh committed
779 780 781 782 783 784 785 786 787 788
                int count=0;
                DO_FIELDS(field_type, lt, fprintf2(cf, hf, "%s%s %s", (count++>0)?", ":"", field_type->type, field_type->name));
                fprintf(hf, ");\n");
                fprintf(cf, ") {\n");
                fprintf(cf, "  return 1 /* the cmd*/\n");
                fprintf(cf, "         + 4 /* the int at the end saying the size */");
                DO_FIELDS(field_type, lt,
                          fprintf(cf, "\n         + toku_logsizeof_%s(%s)", field_type->type, field_type->name));
                fprintf(cf, ";\n}\n");
            });
Yoni Fogel's avatar
Yoni Fogel committed
789
    fprintf2(cf, hf, "uint32_t toku_logger_rollback_fsize(struct roll_entry *item)");
790 791
    fprintf(hf, ";\n");
    fprintf(cf, "{\n  switch(item->cmd) {\n");
792
    DO_ROLLBACKS(lt, {
Leif Walsh's avatar
Leif Walsh committed
793 794 795 796 797
                fprintf(cf, "    case RT_%s: return toku_logger_rollback_fsize_%s(", lt->name, lt->name);
                int count=0;
                DO_FIELDS(field_type, lt, fprintf(cf, "%sitem->u.%s.%s", (count++>0)?", ":"", lt->name, field_type->name));
                fprintf(cf, ");\n");
            });
798 799 800
    fprintf(cf, "  }\n  assert(0);\n  return 0;\n");
    fprintf(cf, "}\n");

Yoni Fogel's avatar
Yoni Fogel committed
801
    fprintf2(cf, hf, "int toku_parse_rollback(unsigned char *buf, uint32_t n_bytes, struct roll_entry **itemp, MEMARENA ma)");
802
    fprintf(hf, ";\n");
803
    fprintf(cf, " {\n  assert(n_bytes>0);\n  struct roll_entry *item;\n  enum rt_cmd cmd = (enum rt_cmd)(buf[0]);\n  size_t mem_needed;\n");
804
    fprintf(cf, "  struct rbuf rc = {buf, n_bytes, 1};\n");
805
    fprintf(cf, "  switch(cmd) {\n");
806
    DO_ROLLBACKS(lt, {
Leif Walsh's avatar
Leif Walsh committed
807 808
                fprintf(cf, "  case RT_%s:\n", lt->name);
                fprintf(cf, "    mem_needed = sizeof(item->u.%s) + __builtin_offsetof(struct roll_entry, u.%s);\n", lt->name, lt->name);
809
                fprintf(cf, "    CAST_FROM_VOIDP(item, malloc_in_memarena(ma, mem_needed));\n");
810
                fprintf(cf, "    item->cmd = cmd;\n");
Leif Walsh's avatar
Leif Walsh committed
811 812 813 814
                DO_FIELDS(field_type, lt, fprintf(cf, "    rbuf_ma_%s(&rc, ma, &item->u.%s.%s);\n", field_type->type, lt->name, field_type->name));
                fprintf(cf, "    *itemp = item;\n");
                fprintf(cf, "    return 0;\n");
        });
815
    fprintf(cf, "  }\n  return EINVAL;\n}\n");
816 817
}

818 819 820 821 822 823 824
static void
generate_log_entry_functions(void) {
    fprintf(hf, "LSN toku_log_entry_get_lsn(struct log_entry *);\n");
    fprintf(cf, "LSN toku_log_entry_get_lsn(struct log_entry *le) {\n");
    fprintf(cf, "    return le->u.begin_checkpoint.lsn;\n");
    fprintf(cf, "}\n");
}
825

826 827
const char codefile[] = "log_code.cc";
const char printfile[] = "log_print.cc";
828
const char headerfile[] = "log_header.h";
829
int main (int argc, const char *const argv[]) {
830 831 832 833 834 835 836 837 838 839 840
    assert(argc==2); // the single argument is the directory into which to put things
    const char *dir = argv[1];
    size_t codepathlen   = sizeof(codefile) + strlen(dir) + 4;
    size_t printpathlen  = sizeof(printfile) + strlen(dir) + 4; 
    size_t headerpathlen = sizeof(headerfile) + strlen(dir) + 4; 
    char codepath[codepathlen];
    char printpath[printpathlen];
    char headerpath[headerpathlen];
    { int r = snprintf(codepath,   codepathlen,   "%s/%s", argv[1], codefile);    assert(r<(int)codepathlen); }
    { int r = snprintf(printpath,  printpathlen,  "%s/%s", argv[1], printfile);   assert(r<(int)printpathlen); }
    { int r = snprintf(headerpath, headerpathlen, "%s/%s", argv[1], headerfile);  assert(r<(int)headerpathlen); }
841 842
    chmod(codepath, S_IRUSR|S_IWUSR);
    chmod(headerpath, S_IRUSR|S_IWUSR);
843 844
    unlink(codepath);
    unlink(headerpath);
845
    cf = fopen(codepath, "w");
846
    if (cf==0) { int r = get_error_errno(); printf("fopen of %s failed because of errno=%d (%s)\n", codepath, r, strerror(r)); } // sometimes this is failing, so let's make a better diagnostic
847
    assert(cf!=0);
848
    hf = fopen(headerpath, "w");     assert(hf!=0);
849
    pf = fopen(printpath, "w");   assert(pf!=0);
850 851
    fprintf2(cf, hf, "/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */\n");
    fprintf2(cf, hf, "// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:\n");
852 853
    fprintf(hf, "#ifndef LOG_HEADER_H\n");
    fprintf(hf, "#define  LOG_HEADER_H\n");
854
    fprintf2(cf, hf, "/* Do not edit this file.  This code generated by logformat.c.  Copyright (c) 2007-2013 Tokutek Inc.    */\n");
855
    fprintf2(cf, hf, "#ident \"Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved.\"\n");
856 857 858 859
    fprintf2(cf, pf, "#include <stdint.h>\n");
    fprintf2(cf, pf, "#include <sys/time.h>\n");
    fprintf2(cf, pf, "#include <ft/fttypes.h>\n");
    fprintf2(cf, pf, "#include <ft/log-internal.h>\n");
860 861
    fprintf(hf, "#include <ft/ft-internal.h>\n");
    fprintf(hf, "#include <ft/memarena.h>\n");
862
    generate_enum();
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
863
    generate_log_struct();
864
    generate_dispatch();
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
865
    generate_log_writer();
866
    generate_log_reader();
867
    generate_rollbacks();
868
    generate_log_entry_functions();
869
    generate_logprint();
870
    fprintf(hf, "#endif\n");
871
    {
Leif Walsh's avatar
Leif Walsh committed
872 873
        int r=fclose(hf); assert(r==0);
        r=fclose(cf); assert(r==0);
874
        r=fclose(pf); assert(r==0);
Leif Walsh's avatar
Leif Walsh committed
875 876 877
        // Make it tougher to modify by mistake
        chmod(codepath, S_IRUSR|S_IRGRP|S_IROTH);
        chmod(headerpath, S_IRUSR|S_IRGRP|S_IROTH);
878
    }
Bradley C. Kuszmaul's avatar
Bradley C. Kuszmaul committed
879 880
    return 0;
}
881