Skip to content

M
mariadb
Commit 4a5571a2 authored by Barry Perlman's avatar Barry Perlman Committed by Yoni Fogel
Browse files
Options

[t:2949] Revert memory status to old design to avoid issues in portability... 

[t:2949] Revert memory status to old design to avoid issues in portability layer, make translation to new system in ydb.c.  Refs #2949.

git-svn-id: file:///svn/toku/tokudb@39389 c7de825b-a66e-492c-adef-691d508d4ae1
parent 975f35da
Hide whitespace changes
Inline Side-by-side
Showing with 136 additions and 111 deletions
linux/memory.c
View file @ 4a5571a2
......@@ -2,8 +2,6 @@
#ident "Copyright (c) 2007-2011 Tokutek Inc. All rights reserved."
#include <toku_portability.h>
#include "db.h" // get Toku-specific version of db.h
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
......@@ -18,51 +16,7 @@ static free_fun_t t_free = 0;
static realloc_fun_t t_realloc = 0;
static realloc_fun_t t_xrealloc = 0;
///////////////////////////////////////////////////////////////////////////////////
// Engine status
//
// Status is intended for display to humans to help understand system behavior.
// It does not need to be perfectly thread-safe.
static MEMORY_STATUS_S memory_status;
static volatile uint64_t max_in_use; // maximum memory footprint (used - freed), approximate (not worth threadsafety overhead for exact, but worth keeping as volatile)
#define STATUS_INIT(k,t,l) { \
memory_status.status[k].keyname = #k; \
memory_status.status[k].type = t; \
memory_status.status[k].legend = "memory: " l; \
}
static void
status_init(void) {
// Note, this function initializes the keyname, type, and legend fields.
// Value fields are initialized to zero by compiler.
STATUS_INIT(MEMORY_MALLOC_COUNT, UINT64, "number of malloc operations");
STATUS_INIT(MEMORY_FREE_COUNT, UINT64, "number of free operations");
STATUS_INIT(MEMORY_REALLOC_COUNT, UINT64, "number of realloc operations");
STATUS_INIT(MEMORY_MALLOC_FAIL, UINT64, "number of malloc operations that failed");
STATUS_INIT(MEMORY_REALLOC_FAIL, UINT64, "number of realloc operations that failed" );
STATUS_INIT(MEMORY_REQUESTED, UINT64, "number of bytes requested");
STATUS_INIT(MEMORY_USED, UINT64, "number of bytes used (requested + overhead)");
STATUS_INIT(MEMORY_FREED, UINT64, "number of bytes freed");
STATUS_INIT(MEMORY_MAX_IN_USE, UINT64, "estimated maximum memory footprint");
STATUS_INIT(MEMORY_MALLOCATOR_VERSION, CHARSTR, "mallocator version");
STATUS_INIT(MEMORY_MMAP_THRESHOLD, UINT64, "mmap threshold");
memory_status.initialized = 1; // TODO 2949 Make this a bool, set to true
}
#undef STATUS_INIT
#define STATUS_VALUE(x) memory_status.status[x].value.num
void
toku_memory_get_status(MEMORY_STATUS statp) {
if (!memory_status.initialized)
status_init();
STATUS_VALUE(MEMORY_MAX_IN_USE) = max_in_use;
*statp = memory_status;
}
#define STATUS_VERSION_STRING memory_status.status[MEMORY_MALLOCATOR_VERSION].value.str
static LOCAL_MEMORY_STATUS_S status;
int
toku_memory_startup(void) {
......@@ -72,8 +26,8 @@ toku_memory_startup(void) {
size_t mmap_threshold = 64 * 1024; // 64K and larger should be malloced with mmap().
int success = mallopt(M_MMAP_THRESHOLD, mmap_threshold);
if (success) {
STATUS_VERSION_STRING = "libc";
STATUS_VALUE(MEMORY_MMAP_THRESHOLD) = mmap_threshold;
status.mallocator_version = "libc";
status.mmap_threshold = mmap_threshold;
} else
result = EINVAL;
......@@ -84,14 +38,14 @@ toku_memory_startup(void) {
mallctl_fun_t mallctl_f;
mallctl_f = (mallctl_fun_t) dlsym(RTLD_DEFAULT, "mallctl");
if (mallctl_f) { // jemalloc is loaded
size_t version_length = sizeof STATUS_VERSION_STRING;
result = mallctl_f("version", &STATUS_VERSION_STRING, &version_length, NULL, 0);
size_t version_length = sizeof status.mallocator_version;
result = mallctl_f("version", &status.mallocator_version, &version_length, NULL, 0);
if (result == 0) {
size_t lg_chunk; // log2 of the mmap threshold
size_t lg_chunk_length = sizeof lg_chunk;
result = mallctl_f("opt.lg_chunk", &lg_chunk, &lg_chunk_length, NULL, 0);
if (result == 0)
STATUS_VALUE(MEMORY_MMAP_THRESHOLD) = 1 << lg_chunk;
status.mmap_threshold = 1 << lg_chunk;
}
}
......@@ -102,6 +56,11 @@ void
toku_memory_shutdown(void) {
}
void
toku_memory_get_status(LOCAL_MEMORY_STATUS s) {
*s = status;
}
// jemalloc's malloc_usable_size does not work with a NULL pointer, so we implement a version that works
static size_t
my_malloc_usable_size(void *p) {
......@@ -112,16 +71,16 @@ my_malloc_usable_size(void *p) {
// It is not worth the overhead to make it completely accurate, but
// this logic is intended to guarantee that it increases monotonically.
// Note that status.sum_used and status.sum_freed increase monotonically
// and that max_in_use is declared volatile.
// and that status.max_in_use is declared volatile.
static inline void
set_max(uint64_t sum_used, uint64_t sum_freed) {
if (sum_used >= sum_freed) {
uint64_t in_use = sum_used - sum_freed;
uint64_t old_max;
do {
old_max = max_in_use;
old_max = status.max_in_use;
} while (old_max < in_use &&
!__sync_bool_compare_and_swap(&max_in_use, old_max, in_use));
!__sync_bool_compare_and_swap(&status.max_in_use, old_max, in_use));
}
}
......@@ -133,7 +92,7 @@ toku_memory_footprint(void * p, size_t touched) {
pagesize = sysconf(_SC_PAGESIZE);
if (p) {
size_t usable = my_malloc_usable_size(p);
if (usable >= STATUS_VALUE(MEMORY_MMAP_THRESHOLD)) {
if (usable >= status.mmap_threshold) {
int num_pages = (touched + pagesize) / pagesize;
rval = num_pages * pagesize;
}
......@@ -149,12 +108,12 @@ toku_malloc(size_t size) {
void *p = t_malloc ? t_malloc(size) : os_malloc(size);
if (p) {
size_t used = my_malloc_usable_size(p);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_MALLOC_COUNT), 1);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REQUESTED), size);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_USED), used);
set_max(STATUS_VALUE(MEMORY_USED), STATUS_VALUE(MEMORY_FREED));
__sync_add_and_fetch(&status.malloc_count, 1);
__sync_add_and_fetch(&status.requested,size);
__sync_add_and_fetch(&status.used, used);
set_max(status.used, status.freed);
} else {
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_MALLOC_FAIL), 1);
__sync_add_and_fetch(&status.malloc_fail, 1);
}
return p;
}
......@@ -173,13 +132,13 @@ toku_realloc(void *p, size_t size) {
void *q = t_realloc ? t_realloc(p, size) : os_realloc(p, size);
if (q) {
size_t used = my_malloc_usable_size(q);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REALLOC_COUNT), 1);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REQUESTED), size);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_USED), used);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_FREED), used_orig);
set_max(STATUS_VALUE(MEMORY_USED), STATUS_VALUE(MEMORY_FREED));
__sync_add_and_fetch(&status.realloc_count, 1);
__sync_add_and_fetch(&status.requested, size);
__sync_add_and_fetch(&status.used, used);
__sync_add_and_fetch(&status.freed, used_orig);
set_max(status.used, status.freed);
} else {
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REALLOC_FAIL), 1);
__sync_add_and_fetch(&status.realloc_fail, 1);
}
return q;
}
......@@ -200,8 +159,8 @@ void
toku_free(void *p) {
if (p) {
size_t used = my_malloc_usable_size(p);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_FREE_COUNT), 1);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_FREED), used);
__sync_add_and_fetch(&status.free_count, 1);
__sync_add_and_fetch(&status.freed, used);
if (t_free)
t_free(p);
else
......@@ -220,10 +179,10 @@ toku_xmalloc(size_t size) {
if (p == NULL) // avoid function call in common case
resource_assert(p);
size_t used = my_malloc_usable_size(p);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_MALLOC_COUNT), 1);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REQUESTED), size);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_USED), used);
set_max(STATUS_VALUE(MEMORY_USED), STATUS_VALUE(MEMORY_FREED));
__sync_add_and_fetch(&status.malloc_count, 1);
__sync_add_and_fetch(&status.requested, size);
__sync_add_and_fetch(&status.used, used);
set_max(status.used, status.freed);
return p;
}
......@@ -242,11 +201,11 @@ toku_xrealloc(void *v, size_t size) {
if (p == 0) // avoid function call in common case
resource_assert(p);
size_t used = my_malloc_usable_size(p);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REALLOC_COUNT), 1);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_REQUESTED), size);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_USED), used);
__sync_add_and_fetch(&STATUS_VALUE(MEMORY_FREED), used_orig);
set_max(STATUS_VALUE(MEMORY_USED), STATUS_VALUE(MEMORY_FREED));
__sync_add_and_fetch(&status.realloc_count, 1);
__sync_add_and_fetch(&status.requested, size);
__sync_add_and_fetch(&status.used, used);
__sync_add_and_fetch(&status.freed, used_orig);
set_max(status.used, status.freed);
return p;
}
......@@ -309,5 +268,5 @@ toku_set_func_free(free_fun_t f) {
void __attribute__((constructor)) toku_memory_drd_ignore(void);
void
toku_memory_drd_ignore(void) {
DRD_IGNORE_VAR(memory_status);
DRD_IGNORE_VAR(status);
}
linux/tests/test-memory-status.c
View file @ 4a5571a2
......@@ -3,7 +3,7 @@
int main(void) {
toku_memory_startup();
MEMORY_STATUS_S s;
LOCAL_MEMORY_STATUS_S s;
toku_memory_get_status(&s);
printf("mallocator: %s\n", s.mallocator_version);
printf("mmap threshold: %" PRIu64 "\n", s.mmap_threshold);
......
src/tests/test.h
View file @ 4a5571a2
......@@ -130,6 +130,7 @@ print_engine_status(DB_ENV * UU(env)) {
static __attribute__((__unused__)) uint64_t
get_engine_status_val(DB_ENV * UU(env), char * keyname) {
uint64_t rval = 0;
#ifdef USE_TDB
uint64_t nrows;
env->get_engine_status_num_rows(env, &nrows);
......@@ -141,7 +142,6 @@ get_engine_status_val(DB_ENV * UU(env), char * keyname) {
int r = env->get_engine_status (env, mystat, nrows, &redzone_state, &panic, panic_string, panic_string_len);
CKERR(r);
int found = 0;
uint64_t rval = 0;
for (uint64_t i = 0; i < nrows && !found; i++) {
if (strcmp(keyname, mystat[i].keyname) == 0) {
found++;
......@@ -149,8 +149,8 @@ get_engine_status_val(DB_ENV * UU(env), char * keyname) {
}
}
CKERR2(found, 1);
return rval;
#endif
return rval;
}
......
src/ydb.c
View file @ 4a5571a2
......@@ -128,7 +128,7 @@ static YDB_LAYER_STATUS_S ydb_layer_status;
ydb_layer_status.status[k].legend = l; \
}
static void
status_init (void) {
ydb_layer_status_init (void) {
// Note, this function initializes the keyname, type, and legend fields.
// Value fields are initialized to zero by compiler.
......@@ -964,7 +964,7 @@ toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mode) {
need_rollback_cachefile = TRUE;
}
status_init(); // do this before possibly upgrading, so upgrade work is counted in status counters
ydb_layer_status_init(); // do this before possibly upgrading, so upgrade work is counted in status counters
LSN last_lsn_of_clean_shutdown_read_from_log = ZERO_LSN;
BOOL upgrade_in_progress = FALSE;
......@@ -1977,7 +1977,11 @@ format_time(const time_t *timer, char *buf) {
}
}
// local status struct, used to concentrate file system information collected from various places
////////////////////////////////////////////////////////////////////////////////////////////////
// Local definition of status information from portability layer, which should not include db.h.
// Local status structs are used to concentrate file system information collected from various places
// and memory information collected from memory.c.
//
typedef enum {
FS_ENOSPC_REDZONE_STATE = 0, // possible values are enumerated by fs_redzone_state
FS_ENOSPC_THREADS_BLOCKED, // how many threads currently blocked on ENOSPC
......@@ -2042,6 +2046,76 @@ fs_get_status(DB_ENV * env, fs_redzone_state * redzone_state) {
}
#undef FS_STATUS_VALUE
// Local status struct used to get information from memory.c
typedef enum {
MEMORY_MALLOC_COUNT = 0,
MEMORY_FREE_COUNT,
MEMORY_REALLOC_COUNT,
MEMORY_MALLOC_FAIL,
MEMORY_REALLOC_FAIL,
MEMORY_REQUESTED,
MEMORY_USED,
MEMORY_FREED,
MEMORY_MAX_IN_USE,
MEMORY_MALLOCATOR_VERSION,
MEMORY_MMAP_THRESHOLD,
MEMORY_STATUS_NUM_ROWS
} memory_status_entry;
typedef struct {
BOOL initialized;
TOKU_ENGINE_STATUS_ROW_S status[MEMORY_STATUS_NUM_ROWS];
} MEMORY_STATUS_S, *MEMORY_STATUS;
static MEMORY_STATUS_S memory_status;
#define STATUS_INIT(k,t,l) { \
memory_status.status[k].keyname = #k; \
memory_status.status[k].type = t; \
memory_status.status[k].legend = "memory: " l; \
}
static void
memory_status_init(void) {
// Note, this function initializes the keyname, type, and legend fields.
// Value fields are initialized to zero by compiler.
STATUS_INIT(MEMORY_MALLOC_COUNT, UINT64, "number of malloc operations");
STATUS_INIT(MEMORY_FREE_COUNT, UINT64, "number of free operations");
STATUS_INIT(MEMORY_REALLOC_COUNT, UINT64, "number of realloc operations");
STATUS_INIT(MEMORY_MALLOC_FAIL, UINT64, "number of malloc operations that failed");
STATUS_INIT(MEMORY_REALLOC_FAIL, UINT64, "number of realloc operations that failed" );
STATUS_INIT(MEMORY_REQUESTED, UINT64, "number of bytes requested");
STATUS_INIT(MEMORY_USED, UINT64, "number of bytes used (requested + overhead)");
STATUS_INIT(MEMORY_FREED, UINT64, "number of bytes freed");
STATUS_INIT(MEMORY_MAX_IN_USE, UINT64, "estimated maximum memory footprint");
STATUS_INIT(MEMORY_MALLOCATOR_VERSION, CHARSTR, "mallocator version");
STATUS_INIT(MEMORY_MMAP_THRESHOLD, UINT64, "mmap threshold");
memory_status.initialized = true;
}
#undef STATUS_INIT
#define MEMORY_STATUS_VALUE(x) memory_status.status[x].value.num
static void
memory_get_status(void) {
if (!memory_status.initialized)
memory_status_init();
LOCAL_MEMORY_STATUS_S local_memstat;
MEMORY_STATUS_VALUE(MEMORY_MALLOC_COUNT) = local_memstat.malloc_count;
MEMORY_STATUS_VALUE(MEMORY_FREE_COUNT) = local_memstat.free_count;
MEMORY_STATUS_VALUE(MEMORY_REALLOC_COUNT) = local_memstat.realloc_count;
MEMORY_STATUS_VALUE(MEMORY_MALLOC_FAIL) = local_memstat.malloc_fail;
MEMORY_STATUS_VALUE(MEMORY_REALLOC_FAIL) = local_memstat.realloc_fail;
MEMORY_STATUS_VALUE(MEMORY_REQUESTED) = local_memstat.requested;
MEMORY_STATUS_VALUE(MEMORY_USED) = local_memstat.used;
MEMORY_STATUS_VALUE(MEMORY_FREED) = local_memstat.freed;
MEMORY_STATUS_VALUE(MEMORY_MAX_IN_USE) = local_memstat.max_in_use;
MEMORY_STATUS_VALUE(MEMORY_MMAP_THRESHOLD) = local_memstat.mmap_threshold;
memory_status.status[MEMORY_MALLOCATOR_VERSION].value.str = local_memstat.mallocator_version;
}
#undef MEMORY_STATUS_VALUE
// how many rows are in engine status?
static int
env_get_engine_status_num_rows (DB_ENV * UU(env), uint64_t * num_rowsp) {
......@@ -2193,10 +2267,10 @@ env_get_engine_status (DB_ENV * env, TOKU_ENGINE_STATUS_ROW engstat, uint64_t ma
}
{
MEMORY_STATUS_S memorystat;
toku_memory_get_status(&memorystat);
// memory_status is local to this file
memory_get_status();
for (int i = 0; i < MEMORY_STATUS_NUM_ROWS && row < maxrows; i++) {
engstat[row++] = memorystat.status[i];
engstat[row++] = memory_status.status[i];
}
}
{
......
toku_include/memory.h
View file @ 4a5571a2
......@@ -5,7 +5,6 @@
#include <stdlib.h>
#include <toku_portability.h>
#include <db.h>
#if defined(__cplusplus) || defined(__cilkplusplus)
extern "C" {
......@@ -98,28 +97,21 @@ void toku_set_func_xrealloc_only(realloc_fun_t f);
void toku_set_func_realloc_only(realloc_fun_t f);
void toku_set_func_free(free_fun_t f);
typedef enum {
MEMORY_MALLOC_COUNT = 0,
MEMORY_FREE_COUNT,
MEMORY_REALLOC_COUNT,
MEMORY_MALLOC_FAIL,
MEMORY_REALLOC_FAIL,
MEMORY_REQUESTED,
MEMORY_USED,
MEMORY_FREED,
MEMORY_MAX_IN_USE,
MEMORY_MALLOCATOR_VERSION,
MEMORY_MMAP_THRESHOLD,
MEMORY_STATUS_NUM_ROWS
} memory_status_entry;
typedef struct {
int initialized; // TODO 2949 make this a bool
TOKU_ENGINE_STATUS_ROW_S status[MEMORY_STATUS_NUM_ROWS];
} MEMORY_STATUS_S, *MEMORY_STATUS;
void toku_memory_get_status(MEMORY_STATUS s);
typedef struct memory_status {
uint64_t malloc_count; // number of malloc operations
uint64_t free_count; // number of free operations
uint64_t realloc_count; // number of realloc operations
uint64_t malloc_fail; // number of malloc operations that failed
uint64_t realloc_fail; // number of realloc operations that failed
uint64_t requested; // number of bytes requested
uint64_t used; // number of bytes used (requested + overhead), obtained from malloc_usable_size()
uint64_t freed; // number of bytes freed;
volatile uint64_t max_in_use; // maximum memory footprint (used - freed), approximate (not worth threadsafety overhead for exact)
char *mallocator_version;
uint64_t mmap_threshold;
} LOCAL_MEMORY_STATUS_S, *LOCAL_MEMORY_STATUS;
void toku_memory_get_status(LOCAL_MEMORY_STATUS s);
size_t toku_memory_footprint(void * p, size_t touched);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7