MDEV-17084 - Optimize append only files for NVDIMM

Append cache implementation. Based on libpmem, which is mostly needed for
effecient data flushing from CPU caches.

When append cache is enabled for particular file, data is first stored in a
mmap()-ed circular buffer on faster persistent storage. Background thread
is flushing this buffer to a file on slower persistent storage.

Append caches are stored in regular append cache files. One append cache file
may contain multiple caches.

CMakeLists.txt

@@ -168,6 +168,7 @@ INCLUDE(mysql_add_executable)
 INCLUDE(symlinks)
 INCLUDE(compile_flags)
 INCLUDE(crc32)
+INCLUDE(pmem)
 
 # Handle options
 OPTION(DISABLE_SHARED 

cmake/pmem.cmake

+INCLUDE(CheckIncludeFiles)
+OPTION(WITH_PMEMAC "Enable persistent memory append cache" OFF)
+IF(WITH_PMEMAC)
+  FIND_LIBRARY(LIBPMEM pmem)
+  CHECK_INCLUDE_FILES(libpmem.h HAVE_LIBPMEM_H)
+  IF (NOT LIBPMEM)
+    MESSAGE(FATAL_ERROR "Can't find libpmem")
+  ELSEIF(NOT HAVE_LIBPMEM_H)
+    MESSAGE(FATAL_ERROR "Can't find libpmem.h")
+  ELSE()
+    INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/extra/libpmemac)
+    ADD_DEFINITIONS(-DHAVE_PMEMAC)
+    SET(LIBPMEMAC "pmemac")
+  ENDIF()
+ENDIF()

extra/CMakeLists.txt

@@ -112,3 +112,7 @@ IF(UNIX)
   MYSQL_ADD_EXECUTABLE(mysqld_safe_helper mysqld_safe_helper.c COMPONENT Server)
   TARGET_LINK_LIBRARIES(mysqld_safe_helper mysys)
 ENDIF()
+
+IF(LIBPMEMAC)
+  ADD_SUBDIRECTORY(libpmemac)
+ENDIF()

extra/libpmemac/CMakeLists.txt

+ADD_CONVENIENCE_LIBRARY(pmemac append_cache.c)
+ADD_EXECUTABLE(pmemac-bin pmemac.c)
+TARGET_LINK_LIBRARIES(pmemac mysys ${LIBPMEM})
+TARGET_LINK_LIBRARIES(pmemac-bin pmemac)
+SET_TARGET_PROPERTIES(pmemac-bin PROPERTIES OUTPUT_NAME pmemac)

extra/libpmemac/append_cache.h

+#ifndef APPEND_CACHE_H_INCLUDED
+#define APPEND_CACHE_H_INCLUDED
+/*
+  Copyright (c) 2019, MariaDB Corporation.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; version 2 of the License.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA
+*/
+
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <my_global.h>
+
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+
+/*
+  Cache file consists of directory header followed by zero or more append
+  caches.
+
+  Directory header:
+  magic            (8 bytes)                file signature
+  n_caches         (8 bytes)                number of caches in directory
+  start_offsets    (8 bytes * n_caches)     array of cache start offsets from
+                                            the beginning of the cache file
+
+  Append cache format:
+  flushed_eof      (8 bytes)                cache is flushed up to this offset
+  cached_eof       (8 bytes)                cache contains data up to this offset
+  file_name_length (8 bytes)                file name length
+  file_name        (file_name_length bytes) target file name
+  buffer           (N)                      circular buffer, lasts up to the
+                                            following cache start offset or eof
+                                            for the last cache
+
+  flushed_eof normally equals to the target file size. It is an error if
+  flushed_eof is smaller than the target file size.
+
+  flushed_eof can be larger than the target file size if crash occured while
+  flushing append cache. In such case data after flushed_eof is overwritten
+  during crash recovery.
+
+  Append cache is holding data, which otherwise would be stored in the target
+  file between flushed_eof and cached_eof.
+
+  flushed_eof is advanced by data size whenever data is added. cached_eof is
+  advanced by flushed data size whenever data is flushed.
+
+  Cached data starts at buffer + flushed_eof % buffer_size and ends at
+  buffer + cached_eof % buffer_size.
+*/
+
+
+/** Fixed-size directory header. */
+typedef struct st_pmem_append_cache_directory_header
+{
+  uint64_t magic;
+  uint64_t n_caches;
+} PMEM_APPEND_CACHE_DIRECTORY_HEADER;
+
+
+/** In-memory cache directory descriptor. */
+typedef struct st_pmem_append_cache_directory
+{
+  PMEM_APPEND_CACHE_DIRECTORY_HEADER *header;
+  uint64_t *start_offsets;
+  size_t mapped_length;
+  bool dummy;
+} PMEM_APPEND_CACHE_DIRECTORY;
+
+
+/** Fixed-size cache header. */
+typedef struct st_pmem_append_cache_header
+{
+  uint64_t flushed_eof;
+  uint64_t cached_eof;
+  uint64_t file_name_length;
+} PMEM_APPEND_CACHE_HEADER;
+
+
+/** In-memory append cache descriptor. */
+typedef struct st_pmem_append_cache
+{
+  pthread_t flusher_thread;
+  PMEM_APPEND_CACHE_HEADER *header;
+  char *file_name;
+  void *buffer;
+  uint64_t buffer_size;
+  File file_fd;
+  int32 stop_flusher;
+
+  size_t (*write)(struct st_pmem_append_cache *cache, const void *data,
+                  size_t length, myf flags);
+  void (*flush)(struct st_pmem_append_cache *cache, uint64_t offset);
+  int (*sync)(struct st_pmem_append_cache *cache, uint64_t offset);
+
+  char pad1[CPU_LEVEL1_DCACHE_LINESIZE];
+  uint64_t flushed_eof;
+  char pad2[CPU_LEVEL1_DCACHE_LINESIZE];
+  uint64_t cached_eof;
+  char pad3[CPU_LEVEL1_DCACHE_LINESIZE];
+  uint64_t reserved_eof;
+  char pad4[CPU_LEVEL1_DCACHE_LINESIZE];
+} PMEM_APPEND_CACHE;
+
+
+int pmem_append_cache_create(const char *path, uint64_t size,
+                             uint64_t n_caches);
+int pmem_append_cache_open(PMEM_APPEND_CACHE_DIRECTORY *dir, const char *path);
+int pmem_append_cache_close(PMEM_APPEND_CACHE_DIRECTORY *dir);
+int pmem_append_cache_flush(PMEM_APPEND_CACHE_DIRECTORY *dir);
+int pmem_append_cache_init(PMEM_APPEND_CACHE_DIRECTORY *dir, const char *path,
+                           uint64_t size, uint64_t n_caches);
+int pmem_append_cache_attach(PMEM_APPEND_CACHE *cache,
+                             PMEM_APPEND_CACHE_DIRECTORY *dir,
+                             uint64_t n,
+                             File file_fd,
+                             const char *file_name);
+int pmem_append_cache_detach(PMEM_APPEND_CACHE *cache);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

extra/libpmemac/pmemac.c

+/*
+  Copyright (c) 2019, MariaDB Corporation.
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; version 2 of the License.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA
+*/
+
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include "append_cache.h"
+#include "my_sys.h"
+
+
+int open_cache(PMEM_APPEND_CACHE *cache, PMEM_APPEND_CACHE_DIRECTORY *dir,
+               uint64_t n);
+
+typedef struct st_command
+{
+  const char *name;
+  const char *description;
+  int (*func)(int argc, char *argv[]);
+} COMMAND;
+
+
+static int help(int argc, char *argv[]);
+
+
+static int info(int argc, char *argv[])
+{
+  PMEM_APPEND_CACHE_DIRECTORY dir;
+
+  if (argc != 3)
+  {
+    help(argc, argv);
+    return 1;
+  }
+  if (pmem_append_cache_open(&dir, argv[2]))
+  {
+    perror("Failed to open cache");
+    return 1;
+  }
+  printf("Number of slots in directory: %" PRIu64 ", mapped size: %zu\n",
+         dir.header->n_caches,
+         dir.mapped_length);
+  for (uint64_t i= 0; i < dir.header->n_caches; i++)
+  {
+    PMEM_APPEND_CACHE cache;
+    printf("  cache %" PRIu64 " at offset %" PRIu64 ": ", i,
+           dir.start_offsets[i]);
+    if (open_cache(&cache, &dir, i))
+      printf("failed to open\n");
+    else
+      printf("buffer size: %" PRIu64
+             ", flushed eof: %" PRIu64
+             ", cached eof: %" PRIu64
+             ", file name length: %" PRIu64
+             ", target file name: %s\n",
+             cache.buffer_size, cache.header->flushed_eof,
+             cache.header->cached_eof, cache.header->file_name_length,
+             cache.header->file_name_length ? cache.file_name : "<not attached>");
+  }
+  if (pmem_append_cache_close(&dir))
+  {
+    perror("Failed to close cache");
+    return 1;
+  }
+  return 0;
+}
+
+
+static int create(int argc, char *argv[])
+{
+  if (argc != 5)
+  {
+    help(argc, argv);
+    return 1;
+  }
+  if (pmem_append_cache_create(argv[2], strtoull(argv[3], 0, 0),
+                               strtoull(argv[4], 0, 0)))
+  {
+    perror("Failed to create cache");
+    return 1;
+  }
+  return 0;
+}
+
+
+static int test(int argc, char *argv[])
+{
+  PMEM_APPEND_CACHE_DIRECTORY dir;
+  PMEM_APPEND_CACHE cache;
+  File fd;
+  int res= 1;
+
+  if (argc != 4)
+  {
+    help(argc, argv);
+    return 1;
+  }
+  if ((fd= my_open(argv[3], O_CREAT | O_WRONLY, MYF(MY_WME))) < 0)
+  {
+    perror("Failed to open target file");
+    return 1;
+  }
+  if (pmem_append_cache_create(argv[2], 64, 1))
+  {
+    perror("Failed to create cache");
+    goto err1;
+  }
+  if (pmem_append_cache_open(&dir, argv[2]))
+  {
+    perror("Failed to open cache");
+    goto err2;
+  }
+  if (pmem_append_cache_attach(&cache, &dir, 0, fd, argv[3]))
+  {
+    perror("Failed to attach to append cache");
+    goto err3;
+  }
+  for (int i= 0; i < 6; i++)
+  {
+    char buf[16];
+    sprintf(buf, "%06d\n", i);
+    cache.write(&cache, buf, 7, MYF(0));
+  }
+  printf("Buffer size: %zd, flushed_eof: %ld, cached_eof: %ld, reserved_eof: %ld\n",
+         cache.buffer_size, cache.flushed_eof, cache.cached_eof,
+         cache.reserved_eof);
+  if (pmem_append_cache_detach(&cache))
+  {
+    perror("Failed to detach from append cache");
+    goto err3;
+  }
+  res= 0;
+err3:
+  if (pmem_append_cache_close(&dir))
+  {
+    perror("Failed to close cache");
+    res= 1;
+  }
+err2:
+  if (my_delete(argv[2], MYF(MY_WME)))
+  {
+    perror("Failed to unlink cache file");
+    res= 1;
+  }
+err1:
+  if (my_close(fd, MYF(MY_WME)))
+  {
+    perror("Failed to close target file");
+    res= 1;
+  }
+  return res;
+}
+
+
+static int flush(int argc, char *argv[])
+{
+  int res;
+  PMEM_APPEND_CACHE_DIRECTORY dir;
+  if (argc != 3)
+  {
+    help(argc, argv);
+    return 1;
+  }
+  if (pmem_append_cache_open(&dir, argv[2]))
+  {
+    perror("Failed to open cache");
+    return 1;
+  }
+  if ((res= pmem_append_cache_flush(&dir)))
+    perror("Failed to flush cache");
+  if (pmem_append_cache_close(&dir))
+  {
+    perror("Failed to close cache");
+    return 1;
+  }
+  return res ? 1 : 0;
+}
+
+
+static COMMAND commands[]=
+{
+  { "help", "", help },
+  { "info", "<path>", info },
+  { "create", "<path> <size> <n_caches>", create },
+  { "flush", "<path>", flush },
+  { "test", "<path> <file_path>", test },
+};
+
+
+static int help(int argc, char *argv[])
+{
+  puts("usage:");
+  for (uint32_t i= 0; i < sizeof(commands) / sizeof(commands[0]); i++)
+    printf("  pmemac %s %s\n", commands[i].name, commands[i].description);
+  return 0;
+}
+
+
+int main(int argc, char *argv[])
+{
+  if (argc > 1)
+  {
+    for (uint32_t i= 0; i < array_elements(commands); i++)
+    {
+      if (!strcmp(commands[i].name, argv[1]))
+      {
+        int res;
+        MY_INIT(argv[0]);
+        res= commands[i].func(argc, argv);
+        my_end(0);
+        return res;
+      }
+    }
+  }
+  help(argc, argv);
+  return 1;
+}

libmysqld/CMakeLists.txt

@@ -158,6 +158,7 @@ SET(LIBS
   ${ZLIB_LIBRARY} ${SSL_LIBRARIES} 
   ${LIBWRAP} ${LIBCRYPT} ${LIBDL}
   ${EMBEDDED_PLUGIN_LIBS}
+  ${LIBPMEMAC}
   sql_embedded
 )
 

mysys/CMakeLists.txt

@@ -69,7 +69,8 @@ ENDIF()
 
 ADD_CONVENIENCE_LIBRARY(mysys ${MYSYS_SOURCES})
 TARGET_LINK_LIBRARIES(mysys dbug strings ${ZLIB_LIBRARY}
- ${LIBNSL} ${LIBM} ${LIBRT} ${LIBDL} ${LIBSOCKET} ${LIBEXECINFO} ${CRC32_LIBRARY})
+ ${LIBNSL} ${LIBM} ${LIBRT} ${LIBDL} ${LIBSOCKET} ${LIBEXECINFO} ${CRC32_LIBRARY}
+ ${LIBPMEMAC})
 DTRACE_INSTRUMENT(mysys)
 
 IF(HAVE_BFD_H)