MDEV-13122 Backup myrocksdb with mariabackup.

extra/mariabackup/backup_copy.cc

@@ -59,6 +59,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include <btr0btr.h>
 #include "xb0xb.h"
 
+#define ROCKSDB_BACKUP_DIR "#rocksdb"
 
 /* list of files to sync for --rsync mode */
 static std::set<std::string> rsync_list;
@@ -68,6 +69,21 @@ static std::map<std::string, std::string> tablespace_locations;
 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
 bool binlog_locked;
 
+static void rocksdb_create_checkpoint();
+static bool has_rocksdb_plugin();
+static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
+static void rocksdb_backup_checkpoint();
+static void rocksdb_copy_back();
+
+static bool is_abs_path(const char *path)
+{
+#ifdef _WIN32
+	return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
+#else
+	return path[0] == '/';
+#endif
+}
+
 /************************************************************************
 Struct represents file or directory. */
 struct datadir_node_t {
@@ -1140,7 +1156,8 @@ bool
 copy_or_move_file(const char *src_file_path,
 		  const char *dst_file_path,
 		  const char *dst_dir,
-		  uint thread_n)
+		  uint thread_n,
+		 bool copy = xtrabackup_copy_back)
 {
 	ds_ctxt_t *datasink = ds_data;		/* copy to datadir by default */
 	char filedir[FN_REFLEN];
@@ -1188,7 +1205,7 @@ copy_or_move_file(const char *src_file_path,
 		free(link_filepath);
 	}
 
-	ret = (xtrabackup_copy_back ?
+	ret = (copy ?
 		copy_file(datasink, src_file_path, dst_file_path, thread_n) :
 		move_file(datasink, src_file_path, dst_file_path,
 			  dst_dir, thread_n));
@@ -1373,6 +1390,10 @@ bool backup_start()
 		return false;
 	}
 
+	if (has_rocksdb_plugin()) {
+		rocksdb_create_checkpoint();
+	}
+
 	// There is no need to stop slave thread before coping non-Innodb data when
 	// --no-lock option is used because --no-lock option requires that no DDL or
 	// DML to non-transaction tables can occur.
@@ -1458,6 +1479,10 @@ bool backup_finish()
 		}
 	}
 
+	if (has_rocksdb_plugin()) {
+		rocksdb_backup_checkpoint();
+	}
+
 	msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir);
 	if (mysql_binlog_position != NULL) {
 		msg("MySQL binlog position: %s\n", mysql_binlog_position);
@@ -1773,6 +1798,16 @@ copy_back()
 		int i_tmp;
 		bool is_ibdata_file;
 
+		if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
+#ifdef _WIN32
+			|| strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
+#endif
+		)
+		{
+			// copied at later step
+			continue;
+		}
+
 		/* create empty directories */
 		if (node.is_empty_dir) {
 			char path[FN_REFLEN];
@@ -1857,6 +1892,8 @@ copy_back()
 		}
 	}
 
+	rocksdb_copy_back();
+
 cleanup:
 	if (it != NULL) {
 		datadir_iter_free(it);
@@ -2033,3 +2070,234 @@ static bool backup_files_from_datadir(const char *dir_path)
 	os_file_closedir(dir);
 	return ret;
 }
+
+
+static int rocksdb_remove_checkpoint_directory()
+{
+	xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
+	return 0;
+}
+
+static bool has_rocksdb_plugin()
+{
+	static bool first_time = true;
+	static bool has_plugin= false;
+	if (!first_time || !xb_backup_rocksdb)
+		return has_plugin;
+
+	const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
+	MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
+	MYSQL_ROW row = mysql_fetch_row(result);
+	if (row)
+		has_plugin = !strcmp(row[0], "1");
+	mysql_free_result(result);
+	first_time = false;
+	return has_plugin;
+}
+
+static char *trim_trailing_dir_sep(char *path)
+{
+	size_t path_len = strlen(path);
+	while (path_len)
+	{
+		char c = path[path_len - 1];
+		if (c == '/' IF_WIN(|| c == '\\', ))
+			path_len--;
+		else
+			break;
+	}
+	path[path_len] = 0;
+	return path;
+}
+
+/*
+Create a file hardlink.
+@return true on success, false on error.
+*/
+static bool make_hardlink(const char *from_path, const char *to_path)
+{
+	DBUG_EXECUTE_IF("no_hardlinks", return false;);
+	char to_path_full[FN_REFLEN];
+	if (!is_abs_path(to_path))
+	{
+		fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
+	}
+	else
+	{
+		strncpy(to_path_full, to_path, sizeof(to_path_full));
+	}
+#ifdef _WIN32
+	return  CreateHardLink(to_path_full, from_path, NULL);
+#else
+	return !link(from_path, to_path_full);
+#endif
+}
+
+/*
+ Copies or moves a directory (non-recursively so far).
+ Helper function used to backup rocksdb checkpoint, or copy-back the
+ rocksdb files.
+
+ Has optimization that allows to use hardlinks when possible
+ (source and destination are directories on the same device)
+*/
+static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
+{
+	datadir_node_t node;
+	datadir_node_init(&node);
+	datadir_iter_t *it = datadir_iter_new(from, false);
+
+	while (datadir_iter_next(it, &node))
+	{
+		char to_path[FN_REFLEN];
+		const char *from_path = node.filepath;
+		snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
+		bool rc = false;
+		if (do_copy && allow_hardlinks)
+		{
+			rc = make_hardlink(from_path, to_path);
+			if (rc)
+			{
+				msg_ts("[%02u] Creating hardlink from %s to %s\n",
+					1, from_path, to_path);
+			}
+			else
+			{
+				allow_hardlinks = false;
+			}
+		}
+
+		if (!rc) 
+		{
+			rc = (do_copy ?
+				copy_file(ds_data, from_path, to_path, 1) :
+				move_file(ds_data, from_path, node.filepath_rel,
+					to, 1));
+		}
+		if (!rc)
+			exit(EXIT_FAILURE);
+	}
+	datadir_iter_free(it);
+	datadir_node_free(&node);
+
+}
+
+/*
+  Obtain user level lock , to protect the checkpoint directory of the server
+  from being  user/overwritten by different backup processes, if backups are
+  running in parallel.
+  
+  This lock will be acquired before rocksdb checkpoint is created,  held
+  while all files from it are being copied to their final backup destination,
+  and finally released after the checkpoint is removed.
+*/
+static void rocksdb_lock_checkpoint()
+{
+	msg_ts("Obtaining rocksdb checkpoint lock.\n");
+	MYSQL_RES *res =
+		xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);
+
+	MYSQL_ROW r = mysql_fetch_row(res);
+	if (r && r[0] && strcmp(r[0], "1"))
+	{
+		msg_ts("Could not obtain rocksdb checkpont lock\n");
+		exit(EXIT_FAILURE);
+	}
+}
+
+static void rocksdb_unlock_checkpoint()
+{
+	xb_mysql_query(mysql_connection, 
+		"SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
+}
+
+
+/*
+  Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
+  directory.
+  A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
+  acquired be this function.
+*/
+#define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
+static 	char rocksdb_checkpoint_dir[FN_REFLEN];
+
+static void rocksdb_create_checkpoint()
+{
+	MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
+	MYSQL_ROW row = mysql_fetch_row(result);
+
+	DBUG_ASSERT(row && row[0] && row[1]);
+
+	char *rocksdbdir = row[0];
+	char *datadir = row[1];
+
+	if (is_abs_path(rocksdbdir))
+	{
+		snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
+			"%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
+	}
+	else 
+	{
+		snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
+			"%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
+			trim_dotslash(rocksdbdir));
+	}
+	mysql_free_result(result);
+
+#ifdef _WIN32
+	for (char *p = rocksdb_checkpoint_dir; *p; p++)
+		if (*p == '\\') *p = '/';
+#endif
+
+	rocksdb_lock_checkpoint();
+
+	if (!access(rocksdb_checkpoint_dir, 0))
+	{
+		msg_ts("Removing rocksdb checkpoint from previous backup attempt.\n");
+		rocksdb_remove_checkpoint_directory();
+	}
+
+	char query[FN_REFLEN + 32];
+	snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
+	xb_mysql_query(mysql_connection, query, false, true);
+}
+
+/*
+  Copy files from rocksdb temporary checkpoint to final destination.
+  remove temp.checkpoint directory (in server's datadir)
+  and release user level lock acquired inside rocksdb_create_checkpoint().
+*/
+static void rocksdb_backup_checkpoint()
+{
+	msg_ts("Backing up rocksdb files.\n");
+	char rocksdb_backup_dir[FN_REFLEN];
+	snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
+	bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
+	if (backup_to_directory) 
+	{
+		if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
+			msg_ts("Can't create rocksdb backup directory %s\n", rocksdb_backup_dir);
+			exit(EXIT_FAILURE);
+		}
+	}
+	copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
+	rocksdb_remove_checkpoint_directory();
+	rocksdb_unlock_checkpoint();
+}
+
+/*
+  Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
+*/
+static void rocksdb_copy_back() {
+	if (access(ROCKSDB_BACKUP_DIR, 0))
+		return;
+	char rocksdb_home_dir[FN_REFLEN];
+        if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
+		strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof(rocksdb_home_dir));
+	} else {
+	   snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home, 
+		xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
+	}
+	mkdirp(rocksdb_home_dir, 0777, MYF(0));
+	copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
+}

extra/mariabackup/xtrabackup.cc

@@ -146,6 +146,8 @@ char *xtrabackup_tmpdir;
 char *xtrabackup_tables;
 char *xtrabackup_tables_file;
 char *xtrabackup_tables_exclude;
+char *xb_rocksdb_datadir;
+my_bool xb_backup_rocksdb = 1;
 
 typedef std::list<regex_t> regex_list_t;
 static regex_list_t regex_include_list;
@@ -687,7 +689,9 @@ enum options_xtrabackup
   OPT_XTRA_TABLES_EXCLUDE,
   OPT_XTRA_DATABASES_EXCLUDE,
   OPT_PROTOCOL,
-  OPT_LOCK_DDL_PER_TABLE
+  OPT_LOCK_DDL_PER_TABLE,
+  OPT_ROCKSDB_DATADIR,
+  OPT_BACKUP_ROCKSDB
 };
 
 struct my_option xb_client_options[] =
@@ -1234,6 +1238,17 @@ struct my_option xb_server_options[] =
    (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
+  {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
+   "This option is only  used with --copy-back or --move-back option",
+  &xb_rocksdb_datadir, &xb_rocksdb_datadir,
+  0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+  { "backup-rocksdb", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
+   "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
+   &xb_backup_rocksdb, &xb_backup_rocksdb,
+   0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
+
+
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 

extra/mariabackup/xtrabackup.h

@@ -44,6 +44,9 @@ extern char		*xtrabackup_incremental_basedir;
 extern char		*innobase_data_home_dir;
 extern char		*innobase_buffer_pool_filename;
 extern char		*xb_plugin_dir;
+extern char		*xb_rocksdb_datadir;
+extern my_bool	xb_backup_rocksdb;
+
 extern uint		opt_protocol;
 extern ds_ctxt_t	*ds_meta;
 extern ds_ctxt_t	*ds_data;

mysql-test/suite/mariabackup/include/have_rocksdb.inc

+if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'rocksdb'`)
+{
+  --skip Requires rocksdb
+}
\ No newline at end of file

mysql-test/suite/mariabackup/xb_rocksdb.opt

+--plugin-load=$HA_ROCKSDB_SO 
\ No newline at end of file

mysql-test/suite/mariabackup/xb_rocksdb.result

+CREATE TABLE t(i INT) ENGINE ROCKSDB;
+INSERT INTO t VALUES(1);
+# xtrabackup backup
+INSERT INTO t VALUES(2);
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT * FROM t;
+i
+1
+# xbstream extract
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT * FROM t;
+i
+1
+DROP TABLE t;

mysql-test/suite/mariabackup/xb_rocksdb.test

+--source include/have_rocksdb.inc
+
+CREATE TABLE t(i INT) ENGINE ROCKSDB;
+INSERT INTO t VALUES(1);
+echo # xtrabackup backup;
+# we'll backup to both directory and to stream to restore that later
+
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+let $stream=$MYSQLTEST_VARDIR/tmp/backup.xb;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir $backup_extra_param;
+--enable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup  --stream=xbstream > $stream 2>$MYSQLTEST_VARDIR/tmp/backup_stream.log;
+
+INSERT INTO t VALUES(2);
+
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+-- source include/restart_and_restore.inc
+--enable_result_log
+
+SELECT * FROM t;
+
+rmdir $targetdir;
+mkdir $targetdir;
+
+
+echo # xbstream extract;
+
+exec $XBSTREAM -x -C $targetdir  < $stream;
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP --prepare --target-dir=$targetdir;
+
+let $_datadir= `SELECT @@datadir`;
+echo # shutdown server;
+--source include/shutdown_mysqld.inc
+echo # remove datadir;
+rmdir $_datadir;
+echo # xtrabackup move back;
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --move-back --datadir=$_datadir --target-dir=$targetdir $copy_back_extra_param;
+echo # restart server;
+--source include/start_mysqld.inc
+
+--enable_result_log
+SELECT * FROM t;
+
+DROP TABLE t;
+rmdir $targetdir;

mysql-test/suite/mariabackup/xb_rocksdb_datadir.opt

+--plugin-load=$HA_ROCKSDB_SO --loose-rocksdb-datadir=$MYSQLTEST_VARDIR/tmp/rocksdb_datadir
\ No newline at end of file

mysql-test/suite/mariabackup/xb_rocksdb_datadir.result

+CREATE TABLE t(i INT) ENGINE ROCKSDB;
+INSERT INTO t VALUES(1);
+# xtrabackup backup
+INSERT INTO t VALUES(2);
+# xtrabackup prepare
+SELECT * FROM t;
+i
+1
+DROP TABLE t;

mysql-test/suite/mariabackup/xb_rocksdb_datadir.test

+if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'rocksdb'`)
+{
+  --skip Requires rocksdb
+}
+
+
+CREATE TABLE t(i INT) ENGINE ROCKSDB;
+INSERT INTO t VALUES(1);
+echo # xtrabackup backup;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+
+INSERT INTO t VALUES(2);
+
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir;
+let $_datadir= `SELECT @@datadir`;
+let $_rocksdb_datadir=`SELECT @@rocksdb_datadir`; 
+--source include/shutdown_mysqld.inc
+rmdir $_datadir;
+rmdir $_rocksdb_datadir;
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --move-back --target-dir=$targetdir --datadir=$_datadir --rocksdb_datadir=$_rocksdb_datadir;
+--enable_result_log
+--source include/start_mysqld.inc
+
+
+SELECT * FROM t;
+DROP TABLE t;
+rmdir $targetdir;
+

mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.opt

+--plugin-load=$HA_ROCKSDB_SO --loose-rocksdb-datadir=$MYSQLTEST_VARDIR/tmp/rocksdb_datadir
\ No newline at end of file

mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.result

+CREATE TABLE t(i INT) ENGINE ROCKSDB;
+INSERT INTO t VALUES(1);
+# xtrabackup backup
+INSERT INTO t VALUES(2);
+# xtrabackup prepare
+SELECT * FROM t;
+i
+1
+DROP TABLE t;

mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.test

+--source include/have_debug.inc
+--source include/have_rocksdb.inc
+
+# Check how rocksdb backup works without hardlinks
+let $backup_extra_param='--dbug=+d,no_hardlinks';
+let $copy_back_extra_param='--dbug=+d,no_hardlinks';
+
+# Pretend that previous backup crashes, and left checkpoint directory
+let $rocksdb_datadir= `SELECT @@rocksdb_datadir`;
+mkdir $rocksdb_datadir/mariadb-checkpoint;
+
+--source xb_rocksdb_datadir.test
+