MDEV-9566 Add Percona Xtrabackup 2.3.7

extra/mariabackup/CMakeLists.txt

@@ -0,0 +1,173 @@
+# Copyright (c) 2013 Percona LLC and/or its affiliates.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+INCLUDE(gcrypt)
+INCLUDE(curl)
+INCLUDE(libev)
+
+ADD_SUBDIRECTORY(libarchive)
+ADD_SUBDIRECTORY(jsmn)
+
+FIND_GCRYPT()
+FIND_CURL()
+FIND_EV()
+
+# xxd is needed to embed version_check script
+FIND_PROGRAM(XXD_PATH xxd)
+
+IF(NOT XXD_PATH)
+  MESSAGE(FATAL_ERROR "xxd not found. Try to install vim-common.")
+ENDIF(NOT XXD_PATH)
+
+INCLUDE_DIRECTORIES(
+  ${CMAKE_SOURCE_DIR}/include
+  ${CMAKE_SOURCE_DIR}/storage/innobase/include
+  ${CMAKE_SOURCE_DIR}/sql
+  ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/libarchive/libarchive
+  ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/quicklz
+  ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/jsmn
+  ${GCRYPT_INCLUDE_DIR}
+  ${CURL_INCLUDE_DIRS}
+  ${LIBEV_INCLUDE_DIRS}
+  ${CMAKE_CURRENT_BINARY_DIR}
+  )
+
+ADD_DEFINITIONS(${SSL_DEFINES})
+
+########################################################################
+# xtrabackup binary
+########################################################################
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/xtrabackup_version.h.in
+               ${CMAKE_CURRENT_BINARY_DIR}/xtrabackup_version.h )
+
+ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h
+                   COMMAND ${XXD_PATH} --include version_check.pl
+                   ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+ADD_CUSTOM_TARGET(GenVersionCheck
+                  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h)
+
+SET_SOURCE_FILES_PROPERTIES(
+  xtrabackup.cc
+  backup_mysql.cc
+  PROPERTIES COMPILE_FLAGS -DMYSQL_CLIENT)
+
+MYSQL_ADD_EXECUTABLE(xtrabackup
+  xtrabackup.cc
+  innobackupex.cc
+  changed_page_bitmap.cc
+  compact.cc
+  datasink.c
+  ds_archive.c
+  ds_buffer.c
+  ds_compress.c
+  ds_encrypt.c
+  ds_local.c
+  ds_stdout.c
+  ds_tmpfile.c
+  ds_xbstream.c
+  fil_cur.cc
+  quicklz/quicklz.c
+  read_filt.cc
+  write_filt.cc
+  wsrep.cc
+  xbcrypt_common.c
+  xbcrypt_write.c
+  xbstream_write.c
+  backup_mysql.cc
+  backup_copy.cc
+  ../../../../sql-common/client_authentication.cc
+  )
+
+SET_TARGET_PROPERTIES(xtrabackup PROPERTIES ENABLE_EXPORTS TRUE)
+
+TARGET_LINK_LIBRARIES(xtrabackup 
+  mysqlserver 
+  ${GCRYPT_LIBS} 
+  archive_static
+  )
+
+ADD_DEPENDENCIES(xtrabackup GenVersionCheck)
+
+########################################################################
+# innobackupex symlink
+########################################################################
+ADD_CUSTOM_COMMAND(TARGET xtrabackup
+                   COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink
+                           xtrabackup innobackupex)
+INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/innobackupex DESTINATION bin)
+
+########################################################################
+# xbstream binary
+########################################################################
+MYSQL_ADD_EXECUTABLE(xbstream
+  ds_buffer.c
+  ds_local.c
+  ds_stdout.c
+  datasink.c
+  xbstream.c
+  xbstream_read.c
+  xbstream_write.c
+  )
+
+SET_TARGET_PROPERTIES(xbstream
+        PROPERTIES LINKER_LANGUAGE CXX
+        )
+
+TARGET_LINK_LIBRARIES(xbstream
+  mysys
+  mysys_ssl
+  )
+
+########################################################################
+# xbcrypt binary
+########################################################################
+MYSQL_ADD_EXECUTABLE(xbcrypt
+  xbcrypt.c
+  xbcrypt_common.c
+  xbcrypt_read.c
+  xbcrypt_write.c
+  )
+
+SET_TARGET_PROPERTIES(xbcrypt
+        PROPERTIES LINKER_LANGUAGE CXX
+        )
+
+TARGET_LINK_LIBRARIES(xbcrypt
+  ${GCRYPT_LIBS}
+  mysys
+  mysys_ssl
+  )
+
+########################################################################
+# xbcloud binary
+########################################################################
+MYSQL_ADD_EXECUTABLE(xbcloud
+  xbcloud.cc
+  )
+
+SET_TARGET_PROPERTIES(xbcloud
+        PROPERTIES LINKER_LANGUAGE CXX
+        )
+
+TARGET_LINK_LIBRARIES(xbcloud
+  ${GCRYPT_LIBS}
+  ${LIBEV_LIBRARIES}
+  ${CURL_LIBRARIES}
+  mysys
+  mysys_ssl
+  jsmn
+  )

extra/mariabackup/backup_copy.h

@@ -0,0 +1,51 @@
+
+#ifndef XTRABACKUP_BACKUP_COPY_H
+#define XTRABACKUP_BACKUP_COPY_H
+
+#include <my_global.h>
+#include "datasink.h"
+
+/* special files */
+#define XTRABACKUP_SLAVE_INFO "xtrabackup_slave_info"
+#define XTRABACKUP_GALERA_INFO "xtrabackup_galera_info"
+#define XTRABACKUP_BINLOG_INFO "xtrabackup_binlog_info"
+#define XTRABACKUP_INFO "xtrabackup_info"
+
+extern bool binlog_locked;
+
+bool
+backup_file_printf(const char *filename, const char *fmt, ...)
+		ATTRIBUTE_FORMAT(printf, 2, 0);
+
+/************************************************************************
+Return true if first and second arguments are the same path. */
+bool
+equal_paths(const char *first, const char *second);
+
+/************************************************************************
+Copy file for backup/restore.
+@return true in case of success. */
+bool
+copy_file(ds_ctxt_t *datasink,
+	  const char *src_file_path,
+	  const char *dst_file_path,
+	  uint thread_n);
+
+bool
+backup_start();
+bool
+backup_finish();
+bool
+apply_log_finish();
+bool
+copy_back();
+bool
+decrypt_decompress();
+void
+version_check();
+bool
+is_path_separator(char);
+bool
+directory_exists(const char *dir, bool create);
+
+#endif

extra/mariabackup/backup_mysql.h

@@ -0,0 +1,92 @@
+#ifndef XTRABACKUP_BACKUP_MYSQL_H
+#define XTRABACKUP_BACKUP_MYSQL_H
+
+#include <mysql.h>
+
+/* mysql flavor and version */
+enum mysql_flavor_t { FLAVOR_UNKNOWN, FLAVOR_MYSQL,
+		      FLAVOR_PERCONA_SERVER, FLAVOR_MARIADB };
+extern mysql_flavor_t server_flavor;
+extern unsigned long mysql_server_version;
+
+/* server capabilities */
+extern bool have_changed_page_bitmaps;
+extern bool have_backup_locks;
+extern bool have_lock_wait_timeout;
+extern bool have_galera_enabled;
+extern bool have_flush_engine_logs;
+extern bool have_multi_threaded_slave;
+extern bool have_gtid_slave;
+
+
+/* History on server */
+extern time_t history_start_time;
+extern time_t history_end_time;
+extern time_t history_lock_time;
+
+
+extern bool sql_thread_started;
+extern char *mysql_slave_position;
+extern char *mysql_binlog_position;
+extern char *buffer_pool_filename;
+
+/** connection to mysql server */
+extern MYSQL *mysql_connection;
+
+void
+capture_tool_command(int argc, char **argv);
+
+bool
+select_history();
+
+bool
+flush_changed_page_bitmaps();
+
+void
+backup_cleanup();
+
+bool
+get_mysql_vars(MYSQL *connection);
+
+bool
+detect_mysql_capabilities_for_backup();
+
+MYSQL *
+xb_mysql_connect();
+
+MYSQL_RES *
+xb_mysql_query(MYSQL *connection, const char *query, bool use_result,
+		bool die_on_error = true);
+
+void
+unlock_all(MYSQL *connection);
+
+bool
+write_current_binlog_file(MYSQL *connection);
+
+bool
+write_binlog_info(MYSQL *connection);
+
+bool
+write_xtrabackup_info(MYSQL *connection);
+
+bool
+write_backup_config_file();
+
+bool
+lock_binlog_maybe(MYSQL *connection);
+
+bool
+lock_tables(MYSQL *connection);
+
+bool
+wait_for_safe_slave(MYSQL *connection);
+
+bool
+write_galera_info(MYSQL *connection);
+
+bool
+write_slave_info(MYSQL *connection);
+
+
+#endif

extra/mariabackup/changed_page_bitmap.h

@@ -0,0 +1,85 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Changed page bitmap interface */
+
+#ifndef XB_CHANGED_PAGE_BITMAP_H
+#define XB_CHANGED_PAGE_BITMAP_H
+
+#include <ut0rbt.h>
+#include <fil0fil.h>
+
+/* The changed page bitmap structure */
+typedef ib_rbt_t xb_page_bitmap;
+
+struct xb_page_bitmap_range_struct;
+
+/* The bitmap range iterator over one space id */
+typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range;
+
+/****************************************************************//**
+Read the disk bitmap and build the changed page bitmap tree for the
+LSN interval incremental_lsn to checkpoint_lsn_start.
+
+@return the built bitmap tree */
+xb_page_bitmap*
+xb_page_bitmap_init(void);
+/*=====================*/
+
+/****************************************************************//**
+Free the bitmap tree. */
+void
+xb_page_bitmap_deinit(
+/*==================*/
+	xb_page_bitmap*	bitmap);	/*!<in/out: bitmap tree */
+
+
+/****************************************************************//**
+Set up a new bitmap range iterator over a given space id changed
+pages in a given bitmap.
+
+@return bitmap range iterator */
+xb_page_bitmap_range*
+xb_page_bitmap_range_init(
+/*======================*/
+	xb_page_bitmap*	bitmap,		/*!< in: bitmap to iterate over */
+	ulint		space_id);	/*!< in: space id */
+
+/****************************************************************//**
+Get the next page id that has its bit set or cleared, i.e. equal to
+bit_value.
+
+@return page id */
+ulint
+xb_page_bitmap_range_get_next_bit(
+/*==============================*/
+	xb_page_bitmap_range*	bitmap_range,	/*!< in/out: bitmap range */
+	ibool			bit_value);	/*!< in: bit value */
+
+/****************************************************************//**
+Free the bitmap range iterator. */
+void
+xb_page_bitmap_range_deinit(
+/*========================*/
+	xb_page_bitmap_range*	bitmap_range);	/*! in/out: bitmap range */
+
+#endif

extra/mariabackup/common.h

@@ -0,0 +1,134 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Common declarations for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XB_COMMON_H
+#define XB_COMMON_H
+
+#include <my_global.h>
+#include <mysql_version.h>
+#include <fcntl.h>
+#include <stdarg.h>
+
+#define xb_a(expr)							\
+	do {								\
+		if (!(expr)) {						\
+			msg("Assertion \"%s\" failed at %s:%lu\n",	\
+			    #expr, __FILE__, (ulong) __LINE__);		\
+			abort();					\
+		}							\
+	} while (0);
+
+#ifdef XB_DEBUG
+#define xb_ad(expr) xb_a(expr)
+#else
+#define xb_ad(expr)
+#endif
+
+#define XB_DELTA_INFO_SUFFIX ".meta"
+
+static inline int msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
+static inline int msg(const char *fmt, ...)
+{
+	int	result;
+	va_list	args;
+
+	va_start(args, fmt);
+	result = vfprintf(stderr, fmt, args);
+	va_end(args);
+
+	return result;
+}
+
+static inline int msg_ts(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
+static inline int msg_ts(const char *fmt, ...)
+{
+	int	result;
+	time_t 	t = time(NULL);
+	char	date[100];
+	char	*line;
+	va_list	args;
+
+	strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t));
+
+	va_start(args, fmt);
+	result = vasprintf(&line, fmt, args);
+	va_end(args);
+
+	if (result != -1) {
+		result = fprintf(stderr, "%s %s", date, line);
+		free(line);
+	}
+
+	return result;
+}
+
+/* Use POSIX_FADV_NORMAL when available */
+
+#ifdef POSIX_FADV_NORMAL
+# define USE_POSIX_FADVISE
+#else
+# define POSIX_FADV_NORMAL
+# define POSIX_FADV_SEQUENTIAL
+# define POSIX_FADV_DONTNEED
+# define posix_fadvise(a,b,c,d) do {} while(0)
+#endif
+
+/***********************************************************************
+Computes bit shift for a given value. If the argument is not a power
+of 2, returns 0.*/
+static inline ulong
+get_bit_shift(ulong value)
+{
+    ulong shift;
+
+    if (value == 0)
+	return 0;
+
+    for (shift = 0; !(value & 1UL); shift++) {
+	value >>= 1;
+    }
+    return (value >> 1) ? 0 : shift;
+}
+
+/****************************************************************************
+Read 'len' bytes from 'fd'. It is identical to my_read(..., MYF(MY_FULL_IO)),
+i.e. tries to combine partial reads into a single block of size 'len', except
+that it bails out on EOF or error, and returns the number of successfully read
+bytes instead. */
+static inline size_t
+xb_read_full(File fd, uchar *buf, size_t len)
+{
+	size_t tlen = 0;
+	size_t tbytes;
+
+	while (tlen < len) {
+		tbytes = my_read(fd, buf, len - tlen, MYF(MY_WME));
+		if (tbytes == 0 || tbytes == MY_FILE_ERROR) {
+			break;
+		}
+
+		buf += tbytes;
+		tlen += tbytes;
+	}
+
+	return tlen;
+}
+
+#endif

extra/mariabackup/compact.h

@@ -0,0 +1,44 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XB_COMPACT_H
+#define XB_COMPACT_H
+
+#include "write_filt.h"
+
+/* Compact page filter context */
+typedef struct {
+	my_bool		 skip;
+	ds_ctxt_t	*ds_buffer;
+	ds_file_t	*buffer;
+	index_id_t	 clustered_index;
+	my_bool		 clustered_index_found;
+	my_bool		 inside_skipped_range;
+	ulint		 free_limit;
+} xb_wf_compact_ctxt_t;
+
+/******************************************************************************
+Expand the data files according to the skipped pages maps created by --compact.
+@return TRUE on success, FALSE on failure. */
+my_bool xb_expand_datafiles(void);
+
+#endif

extra/mariabackup/datasink.c

@@ -0,0 +1,130 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Data sink interface.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#include "common.h"
+#include "datasink.h"
+#include "ds_compress.h"
+#include "ds_archive.h"
+#include "ds_xbstream.h"
+#include "ds_local.h"
+#include "ds_stdout.h"
+#include "ds_tmpfile.h"
+#include "ds_encrypt.h"
+#include "ds_buffer.h"
+
+/************************************************************************
+Create a datasink of the specified type */
+ds_ctxt_t *
+ds_create(const char *root, ds_type_t type)
+{
+	datasink_t	*ds;
+	ds_ctxt_t	*ctxt;
+
+	switch (type) {
+	case DS_TYPE_STDOUT:
+		ds = &datasink_stdout;
+		break;
+	case DS_TYPE_LOCAL:
+		ds = &datasink_local;
+		break;
+	case DS_TYPE_ARCHIVE:
+		ds = &datasink_archive;
+		break;
+	case DS_TYPE_XBSTREAM:
+		ds = &datasink_xbstream;
+		break;
+	case DS_TYPE_COMPRESS:
+		ds = &datasink_compress;
+		break;
+	case DS_TYPE_ENCRYPT:
+		ds = &datasink_encrypt;
+		break;
+	case DS_TYPE_TMPFILE:
+		ds = &datasink_tmpfile;
+		break;
+	case DS_TYPE_BUFFER:
+		ds = &datasink_buffer;
+		break;
+	default:
+		msg("Unknown datasink type: %d\n", type);
+		xb_ad(0);
+		return NULL;
+	}
+
+	ctxt = ds->init(root);
+	if (ctxt != NULL) {
+		ctxt->datasink = ds;
+	} else {
+		msg("Error: failed to initialize datasink.\n");
+		exit(EXIT_FAILURE);
+	}
+
+	return ctxt;
+}
+
+/************************************************************************
+Open a datasink file */
+ds_file_t *
+ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat)
+{
+	ds_file_t	*file;
+
+	file = ctxt->datasink->open(ctxt, path, stat);
+	if (file != NULL) {
+		file->datasink = ctxt->datasink;
+	}
+
+	return file;
+}
+
+/************************************************************************
+Write to a datasink file.
+@return 0 on success, 1 on error. */
+int
+ds_write(ds_file_t *file, const void *buf, size_t len)
+{
+	return file->datasink->write(file, buf, len);
+}
+
+/************************************************************************
+Close a datasink file.
+@return 0 on success, 1, on error. */
+int
+ds_close(ds_file_t *file)
+{
+	return file->datasink->close(file);
+}
+
+/************************************************************************
+Destroy a datasink handle */
+void
+ds_destroy(ds_ctxt_t *ctxt)
+{
+	ctxt->datasink->deinit(ctxt);
+}
+
+/************************************************************************
+Set the destination pipe for a datasink (only makes sense for compress and
+tmpfile). */
+void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt)
+{
+	ctxt->pipe_ctxt = pipe_ctxt;
+}

extra/mariabackup/datasink.h

@@ -0,0 +1,98 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Data sink interface.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XB_DATASINK_H
+#define XB_DATASINK_H
+
+#include <my_global.h>
+#include <my_dir.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct datasink_struct;
+typedef struct datasink_struct datasink_t;
+
+typedef struct ds_ctxt {
+	datasink_t	*datasink;
+	char 		*root;
+	void		*ptr;
+	struct ds_ctxt	*pipe_ctxt;
+} ds_ctxt_t;
+
+typedef struct {
+	void		*ptr;
+	char		*path;
+	datasink_t	*datasink;
+} ds_file_t;
+
+struct datasink_struct {
+	ds_ctxt_t *(*init)(const char *root);
+	ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
+	int (*write)(ds_file_t *file, const void *buf, size_t len);
+	int (*close)(ds_file_t *file);
+	void (*deinit)(ds_ctxt_t *ctxt);
+};
+
+/* Supported datasink types */
+typedef enum {
+	DS_TYPE_STDOUT,
+	DS_TYPE_LOCAL,
+	DS_TYPE_ARCHIVE,
+	DS_TYPE_XBSTREAM,
+	DS_TYPE_COMPRESS,
+	DS_TYPE_ENCRYPT,
+	DS_TYPE_TMPFILE,
+	DS_TYPE_BUFFER
+} ds_type_t;
+
+/************************************************************************
+Create a datasink of the specified type */
+ds_ctxt_t *ds_create(const char *root, ds_type_t type);
+
+/************************************************************************
+Open a datasink file */
+ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
+
+/************************************************************************
+Write to a datasink file.
+@return 0 on success, 1 on error. */
+int ds_write(ds_file_t *file, const void *buf, size_t len);
+
+/************************************************************************
+Close a datasink file.
+@return 0 on success, 1, on error. */
+int ds_close(ds_file_t *file);
+
+/************************************************************************
+Destroy a datasink handle */
+void ds_destroy(ds_ctxt_t *ctxt);
+
+/************************************************************************
+Set the destination pipe for a datasink (only makes sense for compress and
+tmpfile). */
+void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* XB_DATASINK_H */

extra/mariabackup/ds_archive.c

@@ -0,0 +1,275 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Streaming implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#include <archive.h>
+#include <archive_entry.h>
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+	struct archive	*archive;
+	ds_file_t	*dest_file;
+	pthread_mutex_t	mutex;
+} ds_archive_ctxt_t;
+
+typedef struct {
+	struct archive_entry	*entry;
+	ds_archive_ctxt_t	*archive_ctxt;
+} ds_archive_file_t;
+
+
+/***********************************************************************
+General archive interface */
+
+static ds_ctxt_t *archive_init(const char *root);
+static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path,
+			      MY_STAT *mystat);
+static int archive_write(ds_file_t *file, const void *buf, size_t len);
+static int archive_close(ds_file_t *file);
+static void archive_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_archive = {
+	&archive_init,
+	&archive_open,
+	&archive_write,
+	&archive_close,
+	&archive_deinit
+};
+
+static
+int
+my_archive_open_callback(struct archive *a __attribute__((unused)),
+		      void *data __attribute__((unused)))
+{
+	return ARCHIVE_OK;
+}
+
+static
+ssize_t
+my_archive_write_callback(struct archive *a __attribute__((unused)),
+		       void *data, const void *buffer, size_t length)
+{
+	ds_archive_ctxt_t	*archive_ctxt;
+
+	archive_ctxt = (ds_archive_ctxt_t *) data;
+
+	xb_ad(archive_ctxt != NULL);
+	xb_ad(archive_ctxt->dest_file != NULL);
+
+	if (!ds_write(archive_ctxt->dest_file, buffer, length)) {
+		return length;
+	}
+	return -1;
+}
+
+static
+int
+my_archive_close_callback(struct archive *a __attribute__((unused)),
+			  void *data __attribute__((unused)))
+{
+	return ARCHIVE_OK;
+}
+
+static
+ds_ctxt_t *
+archive_init(const char *root __attribute__((unused)))
+{
+	ds_ctxt_t		*ctxt;
+	ds_archive_ctxt_t	*archive_ctxt;
+	struct archive		*a;
+
+	ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t),
+			 MYF(MY_FAE));
+	archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1);
+
+	if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) {
+		msg("archive_init: pthread_mutex_init() failed.\n");
+		goto err;
+	}
+
+	a = archive_write_new();
+	if (a == NULL) {
+		msg("archive_write_new() failed.\n");
+		goto err;
+	}
+
+	archive_ctxt->archive = a;
+	archive_ctxt->dest_file = NULL;
+
+	if (archive_write_set_compression_none(a) != ARCHIVE_OK ||
+	    archive_write_set_format_pax_restricted(a) != ARCHIVE_OK ||
+	    /* disable internal buffering so we don't have to flush the
+	    output in xtrabackup */
+	    archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) {
+		msg("failed to set libarchive archive options: %s\n",
+		    archive_error_string(a));
+		archive_write_finish(a);
+		goto err;
+	}
+
+	if (archive_write_open(a, archive_ctxt, my_archive_open_callback,
+			       my_archive_write_callback,
+			       my_archive_close_callback) != ARCHIVE_OK) {
+		msg("cannot open output archive.\n");
+		return NULL;
+	}
+
+	ctxt->ptr = archive_ctxt;
+
+	return ctxt;
+
+err:
+	my_free(ctxt);
+	return NULL;
+}
+
+static
+ds_file_t *
+archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+	ds_archive_ctxt_t	*archive_ctxt;
+	ds_ctxt_t		*dest_ctxt;
+	ds_file_t		*file;
+	ds_archive_file_t	*archive_file;
+
+	struct archive		*a;
+	struct archive_entry	*entry;
+
+	xb_ad(ctxt->pipe_ctxt != NULL);
+	dest_ctxt = ctxt->pipe_ctxt;
+
+	archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr;
+
+	pthread_mutex_lock(&archive_ctxt->mutex);
+	if (archive_ctxt->dest_file == NULL) {
+		archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
+		if (archive_ctxt->dest_file == NULL) {
+			return NULL;
+		}
+	}
+	pthread_mutex_unlock(&archive_ctxt->mutex);
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_archive_file_t),
+				       MYF(MY_FAE));
+
+	archive_file = (ds_archive_file_t *) (file + 1);
+
+	a = archive_ctxt->archive;
+
+	entry = archive_entry_new();
+	if (entry == NULL) {
+		msg("archive_entry_new() failed.\n");
+		goto err;
+	}
+
+	archive_entry_set_size(entry, mystat->st_size);
+	archive_entry_set_mode(entry, 0660);
+	archive_entry_set_filetype(entry, AE_IFREG);
+	archive_entry_set_pathname(entry, path);
+	archive_entry_set_mtime(entry, mystat->st_mtime, 0);
+
+	archive_file->entry = entry;
+	archive_file->archive_ctxt = archive_ctxt;
+
+	if (archive_write_header(a, entry) != ARCHIVE_OK) {
+		msg("archive_write_header() failed.\n");
+		archive_entry_free(entry);
+		goto err;
+	}
+
+	file->ptr = archive_file;
+	file->path = archive_ctxt->dest_file->path;
+
+	return file;
+
+err:
+	if (archive_ctxt->dest_file) {
+		ds_close(archive_ctxt->dest_file);
+		archive_ctxt->dest_file = NULL;
+	}
+	my_free(file);
+
+	return NULL;
+}
+
+static
+int
+archive_write(ds_file_t *file, const void *buf, size_t len)
+{
+	ds_archive_file_t	*archive_file;
+	struct archive		*a;
+
+	archive_file = (ds_archive_file_t *) file->ptr;
+
+	a = archive_file->archive_ctxt->archive;
+
+	xb_ad(archive_file->archive_ctxt->dest_file != NULL);
+	if (archive_write_data(a, buf, len) < 0) {
+		msg("archive_write_data() failed: %s (errno = %d)\n",
+		    archive_error_string(a), archive_errno(a));
+		return 1;
+	}
+
+	return 0;
+}
+
+static
+int
+archive_close(ds_file_t *file)
+{
+	ds_archive_file_t	*archive_file;
+	int			rc = 0;
+
+	archive_file = (ds_archive_file_t *)file->ptr;
+
+	archive_entry_free(archive_file->entry);
+
+	my_free(file);
+
+	return rc;
+}
+
+static
+void
+archive_deinit(ds_ctxt_t *ctxt)
+{
+	struct archive *a;
+	ds_archive_ctxt_t	*archive_ctxt;
+
+	archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr;
+
+	a = archive_ctxt->archive;
+
+	if (archive_write_close(a) != ARCHIVE_OK) {
+		msg("archive_write_close() failed.\n");
+	}
+	archive_write_finish(a);
+
+	if (archive_ctxt->dest_file) {
+		ds_close(archive_ctxt->dest_file);
+		archive_ctxt->dest_file = NULL;
+	}
+
+	pthread_mutex_destroy(&archive_ctxt->mutex);
+
+	my_free(ctxt);
+}

extra/mariabackup/ds_archive.h

@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Streaming interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_ARCHIVE_H
+#define DS_ARCHIVE_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_archive;
+
+#endif

extra/mariabackup/ds_buffer.c

@@ -0,0 +1,189 @@
+/******************************************************
+Copyright (c) 2012-2013 Percona LLC and/or its affiliates.
+
+buffer datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Does buffered output to a destination datasink set with ds_set_pipe().
+Writes to the destination datasink are guaranteed to not be smaller than a
+specified buffer size (DS_DEFAULT_BUFFER_SIZE by default), with the only
+exception for the last write for a file. */
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include "ds_buffer.h"
+#include "common.h"
+#include "datasink.h"
+
+#define DS_DEFAULT_BUFFER_SIZE (64 * 1024)
+
+typedef struct {
+	ds_file_t	*dst_file;
+	char		*buf;
+	size_t		 pos;
+	size_t		size;
+} ds_buffer_file_t;
+
+typedef struct {
+	size_t	buffer_size;
+} ds_buffer_ctxt_t;
+
+static ds_ctxt_t *buffer_init(const char *root);
+static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path,
+			      MY_STAT *mystat);
+static int buffer_write(ds_file_t *file, const void *buf, size_t len);
+static int buffer_close(ds_file_t *file);
+static void buffer_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_buffer = {
+	&buffer_init,
+	&buffer_open,
+	&buffer_write,
+	&buffer_close,
+	&buffer_deinit
+};
+
+/* Change the default buffer size */
+void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size)
+{
+	ds_buffer_ctxt_t *buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr;
+
+	buffer_ctxt->buffer_size = size;
+}
+
+static ds_ctxt_t *
+buffer_init(const char *root)
+{
+	ds_ctxt_t		*ctxt;
+	ds_buffer_ctxt_t	*buffer_ctxt;
+
+	ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_buffer_ctxt_t),
+			 MYF(MY_FAE));
+	buffer_ctxt = (ds_buffer_ctxt_t *) (ctxt + 1);
+	buffer_ctxt->buffer_size = DS_DEFAULT_BUFFER_SIZE;
+
+	ctxt->ptr = buffer_ctxt;
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static ds_file_t *
+buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+	ds_buffer_ctxt_t	*buffer_ctxt;
+	ds_ctxt_t		*pipe_ctxt;
+	ds_file_t		*dst_file;
+	ds_file_t		*file;
+	ds_buffer_file_t	*buffer_file;
+
+	pipe_ctxt = ctxt->pipe_ctxt;
+	xb_a(pipe_ctxt != NULL);
+
+	dst_file = ds_open(pipe_ctxt, path, mystat);
+	if (dst_file == NULL) {
+		exit(EXIT_FAILURE);
+	}
+
+	buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr;
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_buffer_file_t) +
+				       buffer_ctxt->buffer_size,
+				       MYF(MY_FAE));
+
+	buffer_file = (ds_buffer_file_t *) (file + 1);
+	buffer_file->dst_file = dst_file;
+	buffer_file->buf = (char *) (buffer_file + 1);
+	buffer_file->size = buffer_ctxt->buffer_size;
+	buffer_file->pos = 0;
+
+	file->path = dst_file->path;
+	file->ptr = buffer_file;
+
+	return file;
+}
+
+static int
+buffer_write(ds_file_t *file, const void *buf, size_t len)
+{
+	ds_buffer_file_t	*buffer_file;
+
+	buffer_file = (ds_buffer_file_t *) file->ptr;
+
+	while (len > 0) {
+		if (buffer_file->pos + len > buffer_file->size) {
+			if (buffer_file->pos > 0) {
+				size_t bytes;
+
+				bytes = buffer_file->size - buffer_file->pos;
+				memcpy(buffer_file->buf + buffer_file->pos, buf,
+				       bytes);
+
+				if (ds_write(buffer_file->dst_file,
+					     buffer_file->buf,
+					     buffer_file->size)) {
+					return 1;
+				}
+
+				buffer_file->pos = 0;
+
+				buf = (const char *) buf + bytes;
+				len -= bytes;
+			} else {
+				/* We don't have any buffered bytes, just write
+				the entire source buffer */
+				if (ds_write(buffer_file->dst_file, buf, len)) {
+					return 1;
+				}
+				break;
+			}
+		} else {
+			memcpy(buffer_file->buf + buffer_file->pos, buf, len);
+			buffer_file->pos += len;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int
+buffer_close(ds_file_t *file)
+{
+	ds_buffer_file_t	*buffer_file;
+	int			ret;
+
+	buffer_file = (ds_buffer_file_t *) file->ptr;
+	if (buffer_file->pos > 0) {
+		ds_write(buffer_file->dst_file, buffer_file->buf,
+			 buffer_file->pos);
+	}
+
+	ret = ds_close(buffer_file->dst_file);
+
+	my_free(file);
+
+	return ret;
+}
+
+static void
+buffer_deinit(ds_ctxt_t *ctxt)
+{
+	my_free(ctxt->root);
+	my_free(ctxt);
+}

extra/mariabackup/ds_buffer.h

@@ -0,0 +1,39 @@
+/******************************************************
+Copyright (c) 2012-2013 Percona LLC and/or its affiliates.
+
+buffer datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_BUFFER_H
+#define DS_BUFFER_H
+
+#include "datasink.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern datasink_t datasink_buffer;
+
+/* Change the default buffer size */
+void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

extra/mariabackup/ds_compress.c

@@ -0,0 +1,462 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Compressing datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include <quicklz.h>
+#include <zlib.h>
+#include "common.h"
+#include "datasink.h"
+
+#define COMPRESS_CHUNK_SIZE ((size_t) (xtrabackup_compress_chunk_size))
+#define MY_QLZ_COMPRESS_OVERHEAD 400
+
+typedef struct {
+	pthread_t		id;
+	uint			num;
+	pthread_mutex_t 	ctrl_mutex;
+	pthread_cond_t		ctrl_cond;
+	pthread_mutex_t		data_mutex;
+	pthread_cond_t  	data_cond;
+	my_bool			started;
+	my_bool			data_avail;
+	my_bool			cancelled;
+	const char 		*from;
+	size_t			from_len;
+	char			*to;
+	size_t			to_len;
+	qlz_state_compress	state;
+	ulong			adler;
+} comp_thread_ctxt_t;
+
+typedef struct {
+	comp_thread_ctxt_t	*threads;
+	uint			nthreads;
+} ds_compress_ctxt_t;
+
+typedef struct {
+	ds_file_t		*dest_file;
+	ds_compress_ctxt_t	*comp_ctxt;
+	size_t			bytes_processed;
+} ds_compress_file_t;
+
+/* Compression options */
+extern char		*xtrabackup_compress_alg;
+extern uint		xtrabackup_compress_threads;
+extern ulonglong	xtrabackup_compress_chunk_size;
+
+static ds_ctxt_t *compress_init(const char *root);
+static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path,
+				MY_STAT *mystat);
+static int compress_write(ds_file_t *file, const void *buf, size_t len);
+static int compress_close(ds_file_t *file);
+static void compress_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_compress = {
+	&compress_init,
+	&compress_open,
+	&compress_write,
+	&compress_close,
+	&compress_deinit
+};
+
+static inline int write_uint32_le(ds_file_t *file, ulong n);
+static inline int write_uint64_le(ds_file_t *file, ulonglong n);
+
+static comp_thread_ctxt_t *create_worker_threads(uint n);
+static void destroy_worker_threads(comp_thread_ctxt_t *threads, uint n);
+static void *compress_worker_thread_func(void *arg);
+
+static
+ds_ctxt_t *
+compress_init(const char *root)
+{
+	ds_ctxt_t		*ctxt;
+	ds_compress_ctxt_t	*compress_ctxt;
+	comp_thread_ctxt_t	*threads;
+
+	/* Create and initialize the worker threads */
+	threads = create_worker_threads(xtrabackup_compress_threads);
+	if (threads == NULL) {
+		msg("compress: failed to create worker threads.\n");
+		return NULL;
+	}
+
+	ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
+				       sizeof(ds_compress_ctxt_t),
+				       MYF(MY_FAE));
+
+	compress_ctxt = (ds_compress_ctxt_t *) (ctxt + 1);
+	compress_ctxt->threads = threads;
+	compress_ctxt->nthreads = xtrabackup_compress_threads;
+
+	ctxt->ptr = compress_ctxt;
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static
+ds_file_t *
+compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+	ds_compress_ctxt_t	*comp_ctxt;
+	ds_ctxt_t		*dest_ctxt;
+ 	ds_file_t		*dest_file;
+	char			new_name[FN_REFLEN];
+	size_t			name_len;
+	ds_file_t		*file;
+	ds_compress_file_t	*comp_file;
+
+	xb_ad(ctxt->pipe_ctxt != NULL);
+	dest_ctxt = ctxt->pipe_ctxt;
+
+	comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;
+
+	/* Append the .qp extension to the filename */
+	fn_format(new_name, path, "", ".qp", MYF(MY_APPEND_EXT));
+
+	dest_file = ds_open(dest_ctxt, new_name, mystat);
+	if (dest_file == NULL) {
+		return NULL;
+	}
+
+	/* Write the qpress archive header */
+	if (ds_write(dest_file, "qpress10", 8) ||
+	    write_uint64_le(dest_file, COMPRESS_CHUNK_SIZE)) {
+		goto err;
+	}
+
+	/* We are going to create a one-file "flat" (i.e. with no
+	subdirectories) archive. So strip the directory part from the path and
+	remove the '.qp' suffix. */
+	fn_format(new_name, path, "", "", MYF(MY_REPLACE_DIR));
+
+	/* Write the qpress file header */
+	name_len = strlen(new_name);
+	if (ds_write(dest_file, "F", 1) ||
+	    write_uint32_le(dest_file, name_len) ||
+	    /* we want to write the terminating \0 as well */
+	    ds_write(dest_file, new_name, name_len + 1)) {
+		goto err;
+	}
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_compress_file_t),
+				       MYF(MY_FAE));
+	comp_file = (ds_compress_file_t *) (file + 1);
+	comp_file->dest_file = dest_file;
+	comp_file->comp_ctxt = comp_ctxt;
+	comp_file->bytes_processed = 0;
+
+	file->ptr = comp_file;
+	file->path = dest_file->path;
+
+	return file;
+
+err:
+	ds_close(dest_file);
+	return NULL;
+}
+
+static
+int
+compress_write(ds_file_t *file, const void *buf, size_t len)
+{
+	ds_compress_file_t	*comp_file;
+	ds_compress_ctxt_t	*comp_ctxt;
+	comp_thread_ctxt_t	*threads;
+	comp_thread_ctxt_t	*thd;
+	uint			nthreads;
+	uint			i;
+	const char		*ptr;
+	ds_file_t		*dest_file;
+
+	comp_file = (ds_compress_file_t *) file->ptr;
+	comp_ctxt = comp_file->comp_ctxt;
+	dest_file = comp_file->dest_file;
+
+	threads = comp_ctxt->threads;
+	nthreads = comp_ctxt->nthreads;
+
+	ptr = (const char *) buf;
+	while (len > 0) {
+		uint max_thread;
+
+		/* Send data to worker threads for compression */
+		for (i = 0; i < nthreads; i++) {
+			size_t chunk_len;
+
+			thd = threads + i;
+
+			pthread_mutex_lock(&thd->ctrl_mutex);
+
+			chunk_len = (len > COMPRESS_CHUNK_SIZE) ?
+				COMPRESS_CHUNK_SIZE : len;
+			thd->from = ptr;
+			thd->from_len = chunk_len;
+
+			pthread_mutex_lock(&thd->data_mutex);
+			thd->data_avail = TRUE;
+			pthread_cond_signal(&thd->data_cond);
+			pthread_mutex_unlock(&thd->data_mutex);
+
+			len -= chunk_len;
+			if (len == 0) {
+				break;
+			}
+			ptr += chunk_len;
+		}
+
+		max_thread = (i < nthreads) ? i :  nthreads - 1;
+
+		/* Reap and stream the compressed data */
+		for (i = 0; i <= max_thread; i++) {
+			thd = threads + i;
+
+			pthread_mutex_lock(&thd->data_mutex);
+			while (thd->data_avail == TRUE) {
+				pthread_cond_wait(&thd->data_cond,
+						  &thd->data_mutex);
+			}
+
+			xb_a(threads[i].to_len > 0);
+
+			if (ds_write(dest_file, "NEWBNEWB", 8) ||
+			    write_uint64_le(dest_file,
+					    comp_file->bytes_processed)) {
+				msg("compress: write to the destination stream "
+				    "failed.\n");
+				return 1;
+			}
+
+			comp_file->bytes_processed += threads[i].from_len;
+
+			if (write_uint32_le(dest_file, threads[i].adler) ||
+			    ds_write(dest_file, threads[i].to,
+					   threads[i].to_len)) {
+				msg("compress: write to the destination stream "
+				    "failed.\n");
+				return 1;
+			}
+
+			pthread_mutex_unlock(&threads[i].data_mutex);
+			pthread_mutex_unlock(&threads[i].ctrl_mutex);
+		}
+	}
+
+	return 0;
+}
+
+static
+int
+compress_close(ds_file_t *file)
+{
+	ds_compress_file_t	*comp_file;
+	ds_file_t		*dest_file;
+	int			rc;
+
+	comp_file = (ds_compress_file_t *) file->ptr;
+	dest_file = comp_file->dest_file;
+
+	/* Write the qpress file trailer */
+	ds_write(dest_file, "ENDSENDS", 8);
+
+	/* Supposedly the number of written bytes should be written as a
+	"recovery information" in the file trailer, but in reality qpress
+	always writes 8 zeros here. Let's do the same */
+
+	write_uint64_le(dest_file, 0);
+
+	rc = ds_close(dest_file);
+
+	my_free(file);
+
+	return rc;
+}
+
+static
+void
+compress_deinit(ds_ctxt_t *ctxt)
+{
+	ds_compress_ctxt_t 	*comp_ctxt;
+
+	xb_ad(ctxt->pipe_ctxt != NULL);
+
+	comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;;
+
+	destroy_worker_threads(comp_ctxt->threads, comp_ctxt->nthreads);
+
+	my_free(ctxt->root);
+	my_free(ctxt);
+}
+
+static inline
+int
+write_uint32_le(ds_file_t *file, ulong n)
+{
+	char tmp[4];
+
+	int4store(tmp, n);
+	return ds_write(file, tmp, sizeof(tmp));
+}
+
+static inline
+int
+write_uint64_le(ds_file_t *file, ulonglong n)
+{
+	char tmp[8];
+
+	int8store(tmp, n);
+	return ds_write(file, tmp, sizeof(tmp));
+}
+
+static
+comp_thread_ctxt_t *
+create_worker_threads(uint n)
+{
+	comp_thread_ctxt_t	*threads;
+	uint 			i;
+
+	threads = (comp_thread_ctxt_t *)
+		my_malloc(sizeof(comp_thread_ctxt_t) * n, MYF(MY_FAE));
+
+	for (i = 0; i < n; i++) {
+		comp_thread_ctxt_t *thd = threads + i;
+
+		thd->num = i + 1;
+		thd->started = FALSE;
+		thd->cancelled = FALSE;
+		thd->data_avail = FALSE;
+
+		thd->to = (char *) my_malloc(COMPRESS_CHUNK_SIZE +
+						   MY_QLZ_COMPRESS_OVERHEAD,
+						   MYF(MY_FAE));
+
+		/* Initialize the control mutex and condition var */
+		if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
+		    pthread_cond_init(&thd->ctrl_cond, NULL)) {
+			goto err;
+		}
+
+		/* Initialize and data mutex and condition var */
+		if (pthread_mutex_init(&thd->data_mutex, NULL) ||
+		    pthread_cond_init(&thd->data_cond, NULL)) {
+			goto err;
+		}
+
+		pthread_mutex_lock(&thd->ctrl_mutex);
+
+		if (pthread_create(&thd->id, NULL, compress_worker_thread_func,
+				   thd)) {
+			msg("compress: pthread_create() failed: "
+			    "errno = %d\n", errno);
+			goto err;
+		}
+	}
+
+	/* Wait for the threads to start */
+	for (i = 0; i < n; i++) {
+		comp_thread_ctxt_t *thd = threads + i;
+
+		while (thd->started == FALSE)
+			pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
+		pthread_mutex_unlock(&thd->ctrl_mutex);
+	}
+
+	return threads;
+
+err:
+	return NULL;
+}
+
+static
+void
+destroy_worker_threads(comp_thread_ctxt_t *threads, uint n)
+{
+	uint i;
+
+	for (i = 0; i < n; i++) {
+		comp_thread_ctxt_t *thd = threads + i;
+
+		pthread_mutex_lock(&thd->data_mutex);
+		threads[i].cancelled = TRUE;
+		pthread_cond_signal(&thd->data_cond);
+		pthread_mutex_unlock(&thd->data_mutex);
+
+		pthread_join(thd->id, NULL);
+
+		pthread_cond_destroy(&thd->data_cond);
+		pthread_mutex_destroy(&thd->data_mutex);
+		pthread_cond_destroy(&thd->ctrl_cond);
+		pthread_mutex_destroy(&thd->ctrl_mutex);
+
+		my_free(thd->to);
+	}
+
+	my_free(threads);
+}
+
+static
+void *
+compress_worker_thread_func(void *arg)
+{
+	comp_thread_ctxt_t *thd = (comp_thread_ctxt_t *) arg;
+
+	pthread_mutex_lock(&thd->ctrl_mutex);
+
+	pthread_mutex_lock(&thd->data_mutex);
+
+	thd->started = TRUE;
+	pthread_cond_signal(&thd->ctrl_cond);
+
+	pthread_mutex_unlock(&thd->ctrl_mutex);
+
+	while (1) {
+		thd->data_avail = FALSE;
+		pthread_cond_signal(&thd->data_cond);
+
+		while (!thd->data_avail && !thd->cancelled) {
+			pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
+		}
+
+		if (thd->cancelled)
+			break;
+
+		thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len,
+					   &thd->state);
+
+		/* qpress uses 0x00010000 as the initial value, but its own
+		Adler-32 implementation treats the value differently:
+		  1. higher order bits are the sum of all bytes in the sequence
+		  2. lower order bits are the sum of resulting values at every
+		     step.
+		So it's the other way around as compared to zlib's adler32().
+		That's why  0x00000001 is being passed here to be compatible
+		with qpress implementation. */
+
+		thd->adler = adler32(0x00000001, (uchar *) thd->to,
+				     thd->to_len);
+	}
+
+	pthread_mutex_unlock(&thd->data_mutex);
+
+	return NULL;
+}

extra/mariabackup/ds_compress.h

@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Compression interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_COMPRESS_H
+#define DS_COMPRESS_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_compress;
+
+#endif

extra/mariabackup/ds_encrypt.c

@@ -0,0 +1,617 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Encryption datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+
+#include <my_base.h>
+#include "common.h"
+#include "datasink.h"
+
+#if GCC_VERSION >= 4002
+/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
+#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+#include <gcrypt.h>
+
+#if GCC_VERSION >= 4002
+#  pragma GCC diagnostic warning "-Wdeprecated-declarations"
+#endif
+
+#include "xbcrypt.h"
+
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+GCRY_THREAD_OPTION_PTHREAD_IMPL;
+#endif
+
+#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size))
+
+typedef struct {
+	pthread_t		id;
+	uint			num;
+	pthread_mutex_t 	ctrl_mutex;
+	pthread_cond_t		ctrl_cond;
+	pthread_mutex_t		data_mutex;
+	pthread_cond_t  	data_cond;
+	my_bool			started;
+	my_bool			data_avail;
+	my_bool			cancelled;
+	const char 		*from;
+	size_t			from_len;
+	char			*to;
+	char			*iv;
+	size_t			to_len;
+	gcry_cipher_hd_t	cipher_handle;
+} crypt_thread_ctxt_t;
+
+typedef struct {
+	crypt_thread_ctxt_t	*threads;
+	uint			nthreads;
+} ds_encrypt_ctxt_t;
+
+typedef struct {
+	xb_wcrypt_t		*xbcrypt_file;
+	ds_encrypt_ctxt_t	*crypt_ctxt;
+	size_t			bytes_processed;
+	ds_file_t		*dest_file;
+} ds_encrypt_file_t;
+
+/* Encryption options */
+extern ulong		xtrabackup_encrypt_algo;
+extern char		*xtrabackup_encrypt_key;
+extern char		*xtrabackup_encrypt_key_file;
+extern uint		xtrabackup_encrypt_threads;
+extern ulonglong	xtrabackup_encrypt_chunk_size;
+
+static ds_ctxt_t *encrypt_init(const char *root);
+static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path,
+				MY_STAT *mystat);
+static int encrypt_write(ds_file_t *file, const void *buf, size_t len);
+static int encrypt_close(ds_file_t *file);
+static void encrypt_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_encrypt = {
+	&encrypt_init,
+	&encrypt_open,
+	&encrypt_write,
+	&encrypt_close,
+	&encrypt_deinit
+};
+
+static crypt_thread_ctxt_t *create_worker_threads(uint n);
+static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
+static void *encrypt_worker_thread_func(void *arg);
+
+static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
+				GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
+static uint encrypt_algo;
+static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
+static uint encrypt_key_len = 0;
+static size_t encrypt_iv_len = 0;
+
+static
+ssize_t
+my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len)
+{
+	ds_encrypt_file_t		*encrypt_file;
+
+	encrypt_file = (ds_encrypt_file_t *) userdata;
+
+	xb_ad(encrypt_file != NULL);
+	xb_ad(encrypt_file->dest_file != NULL);
+
+	if (!ds_write(encrypt_file->dest_file, buf, len)) {
+		return len;
+	}
+	return -1;
+}
+
+static
+ds_ctxt_t *
+encrypt_init(const char *root)
+{
+	ds_ctxt_t		*ctxt;
+	ds_encrypt_ctxt_t	*encrypt_ctxt;
+	crypt_thread_ctxt_t	*threads;
+	gcry_error_t 		gcry_error;
+
+	/* Acording to gcrypt docs (and my testing), setting up the threading
+	   callbacks must be done first, so, lets give it a shot */
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+	gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
+	if (gcry_error) {
+		msg("encrypt: unable to set libgcrypt thread cbs - "
+		    "%s : %s\n",
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return NULL;
+	}
+#endif
+
+	/* Version check should be the very next call because it
+	makes sure that important subsystems are intialized. */
+	if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
+		const char	*gcrypt_version;
+		gcrypt_version = gcry_check_version(NULL);
+		/* No other library has already initialized libgcrypt. */
+		if (!gcrypt_version) {
+			msg("encrypt: failed to initialize libgcrypt\n");
+			return NULL;
+		} else {
+			msg("encrypt: using gcrypt %s\n", gcrypt_version);
+		}
+	}
+
+	/* Disable the gcry secure memory, not dealing with this for now */
+	gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
+	if (gcry_error) {
+		msg("encrypt: unable to disable libgcrypt secmem - "
+		    "%s : %s\n",
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return NULL;
+	}
+
+	/* Finalize gcry initialization. */
+	gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
+	if (gcry_error) {
+		msg("encrypt: unable to finish libgcrypt initialization - "
+		    "%s : %s\n",
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return NULL;
+	}
+
+	/* Determine the algorithm */
+	encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo];
+
+	/* Set up the iv length */
+	encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
+	xb_a(encrypt_iv_len > 0);
+
+	/* Now set up the key */
+	if (xtrabackup_encrypt_key == NULL &&
+	    xtrabackup_encrypt_key_file == NULL) {
+		msg("encrypt: no encryption key or key file specified.\n");
+		return NULL;
+	} else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) {
+		msg("encrypt: both encryption key and key file specified.\n");
+		return NULL;
+	} else if (xtrabackup_encrypt_key_file) {
+		if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file,
+					    (void**)&xtrabackup_encrypt_key,
+					    &encrypt_key_len)) {
+			msg("encrypt: unable to read encryption key file"
+			    " \"%s\".\n", xtrabackup_encrypt_key_file);
+			return NULL;
+		}
+	} else if (xtrabackup_encrypt_key) {
+		encrypt_key_len = strlen(xtrabackup_encrypt_key);
+	} else {
+		msg("encrypt: no encryption key or key file specified.\n");
+		return NULL;
+	}
+
+	/* Create and initialize the worker threads */
+	threads = create_worker_threads(xtrabackup_encrypt_threads);
+	if (threads == NULL) {
+		msg("encrypt: failed to create worker threads.\n");
+		return NULL;
+	}
+
+	ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
+				       sizeof(ds_encrypt_ctxt_t),
+				       MYF(MY_FAE));
+
+	encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1);
+	encrypt_ctxt->threads = threads;
+	encrypt_ctxt->nthreads = xtrabackup_encrypt_threads;
+
+	ctxt->ptr = encrypt_ctxt;
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static
+ds_file_t *
+encrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+	ds_ctxt_t		*dest_ctxt;
+
+	ds_encrypt_ctxt_t	*crypt_ctxt;
+	ds_encrypt_file_t	*crypt_file;
+
+	char			new_name[FN_REFLEN];
+	ds_file_t		*file;
+
+	xb_ad(ctxt->pipe_ctxt != NULL);
+	dest_ctxt = ctxt->pipe_ctxt;
+
+	crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr;
+
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_encrypt_file_t),
+				       MYF(MY_FAE|MY_ZEROFILL));
+
+	crypt_file = (ds_encrypt_file_t *) (file + 1);
+
+	/* Append the .xbcrypt extension to the filename */
+	fn_format(new_name, path, "", ".xbcrypt", MYF(MY_APPEND_EXT));
+	crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat);
+	if (crypt_file->dest_file == NULL) {
+		msg("encrypt: ds_open(\"%s\") failed.\n", new_name);
+		goto err;
+	}
+
+	crypt_file->crypt_ctxt = crypt_ctxt;
+	crypt_file->xbcrypt_file = xb_crypt_write_open(crypt_file,
+					   my_xb_crypt_write_callback);
+
+	if (crypt_file->xbcrypt_file == NULL) {
+		msg("encrypt: xb_crypt_write_open() failed.\n");
+		goto err;
+	}
+
+
+	file->ptr = crypt_file;
+	file->path = crypt_file->dest_file->path;
+
+	return file;
+
+err:
+	if (crypt_file->dest_file) {
+		ds_close(crypt_file->dest_file);
+	}
+	my_free(file);
+	return NULL;
+}
+
+static
+int
+encrypt_write(ds_file_t *file, const void *buf, size_t len)
+{
+	ds_encrypt_file_t	*crypt_file;
+	ds_encrypt_ctxt_t	*crypt_ctxt;
+	crypt_thread_ctxt_t	*threads;
+	crypt_thread_ctxt_t	*thd;
+	uint			nthreads;
+	uint			i;
+	const char		*ptr;
+
+	crypt_file = (ds_encrypt_file_t *) file->ptr;
+	crypt_ctxt = crypt_file->crypt_ctxt;
+
+	threads = crypt_ctxt->threads;
+	nthreads = crypt_ctxt->nthreads;
+
+	ptr = (const char *) buf;
+	while (len > 0) {
+		uint max_thread;
+
+		/* Send data to worker threads for encryption */
+		for (i = 0; i < nthreads; i++) {
+			size_t chunk_len;
+
+			thd = threads + i;
+
+			pthread_mutex_lock(&thd->ctrl_mutex);
+
+			chunk_len = (len > XB_CRYPT_CHUNK_SIZE) ?
+				XB_CRYPT_CHUNK_SIZE : len;
+			thd->from = ptr;
+			thd->from_len = chunk_len;
+
+			pthread_mutex_lock(&thd->data_mutex);
+			thd->data_avail = TRUE;
+			pthread_cond_signal(&thd->data_cond);
+			pthread_mutex_unlock(&thd->data_mutex);
+
+			len -= chunk_len;
+			if (len == 0) {
+				break;
+			}
+			ptr += chunk_len;
+		}
+
+		max_thread = (i < nthreads) ? i :  nthreads - 1;
+
+		/* Reap and stream the encrypted data */
+		for (i = 0; i <= max_thread; i++) {
+			thd = threads + i;
+
+			pthread_mutex_lock(&thd->data_mutex);
+			while (thd->data_avail == TRUE) {
+				pthread_cond_wait(&thd->data_cond,
+						  &thd->data_mutex);
+			}
+
+			xb_a(threads[i].to_len > 0);
+
+			if (xb_crypt_write_chunk(crypt_file->xbcrypt_file,
+						 threads[i].to,
+						 threads[i].from_len +
+							XB_CRYPT_HASH_LEN,
+						 threads[i].to_len,
+						 threads[i].iv,
+						 encrypt_iv_len)) {
+				msg("encrypt: write to the destination file "
+				    "failed.\n");
+				return 1;
+			}
+
+			crypt_file->bytes_processed += threads[i].from_len;
+
+			pthread_mutex_unlock(&threads[i].data_mutex);
+			pthread_mutex_unlock(&threads[i].ctrl_mutex);
+		}
+	}
+
+	return 0;
+}
+
+static
+int
+encrypt_close(ds_file_t *file)
+{
+	ds_encrypt_file_t	*crypt_file;
+	ds_file_t		*dest_file;
+	int			rc = 0;
+
+	crypt_file = (ds_encrypt_file_t *) file->ptr;
+	dest_file = crypt_file->dest_file;
+
+	rc = xb_crypt_write_close(crypt_file->xbcrypt_file);
+
+	if (ds_close(dest_file)) {
+		rc = 1;
+	}
+
+	my_free(file);
+
+	return rc;
+}
+
+static
+void
+encrypt_deinit(ds_ctxt_t *ctxt)
+{
+	ds_encrypt_ctxt_t 	*crypt_ctxt;
+
+	xb_ad(ctxt->pipe_ctxt != NULL);
+
+	crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr;
+
+	destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads);
+
+	my_free(ctxt->root);
+	my_free(ctxt);
+	if (xtrabackup_encrypt_key)
+		my_free(xtrabackup_encrypt_key);
+	if (xtrabackup_encrypt_key_file)
+		my_free(xtrabackup_encrypt_key_file);
+}
+
+static
+crypt_thread_ctxt_t *
+create_worker_threads(uint n)
+{
+	crypt_thread_ctxt_t	*threads;
+	uint 			i;
+
+	threads = (crypt_thread_ctxt_t *)
+		my_malloc(sizeof(crypt_thread_ctxt_t) * n, MYF(MY_FAE));
+
+	for (i = 0; i < n; i++) {
+		crypt_thread_ctxt_t *thd = threads + i;
+
+		thd->num = i + 1;
+		thd->started = FALSE;
+		thd->cancelled = FALSE;
+		thd->data_avail = FALSE;
+
+		thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE +
+					     XB_CRYPT_HASH_LEN, MYF(MY_FAE));
+
+		thd->iv = (char *) my_malloc(encrypt_iv_len,
+						   MYF(MY_FAE));
+
+		/* Initialize the control mutex and condition var */
+		if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
+		    pthread_cond_init(&thd->ctrl_cond, NULL)) {
+			goto err;
+		}
+
+		/* Initialize and data mutex and condition var */
+		if (pthread_mutex_init(&thd->data_mutex, NULL) ||
+		    pthread_cond_init(&thd->data_cond, NULL)) {
+			goto err;
+		}
+
+		if (encrypt_algo != GCRY_CIPHER_NONE) {
+			gcry_error_t 		gcry_error;
+
+			gcry_error = gcry_cipher_open(&thd->cipher_handle,
+						      encrypt_algo,
+						      encrypt_mode, 0);
+			if (gcry_error) {
+				msg("encrypt: unable to open libgcrypt"
+				    " cipher - %s : %s\n",
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				gcry_cipher_close(thd->cipher_handle);
+				goto err;
+			}
+
+			gcry_error = gcry_cipher_setkey(thd->cipher_handle,
+							xtrabackup_encrypt_key,
+							encrypt_key_len);
+			if (gcry_error) {
+				msg("encrypt: unable to set libgcrypt"
+				    " cipher key - %s : %s\n",
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				gcry_cipher_close(thd->cipher_handle);
+				goto err;
+			}
+		}
+
+		pthread_mutex_lock(&thd->ctrl_mutex);
+
+		if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func,
+				   thd)) {
+			msg("encrypt: pthread_create() failed: "
+			    "errno = %d\n", errno);
+			goto err;
+		}
+	}
+
+	/* Wait for the threads to start */
+	for (i = 0; i < n; i++) {
+		crypt_thread_ctxt_t *thd = threads + i;
+
+		while (thd->started == FALSE)
+			pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
+		pthread_mutex_unlock(&thd->ctrl_mutex);
+	}
+
+	return threads;
+
+err:
+	return NULL;
+}
+
+static
+void
+destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
+{
+	uint i;
+
+	for (i = 0; i < n; i++) {
+		crypt_thread_ctxt_t *thd = threads + i;
+
+		pthread_mutex_lock(&thd->data_mutex);
+		threads[i].cancelled = TRUE;
+		pthread_cond_signal(&thd->data_cond);
+		pthread_mutex_unlock(&thd->data_mutex);
+
+		pthread_join(thd->id, NULL);
+
+		pthread_cond_destroy(&thd->data_cond);
+		pthread_mutex_destroy(&thd->data_mutex);
+		pthread_cond_destroy(&thd->ctrl_cond);
+		pthread_mutex_destroy(&thd->ctrl_mutex);
+
+		if (encrypt_algo != GCRY_CIPHER_NONE)
+			gcry_cipher_close(thd->cipher_handle);
+
+		my_free(thd->to);
+		my_free(thd->iv);
+	}
+
+	my_free(threads);
+}
+
+static
+void *
+encrypt_worker_thread_func(void *arg)
+{
+	crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg;
+
+	pthread_mutex_lock(&thd->ctrl_mutex);
+
+	pthread_mutex_lock(&thd->data_mutex);
+
+	thd->started = TRUE;
+	pthread_cond_signal(&thd->ctrl_cond);
+
+	pthread_mutex_unlock(&thd->ctrl_mutex);
+
+	while (1) {
+		thd->data_avail = FALSE;
+		pthread_cond_signal(&thd->data_cond);
+
+		while (!thd->data_avail && !thd->cancelled) {
+			pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
+		}
+
+		if (thd->cancelled)
+			break;
+
+		/* ensure that XB_CRYPT_HASH_LEN is the correct length
+		of XB_CRYPT_HASH hashing algorithm output */
+		assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+		       XB_CRYPT_HASH_LEN);
+
+		memcpy(thd->to, thd->from, thd->from_len);
+		gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len,
+				    thd->from, thd->from_len);
+		thd->to_len = thd->from_len;
+
+		if (encrypt_algo != GCRY_CIPHER_NONE) {
+			gcry_error_t 		gcry_error;
+
+			gcry_error = gcry_cipher_reset(thd->cipher_handle);
+			if (gcry_error) {
+				msg("encrypt: unable to reset cipher - "
+				    "%s : %s\n",
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				thd->to_len = 0;
+				continue;
+			}
+
+			xb_crypt_create_iv(thd->iv, encrypt_iv_len);
+			gcry_error = gcry_cipher_setctr(thd->cipher_handle,
+							thd->iv,
+							encrypt_iv_len);
+			if (gcry_error) {
+				msg("encrypt: unable to set cipher ctr - "
+				    "%s : %s\n",
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				thd->to_len = 0;
+				continue;
+			}
+
+			gcry_error = gcry_cipher_encrypt(thd->cipher_handle,
+							 thd->to,
+							 thd->to_len +
+							 XB_CRYPT_HASH_LEN,
+							 thd->to,
+							 thd->from_len +
+							 XB_CRYPT_HASH_LEN);
+			if (gcry_error) {
+				msg("encrypt: unable to encrypt buffer - "
+				    "%s : %s\n", gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				thd->to_len = 0;
+			}
+		} else {
+			memcpy(thd->to, thd->from,
+			       thd->from_len + XB_CRYPT_HASH_LEN);
+		}
+		thd->to_len += XB_CRYPT_HASH_LEN;
+	}
+
+	pthread_mutex_unlock(&thd->data_mutex);
+
+	return NULL;
+}

extra/mariabackup/ds_encrypt.h

@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Encryption interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_ENCRYPT_H
+#define DS_ENCRYPT_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_encrypt;
+
+#endif

extra/mariabackup/ds_local.c

@@ -0,0 +1,151 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Local datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include <mysys_err.h>
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+	File fd;
+} ds_local_file_t;
+
+static ds_ctxt_t *local_init(const char *root);
+static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path,
+			     MY_STAT *mystat);
+static int local_write(ds_file_t *file, const void *buf, size_t len);
+static int local_close(ds_file_t *file);
+static void local_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_local = {
+	&local_init,
+	&local_open,
+	&local_write,
+	&local_close,
+	&local_deinit
+};
+
+static
+ds_ctxt_t *
+local_init(const char *root)
+{
+	ds_ctxt_t *ctxt;
+
+	if (my_mkdir(root, 0777, MYF(0)) < 0
+	    && my_errno != EEXIST && my_errno != EISDIR)
+	{
+		char errbuf[MYSYS_STRERROR_SIZE];
+		my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
+			 root, my_errno, my_strerror(errbuf, sizeof(errbuf),
+			 my_errno));
+		return NULL;
+	}
+
+	ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE));
+
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static
+ds_file_t *
+local_open(ds_ctxt_t *ctxt, const char *path,
+	   MY_STAT *mystat __attribute__((unused)))
+{
+	char 		fullpath[FN_REFLEN];
+	char		dirpath[FN_REFLEN];
+	size_t		dirpath_len;
+	size_t		path_len;
+	ds_local_file_t *local_file;
+	ds_file_t	*file;
+	File 		fd;
+
+	fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
+
+	/* Create the directory if needed */
+	dirname_part(dirpath, fullpath, &dirpath_len);
+	if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) {
+		char errbuf[MYSYS_STRERROR_SIZE];
+		my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
+			 dirpath, my_errno, my_strerror(errbuf, sizeof(errbuf),
+			 my_errno));
+		return NULL;
+	}
+
+	fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW,
+		     MYF(MY_WME));
+	if (fd < 0) {
+		return NULL;
+	}
+
+	path_len = strlen(fullpath) + 1; /* terminating '\0' */
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_local_file_t) +
+				       path_len,
+				       MYF(MY_FAE));
+	local_file = (ds_local_file_t *) (file + 1);
+
+	local_file->fd = fd;
+
+	file->path = (char *) local_file + sizeof(ds_local_file_t);
+	memcpy(file->path, fullpath, path_len);
+
+	file->ptr = local_file;
+
+	return file;
+}
+
+static
+int
+local_write(ds_file_t *file, const void *buf, size_t len)
+{
+	File fd = ((ds_local_file_t *) file->ptr)->fd;
+
+	if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
+		posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+		return 0;
+	}
+
+	return 1;
+}
+
+static
+int
+local_close(ds_file_t *file)
+{
+	File fd = ((ds_local_file_t *) file->ptr)->fd;
+
+	my_free(file);
+
+	my_sync(fd, MYF(MY_WME));
+
+	return my_close(fd, MYF(MY_WME));
+}
+
+static
+void
+local_deinit(ds_ctxt_t *ctxt)
+{
+	my_free(ctxt->root);
+	my_free(ctxt);
+}

extra/mariabackup/ds_local.h

@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Local datasink interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_LOCAL_H
+#define DS_LOCAL_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_local;
+
+#endif

extra/mariabackup/ds_stdout.c

@@ -0,0 +1,121 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Local datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#include <mysys_err.h>
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+	File fd;
+} ds_stdout_file_t;
+
+static ds_ctxt_t *stdout_init(const char *root);
+static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path,
+			     MY_STAT *mystat);
+static int stdout_write(ds_file_t *file, const void *buf, size_t len);
+static int stdout_close(ds_file_t *file);
+static void stdout_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_stdout = {
+	&stdout_init,
+	&stdout_open,
+	&stdout_write,
+	&stdout_close,
+	&stdout_deinit
+};
+
+static
+ds_ctxt_t *
+stdout_init(const char *root)
+{
+	ds_ctxt_t *ctxt;
+
+	ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE));
+
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static
+ds_file_t *
+stdout_open(ds_ctxt_t *ctxt __attribute__((unused)),
+	    const char *path __attribute__((unused)),
+	    MY_STAT *mystat __attribute__((unused)))
+{
+	ds_stdout_file_t 	*stdout_file;
+	ds_file_t		*file;
+	size_t			pathlen;
+	const char		*fullpath = "<STDOUT>";
+
+	pathlen = strlen(fullpath) + 1;
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_stdout_file_t) +
+				       pathlen,
+				       MYF(MY_FAE));
+	stdout_file = (ds_stdout_file_t *) (file + 1);
+
+
+#ifdef __WIN__
+	setmode(fileno(stdout), _O_BINARY);
+#endif
+
+	stdout_file->fd = fileno(stdout);
+
+	file->path = (char *) stdout_file + sizeof(ds_stdout_file_t);
+	memcpy(file->path, fullpath, pathlen);
+
+	file->ptr = stdout_file;
+
+	return file;
+}
+
+static
+int
+stdout_write(ds_file_t *file, const void *buf, size_t len)
+{
+	File fd = ((ds_stdout_file_t *) file->ptr)->fd;
+
+	if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
+		posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+		return 0;
+	}
+
+	return 1;
+}
+
+static
+int
+stdout_close(ds_file_t *file)
+{
+	my_free(file);
+
+	return 1;
+}
+
+static
+void
+stdout_deinit(ds_ctxt_t *ctxt)
+{
+	my_free(ctxt->root);
+	my_free(ctxt);
+}

extra/mariabackup/ds_stdout.h

@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Local datasink interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_STDOUT_H
+#define DS_STDOUT_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_stdout;
+
+#endif

extra/mariabackup/ds_tmpfile.c

@@ -0,0 +1,248 @@
+/******************************************************
+Copyright (c) 2012 Percona LLC and/or its affiliates.
+
+tmpfile datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Do all writes to temporary files first, then pipe them to the specified
+datasink in a serialized way in deinit(). */
+
+#include <my_base.h>
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+	pthread_mutex_t	 mutex;
+	LIST		*file_list;
+} ds_tmpfile_ctxt_t;
+
+typedef struct {
+	LIST		 list;
+	File		 fd;
+	char		*orig_path;
+	MY_STAT		 mystat;
+	ds_file_t	*file;
+} ds_tmp_file_t;
+
+static ds_ctxt_t *tmpfile_init(const char *root);
+static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path,
+			       MY_STAT *mystat);
+static int tmpfile_write(ds_file_t *file, const void *buf, size_t len);
+static int tmpfile_close(ds_file_t *file);
+static void tmpfile_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_tmpfile = {
+	&tmpfile_init,
+	&tmpfile_open,
+	&tmpfile_write,
+	&tmpfile_close,
+	&tmpfile_deinit
+};
+
+MY_TMPDIR mysql_tmpdir_list;
+
+static ds_ctxt_t *
+tmpfile_init(const char *root)
+{
+	ds_ctxt_t		*ctxt;
+	ds_tmpfile_ctxt_t	*tmpfile_ctxt;
+
+	ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_tmpfile_ctxt_t),
+			 MYF(MY_FAE));
+	tmpfile_ctxt = (ds_tmpfile_ctxt_t *) (ctxt + 1);
+	tmpfile_ctxt->file_list = NULL;
+	if (pthread_mutex_init(&tmpfile_ctxt->mutex, NULL)) {
+
+		my_free(ctxt);
+		return NULL;
+	}
+
+	ctxt->ptr = tmpfile_ctxt;
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static ds_file_t *
+tmpfile_open(ds_ctxt_t *ctxt, const char *path,
+			       MY_STAT *mystat)
+{
+	ds_tmpfile_ctxt_t	*tmpfile_ctxt;
+	char			 tmp_path[FN_REFLEN];
+	ds_tmp_file_t		*tmp_file;
+	ds_file_t		*file;
+	size_t			 path_len;
+	File			 fd;
+
+	/* Create a temporary file in tmpdir. The file will be automatically
+	removed on close. Code copied from mysql_tmpfile(). */
+	fd = create_temp_file(tmp_path, my_tmpdir(&mysql_tmpdir_list),
+			      "xbtemp",
+#ifdef __WIN__
+			      O_BINARY | O_TRUNC | O_SEQUENTIAL |
+			      O_TEMPORARY | O_SHORT_LIVED |
+#endif /* __WIN__ */
+			      O_CREAT | O_EXCL | O_RDWR,
+			      MYF(MY_WME));
+
+#ifndef __WIN__
+	if (fd >= 0) {
+		/* On Windows, open files cannot be removed, but files can be
+		created with the O_TEMPORARY flag to the same effect
+		("delete on close"). */
+		unlink(tmp_path);
+	}
+#endif /* !__WIN__ */
+
+	if (fd < 0) {
+		return NULL;
+	}
+
+	path_len = strlen(path) + 1; /* terminating '\0' */
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_tmp_file_t) + path_len,
+				       MYF(MY_FAE));
+
+	tmp_file = (ds_tmp_file_t *) (file + 1);
+	tmp_file->file = file;
+	memcpy(&tmp_file->mystat, mystat, sizeof(MY_STAT));
+	/* Save a copy of 'path', since it may not be accessible later */
+	tmp_file->orig_path = (char *) tmp_file + sizeof(ds_tmp_file_t);
+
+	tmp_file->fd = fd;
+	memcpy(tmp_file->orig_path, path, path_len);
+
+	/* Store the real temporary file name in file->path */
+	file->path = my_strdup(tmp_path, MYF(MY_FAE));
+	file->ptr = tmp_file;
+
+	/* Store the file object in the list to be piped later */
+	tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr;
+	tmp_file->list.data = tmp_file;
+
+	pthread_mutex_lock(&tmpfile_ctxt->mutex);
+	tmpfile_ctxt->file_list = list_add(tmpfile_ctxt->file_list,
+					   &tmp_file->list);
+	pthread_mutex_unlock(&tmpfile_ctxt->mutex);
+
+	return file;
+}
+
+static int
+tmpfile_write(ds_file_t *file, const void *buf, size_t len)
+{
+	File fd = ((ds_tmp_file_t *) file->ptr)->fd;
+
+	if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
+		posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int
+tmpfile_close(ds_file_t *file)
+{
+	/* Do nothing -- we will close (and thus remove) the file after piping
+	it to the destination datasink in tmpfile_deinit(). */
+
+	my_free(file->path);
+
+	return 0;
+}
+
+static void
+tmpfile_deinit(ds_ctxt_t *ctxt)
+{
+	LIST			*list;
+	ds_tmpfile_ctxt_t	*tmpfile_ctxt;
+	MY_STAT			 mystat;
+	ds_tmp_file_t 		*tmp_file;
+	ds_file_t		*dst_file;
+	ds_ctxt_t		*pipe_ctxt;
+	void			*buf = NULL;
+	const size_t		 buf_size = 10 * 1024 * 1024;
+	size_t			 bytes;
+	size_t			 offset;
+
+	pipe_ctxt = ctxt->pipe_ctxt;
+	xb_a(pipe_ctxt != NULL);
+
+	buf = my_malloc(buf_size, MYF(MY_FAE));
+
+	tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr;
+	list = tmpfile_ctxt->file_list;
+
+	/* Walk the files in the order they have been added */
+	list = list_reverse(list);
+	while (list != NULL) {
+		tmp_file = list->data;
+		/* Stat the file to replace size and mtime on the original
+		* mystat struct */
+		if (my_fstat(tmp_file->fd, &mystat, MYF(0))) {
+			msg("error: my_fstat() failed.\n");
+			exit(EXIT_FAILURE);
+		}
+		tmp_file->mystat.st_size = mystat.st_size;
+		tmp_file->mystat.st_mtime = mystat.st_mtime;
+
+		dst_file = ds_open(pipe_ctxt, tmp_file->orig_path,
+				   &tmp_file->mystat);
+		if (dst_file == NULL) {
+			msg("error: could not stream a temporary file to "
+			    "'%s'\n", tmp_file->orig_path);
+			exit(EXIT_FAILURE);
+		}
+
+		/* copy to the destination datasink */
+		posix_fadvise(tmp_file->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+		if (my_seek(tmp_file->fd, 0, SEEK_SET, MYF(0)) ==
+		    MY_FILEPOS_ERROR) {
+			msg("error: my_seek() failed for '%s', errno = %d.\n",
+			    tmp_file->file->path, my_errno);
+			exit(EXIT_FAILURE);
+		}
+		offset = 0;
+		while ((bytes = my_read(tmp_file->fd, buf, buf_size,
+					MYF(MY_WME))) > 0) {
+			posix_fadvise(tmp_file->fd, offset, buf_size, POSIX_FADV_DONTNEED);
+			offset += buf_size;
+			if (ds_write(dst_file, buf, bytes)) {
+				msg("error: cannot write to stream for '%s'.\n",
+				    tmp_file->orig_path);
+				exit(EXIT_FAILURE);
+			}
+		}
+		if (bytes == (size_t) -1) {
+			exit(EXIT_FAILURE);
+		}
+
+		my_close(tmp_file->fd, MYF(MY_WME));
+		ds_close(dst_file);
+
+		list = list_rest(list);
+		my_free(tmp_file->file);
+	}
+
+	pthread_mutex_destroy(&tmpfile_ctxt->mutex);
+
+	my_free(buf);
+	my_free(ctxt->root);
+	my_free(ctxt);
+}

extra/mariabackup/ds_tmpfile.h

@@ -0,0 +1,30 @@
+/******************************************************
+Copyright (c) 2012 Percona LLC and/or its affiliates.
+
+tmpfile datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_TMPFILE_H
+#define DS_TMPFILE_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_tmpfile;
+
+extern MY_TMPDIR mysql_tmpdir_list;
+
+#endif

extra/mariabackup/ds_xbstream.c

@@ -0,0 +1,223 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Streaming implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include "common.h"
+#include "datasink.h"
+#include "xbstream.h"
+
+typedef struct {
+	xb_wstream_t	*xbstream;
+	ds_file_t	*dest_file;
+	pthread_mutex_t	mutex;
+} ds_stream_ctxt_t;
+
+typedef struct {
+	xb_wstream_file_t	*xbstream_file;
+	ds_stream_ctxt_t	*stream_ctxt;
+} ds_stream_file_t;
+
+/***********************************************************************
+General streaming interface */
+
+static ds_ctxt_t *xbstream_init(const char *root);
+static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path,
+			      MY_STAT *mystat);
+static int xbstream_write(ds_file_t *file, const void *buf, size_t len);
+static int xbstream_close(ds_file_t *file);
+static void xbstream_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_xbstream = {
+	&xbstream_init,
+	&xbstream_open,
+	&xbstream_write,
+	&xbstream_close,
+	&xbstream_deinit
+};
+
+static
+ssize_t
+my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)),
+		       void *userdata, const void *buf, size_t len)
+{
+	ds_stream_ctxt_t	*stream_ctxt;
+
+	stream_ctxt = (ds_stream_ctxt_t *) userdata;
+
+	xb_ad(stream_ctxt != NULL);
+	xb_ad(stream_ctxt->dest_file != NULL);
+
+	if (!ds_write(stream_ctxt->dest_file, buf, len)) {
+		return len;
+	}
+	return -1;
+}
+
+static
+ds_ctxt_t *
+xbstream_init(const char *root __attribute__((unused)))
+{
+	ds_ctxt_t		*ctxt;
+	ds_stream_ctxt_t	*stream_ctxt;
+	xb_wstream_t *xbstream;
+
+	ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_stream_ctxt_t),
+			 MYF(MY_FAE));
+	stream_ctxt = (ds_stream_ctxt_t *)(ctxt + 1);
+
+	if (pthread_mutex_init(&stream_ctxt->mutex, NULL)) {
+		msg("xbstream_init: pthread_mutex_init() failed.\n");
+		goto err;
+	}
+
+	xbstream = xb_stream_write_new();
+	if (xbstream == NULL) {
+		msg("xb_stream_write_new() failed.\n");
+		goto err;
+	}
+	stream_ctxt->xbstream = xbstream;
+	stream_ctxt->dest_file = NULL;
+
+	ctxt->ptr = stream_ctxt;
+
+	return ctxt;
+
+err:
+	my_free(ctxt);
+	return NULL;
+}
+
+static
+ds_file_t *
+xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+	ds_file_t		*file;
+	ds_stream_file_t	*stream_file;
+	ds_stream_ctxt_t	*stream_ctxt;
+	ds_ctxt_t		*dest_ctxt;
+	xb_wstream_t		*xbstream;
+	xb_wstream_file_t	*xbstream_file;
+
+
+	xb_ad(ctxt->pipe_ctxt != NULL);
+	dest_ctxt = ctxt->pipe_ctxt;
+
+	stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr;
+
+	pthread_mutex_lock(&stream_ctxt->mutex);
+	if (stream_ctxt->dest_file == NULL) {
+		stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
+		if (stream_ctxt->dest_file == NULL) {
+			return NULL;
+		}
+	}
+	pthread_mutex_unlock(&stream_ctxt->mutex);
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_stream_file_t),
+				       MYF(MY_FAE));
+	stream_file = (ds_stream_file_t *) (file + 1);
+
+	xbstream = stream_ctxt->xbstream;
+
+	xbstream_file = xb_stream_write_open(xbstream, path, mystat,
+		                             stream_ctxt,
+					     my_xbstream_write_callback);
+
+	if (xbstream_file == NULL) {
+		msg("xb_stream_write_open() failed.\n");
+		goto err;
+	}
+
+	stream_file->xbstream_file = xbstream_file;
+	stream_file->stream_ctxt = stream_ctxt;
+	file->ptr = stream_file;
+	file->path = stream_ctxt->dest_file->path;
+
+	return file;
+
+err:
+	if (stream_ctxt->dest_file) {
+		ds_close(stream_ctxt->dest_file);
+		stream_ctxt->dest_file = NULL;
+	}
+	my_free(file);
+
+	return NULL;
+}
+
+static
+int
+xbstream_write(ds_file_t *file, const void *buf, size_t len)
+{
+	ds_stream_file_t	*stream_file;
+	xb_wstream_file_t	*xbstream_file;
+
+
+	stream_file = (ds_stream_file_t *) file->ptr;
+
+	xbstream_file = stream_file->xbstream_file;
+
+	if (xb_stream_write_data(xbstream_file, buf, len)) {
+		msg("xb_stream_write_data() failed.\n");
+		return 1;
+	}
+
+	return 0;
+}
+
+static
+int
+xbstream_close(ds_file_t *file)
+{
+	ds_stream_file_t	*stream_file;
+	int			rc = 0;
+
+	stream_file = (ds_stream_file_t *)file->ptr;
+
+	rc = xb_stream_write_close(stream_file->xbstream_file);
+
+	my_free(file);
+
+	return rc;
+}
+
+static
+void
+xbstream_deinit(ds_ctxt_t *ctxt)
+{
+	ds_stream_ctxt_t	*stream_ctxt;
+
+	stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr;
+
+	if (xb_stream_write_done(stream_ctxt->xbstream)) {
+		msg("xb_stream_done() failed.\n");
+	}
+
+	if (stream_ctxt->dest_file) {
+		ds_close(stream_ctxt->dest_file);
+		stream_ctxt->dest_file = NULL;
+	}
+
+	pthread_mutex_destroy(&stream_ctxt->mutex);
+
+	my_free(ctxt);
+}

extra/mariabackup/ds_xbstream.h

@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Streaming interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_XBSTREAM_H
+#define DS_XBSTREAM_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_xbstream;
+
+#endif

extra/mariabackup/fil_cur.cc

@@ -0,0 +1,402 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Source file cursor implementation */
+
+#include <my_base.h>
+
+#include <univ.i>
+#include <fil0fil.h>
+#include <srv0start.h>
+#include <trx0sys.h>
+
+#include "fil_cur.h"
+#include "common.h"
+#include "read_filt.h"
+#include "xtrabackup.h"
+
+/* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */
+#define XB_FIL_CUR_PAGES 640
+
+/***********************************************************************
+Extracts the relative path ("database/table.ibd") of a tablespace from a
+specified possibly absolute path.
+
+For user tablespaces both "./database/table.ibd" and
+"/remote/dir/database/table.ibd" result in "database/table.ibd".
+
+For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1"
+and "./ibdata1" yield "ibdata1" in the output. */
+const char *
+xb_get_relative_path(
+/*=================*/
+	const char*	path,		/*!< in: tablespace path (either
+			  		relative or absolute) */
+	ibool		is_system)	/*!< in: TRUE for system tablespaces,
+					i.e. when only the filename must be
+					returned. */
+{
+	const char *next;
+	const char *cur;
+	const char *prev;
+
+	prev = NULL;
+	cur = path;
+
+	while ((next = strchr(cur, SRV_PATH_SEPARATOR)) != NULL) {
+
+		prev = cur;
+		cur = next + 1;
+	}
+
+	if (is_system) {
+
+		return(cur);
+	} else {
+
+		return((prev == NULL) ? cur : prev);
+	}
+
+}
+
+/**********************************************************************//**
+Closes a file. */
+static
+void
+xb_fil_node_close_file(
+/*===================*/
+	fil_node_t*	node)	/*!< in: file node */
+{
+	ibool	ret;
+
+	mutex_enter(&fil_system->mutex);
+
+	ut_ad(node);
+	ut_a(node->n_pending == 0);
+	ut_a(node->n_pending_flushes == 0);
+	ut_a(!node->being_extended);
+
+	if (!node->open) {
+
+		mutex_exit(&fil_system->mutex);
+
+		return;
+	}
+
+	ret = os_file_close(node->handle);
+	ut_a(ret);
+
+	node->open = FALSE;
+
+	ut_a(fil_system->n_open > 0);
+	fil_system->n_open--;
+	fil_n_file_opened--;
+
+	if (node->space->purpose == FIL_TABLESPACE &&
+	    fil_is_user_tablespace_id(node->space->id)) {
+
+		ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0);
+
+		/* The node is in the LRU list, remove it */
+		UT_LIST_REMOVE(LRU, fil_system->LRU, node);
+	}
+
+	mutex_exit(&fil_system->mutex);
+}
+
+/************************************************************************
+Open a source file cursor and initialize the associated read filter.
+
+@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must
+be skipped and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_open(
+/*============*/
+	xb_fil_cur_t*	cursor,		/*!< out: source file cursor */
+	xb_read_filt_t*	read_filter,	/*!< in/out: the read filter */
+	fil_node_t*	node,		/*!< in: source tablespace node */
+	uint		thread_n)	/*!< thread number for diagnostics */
+{
+	ulint	page_size;
+	ulint	page_size_shift;
+	ulint	zip_size;
+	ibool	success;
+
+	/* Initialize these first so xb_fil_cur_close() handles them correctly
+	in case of error */
+	cursor->orig_buf = NULL;
+	cursor->node = NULL;
+
+	cursor->space_id = node->space->id;
+	cursor->is_system = !fil_is_user_tablespace_id(node->space->id);
+
+	strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path));
+
+	/* Get the relative path for the destination tablespace name, i.e. the
+	one that can be appended to the backup root directory. Non-system
+	tablespaces may have absolute paths for remote tablespaces in MySQL
+	5.6+. We want to make "local" copies for the backup. */
+	strncpy(cursor->rel_path,
+		xb_get_relative_path(cursor->abs_path, cursor->is_system),
+		sizeof(cursor->rel_path));
+
+	/* In the backup mode we should already have a tablespace handle created
+	by fil_load_single_table_tablespace() unless it is a system
+	tablespace. Otherwise we open the file here. */
+	if (cursor->is_system || !srv_backup_mode || srv_close_files) {
+		node->handle =
+			os_file_create_simple_no_error_handling(0, node->name,
+								OS_FILE_OPEN,
+								OS_FILE_READ_ONLY,
+								&success);
+		if (!success) {
+			/* The following call prints an error message */
+			os_file_get_last_error(TRUE);
+
+			msg("[%02u] xtrabackup: error: cannot open "
+			    "tablespace %s\n",
+			    thread_n, cursor->abs_path);
+
+			return(XB_FIL_CUR_ERROR);
+		}
+		mutex_enter(&fil_system->mutex);
+
+		node->open = TRUE;
+
+		fil_system->n_open++;
+		fil_n_file_opened++;
+
+		if (node->space->purpose == FIL_TABLESPACE &&
+		    fil_is_user_tablespace_id(node->space->id)) {
+
+			/* Put the node to the LRU list */
+			UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node);
+		}
+
+		mutex_exit(&fil_system->mutex);
+	}
+
+	ut_ad(node->open);
+
+	cursor->node = node;
+	cursor->file = node->handle;
+
+	if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) {
+		msg("[%02u] xtrabackup: error: cannot stat %s\n",
+		    thread_n, cursor->abs_path);
+
+		xb_fil_cur_close(cursor);
+
+		return(XB_FIL_CUR_ERROR);
+	}
+
+	if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT
+	    || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) {
+
+		os_file_set_nocache(cursor->file, node->name, "OPEN");
+	}
+
+	posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+	/* Determine the page size */
+	zip_size = xb_get_zip_size(cursor->file);
+	if (zip_size == ULINT_UNDEFINED) {
+		xb_fil_cur_close(cursor);
+		return(XB_FIL_CUR_SKIP);
+	} else if (zip_size) {
+		page_size = zip_size;
+		page_size_shift = get_bit_shift(page_size);
+		msg("[%02u] %s is compressed with page size = "
+		    "%lu bytes\n", thread_n, node->name, page_size);
+		if (page_size_shift < 10 || page_size_shift > 14) {
+			msg("[%02u] xtrabackup: Error: Invalid "
+			    "page size: %lu.\n", thread_n, page_size);
+			ut_error;
+		}
+	} else {
+		page_size = UNIV_PAGE_SIZE;
+		page_size_shift = UNIV_PAGE_SIZE_SHIFT;
+	}
+	cursor->page_size = page_size;
+	cursor->page_size_shift = page_size_shift;
+	cursor->zip_size = zip_size;
+
+	/* Allocate read buffer */
+	cursor->buf_size = XB_FIL_CUR_PAGES * page_size;
+	cursor->orig_buf = static_cast<byte *>
+		(ut_malloc(cursor->buf_size + UNIV_PAGE_SIZE));
+	cursor->buf = static_cast<byte *>
+		(ut_align(cursor->orig_buf, UNIV_PAGE_SIZE));
+
+	cursor->buf_read = 0;
+	cursor->buf_npages = 0;
+	cursor->buf_offset = 0;
+	cursor->buf_page_no = 0;
+	cursor->thread_n = thread_n;
+
+	cursor->space_size = cursor->statinfo.st_size / page_size;
+
+	cursor->read_filter = read_filter;
+	cursor->read_filter->init(&cursor->read_filter_ctxt, cursor,
+				  node->space->id);
+
+	return(XB_FIL_CUR_SUCCESS);
+}
+
+/************************************************************************
+Reads and verifies the next block of pages from the source
+file. Positions the cursor after the last read non-corrupted page.
+
+@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF
+if there are no more pages to read and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_read(
+/*============*/
+	xb_fil_cur_t*	cursor)	/*!< in/out: source file cursor */
+{
+	ibool			success;
+	byte*			page;
+	ulint			i;
+	ulint			npages;
+	ulint			retry_count;
+	xb_fil_cur_result_t	ret;
+	ib_int64_t		offset;
+	ib_int64_t		to_read;
+
+	cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt,
+					    &offset, &to_read);
+
+	if (to_read == 0LL) {
+		return(XB_FIL_CUR_EOF);
+	}
+
+	if (to_read > (ib_int64_t) cursor->buf_size) {
+		to_read = (ib_int64_t) cursor->buf_size;
+	}
+
+	xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL);
+
+	if (to_read % cursor->page_size != 0 &&
+	    offset + to_read == cursor->statinfo.st_size) {
+
+		if (to_read < (ib_int64_t) cursor->page_size) {
+			msg("[%02u] xtrabackup: Warning: junk at the end of "
+			    "%s:\n", cursor->thread_n, cursor->abs_path);
+			msg("[%02u] xtrabackup: Warning: offset = %llu, "
+			    "to_read = %llu\n",
+			    cursor->thread_n,
+			    (unsigned long long) offset,
+			    (unsigned long long) to_read);
+
+			return(XB_FIL_CUR_EOF);
+		}
+
+		to_read = (ib_int64_t) (((ulint) to_read) &
+					~(cursor->page_size - 1));
+	}
+
+	xb_a(to_read % cursor->page_size == 0);
+
+	npages = (ulint) (to_read >> cursor->page_size_shift);
+
+	retry_count = 10;
+	ret = XB_FIL_CUR_SUCCESS;
+
+read_retry:
+	xtrabackup_io_throttling();
+
+	cursor->buf_read = 0;
+	cursor->buf_npages = 0;
+	cursor->buf_offset = offset;
+	cursor->buf_page_no = (ulint) (offset >> cursor->page_size_shift);
+
+	success = os_file_read(cursor->file, cursor->buf, offset,
+			       to_read);
+	if (!success) {
+		return(XB_FIL_CUR_ERROR);
+	}
+
+	/* check pages for corruption and re-read if necessary. i.e. in case of
+	partially written pages */
+	for (page = cursor->buf, i = 0; i < npages;
+	     page += cursor->page_size, i++) {
+
+		if (buf_page_is_corrupted(TRUE, page, cursor->zip_size)) {
+
+			ulint page_no = cursor->buf_page_no + i;
+
+			if (cursor->is_system &&
+			    page_no >= FSP_EXTENT_SIZE &&
+			    page_no < FSP_EXTENT_SIZE * 3) {
+				/* skip doublewrite buffer pages */
+				xb_a(cursor->page_size == UNIV_PAGE_SIZE);
+				msg("[%02u] xtrabackup: "
+				    "Page %lu is a doublewrite buffer page, "
+				    "skipping.\n", cursor->thread_n, page_no);
+			} else {
+				retry_count--;
+				if (retry_count == 0) {
+					msg("[%02u] xtrabackup: "
+					    "Error: failed to read page after "
+					    "10 retries. File %s seems to be "
+					    "corrupted.\n", cursor->thread_n,
+					    cursor->abs_path);
+					ret = XB_FIL_CUR_ERROR;
+					break;
+				}
+				msg("[%02u] xtrabackup: "
+				    "Database page corruption detected at page "
+				    "%lu, retrying...\n", cursor->thread_n,
+				    page_no);
+
+				os_thread_sleep(100000);
+
+				goto read_retry;
+			}
+		}
+		cursor->buf_read += cursor->page_size;
+		cursor->buf_npages++;
+	}
+
+	posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED);
+
+	return(ret);
+}
+
+/************************************************************************
+Close the source file cursor opened with xb_fil_cur_open() and its
+associated read filter. */
+void
+xb_fil_cur_close(
+/*=============*/
+	xb_fil_cur_t *cursor)	/*!< in/out: source file cursor */
+{
+	cursor->read_filter->deinit(&cursor->read_filter_ctxt);
+
+	if (cursor->orig_buf != NULL) {
+		ut_free(cursor->orig_buf);
+	}
+	if (cursor->node != NULL) {
+		xb_fil_node_close_file(cursor->node);
+		cursor->file = XB_FILE_UNDEFINED;
+	}
+}

extra/mariabackup/fil_cur.h

@@ -0,0 +1,123 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Source file cursor interface */
+
+#ifndef FIL_CUR_H
+#define FIL_CUR_H
+
+#include <my_dir.h>
+#include "read_filt.h"
+
+struct xb_fil_cur_t {
+	os_file_t	file;		/*!< source file handle */
+	fil_node_t*	node;		/*!< source tablespace node */
+	char		rel_path[FN_REFLEN];
+					/*!< normalized file path */
+	char		abs_path[FN_REFLEN];
+					/*!< absolute file path */
+	MY_STAT		statinfo;	/*!< information about the file */
+	ulint		zip_size;	/*!< compressed page size in bytes or 0
+					for uncompressed pages */
+	ulint		page_size;	/*!< = zip_size for compressed pages or
+					UNIV_PAGE_SIZE for uncompressed ones */
+	ulint		page_size_shift;/*!< bit shift corresponding to
+					page_size */
+	my_bool		is_system;	/*!< TRUE for system tablespace, FALSE
+					otherwise */
+	xb_read_filt_t*	read_filter;	/*!< read filter */
+	xb_read_filt_ctxt_t	read_filter_ctxt;
+					/*!< read filter context */
+	byte*		orig_buf;	/*!< read buffer */
+	byte*		buf;		/*!< aligned pointer for orig_buf */
+	ulint		buf_size;	/*!< buffer size in bytes */
+	ulint		buf_read;	/*!< number of read bytes in buffer
+					after the last cursor read */
+	ulint		buf_npages;	/*!< number of pages in buffer after the
+					last cursor read */
+	ib_int64_t	buf_offset;	/*!< file offset of the first page in
+					buffer */
+	ulint		buf_page_no;	/*!< number of the first page in
+					buffer */
+	uint		thread_n;	/*!< thread number for diagnostics */
+	ulint		space_id;	/*!< ID of tablespace */
+	ulint		space_size;	/*!< space size in pages */
+};
+
+typedef enum {
+	XB_FIL_CUR_SUCCESS,
+	XB_FIL_CUR_SKIP,
+	XB_FIL_CUR_ERROR,
+	XB_FIL_CUR_EOF
+} xb_fil_cur_result_t;
+
+/************************************************************************
+Open a source file cursor and initialize the associated read filter.
+
+@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must
+be skipped and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_open(
+/*============*/
+	xb_fil_cur_t*	cursor,		/*!< out: source file cursor */
+	xb_read_filt_t*	read_filter,	/*!< in/out: the read filter */
+	fil_node_t*	node,		/*!< in: source tablespace node */
+	uint		thread_n);	/*!< thread number for diagnostics */
+
+/************************************************************************
+Reads and verifies the next block of pages from the source
+file. Positions the cursor after the last read non-corrupted page.
+
+@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF
+if there are no more pages to read and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_read(
+/*============*/
+	xb_fil_cur_t*	cursor);	/*!< in/out: source file cursor */
+
+/************************************************************************
+Close the source file cursor opened with xb_fil_cur_open() and its
+associated read filter. */
+void
+xb_fil_cur_close(
+/*=============*/
+	xb_fil_cur_t *cursor);	/*!< in/out: source file cursor */
+
+/***********************************************************************
+Extracts the relative path ("database/table.ibd") of a tablespace from a
+specified possibly absolute path.
+
+For user tablespaces both "./database/table.ibd" and
+"/remote/dir/database/table.ibd" result in "database/table.ibd".
+
+For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1"
+and "./ibdata1" yield "ibdata1" in the output. */
+const char *
+xb_get_relative_path(
+/*=================*/
+	const char*	path,		/*!< in: tablespace path (either
+			  		relative or absolute) */
+	ibool		is_system);	/*!< in: TRUE for system tablespaces,
+					i.e. when only the filename must be
+					returned. */
+
+#endif

extra/mariabackup/innobackupex.h

@@ -0,0 +1,45 @@
+/******************************************************
+Copyright (c) 2011-2014 Percona LLC and/or its affiliates.
+
+Declarations for innobackupex.cc
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef INNOBACKUPEX_H
+#define INNOBACKUPEX_H
+
+#define INNOBACKUPEX_BIN_NAME "innobackupex"
+
+enum ibx_mode_t {
+	IBX_MODE_BACKUP,
+	IBX_MODE_APPLY_LOG,
+	IBX_MODE_COPY_BACK,
+	IBX_MODE_MOVE_BACK,
+	IBX_MODE_DECRYPT_DECOMPRESS
+};
+
+extern ibx_mode_t ibx_mode;
+
+bool
+ibx_handle_options(int *argc, char ***argv);
+
+bool
+ibx_init();
+
+void
+ibx_cleanup();
+
+#endif

extra/mariabackup/quicklz/quicklz.c

@@ -0,0 +1,848 @@
+// Fast data compression library
+// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
+// lar@quicklz.com
+//
+// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything 
+// released into public must be open source) or under a commercial license if such 
+// has been acquired (see http://www.quicklz.com/order.html). The commercial license 
+// does not cover derived or ported versions created by third parties under GPL.
+
+// 1.5.0 final
+
+#include "quicklz.h"
+
+#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
+	#error quicklz.c and quicklz.h have different versions
+#endif
+
+#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64))
+	#define X86X64
+#endif
+
+#define MINOFFSET 2
+#define UNCONDITIONAL_MATCHLEN 6
+#define UNCOMPRESSED_END 4
+#define CWORD_LEN 4
+
+#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
+	#define OFFSET_BASE source
+	#define CAST (ui32)(size_t)
+#else
+	#define OFFSET_BASE 0
+	#define CAST
+#endif
+
+int qlz_get_setting(int setting)
+{
+	switch (setting)
+	{
+		case 0: return QLZ_COMPRESSION_LEVEL;
+		case 1: return sizeof(qlz_state_compress);
+		case 2: return sizeof(qlz_state_decompress);
+		case 3: return QLZ_STREAMING_BUFFER;
+#ifdef QLZ_MEMORY_SAFE
+		case 6: return 1;
+#else
+		case 6: return 0;
+#endif
+		case 7: return QLZ_VERSION_MAJOR;
+		case 8: return QLZ_VERSION_MINOR;
+		case 9: return QLZ_VERSION_REVISION;
+	}
+	return -1;
+}
+
+#if QLZ_COMPRESSION_LEVEL == 1
+static int same(const unsigned char *src, size_t n)
+{
+	while(n > 0 && *(src + n) == *src)
+		n--;
+	return n == 0 ? 1 : 0;
+}
+#endif
+
+static void reset_table_compress(qlz_state_compress *state)
+{
+	int i;
+	for(i = 0; i < QLZ_HASH_VALUES; i++)
+	{
+#if QLZ_COMPRESSION_LEVEL == 1
+		state->hash[i].offset = 0;
+#else
+		state->hash_counter[i] = 0;
+#endif
+	}
+}
+
+static void reset_table_decompress(qlz_state_decompress *state)
+{
+	int i;
+	(void)state;
+	(void)i;
+#if QLZ_COMPRESSION_LEVEL == 2
+	for(i = 0; i < QLZ_HASH_VALUES; i++)
+	{
+		state->hash_counter[i] = 0;
+	}
+#endif
+}
+
+static __inline ui32 hash_func(ui32 i)
+{
+#if QLZ_COMPRESSION_LEVEL == 2
+	return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
+#else
+	return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
+#endif
+}
+
+static __inline ui32 fast_read(void const *src, ui32 bytes)
+{
+#ifndef X86X64
+	unsigned char *p = (unsigned char*)src;
+	switch (bytes)
+	{
+		case 4:
+			return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
+		case 3: 
+			return(*p | *(p + 1) << 8 | *(p + 2) << 16);
+		case 2:
+			return(*p | *(p + 1) << 8);
+		case 1: 
+			return(*p);
+	}
+	return 0;
+#else
+	if (bytes >= 1 && bytes <= 4)
+		return *((ui32*)src);
+	else
+		return 0;
+#endif
+}
+
+static __inline ui32 hashat(const unsigned char *src)
+{
+	ui32 fetch, hash;
+	fetch = fast_read(src, 3);
+	hash = hash_func(fetch);
+	return hash;
+}
+
+static __inline void fast_write(ui32 f, void *dst, size_t bytes)
+{
+#ifndef X86X64
+	unsigned char *p = (unsigned char*)dst;
+
+	switch (bytes)
+	{
+		case 4: 
+			*p = (unsigned char)f;
+			*(p + 1) = (unsigned char)(f >> 8);
+			*(p + 2) = (unsigned char)(f >> 16);
+			*(p + 3) = (unsigned char)(f >> 24);
+			return;
+		case 3:
+			*p = (unsigned char)f;
+			*(p + 1) = (unsigned char)(f >> 8);
+			*(p + 2) = (unsigned char)(f >> 16);
+			return;
+		case 2:
+			*p = (unsigned char)f;
+			*(p + 1) = (unsigned char)(f >> 8);
+			return;
+		case 1:
+			*p = (unsigned char)f;
+			return;
+	}
+#else
+	switch (bytes)
+	{
+		case 4: 
+			*((ui32*)dst) = f;
+			return;
+		case 3:
+			*((ui32*)dst) = f;
+			return;
+		case 2:
+			*((ui16 *)dst) = (ui16)f;
+			return;
+		case 1:
+			*((unsigned char*)dst) = (unsigned char)f;
+			return;
+	}
+#endif
+}
+
+
+size_t qlz_size_decompressed(const char *source)
+{
+	ui32 n, r;
+	n = (((*source) & 2) == 2) ? 4 : 1;
+	r = fast_read(source + 1 + n, n);
+	r = r & (0xffffffff >> ((4 - n)*8));
+	return r;
+}
+
+size_t qlz_size_compressed(const char *source)
+{
+	ui32 n, r;
+	n = (((*source) & 2) == 2) ? 4 : 1;
+	r = fast_read(source + 1, n);
+	r = r & (0xffffffff >> ((4 - n)*8));
+	return r;
+}
+
+size_t qlz_size_header(const char *source)
+{
+	size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1;
+	return n;
+}
+
+
+static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
+{
+	// Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
+#ifndef X86X64
+	unsigned char *end = dst + n;
+	while(dst < end)
+	{
+		*dst = *src;
+		dst++;
+		src++;
+	}
+#else
+	ui32 f = 0;
+	do
+	{
+		*(ui32 *)(dst + f) = *(ui32 *)(src + f);
+		f += MINOFFSET + 1;
+	}
+	while (f < n);
+#endif
+}
+
+static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
+{
+#if QLZ_COMPRESSION_LEVEL == 1
+	ui32 hash;
+	hash = hashat(s);
+	state->hash[hash].offset = s;
+	state->hash_counter[hash] = 1;
+#elif QLZ_COMPRESSION_LEVEL == 2
+	ui32 hash;
+	unsigned char c;
+	hash = hashat(s);
+	c = state->hash_counter[hash];
+	state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
+	c++;
+	state->hash_counter[hash] = c;
+#endif
+	(void)state;
+	(void)s;
+}
+
+#if QLZ_COMPRESSION_LEVEL <= 2
+static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
+{
+	while(*lh < max)
+	{
+		(*lh)++;
+		update_hash(state, *lh);
+	}
+}
+#endif
+
+static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
+{
+	const unsigned char *last_byte = source + size - 1;
+	const unsigned char *src = source;
+	unsigned char *cword_ptr = destination;
+	unsigned char *dst = destination + CWORD_LEN;
+	ui32 cword_val = 1U << 31;
+	const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; 
+	ui32 fetch = 0;
+	unsigned int lits = 0;
+
+	(void) lits;
+
+	if(src <= last_matchstart)
+		fetch = fast_read(src, 3);
+	
+	while(src <= last_matchstart)
+	{
+		if ((cword_val & 1) == 1)
+		{
+			// store uncompressed if compression ratio is too low
+			if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
+				return 0;
+
+			fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
+
+			cword_ptr = dst;
+			dst += CWORD_LEN;
+			cword_val = 1U << 31;
+			fetch = fast_read(src, 3);
+		}
+#if QLZ_COMPRESSION_LEVEL == 1
+		{
+			const unsigned char *o;
+			ui32 hash, cached;
+
+			hash = hash_func(fetch);
+			cached = fetch ^ state->hash[hash].cache;
+			state->hash[hash].cache = fetch;
+
+			o = state->hash[hash].offset + OFFSET_BASE;
+			state->hash[hash].offset = CAST(src - OFFSET_BASE);
+
+#ifdef X86X64
+			if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
+			{
+				if(cached != 0)
+				{
+#else
+			if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
+			{
+				if (*(o + 3) != *(src + 3))
+				{
+#endif
+					hash <<= 4;
+					cword_val = (cword_val >> 1) | (1U << 31);
+					fast_write((3 - 2) | hash, dst, 2);
+					src += 3;
+					dst += 2;
+				}
+				else
+				{
+					const unsigned char *old_src = src;
+					size_t matchlen;
+					hash <<= 4;
+
+					cword_val = (cword_val >> 1) | (1U << 31);
+					src += 4;
+
+					if(*(o + (src - old_src)) == *src)
+					{
+						src++;
+						if(*(o + (src - old_src)) == *src)
+						{
+							size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
+							size_t remaining = q > 255 ? 255 : q;
+							src++;	
+							while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
+								src++;
+						}
+					}
+
+					matchlen = src - old_src;
+					if (matchlen < 18)
+					{
+						fast_write((ui32)(matchlen - 2) | hash, dst, 2);
+						dst += 2;
+					} 
+					else
+					{
+						fast_write((ui32)(matchlen << 16) | hash, dst, 3);
+						dst += 3;
+					}
+				}
+				fetch = fast_read(src, 3);
+				lits = 0;
+			}
+			else
+			{
+				lits++;
+				*dst = *src;
+				src++;
+				dst++;
+				cword_val = (cword_val >> 1);
+#ifdef X86X64
+				fetch = fast_read(src, 3);
+#else
+				fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
+#endif
+			}
+		}
+#elif QLZ_COMPRESSION_LEVEL >= 2
+		{
+			const unsigned char *o, *offset2;
+			ui32 hash, matchlen, k, m, best_k = 0;
+			unsigned char c;
+			size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
+			(void)best_k;
+		
+
+			//hash = hashat(src);
+			fetch = fast_read(src, 3);
+			hash = hash_func(fetch);
+
+			c = state->hash_counter[hash];
+
+			offset2 = state->hash[hash].offset[0];
+			if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
+			{	
+				matchlen = 3;
+				if(*(offset2 + matchlen) == *(src + matchlen))
+				{
+					matchlen = 4;
+					while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
+						matchlen++;
+				}
+			}
+			else
+				matchlen = 0;
+			for(k = 1; k < QLZ_POINTERS && c > k; k++)
+			{
+				o = state->hash[hash].offset[k];
+#if QLZ_COMPRESSION_LEVEL == 3
+				if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
+#elif QLZ_COMPRESSION_LEVEL == 2
+				if(*(src + matchlen) == *(o + matchlen)	&& ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
+#endif
+				{	
+					m = 3;
+					while(*(o + m) == *(src + m) && m < remaining)
+						m++;
+#if QLZ_COMPRESSION_LEVEL == 3
+					if ((m > matchlen) || (m == matchlen && o > offset2))
+#elif QLZ_COMPRESSION_LEVEL == 2
+					if (m > matchlen)
+#endif
+					{
+						offset2 = o;
+						matchlen = m;
+						best_k = k;
+					}
+				}
+			}
+			o = offset2;
+			state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
+			c++;
+			state->hash_counter[hash] = c;
+
+#if QLZ_COMPRESSION_LEVEL == 3
+			if(matchlen > 2 && src - o < 131071)
+			{
+				ui32 u;
+				size_t offset = src - o;
+
+				for(u = 1; u < matchlen; u++)
+				{	
+					hash = hashat(src + u);
+					c = state->hash_counter[hash]++;
+					state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
+				}
+
+				cword_val = (cword_val >> 1) | (1U << 31);
+				src += matchlen;
+
+				if(matchlen == 3 && offset <= 63)
+				{
+					*dst = (unsigned char)(offset << 2);
+					dst++;
+				}
+				else if (matchlen == 3 && offset <= 16383)
+				{
+					ui32 f = (ui32)((offset << 2) | 1);
+					fast_write(f, dst, 2);
+					dst += 2;
+				}		
+				else if (matchlen <= 18 && offset <= 1023)
+				{
+					ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
+					fast_write(f, dst, 2);
+					dst += 2;
+				}
+
+				else if(matchlen <= 33)
+				{
+					ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
+					fast_write(f, dst, 3);
+					dst += 3;
+				}
+				else
+				{
+					ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
+					fast_write(f, dst, 4);
+					dst += 4;
+				}
+			}
+			else
+			{
+				*dst = *src;
+				src++;
+				dst++;
+				cword_val = (cword_val >> 1);
+			}
+#elif QLZ_COMPRESSION_LEVEL == 2
+
+			if(matchlen > 2)
+			{
+				cword_val = (cword_val >> 1) | (1U << 31);
+				src += matchlen;			
+
+				if (matchlen < 10)
+				{			
+					ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
+					fast_write(f, dst, 2);
+					dst += 2;
+				}
+				else
+				{
+					ui32 f = best_k | (matchlen << 16) | (hash << 5);
+					fast_write(f, dst, 3);
+					dst += 3;
+				}
+			}
+			else
+			{
+				*dst = *src;
+				src++;
+				dst++;
+				cword_val = (cword_val >> 1);
+			}
+#endif
+		}
+#endif
+	}
+	while (src <= last_byte)
+	{
+		if ((cword_val & 1) == 1)
+		{
+			fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
+			cword_ptr = dst;
+			dst += CWORD_LEN;
+			cword_val = 1U << 31;
+		}
+#if QLZ_COMPRESSION_LEVEL < 3
+		if (src <= last_byte - 3)
+		{
+#if QLZ_COMPRESSION_LEVEL == 1
+			ui32 hash, fetch;
+			fetch = fast_read(src, 3);
+			hash = hash_func(fetch);
+			state->hash[hash].offset = CAST(src - OFFSET_BASE);
+			state->hash[hash].cache = fetch;
+#elif QLZ_COMPRESSION_LEVEL == 2
+			ui32 hash;
+			unsigned char c;
+			hash = hashat(src);
+			c = state->hash_counter[hash];
+			state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
+			c++;
+			state->hash_counter[hash] = c;
+#endif
+		}
+#endif
+		*dst = *src;
+		src++;
+		dst++;
+		cword_val = (cword_val >> 1);
+	}
+
+	while((cword_val & 1) != 1)
+		cword_val = (cword_val >> 1);
+
+	fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
+
+	// min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
+	return dst - destination < 9 ? 9 : dst - destination;
+}
+
+static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
+{
+	const unsigned char *src = source + qlz_size_header((const char *)source);
+	unsigned char *dst = destination;
+	const unsigned char *last_destination_byte = destination + size - 1;
+	ui32 cword_val = 1;
+	const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
+	unsigned char *last_hashed = destination - 1;
+	const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
+	static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
+
+	(void) last_source_byte;
+	(void) last_hashed;
+	(void) state;
+	(void) history;
+
+	for(;;) 
+	{
+		ui32 fetch;
+
+		if (cword_val == 1)
+		{
+#ifdef QLZ_MEMORY_SAFE
+			if(src + CWORD_LEN - 1 > last_source_byte)
+				return 0;
+#endif
+			cword_val = fast_read(src, CWORD_LEN);
+			src += CWORD_LEN;
+		}
+
+#ifdef QLZ_MEMORY_SAFE
+			if(src + 4 - 1 > last_source_byte)
+				return 0;
+#endif
+
+		fetch = fast_read(src, 4);
+
+		if ((cword_val & 1) == 1)
+		{
+			ui32 matchlen;
+			const unsigned char *offset2;
+
+#if QLZ_COMPRESSION_LEVEL == 1
+			ui32 hash;
+			cword_val = cword_val >> 1;
+			hash = (fetch >> 4) & 0xfff;
+			offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;
+
+			if((fetch & 0xf) != 0)
+			{
+				matchlen = (fetch & 0xf) + 2;
+				src += 2;
+			}
+			else
+			{
+				matchlen = *(src + 2);
+				src += 3;							
+			}	
+
+#elif QLZ_COMPRESSION_LEVEL == 2
+			ui32 hash;
+			unsigned char c;
+			cword_val = cword_val >> 1;
+			hash = (fetch >> 5) & 0x7ff;
+			c = (unsigned char)(fetch & 0x3);
+			offset2 = state->hash[hash].offset[c];
+
+			if((fetch & (28)) != 0)
+			{
+				matchlen = ((fetch >> 2) & 0x7) + 2;
+				src += 2;
+			}
+			else
+			{
+				matchlen = *(src + 2);
+				src += 3;							
+			}	
+
+#elif QLZ_COMPRESSION_LEVEL == 3
+			ui32 offset;
+			cword_val = cword_val >> 1;
+			if ((fetch & 3) == 0)
+			{
+				offset = (fetch & 0xff) >> 2;
+				matchlen = 3;
+				src++;
+			}
+			else if ((fetch & 2) == 0)
+			{
+				offset = (fetch & 0xffff) >> 2;
+				matchlen = 3;
+				src += 2;
+			}
+			else if ((fetch & 1) == 0)
+			{
+				offset = (fetch & 0xffff) >> 6;
+				matchlen = ((fetch >> 2) & 15) + 3;
+				src += 2;
+			}
+			else if ((fetch & 127) != 3)
+			{
+				offset = (fetch >> 7) & 0x1ffff;
+				matchlen = ((fetch >> 2) & 0x1f) + 2;
+				src += 3;
+			}
+			else
+			{
+				offset = (fetch >> 15);
+				matchlen = ((fetch >> 7) & 255) + 3;
+				src += 4;
+			}
+
+			offset2 = dst - offset;
+#endif
+	
+#ifdef QLZ_MEMORY_SAFE
+			if(offset2 < history || offset2 > dst - MINOFFSET - 1)
+				return 0;
+
+			if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
+				return 0;
+#endif
+
+			memcpy_up(dst, offset2, matchlen);
+			dst += matchlen;
+
+#if QLZ_COMPRESSION_LEVEL <= 2
+			update_hash_upto(state, &last_hashed, dst - matchlen);
+			last_hashed = dst - 1;
+#endif
+		}
+		else
+		{
+			if (dst < last_matchstart)
+			{
+				unsigned int n = bitlut[cword_val & 0xf];
+#ifdef X86X64
+				*(ui32 *)dst = *(ui32 *)src;
+#else
+				memcpy_up(dst, src, 4);
+#endif
+				cword_val = cword_val >> n;
+				dst += n;
+				src += n;
+#if QLZ_COMPRESSION_LEVEL <= 2
+				update_hash_upto(state, &last_hashed, dst - 3);		
+#endif
+			}
+			else
+			{			
+				while(dst <= last_destination_byte)
+				{
+					if (cword_val == 1)
+					{
+						src += CWORD_LEN;
+						cword_val = 1U << 31;
+					}
+#ifdef QLZ_MEMORY_SAFE
+					if(src >= last_source_byte + 1)
+						return 0;
+#endif
+					*dst = *src;
+					dst++;
+					src++;
+					cword_val = cword_val >> 1;
+				}
+
+#if QLZ_COMPRESSION_LEVEL <= 2
+				update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
+#endif
+				return size;
+			}
+
+		}
+	}
+}
+
+size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
+{
+	size_t r;
+	ui32 compressed;
+	size_t base;
+
+	if(size == 0 || size > 0xffffffff - 400)
+		return 0;
+
+	if(size < 216)
+		base = 3;
+	else
+		base = 9;
+
+#if QLZ_STREAMING_BUFFER > 0
+	if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
+#endif
+	{
+		reset_table_compress(state);
+		r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state);
+#if QLZ_STREAMING_BUFFER > 0
+		reset_table_compress(state);
+#endif
+		if(r == base)
+		{
+			memcpy(destination + base, source, size);
+			r = size + base;
+			compressed = 0;
+		}
+		else
+		{
+			compressed = 1;
+		}
+		state->stream_counter = 0;
+	}
+#if QLZ_STREAMING_BUFFER > 0
+	else
+	{
+		unsigned char *src = state->stream_buffer + state->stream_counter;
+
+		memcpy(src, source, size);
+		r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state);
+
+ 		if(r == base)
+		{
+			memcpy(destination + base, src, size);
+			r = size + base;
+			compressed = 0;
+			reset_table_compress(state);
+		}
+		else
+		{
+			compressed = 1;
+		}
+		state->stream_counter += size;
+	}
+#endif
+	if(base == 3)
+	{
+		*destination = (unsigned char)(0 | compressed);
+		*(destination + 1) = (unsigned char)r;
+		*(destination + 2) = (unsigned char)size;
+	}
+	else
+	{
+		*destination = (unsigned char)(2 | compressed);
+		fast_write((ui32)r, destination + 1, 4);
+		fast_write((ui32)size, destination + 5, 4);
+	}
+	
+	*destination |= (QLZ_COMPRESSION_LEVEL << 2);
+	*destination |= (1 << 6);
+	*destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);
+
+// 76543210
+// 01SSLLHC
+
+	return r;
+}
+
+size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
+{
+	size_t dsiz = qlz_size_decompressed(source);
+
+#if QLZ_STREAMING_BUFFER > 0
+	if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) 
+#endif
+	{
+		if((*source & 1) == 1)
+		{
+			reset_table_decompress(state);
+			dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
+		}
+		else
+		{
+			memcpy(destination, source + qlz_size_header(source), dsiz);
+		}
+		state->stream_counter = 0;
+		reset_table_decompress(state);
+	}
+#if QLZ_STREAMING_BUFFER > 0
+	else
+	{
+		unsigned char *dst = state->stream_buffer + state->stream_counter;
+		if((*source & 1) == 1)
+		{
+			dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
+		}
+		else
+		{
+			memcpy(dst, source + qlz_size_header(source), dsiz);
+			reset_table_decompress(state);
+		}
+		memcpy(destination, dst, dsiz);
+		state->stream_counter += dsiz;
+	}
+#endif
+	return dsiz;
+}
+

extra/mariabackup/quicklz/quicklz.h

@@ -0,0 +1,144 @@
+#ifndef QLZ_HEADER
+#define QLZ_HEADER
+
+// Fast data compression library
+// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
+// lar@quicklz.com
+//
+// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything 
+// released into public must be open source) or under a commercial license if such 
+// has been acquired (see http://www.quicklz.com/order.html). The commercial license 
+// does not cover derived or ported versions created by third parties under GPL.
+
+// You can edit following user settings. Data must be decompressed with the same 
+// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed
+// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially
+// zeroed out (see manual). First #ifndef makes it possible to define settings from 
+// the outside like the compiler command line.
+
+// 1.5.0 final
+
+#ifndef QLZ_COMPRESSION_LEVEL
+	#define QLZ_COMPRESSION_LEVEL 1
+	//#define QLZ_COMPRESSION_LEVEL 2
+	//#define QLZ_COMPRESSION_LEVEL 3
+
+	#define QLZ_STREAMING_BUFFER 0
+	//#define QLZ_STREAMING_BUFFER 100000
+	//#define QLZ_STREAMING_BUFFER 1000000
+
+	//#define QLZ_MEMORY_SAFE
+#endif
+
+#define QLZ_VERSION_MAJOR 1
+#define QLZ_VERSION_MINOR 5
+#define QLZ_VERSION_REVISION 0
+
+// Using size_t, memset() and memcpy()
+#include <string.h>
+
+// Verify compression level
+#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3
+#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3
+#endif
+
+typedef unsigned int ui32;
+typedef unsigned short int ui16;
+
+// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values!
+#if QLZ_COMPRESSION_LEVEL == 1
+#define QLZ_POINTERS 1
+#define QLZ_HASH_VALUES 4096
+#elif QLZ_COMPRESSION_LEVEL == 2
+#define QLZ_POINTERS 4
+#define QLZ_HASH_VALUES 2048
+#elif QLZ_COMPRESSION_LEVEL == 3
+#define QLZ_POINTERS 16
+#define QLZ_HASH_VALUES 4096
+#endif
+
+// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization.
+#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__
+	#define QLZ_PTR_64
+#endif
+
+// hash entry
+typedef struct 
+{
+#if QLZ_COMPRESSION_LEVEL == 1
+	ui32 cache;
+#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
+	unsigned int offset;
+#else
+	const unsigned char *offset;
+#endif
+#else
+	const unsigned char *offset[QLZ_POINTERS];
+#endif
+
+} qlz_hash_compress;
+
+typedef struct 
+{
+#if QLZ_COMPRESSION_LEVEL == 1
+	const unsigned char *offset;
+#else
+	const unsigned char *offset[QLZ_POINTERS];
+#endif
+} qlz_hash_decompress;
+
+
+// states
+typedef struct
+{
+	#if QLZ_STREAMING_BUFFER > 0
+		unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
+	#endif
+	size_t stream_counter;
+	qlz_hash_compress hash[QLZ_HASH_VALUES];
+	unsigned char hash_counter[QLZ_HASH_VALUES];
+} qlz_state_compress;
+
+
+#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2
+	typedef struct
+	{
+#if QLZ_STREAMING_BUFFER > 0
+		unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
+#endif
+		qlz_hash_decompress hash[QLZ_HASH_VALUES];
+		unsigned char hash_counter[QLZ_HASH_VALUES];
+		size_t stream_counter;
+	} qlz_state_decompress;
+#elif QLZ_COMPRESSION_LEVEL == 3
+	typedef struct
+	{
+#if QLZ_STREAMING_BUFFER > 0
+		unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
+#endif
+#if QLZ_COMPRESSION_LEVEL <= 2
+		qlz_hash_decompress hash[QLZ_HASH_VALUES];
+#endif
+		size_t stream_counter;
+	} qlz_state_decompress;
+#endif
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+// Public functions of QuickLZ
+size_t qlz_size_decompressed(const char *source);
+size_t qlz_size_compressed(const char *source);
+size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state);
+size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state);
+int qlz_get_setting(int setting);
+size_t qlz_size_header(const char *source);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif
+

extra/mariabackup/read_filt.cc

@@ -0,0 +1,206 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Data file read filter implementation */
+
+#include "read_filt.h"
+#include "common.h"
+#include "fil_cur.h"
+#include "xtrabackup.h"
+
+/****************************************************************//**
+Perform read filter context initialization that is common to all read
+filters.  */
+static
+void
+common_init(
+/*========*/
+	xb_read_filt_ctxt_t*	ctxt,	/*!<in/out: read filter context */
+	const xb_fil_cur_t*	cursor)	/*!<in: file cursor */
+{
+	ctxt->offset = 0;
+	ctxt->data_file_size = cursor->statinfo.st_size;
+	ctxt->buffer_capacity = cursor->buf_size;
+	ctxt->page_size = cursor->page_size;
+}
+
+/****************************************************************//**
+Initialize the pass-through read filter. */
+static
+void
+rf_pass_through_init(
+/*=================*/
+	xb_read_filt_ctxt_t*	ctxt,	/*!<in/out: read filter context */
+	const xb_fil_cur_t*	cursor,	/*!<in: file cursor */
+	ulint			space_id __attribute__((unused)))
+					/*!<in: space id we are reading */
+{
+	common_init(ctxt, cursor);
+}
+
+/****************************************************************//**
+Get the next batch of pages for the pass-through read filter.  */
+static
+void
+rf_pass_through_get_next_batch(
+/*===========================*/
+	xb_read_filt_ctxt_t*	ctxt,			/*!<in/out: read filter
+							context */
+	ib_int64_t*		read_batch_start,	/*!<out: starting read
+							offset in bytes for the
+							next batch of pages */
+	ib_int64_t*		read_batch_len)		/*!<out: length in
+							bytes of the next batch
+							of pages */
+{
+	*read_batch_start = ctxt->offset;
+	*read_batch_len = ctxt->data_file_size - ctxt->offset;
+
+	if (*read_batch_len > ctxt->buffer_capacity) {
+		*read_batch_len = ctxt->buffer_capacity;
+	}
+
+	ctxt->offset += *read_batch_len;
+}
+
+/****************************************************************//**
+Deinitialize the pass-through read filter.  */
+static
+void
+rf_pass_through_deinit(
+/*===================*/
+	xb_read_filt_ctxt_t*	ctxt __attribute__((unused)))
+					/*!<in: read filter context */
+{
+}
+
+/****************************************************************//**
+Initialize the changed page bitmap-based read filter.  Assumes that
+the bitmap is already set up in changed_page_bitmap.  */
+static
+void
+rf_bitmap_init(
+/*===========*/
+	xb_read_filt_ctxt_t*	ctxt,		/*!<in/out: read filter
+						context */
+	const xb_fil_cur_t*	cursor,		/*!<in: read cursor */
+	ulint			space_id)	/*!<in: space id  */
+{
+	common_init(ctxt, cursor);
+	ctxt->bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap,
+						       space_id);
+	ctxt->filter_batch_end = 0;
+}
+
+/****************************************************************//**
+Get the next batch of pages for the bitmap read filter.  */
+static
+void
+rf_bitmap_get_next_batch(
+/*=====================*/
+	xb_read_filt_ctxt_t*	ctxt,			/*!<in/out: read filter
+							context */
+	ib_int64_t*		read_batch_start,	/*!<out: starting read
+							offset in bytes for the
+							next batch of pages */
+	ib_int64_t*		read_batch_len)		/*!<out: length in
+							bytes of the next batch
+							of pages */
+{
+	ulint	start_page_id;
+
+	start_page_id = ctxt->offset / ctxt->page_size;
+
+	xb_a (ctxt->offset % ctxt->page_size == 0);
+
+	if (start_page_id == ctxt->filter_batch_end) {
+
+		/* Used up all the previous bitmap range, get some more */
+		ulint next_page_id;
+
+		/* Find the next changed page using the bitmap */
+		next_page_id = xb_page_bitmap_range_get_next_bit
+			(ctxt->bitmap_range, TRUE);
+
+		if (next_page_id == ULINT_UNDEFINED) {
+			*read_batch_len = 0;
+			return;
+		}
+
+		ctxt->offset = next_page_id * ctxt->page_size;
+
+		/* Find the end of the current changed page block by searching
+		for the next cleared bitmap bit */
+		ctxt->filter_batch_end
+			= xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range,
+							    FALSE);
+		xb_a(next_page_id < ctxt->filter_batch_end);
+	}
+
+	*read_batch_start = ctxt->offset;
+	if (ctxt->filter_batch_end == ULINT_UNDEFINED) {
+		/* No more cleared bits in the bitmap, need to copy all the
+		remaining pages.  */
+		*read_batch_len = ctxt->data_file_size - ctxt->offset;
+	} else {
+		*read_batch_len = ctxt->filter_batch_end * ctxt->page_size
+			- ctxt->offset;
+	}
+
+	/* If the page block is larger than the buffer capacity, limit it to
+	buffer capacity.  The subsequent invocations will continue returning
+	the current block in buffer-sized pieces until ctxt->filter_batch_end
+	is reached, trigerring the next bitmap query.  */
+	if (*read_batch_len > ctxt->buffer_capacity) {
+		*read_batch_len = ctxt->buffer_capacity;
+	}
+
+	ctxt->offset += *read_batch_len;
+	xb_a (ctxt->offset % ctxt->page_size == 0);
+	xb_a (*read_batch_start % ctxt->page_size == 0);
+	xb_a (*read_batch_len % ctxt->page_size == 0);
+}
+
+/****************************************************************//**
+Deinitialize the changed page bitmap-based read filter.  */
+static
+void
+rf_bitmap_deinit(
+/*=============*/
+	xb_read_filt_ctxt_t*	ctxt)	/*!<in/out: read filter context */
+{
+	xb_page_bitmap_range_deinit(ctxt->bitmap_range);
+}
+
+/* The pass-through read filter */
+xb_read_filt_t rf_pass_through = {
+	&rf_pass_through_init,
+	&rf_pass_through_get_next_batch,
+	&rf_pass_through_deinit
+};
+
+/* The changed page bitmap-based read filter */
+xb_read_filt_t rf_bitmap = {
+	&rf_bitmap_init,
+	&rf_bitmap_get_next_batch,
+	&rf_bitmap_deinit
+};

extra/mariabackup/read_filt.h

@@ -0,0 +1,62 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Data file read filter interface */
+
+#ifndef XB_READ_FILT_H
+#define XB_READ_FILT_H
+
+#include "changed_page_bitmap.h"
+
+struct xb_fil_cur_t;
+
+/* The read filter context */
+struct xb_read_filt_ctxt_t {
+	ib_int64_t		offset;		/*!< current file offset */
+	ib_int64_t		data_file_size;	/*!< data file size */
+	ib_int64_t		buffer_capacity;/*!< read buffer capacity */
+	ulint			space_id;	/*!< space id */
+	/* The following fields used only in bitmap filter */
+	/* Move these to union if any other filters are added in future */
+	xb_page_bitmap_range	*bitmap_range;	/*!< changed page bitmap range
+						iterator for space_id */
+	ulint			page_size;	/*!< page size */
+	ulint			filter_batch_end;/*!< the ending page id of the
+						 current changed page block in
+						 the bitmap */
+};
+
+/* The read filter */
+struct xb_read_filt_t {
+	void (*init)(xb_read_filt_ctxt_t* ctxt,
+		     const xb_fil_cur_t* cursor,
+		     ulint space_id);
+	void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt,
+			       ib_int64_t* read_batch_start,
+			       ib_int64_t* read_batch_len);
+	void (*deinit)(xb_read_filt_ctxt_t* ctxt);
+};
+
+extern xb_read_filt_t rf_pass_through;
+extern xb_read_filt_t rf_bitmap;
+
+#endif

extra/mariabackup/write_filt.cc

@@ -0,0 +1,219 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Page write filters implementation */
+
+#include <my_base.h>
+#include "common.h"
+#include "write_filt.h"
+#include "fil_cur.h"
+#include "xtrabackup.h"
+
+/************************************************************************
+Write-through page write filter. */
+static my_bool wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+			  xb_fil_cur_t *cursor);
+static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile);
+
+xb_write_filt_t wf_write_through = {
+	&wf_wt_init,
+	&wf_wt_process,
+	NULL,
+	NULL
+};
+
+/************************************************************************
+Incremental page write filter. */
+static my_bool wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+				   xb_fil_cur_t *cursor);
+static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt,
+				      ds_file_t *dstfile);
+static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt,
+				       ds_file_t *dstfile);
+static void wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt);
+
+xb_write_filt_t wf_incremental = {
+	&wf_incremental_init,
+	&wf_incremental_process,
+	&wf_incremental_finalize,
+	&wf_incremental_deinit
+};
+
+/************************************************************************
+Initialize incremental page write filter.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+		    xb_fil_cur_t *cursor)
+{
+	char				meta_name[FN_REFLEN];
+	xb_delta_info_t			info;
+	ulint				buf_size;
+	xb_wf_incremental_ctxt_t	*cp =
+		&(ctxt->u.wf_incremental_ctxt);
+
+	ctxt->cursor = cursor;
+
+	/* allocate buffer for incremental backup (4096 pages) */
+	buf_size = (UNIV_PAGE_SIZE_MAX / 4 + 1) * UNIV_PAGE_SIZE_MAX;
+	cp->delta_buf_base = static_cast<byte *>(ut_malloc(buf_size));
+	memset(cp->delta_buf_base, 0, buf_size);
+	cp->delta_buf = static_cast<byte *>
+		(ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX));
+
+	/* write delta meta info */
+	snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name,
+		 XB_DELTA_INFO_SUFFIX);
+	info.page_size = cursor->page_size;
+	info.zip_size = cursor->zip_size;
+	info.space_id = cursor->space_id;
+	if (!xb_write_delta_metadata(meta_name, &info)) {
+		msg("[%02u] xtrabackup: Error: "
+		    "failed to write meta info for %s\n",
+		    cursor->thread_n, cursor->rel_path);
+		return(FALSE);
+	}
+
+	/* change the target file name, since we are only going to write
+	delta pages */
+	strcat(dst_name, ".delta");
+
+	mach_write_to_4(cp->delta_buf, 0x78747261UL); /*"xtra"*/
+	cp->npages = 1;
+
+	return(TRUE);
+}
+
+/************************************************************************
+Run the next batch of pages through incremental page write filter.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
+{
+	ulint				i;
+	xb_fil_cur_t			*cursor = ctxt->cursor;
+	ulint				page_size = cursor->page_size;
+	byte				*page;
+	xb_wf_incremental_ctxt_t	*cp = &(ctxt->u.wf_incremental_ctxt);
+
+	for (i = 0, page = cursor->buf; i < cursor->buf_npages;
+	     i++, page += page_size) {
+
+		if (incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN)) {
+
+			continue;
+		}
+
+		/* updated page */
+		if (cp->npages == page_size / 4) {
+			/* flush buffer */
+			if (ds_write(dstfile, cp->delta_buf,
+				     cp->npages * page_size)) {
+				return(FALSE);
+			}
+
+			/* clear buffer */
+			memset(cp->delta_buf, 0, page_size / 4 * page_size);
+			/*"xtra"*/
+			mach_write_to_4(cp->delta_buf, 0x78747261UL);
+			cp->npages = 1;
+		}
+
+		mach_write_to_4(cp->delta_buf + cp->npages * 4,
+				cursor->buf_page_no + i);
+		memcpy(cp->delta_buf + cp->npages * page_size, page,
+		       page_size);
+
+		cp->npages++;
+	}
+
+	return(TRUE);
+}
+
+/************************************************************************
+Flush the incremental page write filter's buffer.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
+{
+	xb_fil_cur_t			*cursor = ctxt->cursor;
+	ulint				page_size = cursor->page_size;
+	xb_wf_incremental_ctxt_t	*cp = &(ctxt->u.wf_incremental_ctxt);
+
+	if (cp->npages != page_size / 4) {
+		mach_write_to_4(cp->delta_buf + cp->npages * 4, 0xFFFFFFFFUL);
+	}
+
+	/* Mark the final block */
+	mach_write_to_4(cp->delta_buf, 0x58545241UL); /*"XTRA"*/
+
+	/* flush buffer */
+	if (ds_write(dstfile, cp->delta_buf, cp->npages * page_size)) {
+		return(FALSE);
+	}
+
+	return(TRUE);
+}
+
+/************************************************************************
+Free the incremental page write filter's buffer. */
+static void
+wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt)
+{
+	xb_wf_incremental_ctxt_t	*cp = &(ctxt->u.wf_incremental_ctxt);
+
+	if (cp->delta_buf_base != NULL) {
+		ut_free(cp->delta_buf_base);
+	}
+}
+
+/************************************************************************
+Initialize the write-through page write filter.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)),
+	   xb_fil_cur_t *cursor)
+{
+	ctxt->cursor = cursor;
+
+	return(TRUE);
+}
+
+/************************************************************************
+Write the next batch of pages to the destination datasink.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
+{
+	xb_fil_cur_t			*cursor = ctxt->cursor;
+
+	if (ds_write(dstfile, cursor->buf, cursor->buf_read)) {
+		return(FALSE);
+	}
+
+	return(TRUE);
+}

extra/mariabackup/write_filt.h

@@ -0,0 +1,61 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* Page write filter interface */
+
+#ifndef XB_WRITE_FILT_H
+#define XB_WRITE_FILT_H
+
+#include "fil_cur.h"
+#include "datasink.h"
+#include "compact.h"
+
+/* Incremental page filter context */
+typedef struct {
+	byte		*delta_buf_base;
+	byte		*delta_buf;
+	ulint		 npages;
+} xb_wf_incremental_ctxt_t;
+
+/* Page filter context used as an opaque structure by callers */
+typedef struct {
+	xb_fil_cur_t	*cursor;
+	union {
+		xb_wf_incremental_ctxt_t	wf_incremental_ctxt;
+		xb_wf_compact_ctxt_t		wf_compact_ctxt;
+	} u;
+} xb_write_filt_ctxt_t;
+
+
+typedef struct {
+	my_bool	(*init)(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+			xb_fil_cur_t *cursor);
+	my_bool	(*process)(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile);
+	my_bool	(*finalize)(xb_write_filt_ctxt_t *, ds_file_t *dstfile);
+	void (*deinit)(xb_write_filt_ctxt_t *);
+} xb_write_filt_t;
+
+extern xb_write_filt_t wf_write_through;
+extern xb_write_filt_t wf_incremental;
+extern xb_write_filt_t wf_compact;
+
+#endif /* XB_WRITE_FILT_H */

extra/mariabackup/wsrep.cc

@@ -0,0 +1,219 @@
+/******************************************************
+Percona XtraBackup: hot backup tool for InnoDB
+(c) 2009-2014 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************
+
+This file incorporates work covered by the following copyright and
+permission notice:
+
+   Copyright 2010 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include <handler.h>
+#include <trx0sys.h>
+
+#include "common.h"
+
+#define WSREP_XID_PREFIX "WSREPXid"
+#define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN
+#define WSREP_XID_UUID_OFFSET 8
+#define WSREP_XID_SEQNO_OFFSET (WSREP_XID_UUID_OFFSET + sizeof(wsrep_uuid_t))
+#define WSREP_XID_GTRID_LEN (WSREP_XID_SEQNO_OFFSET + sizeof(wsrep_seqno_t))
+
+/*! undefined seqno */
+#define WSREP_SEQNO_UNDEFINED (-1)
+
+/*! Name of file where Galera info is stored on recovery */
+#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info"
+
+/* Galera UUID type - for all unique IDs */
+typedef struct wsrep_uuid {
+    uint8_t data[16];
+} wsrep_uuid_t;
+
+/* sequence number of a writeset, etc. */
+typedef int64_t  wsrep_seqno_t;
+
+/* Undefined UUID */
+static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}};
+
+/***********************************************************************//**
+Check if a given WSREP XID is valid.
+
+@return true if valid.
+*/
+static
+bool
+wsrep_is_wsrep_xid(
+/*===============*/
+	const void*	xid_ptr)
+{
+	const XID*	xid = reinterpret_cast<const XID*>(xid_ptr);
+
+	return((xid->formatID      == 1                   &&
+		xid->gtrid_length  == WSREP_XID_GTRID_LEN &&
+		xid->bqual_length  == 0                   &&
+		!memcmp(xid->data, WSREP_XID_PREFIX, WSREP_XID_PREFIX_LEN)));
+}
+
+/***********************************************************************//**
+Retrieve binary WSREP UUID from XID.
+
+@return binary WSREP UUID represenataion, if UUID is valid, or
+	WSREP_UUID_UNDEFINED otherwise.
+*/
+static
+const wsrep_uuid_t*
+wsrep_xid_uuid(
+/*===========*/
+	const XID*	xid)
+{
+	if (wsrep_is_wsrep_xid(xid)) {
+		return(reinterpret_cast<const wsrep_uuid_t*>
+		       (xid->data + WSREP_XID_UUID_OFFSET));
+	} else {
+		return(&WSREP_UUID_UNDEFINED);
+	}
+}
+
+/***********************************************************************//**
+Retrieve WSREP seqno from XID.
+
+@return WSREP seqno, if it is valid, or WSREP_SEQNO_UNDEFINED otherwise.
+*/
+wsrep_seqno_t wsrep_xid_seqno(
+/*==========================*/
+	const XID*	xid)
+{
+	if (wsrep_is_wsrep_xid(xid)) {
+		wsrep_seqno_t seqno;
+		memcpy(&seqno, xid->data + WSREP_XID_SEQNO_OFFSET,
+		       sizeof(wsrep_seqno_t));
+
+		return(seqno);
+	} else {
+		return(WSREP_SEQNO_UNDEFINED);
+	}
+}
+
+/***********************************************************************//**
+Write UUID to string.
+
+@return length of UUID string representation or -EMSGSIZE if string is too
+short.
+*/
+static
+int
+wsrep_uuid_print(
+/*=============*/
+	const wsrep_uuid_t*	uuid,
+	char*			str,
+	size_t			str_len)
+{
+	if (str_len > 36) {
+		const unsigned char* u = uuid->data;
+		return snprintf(str, str_len,
+				"%02x%02x%02x%02x-%02x%02x-%02x%02x-"
+				"%02x%02x-%02x%02x%02x%02x%02x%02x",
+				u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6],
+				u[ 7], u[ 8], u[ 9], u[10], u[11], u[12], u[13],
+				u[14], u[15]);
+	}
+	else {
+		return -EMSGSIZE;
+	}
+}
+
+/***********************************************************************
+Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that
+information is present in the trx system header. Otherwise, do nothing. */
+void
+xb_write_galera_info(bool incremental_prepare)
+/*==================*/
+{
+	FILE*		fp;
+	XID		xid;
+	char		uuid_str[40];
+	wsrep_seqno_t	seqno;
+	MY_STAT		statinfo;
+
+	/* Do not overwrite existing an existing file to be compatible with
+	servers with older server versions */
+	if (!incremental_prepare &&
+		my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) {
+
+		return;
+	}
+
+	memset(&xid, 0, sizeof(xid));
+	xid.formatID = -1;
+
+	if (!trx_sys_read_wsrep_checkpoint(&xid)) {
+
+		return;
+	}
+
+	if (wsrep_uuid_print(wsrep_xid_uuid(&xid), uuid_str,
+			     sizeof(uuid_str)) < 0) {
+		return;
+	}
+
+	fp = fopen(XB_GALERA_INFO_FILENAME, "w");
+	if (fp == NULL) {
+
+		msg("xtrabackup: error: "
+		    "could not create " XB_GALERA_INFO_FILENAME
+		    ", errno = %d\n",
+		    errno);
+		exit(EXIT_FAILURE);
+	}
+
+	seqno = wsrep_xid_seqno(&xid);
+
+	msg("xtrabackup: Recovered WSREP position: %s:%lld\n",
+	    uuid_str, (long long) seqno);
+
+	if (fprintf(fp, "%s:%lld", uuid_str, (long long) seqno) < 0) {
+
+		msg("xtrabackup: error: "
+		    "could not write to " XB_GALERA_INFO_FILENAME
+		    ", errno = %d\n",
+		    errno);
+		exit(EXIT_FAILURE);
+	}
+
+	fclose(fp);
+}

extra/mariabackup/wsrep.h

@@ -0,0 +1,32 @@
+/******************************************************
+Percona XtraBackup: hot backup tool for InnoDB
+(c) 2009-2014 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*******************************************************/
+
+#ifndef WSREP_H
+#define WSREP_H
+
+/***********************************************************************
+Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that
+information is present in the trx system header. Otherwise, do nothing. */
+void
+xb_write_galera_info(bool incremental_prepare);
+/*==================*/
+
+#endif

extra/mariabackup/xb_regex.h

@@ -0,0 +1,71 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+/* This file is required to abstract away regex(3) calls so that
+my_regex is used on Windows and native calls are used on POSIX platforms. */
+
+#ifndef XB_REGEX_H
+#define XB_REGEX_H
+
+#ifdef _WIN32
+
+#include <my_regex.h>
+
+typedef my_regex_t xb_regex_t;
+typedef my_regmatch_t xb_regmatch_t;
+
+#define xb_regex_init() my_regex_init(&my_charset_latin1)
+
+#define xb_regexec(preg,string,nmatch,pmatch,eflags) \
+	my_regexec(preg, string, nmatch, pmatch, eflags)
+
+#define xb_regerror(errcode,preg,errbuf,errbuf_size) \
+	my_regerror(errcode, preg, errbuf, errbuf_size)
+
+#define xb_regcomp(preg,regex,cflags) \
+	my_regcomp(preg, regex, cflags, &my_charset_latin1)
+
+#define xb_regfree(preg) my_regfree(preg)
+
+#define xb_regex_end() my_regex_end()
+
+#else /* ! _WIN32 */
+
+#include <regex.h>
+
+typedef regex_t xb_regex_t;
+typedef regmatch_t xb_regmatch_t;
+
+#define xb_regex_init() do { } while(0)
+
+#define xb_regexec(preg,string,nmatch,pmatch,eflags)	\
+	regexec(preg, string, nmatch, pmatch, eflags)
+
+#define xb_regerror(errcode,preg,errbuf,errbuf_size)	\
+	regerror(errcode, preg, errbuf, errbuf_size)
+
+#define xb_regcomp(preg,regex,cflags)				\
+	regcomp(preg, regex, cflags)
+
+#define xb_regfree(preg) regfree(preg)
+
+#define xb_regex_end() do { } while (0)
+
+#endif /* _WIN32 */
+
+#endif /* XB_REGEX_H */

extra/mariabackup/xbcrypt.c

@@ -0,0 +1,694 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+The xbcrypt utility: decrypt files in the XBCRYPT format.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#include <my_getopt.h>
+#include "common.h"
+#include "xbcrypt.h"
+#include <gcrypt.h>
+
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+GCRY_THREAD_OPTION_PTHREAD_IMPL;
+#endif
+
+#define XBCRYPT_VERSION "1.1"
+
+typedef enum {
+	RUN_MODE_NONE,
+	RUN_MODE_ENCRYPT,
+	RUN_MODE_DECRYPT
+} run_mode_t;
+
+const char *xbcrypt_encrypt_algo_names[] =
+{ "NONE", "AES128", "AES192", "AES256", NullS};
+TYPELIB xbcrypt_encrypt_algo_typelib=
+{array_elements(xbcrypt_encrypt_algo_names)-1,"",
+	xbcrypt_encrypt_algo_names, NULL};
+
+static run_mode_t	opt_run_mode = RUN_MODE_ENCRYPT;
+static char 		*opt_input_file = NULL;
+static char 		*opt_output_file = NULL;
+static ulong 		opt_encrypt_algo;
+static char 		*opt_encrypt_key_file = NULL;
+static void 		*opt_encrypt_key = NULL;
+static ulonglong	opt_encrypt_chunk_size = 0;
+static my_bool		opt_verbose = FALSE;
+
+static uint 		encrypt_algos[] = { GCRY_CIPHER_NONE,
+					    GCRY_CIPHER_AES128,
+					    GCRY_CIPHER_AES192,
+					    GCRY_CIPHER_AES256 };
+static int		encrypt_algo = 0;
+static int		encrypt_mode = GCRY_CIPHER_MODE_CTR;
+static uint 		encrypt_key_len = 0;
+static size_t 		encrypt_iv_len = 0;
+
+static struct my_option my_long_options[] =
+{
+	{"help", '?', "Display this help and exit.",
+	 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"decrypt", 'd', "Decrypt data input to output.",
+	 0, 0, 0,
+	 GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"input", 'i', "Optional input file. If not specified, input"
+	 " will be read from standard input.",
+	 &opt_input_file, &opt_input_file, 0,
+	 GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"output", 'o', "Optional output file. If not specified, output"
+	 " will be written to standard output.",
+	 &opt_output_file, &opt_output_file, 0,
+	 GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"encrypt-algo", 'a', "Encryption algorithm.",
+	 &opt_encrypt_algo, &opt_encrypt_algo, &xbcrypt_encrypt_algo_typelib,
+	 GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"encrypt-key", 'k', "Encryption key.",
+	 &opt_encrypt_key, &opt_encrypt_key, 0,
+	 GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"encrypt-key-file", 'f', "File which contains encryption key.",
+	 &opt_encrypt_key_file, &opt_encrypt_key_file, 0,
+	 GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+	{"encrypt-chunk-size", 's', "Size of working buffer for encryption in"
+	 " bytes. The default value is 64K.",
+	 &opt_encrypt_chunk_size, &opt_encrypt_chunk_size, 0,
+	GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
+
+	{"verbose", 'v', "Display verbose status output.",
+	 &opt_verbose, &opt_verbose,
+	  0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+	{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
+};
+
+static
+int
+get_options(int *argc, char ***argv);
+
+static
+my_bool
+get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
+	       char *argument __attribute__((unused)));
+
+static
+void
+print_version(void);
+
+static
+void
+usage(void);
+
+static
+int
+mode_decrypt(File filein, File fileout);
+
+static
+int
+mode_encrypt(File filein, File fileout);
+
+int
+main(int argc, char **argv)
+{
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+	gcry_error_t 		gcry_error;
+#endif
+	File filein = 0;
+	File fileout = 0;
+
+	MY_INIT(argv[0]);
+
+	if (get_options(&argc, &argv)) {
+		goto err;
+	}
+
+	/* Acording to gcrypt docs (and my testing), setting up the threading
+	   callbacks must be done first, so, lets give it a shot */
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+	gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
+	if (gcry_error) {
+		msg("%s: unable to set libgcrypt thread cbs - "
+		    "%s : %s\n", my_progname,
+		    gcry_strsource(gcry_error),
+		    gcry_strerror(gcry_error));
+		return 1;
+	}
+#endif
+
+	/* Version check should be the very first call because it
+	makes sure that important subsystems are intialized. */
+	if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
+		const char	*gcrypt_version;
+		gcrypt_version = gcry_check_version(NULL);
+		/* No other library has already initialized libgcrypt. */
+		if (!gcrypt_version) {
+			msg("%s: failed to initialize libgcrypt\n",
+			    my_progname);
+			return 1;
+		} else if (opt_verbose) {
+			msg("%s: using gcrypt %s\n", my_progname,
+			    gcrypt_version);
+		}
+	}
+	gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
+	gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
+
+	/* Determine the algorithm */
+	encrypt_algo = encrypt_algos[opt_encrypt_algo];
+
+	/* Set up the iv length */
+	encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
+
+	/* Now set up the key */
+	if (opt_encrypt_key == NULL && opt_encrypt_key_file == NULL) {
+		msg("%s: no encryption key or key file specified.\n",
+		    my_progname);
+		return 1;
+	} else if (opt_encrypt_key && opt_encrypt_key_file) {
+		msg("%s: both encryption key and key file specified.\n",
+		    my_progname);
+		return 1;
+	} else if (opt_encrypt_key_file) {
+		if (!xb_crypt_read_key_file(opt_encrypt_key_file,
+					    &opt_encrypt_key,
+					    &encrypt_key_len)) {
+			msg("%s: unable to read encryption key file \"%s\".\n",
+			    opt_encrypt_key_file, my_progname);
+			return 1;
+		}
+	} else {
+		encrypt_key_len = strlen(opt_encrypt_key);
+	}
+
+	if (opt_input_file) {
+		MY_STAT 	mystat;
+
+		if (opt_verbose)
+			msg("%s: input file \"%s\".\n", my_progname,
+			    opt_input_file);
+
+		if (my_stat(opt_input_file, &mystat, MYF(MY_WME)) == NULL) {
+			goto err;
+		}
+		if (!MY_S_ISREG(mystat.st_mode)) {
+			msg("%s: \"%s\" is not a regular file, exiting.\n",
+			    my_progname, opt_input_file);
+			goto err;
+		}
+		if ((filein = my_open(opt_input_file, O_RDONLY, MYF(MY_WME)))
+		     < 0) {
+			msg("%s: failed to open \"%s\".\n", my_progname,
+			     opt_input_file);
+			goto err;
+		}
+	} else {
+		if (opt_verbose)
+			msg("%s: input from standard input.\n", my_progname);
+		filein = fileno(stdin);
+	}
+
+	if (opt_output_file) {
+		if (opt_verbose)
+			msg("%s: output file \"%s\".\n", my_progname,
+			    opt_output_file);
+
+		if ((fileout = my_create(opt_output_file, 0,
+					 O_WRONLY|O_BINARY|O_EXCL|O_NOFOLLOW,
+					 MYF(MY_WME))) < 0) {
+			msg("%s: failed to create output file \"%s\".\n",
+			    my_progname, opt_output_file);
+			goto err;
+		}
+	} else {
+		if (opt_verbose)
+			msg("%s: output to standard output.\n", my_progname);
+		fileout = fileno(stdout);
+	}
+
+	if (opt_run_mode == RUN_MODE_DECRYPT
+	    && mode_decrypt(filein, fileout)) {
+		goto err;
+	} else if (opt_run_mode == RUN_MODE_ENCRYPT
+		   && mode_encrypt(filein, fileout)) {
+		goto err;
+	}
+
+	if (opt_input_file && filein) {
+		my_close(filein, MYF(MY_WME));
+	}
+	if (opt_output_file && fileout) {
+		my_close(fileout, MYF(MY_WME));
+	}
+
+	my_cleanup_options(my_long_options);
+
+	my_end(0);
+
+	return EXIT_SUCCESS;
+err:
+	if (opt_input_file && filein) {
+		my_close(filein, MYF(MY_WME));
+	}
+	if (opt_output_file && fileout) {
+		my_close(fileout, MYF(MY_WME));
+	}
+
+	my_cleanup_options(my_long_options);
+
+	my_end(0);
+
+	exit(EXIT_FAILURE);
+
+}
+
+
+static
+size_t
+my_xb_crypt_read_callback(void *userdata, void *buf, size_t len)
+{
+	File* file = (File *) userdata;
+	return xb_read_full(*file, buf, len);
+}
+
+static
+int
+mode_decrypt(File filein, File fileout)
+{
+	xb_rcrypt_t		*xbcrypt_file = NULL;
+	void			*chunkbuf = NULL;
+	size_t			chunksize;
+	size_t			originalsize;
+	void			*ivbuf = NULL;
+	size_t			ivsize;
+	void			*decryptbuf = NULL;
+	size_t			decryptbufsize = 0;
+	ulonglong		ttlchunksread = 0;
+	ulonglong		ttlbytesread = 0;
+	xb_rcrypt_result_t	result;
+	gcry_cipher_hd_t	cipher_handle;
+	gcry_error_t		gcry_error;
+	my_bool			hash_appended;
+
+	if (encrypt_algo != GCRY_CIPHER_NONE) {
+		gcry_error = gcry_cipher_open(&cipher_handle,
+					      encrypt_algo,
+					      encrypt_mode, 0);
+		if (gcry_error) {
+			msg("%s:decrypt: unable to open libgcrypt"
+			    " cipher - %s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return 1;
+		}
+
+		gcry_error = gcry_cipher_setkey(cipher_handle,
+						opt_encrypt_key,
+						encrypt_key_len);
+		if (gcry_error) {
+			msg("%s:decrypt: unable to set libgcrypt cipher"
+			    "key - %s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			goto err;
+		}
+	}
+
+	/* Initialize the xb_crypt format reader */
+	xbcrypt_file = xb_crypt_read_open(&filein, my_xb_crypt_read_callback);
+	if (xbcrypt_file == NULL) {
+		msg("%s:decrypt: xb_crypt_read_open() failed.\n", my_progname);
+		goto err;
+	}
+
+	/* Walk the encrypted chunks, decrypting them and writing out */
+	while ((result = xb_crypt_read_chunk(xbcrypt_file, &chunkbuf,
+					     &originalsize, &chunksize,
+					     &ivbuf, &ivsize, &hash_appended))
+		== XB_CRYPT_READ_CHUNK) {
+
+		if (encrypt_algo != GCRY_CIPHER_NONE) {
+			gcry_error = gcry_cipher_reset(cipher_handle);
+			if (gcry_error) {
+				msg("%s:decrypt: unable to reset libgcrypt"
+				    " cipher - %s : %s\n", my_progname,
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				goto err;
+			}
+
+			if (ivsize) {
+				gcry_error = gcry_cipher_setctr(cipher_handle,
+								ivbuf,
+								ivsize);
+			}
+			if (gcry_error) {
+				msg("%s:decrypt: unable to set cipher iv - "
+				    "%s : %s\n", my_progname,
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				continue;
+			}
+
+			if (decryptbufsize < originalsize) {
+				decryptbuf = my_realloc(decryptbuf,
+					originalsize,
+					MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+				decryptbufsize = originalsize;
+			}
+
+			/* Try to decrypt it */
+			gcry_error = gcry_cipher_decrypt(cipher_handle,
+							 decryptbuf,
+							 originalsize,
+							 chunkbuf,
+							 chunksize);
+			if (gcry_error) {
+				msg("%s:decrypt: unable to decrypt chunk - "
+				    "%s : %s\n", my_progname,
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				gcry_cipher_close(cipher_handle);
+					goto err;
+			}
+
+		} else {
+			decryptbuf = chunkbuf;
+		}
+
+		if (hash_appended) {
+			uchar hash[XB_CRYPT_HASH_LEN];
+
+			originalsize -= XB_CRYPT_HASH_LEN;
+
+			/* ensure that XB_CRYPT_HASH_LEN is the correct length
+			of XB_CRYPT_HASH hashing algorithm output */
+			assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+			       XB_CRYPT_HASH_LEN);
+			gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf,
+					    originalsize);
+			if (memcmp(hash, (char *) decryptbuf + originalsize,
+				   XB_CRYPT_HASH_LEN) != 0) {
+				msg("%s:%s invalid plaintext hash. "
+				    "Wrong encrytion key specified?\n",
+				    my_progname, __FUNCTION__);
+				result = XB_CRYPT_READ_ERROR;
+				goto err;
+			}
+		}
+
+		/* Write it out */
+		if (my_write(fileout, (const uchar *) decryptbuf, originalsize,
+			     MYF(MY_WME | MY_NABP))) {
+			msg("%s:decrypt: unable to write output chunk.\n",
+			    my_progname);
+			goto err;
+		}
+		ttlchunksread++;
+		ttlbytesread += chunksize;
+		if (opt_verbose)
+			msg("%s:decrypt: %llu chunks read, %llu bytes read\n.",
+		    	    my_progname, ttlchunksread, ttlbytesread);
+	}
+
+	xb_crypt_read_close(xbcrypt_file);
+
+	if (encrypt_algo != GCRY_CIPHER_NONE)
+		gcry_cipher_close(cipher_handle);
+
+	if (decryptbuf && decryptbufsize)
+		my_free(decryptbuf);
+
+	if (opt_verbose)
+		msg("\n%s:decrypt: done\n", my_progname);
+
+	return 0;
+err:
+	if (xbcrypt_file)
+		xb_crypt_read_close(xbcrypt_file);
+
+	if (encrypt_algo != GCRY_CIPHER_NONE)
+		gcry_cipher_close(cipher_handle);
+
+	if (decryptbuf && decryptbufsize)
+		my_free(decryptbuf);
+
+	return 1;
+}
+
+static
+ssize_t
+my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len)
+{
+	File* file = (File *) userdata;
+
+	ssize_t ret = my_write(*file, buf, len, MYF(MY_WME));
+	posix_fadvise(*file, 0, 0, POSIX_FADV_DONTNEED);
+	return ret;
+}
+
+static
+int
+mode_encrypt(File filein, File fileout)
+{
+	size_t			bytesread;
+	size_t			chunkbuflen;
+	uchar			*chunkbuf = NULL;
+	void			*ivbuf = NULL;
+	size_t			encryptbuflen = 0;
+	size_t			encryptedlen = 0;
+	void			*encryptbuf = NULL;
+	ulonglong		ttlchunkswritten = 0;
+	ulonglong		ttlbyteswritten = 0;
+	xb_wcrypt_t		*xbcrypt_file = NULL;
+	gcry_cipher_hd_t	cipher_handle;
+	gcry_error_t		gcry_error;
+
+	if (encrypt_algo != GCRY_CIPHER_NONE) {
+		gcry_error = gcry_cipher_open(&cipher_handle,
+					      encrypt_algo,
+					      encrypt_mode, 0);
+		if (gcry_error) {
+			msg("%s:encrypt: unable to open libgcrypt cipher - "
+			    "%s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			return 1;
+		}
+
+		gcry_error = gcry_cipher_setkey(cipher_handle,
+						opt_encrypt_key,
+						encrypt_key_len);
+		if (gcry_error) {
+			msg("%s:encrypt: unable to set libgcrypt cipher key - "
+			    "%s : %s\n", my_progname,
+			    gcry_strsource(gcry_error),
+			    gcry_strerror(gcry_error));
+			goto err;
+		}
+	}
+
+	posix_fadvise(filein, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+	xbcrypt_file = xb_crypt_write_open(&fileout,
+					   my_xb_crypt_write_callback);
+	if (xbcrypt_file == NULL) {
+		msg("%s:encrypt: xb_crypt_write_open() failed.\n",
+		    my_progname);
+		goto err;
+	}
+
+	ivbuf = my_malloc(encrypt_iv_len, MYF(MY_FAE));
+
+	/* now read in data in chunk size, encrypt and write out */
+	chunkbuflen = opt_encrypt_chunk_size + XB_CRYPT_HASH_LEN;
+	chunkbuf = (uchar *) my_malloc(chunkbuflen, MYF(MY_FAE));
+	while ((bytesread = my_read(filein, chunkbuf, opt_encrypt_chunk_size,
+				    MYF(MY_WME))) > 0) {
+
+		size_t origbuflen = bytesread + XB_CRYPT_HASH_LEN;
+
+		/* ensure that XB_CRYPT_HASH_LEN is the correct length
+		of XB_CRYPT_HASH hashing algorithm output */
+		assert(XB_CRYPT_HASH_LEN ==
+		       gcry_md_get_algo_dlen(XB_CRYPT_HASH));
+		gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread,
+				    chunkbuf, bytesread);
+
+		if (encrypt_algo != GCRY_CIPHER_NONE) {
+			gcry_error = gcry_cipher_reset(cipher_handle);
+
+			if (gcry_error) {
+				msg("%s:encrypt: unable to reset cipher - "
+				    "%s : %s\n", my_progname,
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				goto err;
+			}
+
+			xb_crypt_create_iv(ivbuf, encrypt_iv_len);
+			gcry_error = gcry_cipher_setctr(cipher_handle,
+							ivbuf,
+							encrypt_iv_len);
+
+			if (gcry_error) {
+				msg("%s:encrypt: unable to set cipher iv - "
+				    "%s : %s\n", my_progname,
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				continue;
+			}
+
+			if (encryptbuflen < origbuflen) {
+				encryptbuf = my_realloc(encryptbuf, origbuflen,
+					MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+				encryptbuflen = origbuflen;
+			}
+
+			gcry_error = gcry_cipher_encrypt(cipher_handle,
+							 encryptbuf,
+							 encryptbuflen,
+							 chunkbuf,
+							 origbuflen);
+
+			encryptedlen = origbuflen;
+
+			if (gcry_error) {
+				msg("%s:encrypt: unable to encrypt chunk - "
+				    "%s : %s\n", my_progname,
+				    gcry_strsource(gcry_error),
+				    gcry_strerror(gcry_error));
+				gcry_cipher_close(cipher_handle);
+				goto err;
+			}
+		} else {
+			encryptedlen = origbuflen;
+			encryptbuf = chunkbuf;
+		}
+
+		if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf,
+					 bytesread + XB_CRYPT_HASH_LEN,
+					 encryptedlen, ivbuf, encrypt_iv_len)) {
+			msg("%s:encrypt: abcrypt_write_chunk() failed.\n",
+			    my_progname);
+			goto err;
+		}
+
+		ttlchunkswritten++;
+		ttlbyteswritten += encryptedlen;
+
+		if (opt_verbose)
+			msg("%s:encrypt: %llu chunks written, %llu bytes "
+			    "written\n.", my_progname, ttlchunkswritten,
+			    ttlbyteswritten);
+	}
+
+	my_free(ivbuf);
+	my_free(chunkbuf);
+
+	if (encryptbuf && encryptbuflen)
+		my_free(encryptbuf);
+
+	xb_crypt_write_close(xbcrypt_file);
+
+	if (encrypt_algo != GCRY_CIPHER_NONE)
+		gcry_cipher_close(cipher_handle);
+
+	if (opt_verbose)
+		msg("\n%s:encrypt: done\n", my_progname);
+
+	return 0;
+err:
+	if (chunkbuf)
+		my_free(chunkbuf);
+
+	if (encryptbuf && encryptbuflen)
+		my_free(encryptbuf);
+
+	if (xbcrypt_file)
+		xb_crypt_write_close(xbcrypt_file);
+
+	if (encrypt_algo != GCRY_CIPHER_NONE)
+		gcry_cipher_close(cipher_handle);
+
+	return 1;
+}
+
+static
+int
+get_options(int *argc, char ***argv)
+{
+	int ho_error;
+
+	if ((ho_error= handle_options(argc, argv, my_long_options,
+				      get_one_option))) {
+		exit(EXIT_FAILURE);
+	}
+
+	return 0;
+}
+
+static
+my_bool
+get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
+	       char *argument __attribute__((unused)))
+{
+	switch (optid) {
+	case 'd':
+		opt_run_mode = RUN_MODE_DECRYPT;
+		break;
+	case '?':
+		usage();
+		exit(0);
+	}
+
+	return FALSE;
+}
+
+static
+void
+print_version(void)
+{
+	printf("%s  Ver %s for %s (%s)\n", my_progname, XBCRYPT_VERSION,
+	       SYSTEM_TYPE, MACHINE_TYPE);
+}
+
+static
+void
+usage(void)
+{
+	print_version();
+	puts("Copyright (C) 2011 Percona Inc.");
+	puts("This software comes with ABSOLUTELY NO WARRANTY. "
+	     "This is free software,\nand you are welcome to modify and "
+	     "redistribute it under the GPL license.\n");
+
+	puts("Encrypt or decrypt files in the XBCRYPT format.\n");
+
+	puts("Usage: ");
+	printf("  %s [OPTIONS...]"
+ 	       " # read data from specified input, encrypting or decrypting "
+	       " and writing the result to the specified output.\n",
+	       my_progname);
+	puts("\nOptions:");
+	my_print_help(my_long_options);
+}

extra/mariabackup/xbcrypt.h

@@ -0,0 +1,84 @@
+/******************************************************
+Copyright (c) 2011 Percona LLC and/or its affiliates.
+
+Encryption interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XBCRYPT_H
+#define XBCRYPT_H
+
+#include <my_base.h>
+#include "common.h"
+
+#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01"
+#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02"
+#define XB_CRYPT_CHUNK_MAGIC3 "XBCRYP03" /* must be same size as ^^ */
+#define XB_CRYPT_CHUNK_MAGIC_CURRENT XB_CRYPT_CHUNK_MAGIC3
+#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1)
+
+#define XB_CRYPT_HASH GCRY_MD_SHA256
+#define XB_CRYPT_HASH_LEN 32
+
+/******************************************************************************
+Write interface */
+typedef struct xb_wcrypt_struct xb_wcrypt_t;
+
+/* Callback on write for i/o, must return # of bytes written or -1 on error */
+typedef ssize_t xb_crypt_write_callback(void *userdata,
+					const void *buf, size_t len);
+
+xb_wcrypt_t *xb_crypt_write_open(void *userdata,
+				 xb_crypt_write_callback *onwrite);
+
+/* Takes buffer, original length, encrypted length iv and iv length, formats
+   output buffer and calls write callback.
+   Returns 0 on success, 1 on error */
+int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
+			 size_t elen, const void *iv, size_t ivlen);
+
+/* Returns 0 on success, 1 on error */
+int xb_crypt_write_close(xb_wcrypt_t *crypt);
+
+/******************************************************************************
+Read interface */
+typedef struct xb_rcrypt_struct xb_rcrypt_t;
+
+/* Callback on read for i/o, must return # of bytes read or -1 on error */
+typedef size_t xb_crypt_read_callback(void *userdata, void *buf, size_t len);
+
+xb_rcrypt_t *xb_crypt_read_open(void *userdata,
+				  xb_crypt_read_callback *onread);
+
+typedef enum {
+	XB_CRYPT_READ_CHUNK,
+	XB_CRYPT_READ_EOF,
+	XB_CRYPT_READ_ERROR
+} xb_rcrypt_result_t;
+
+xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
+				       size_t *olen, size_t *elen, void **iv,
+				       size_t *ivlen, my_bool *hash_appended);
+
+int xb_crypt_read_close(xb_rcrypt_t *crypt);
+
+/******************************************************************************
+Utility interface */
+my_bool xb_crypt_read_key_file(const char *filename,
+			       void** key, uint *keylength);
+
+void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
+#endif

extra/mariabackup/xbcrypt_common.c

@@ -0,0 +1,60 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Encryption configuration file interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#include "common.h"
+#include "xbcrypt.h"
+
+#if GCC_VERSION >= 4002
+/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
+#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+#include <gcrypt.h>
+
+#if GCC_VERSION >= 4002
+#  pragma GCC diagnostic warning "-Wdeprecated-declarations"
+#endif
+
+my_bool
+xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
+{
+	FILE *fp;
+
+	if (!(fp = my_fopen(filename, O_RDONLY, MYF(0)))) {
+		msg("%s:%s: unable to open config file \"%s\", errno(%d)\n",
+			my_progname, __FUNCTION__, filename, my_errno);
+		return FALSE;
+	}
+
+	fseek(fp, 0 , SEEK_END);
+	*keylength = ftell(fp);
+	rewind(fp);
+	*key = my_malloc(*keylength, MYF(MY_FAE));
+	*keylength = fread(*key, 1, *keylength, fp);
+	my_fclose(fp, MYF(0));
+	return TRUE;
+}
+
+void
+xb_crypt_create_iv(void* ivbuf, size_t ivlen)
+{
+	gcry_create_nonce(ivbuf, ivlen);
+}

extra/mariabackup/xbcrypt_read.c

@@ -0,0 +1,251 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+The xbcrypt format reader implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include "xbcrypt.h"
+
+struct xb_rcrypt_struct {
+	void				*userdata;
+	xb_crypt_read_callback		*read;
+	void				*buffer;
+	size_t				bufsize;
+	void				*ivbuffer;
+	size_t				ivbufsize;
+	ulonglong			offset;
+};
+
+xb_rcrypt_t *
+xb_crypt_read_open(void *userdata, xb_crypt_read_callback *onread)
+{
+	xb_rcrypt_t	*crypt;
+
+	xb_ad(onread);
+
+	crypt = (xb_rcrypt_t *) my_malloc(sizeof(xb_rcrypt_t), MYF(MY_FAE));
+
+	crypt->userdata = userdata;
+	crypt->read = onread;
+	crypt->buffer = NULL;
+	crypt->bufsize = 0;
+	crypt->offset = 0;
+	crypt->ivbuffer = NULL;
+	crypt->ivbufsize = 0;
+	return crypt;
+}
+
+xb_rcrypt_result_t
+xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
+		    void **iv, size_t *ivlen, my_bool *hash_appended)
+
+{
+	uchar		tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4];
+	uchar		*ptr;
+	ulonglong	tmp;
+	ulong		checksum, checksum_exp, version;
+	size_t		bytesread;
+	xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
+
+	if ((bytesread = crypt->read(crypt->userdata, tmpbuf, sizeof(tmpbuf)))
+	    != sizeof(tmpbuf)) {
+		if (bytesread == 0) {
+			result = XB_CRYPT_READ_EOF;
+			goto err;
+		} else {
+			msg("%s:%s: unable to read chunk header data at "
+			    "offset 0x%llx.\n",
+			    my_progname, __FUNCTION__, crypt->offset);
+			result =  XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+	}
+
+	ptr = tmpbuf;
+
+	if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
+		   XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+		version = 3;
+	} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
+			  XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+		version = 2;
+	} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
+			  XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+		version = 1;
+	} else {
+		msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
+		    my_progname, __FUNCTION__, crypt->offset);
+		result = XB_CRYPT_READ_ERROR;
+		goto err;
+	}
+
+	ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
+	crypt->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
+
+	tmp = uint8korr(ptr);	/* reserved */
+	ptr += 8;
+	crypt->offset += 8;
+
+	tmp = uint8korr(ptr);	/* original size */
+	ptr += 8;
+	if (tmp > INT_MAX) {
+		msg("%s:%s: invalid original size at offset 0x%llx.\n",
+		    my_progname, __FUNCTION__, crypt->offset);
+		result = XB_CRYPT_READ_ERROR;
+		goto err;
+	}
+	crypt->offset += 8;
+	*olen = (size_t)tmp;
+
+	tmp = uint8korr(ptr);	/* encrypted size */
+	ptr += 8;
+	if (tmp > INT_MAX) {
+		msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
+		    my_progname, __FUNCTION__, crypt->offset);
+		result = XB_CRYPT_READ_ERROR;
+		goto err;
+	}
+	crypt->offset += 8;
+	*elen = (size_t)tmp;
+
+	checksum_exp = uint4korr(ptr);	/* checksum */
+	ptr += 4;
+	crypt->offset += 4;
+
+	/* iv size */
+	if (version == 1) {
+		*ivlen = 0;
+		*iv = 0;
+	} else {
+		if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8))
+		    != 8) {
+			if (bytesread == 0) {
+				result = XB_CRYPT_READ_EOF;
+				goto err;
+			} else {
+				msg("%s:%s: unable to read chunk iv size at "
+				    "offset 0x%llx.\n",
+				    my_progname, __FUNCTION__, crypt->offset);
+				result =  XB_CRYPT_READ_ERROR;
+				goto err;
+			}
+		}
+
+		tmp = uint8korr(tmpbuf);
+		if (tmp > INT_MAX) {
+			msg("%s:%s: invalid iv size at offset 0x%llx.\n",
+			    my_progname, __FUNCTION__, crypt->offset);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+		crypt->offset += 8;
+		*ivlen = (size_t)tmp;
+	}
+
+	if (*ivlen > crypt->ivbufsize) {
+		crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen,
+					     MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+		if (crypt->ivbuffer == NULL) {
+			msg("%s:%s: failed to increase iv buffer to "
+			    "%llu bytes.\n", my_progname, __FUNCTION__,
+			    (ulonglong)*ivlen);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+		crypt->ivbufsize = *ivlen;
+	}
+
+	if (*ivlen > 0) {
+		if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen)
+		    != *ivlen) {
+			msg("%s:%s: failed to read %lld bytes for chunk iv "
+			    "at offset 0x%llx.\n", my_progname, __FUNCTION__,
+			    (ulonglong)*ivlen, crypt->offset);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+		*iv = crypt->ivbuffer;
+	}
+
+	/* for version euqals 2 we need to read in the iv data but do not init
+	CTR with it */
+	if (version == 2) {
+		*ivlen = 0;
+		*iv = 0;
+	}
+
+	if (*olen > crypt->bufsize) {
+		crypt->buffer = my_realloc(crypt->buffer, *olen,
+					   MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+		if (crypt->buffer == NULL) {
+			msg("%s:%s: failed to increase buffer to "
+			    "%llu bytes.\n", my_progname, __FUNCTION__,
+			    (ulonglong)*olen);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+		crypt->bufsize = *olen;
+	}
+
+	if (*elen > 0) {
+		if (crypt->read(crypt->userdata, crypt->buffer, *elen)
+		    != *elen) {
+			msg("%s:%s: failed to read %lld bytes for chunk payload "
+			    "at offset 0x%llx.\n", my_progname, __FUNCTION__,
+			    (ulonglong)*elen, crypt->offset);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+	}
+
+	checksum = crc32(0, crypt->buffer, *elen);
+	if (checksum != checksum_exp) {
+		msg("%s:%s invalid checksum at offset 0x%llx, "
+		    "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,
+		    crypt->offset, checksum_exp, checksum);
+		result = XB_CRYPT_READ_ERROR;
+		goto err;
+	}
+
+	crypt->offset += *elen;
+	*buf = crypt->buffer;
+
+	*hash_appended = version > 2;
+
+	goto exit;
+
+err:
+	*buf = NULL;
+	*olen = 0;
+	*elen = 0;
+	*ivlen = 0;
+	*iv = 0;
+exit:
+	return result;
+}
+
+int xb_crypt_read_close(xb_rcrypt_t *crypt)
+{
+	if (crypt->buffer)
+		my_free(crypt->buffer);
+	if (crypt->ivbuffer)
+		my_free(crypt->ivbuffer);
+	my_free(crypt);
+
+	return 0;
+}
+

extra/mariabackup/xbcrypt_write.c

@@ -0,0 +1,104 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+The xbcrypt format writer implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include "xbcrypt.h"
+
+struct xb_wcrypt_struct {
+	void				*userdata;
+	xb_crypt_write_callback		*write;
+};
+
+xb_wcrypt_t *
+xb_crypt_write_open(void *userdata, xb_crypt_write_callback *onwrite)
+{
+	xb_wcrypt_t	*crypt;
+
+	xb_ad(onwrite);
+
+	crypt = (xb_wcrypt_t *) my_malloc(sizeof(xb_wcrypt_t), MYF(MY_FAE));
+
+	crypt->userdata = userdata;
+	crypt->write = onwrite;
+
+	return crypt;
+}
+
+int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
+			 size_t elen, const void *iv, size_t ivlen)
+{
+	uchar		tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8];
+	uchar		*ptr;
+	ulong		checksum;
+
+	xb_ad(olen <= INT_MAX);
+	if (olen > INT_MAX)
+		return 0;
+
+	xb_ad(elen <= INT_MAX);
+	if (elen > INT_MAX)
+		return 0;
+
+	xb_ad(ivlen <= INT_MAX);
+	if (ivlen > INT_MAX)
+		return 0;
+
+	ptr = tmpbuf;
+
+	memcpy(ptr, XB_CRYPT_CHUNK_MAGIC_CURRENT, XB_CRYPT_CHUNK_MAGIC_SIZE);
+	ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
+
+	int8store(ptr, (ulonglong)0);	/* reserved */
+	ptr += 8;
+
+	int8store(ptr, (ulonglong)olen); /* original size */
+	ptr += 8;
+
+	int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
+	ptr += 8;
+
+	checksum = crc32(0, buf, elen);
+	int4store(ptr, checksum);	/* checksum */
+	ptr += 4;
+
+	int8store(ptr, (ulonglong)ivlen); /* iv size */
+	ptr += 8;
+
+	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
+
+	if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1)
+		return 1;
+
+	if (crypt->write(crypt->userdata, iv, ivlen) == -1)
+		return 1;
+
+	if (crypt->write(crypt->userdata, buf, elen) == -1)
+		return 1;
+
+	return 0;
+}
+
+int xb_crypt_write_close(xb_wcrypt_t *crypt)
+{
+	my_free(crypt);
+
+	return 0;
+}
+
+

extra/mariabackup/xbstream.c

@@ -0,0 +1,456 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+The xbstream utility: serialize/deserialize files in the XBSTREAM format.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include <my_getopt.h>
+#include <hash.h>
+#include "common.h"
+#include "xbstream.h"
+#include "ds_local.h"
+#include "ds_stdout.h"
+
+#define XBSTREAM_VERSION "1.0"
+#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL)
+
+#define START_FILE_HASH_SIZE 16
+
+typedef enum {
+	RUN_MODE_NONE,
+	RUN_MODE_CREATE,
+	RUN_MODE_EXTRACT
+} run_mode_t;
+
+/* Need the following definitions to avoid linking with ds_*.o and their link
+dependencies */
+datasink_t datasink_archive;
+datasink_t datasink_xbstream;
+datasink_t datasink_compress;
+datasink_t datasink_tmpfile;
+datasink_t datasink_encrypt;
+datasink_t datasink_buffer;
+
+static run_mode_t 	opt_mode;
+static char *		opt_directory = NULL;
+static my_bool		opt_verbose = 0;
+
+static struct my_option my_long_options[] =
+{
+	{"help", '?', "Display this help and exit.",
+	 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+	{"create", 'c', "Stream the specified files to the standard output.",
+	 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+	{"extract", 'x', "Extract to disk files from the stream on the "
+	 "standard input.",
+	 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+	{"directory", 'C', "Change the current directory to the specified one "
+	 "before streaming or extracting.", &opt_directory, &opt_directory, 0,
+	 GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+	{"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose,
+	 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+	{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
+};
+
+typedef struct {
+	char 		*path;
+	uint		pathlen;
+	my_off_t	offset;
+	ds_ctxt_t	*ds_ctxt;
+	ds_file_t	*file;
+} file_entry_t;
+
+static int get_options(int *argc, char ***argv);
+static int mode_create(int argc, char **argv);
+static int mode_extract(int argc, char **argv);
+static my_bool get_one_option(int optid, const struct my_option *opt,
+			      char *argument);
+
+int
+main(int argc, char **argv)
+{
+	MY_INIT(argv[0]);
+
+	if (get_options(&argc, &argv)) {
+		goto err;
+	}
+
+	if (opt_mode == RUN_MODE_NONE) {
+		msg("%s: either -c or -x must be specified.\n", my_progname);
+		goto err;
+	}
+
+	/* Change the current directory if -C is specified */
+	if (opt_directory && my_setwd(opt_directory, MYF(MY_WME))) {
+		goto err;
+	}
+
+	if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) {
+		goto err;
+	} else if (opt_mode == RUN_MODE_EXTRACT && mode_extract(argc, argv)) {
+		goto err;
+	}
+
+	my_cleanup_options(my_long_options);
+
+	my_end(0);
+
+	return EXIT_SUCCESS;
+err:
+	my_cleanup_options(my_long_options);
+
+	my_end(0);
+
+	exit(EXIT_FAILURE);
+}
+
+static
+int
+get_options(int *argc, char ***argv)
+{
+	int ho_error;
+
+	if ((ho_error= handle_options(argc, argv, my_long_options,
+				      get_one_option))) {
+		exit(EXIT_FAILURE);
+	}
+
+	return 0;
+}
+
+static
+void
+print_version(void)
+{
+	printf("%s  Ver %s for %s (%s)\n", my_progname, XBSTREAM_VERSION,
+	       SYSTEM_TYPE, MACHINE_TYPE);
+}
+
+static
+void
+usage(void)
+{
+	print_version();
+	puts("Copyright (C) 2011-2013 Percona LLC and/or its affiliates.");
+	puts("This software comes with ABSOLUTELY NO WARRANTY. "
+	     "This is free software,\nand you are welcome to modify and "
+	     "redistribute it under the GPL license.\n");
+
+	puts("Serialize/deserialize files in the XBSTREAM format.\n");
+
+	puts("Usage: ");
+	printf("  %s -c [OPTIONS...] FILES...	# stream specified files to "
+	       "standard output.\n", my_progname);
+	printf("  %s -x [OPTIONS...]		# extract files from the stream"
+	       "on the standard input.\n", my_progname);
+
+	puts("\nOptions:");
+	my_print_help(my_long_options);
+}
+
+static
+int
+set_run_mode(run_mode_t mode)
+{
+	if (opt_mode != RUN_MODE_NONE) {
+		msg("%s: can't set specify both -c and -x.\n", my_progname);
+		return 1;
+	}
+
+	opt_mode = mode;
+
+	return 0;
+}
+
+static
+my_bool
+get_one_option(int optid, const struct my_option *opt __attribute__((unused)),
+	       char *argument __attribute__((unused)))
+{
+	switch (optid) {
+	case 'c':
+		if (set_run_mode(RUN_MODE_CREATE)) {
+			return TRUE;
+		}
+		break;
+	case 'x':
+		if (set_run_mode(RUN_MODE_EXTRACT)) {
+			return TRUE;
+		}
+		break;
+	case '?':
+		usage();
+		exit(0);
+	}
+
+	return FALSE;
+}
+
+static
+int
+stream_one_file(File file, xb_wstream_file_t *xbfile)
+{
+	uchar	*buf;
+	size_t	bytes;
+	size_t	offset;
+
+	posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL);
+	offset = my_tell(file, MYF(MY_WME));
+
+	buf = (uchar*)(my_malloc(XBSTREAM_BUFFER_SIZE, MYF(MY_FAE)));
+
+	while ((bytes = my_read(file, buf, XBSTREAM_BUFFER_SIZE,
+				MYF(MY_WME))) > 0) {
+		if (xb_stream_write_data(xbfile, buf, bytes)) {
+			msg("%s: xb_stream_write_data() failed.\n",
+			    my_progname);
+			my_free(buf);
+			return 1;
+		}
+		posix_fadvise(file, offset, XBSTREAM_BUFFER_SIZE,
+			      POSIX_FADV_DONTNEED);
+		offset += XBSTREAM_BUFFER_SIZE;
+
+	}
+
+	my_free(buf);
+
+	if (bytes == (size_t) -1) {
+		return 1;
+	}
+
+	return 0;
+}
+
+static
+int
+mode_create(int argc, char **argv)
+{
+	int		i;
+	MY_STAT		mystat;
+	xb_wstream_t	*stream;
+
+	if (argc < 1) {
+		msg("%s: no files are specified.\n", my_progname);
+		return 1;
+	}
+
+	stream = xb_stream_write_new();
+	if (stream == NULL) {
+		msg("%s: xb_stream_write_new() failed.\n", my_progname);
+		return 1;
+	}
+
+	for (i = 0; i < argc; i++) {
+		char			*filepath = argv[i];
+		File			src_file;
+		xb_wstream_file_t	*file;
+
+		if (my_stat(filepath, &mystat, MYF(MY_WME)) == NULL) {
+			goto err;
+		}
+		if (!MY_S_ISREG(mystat.st_mode)) {
+			msg("%s: %s is not a regular file, exiting.\n",
+			    my_progname, filepath);
+			goto err;
+		}
+
+		if ((src_file = my_open(filepath, O_RDONLY, MYF(MY_WME))) < 0) {
+			msg("%s: failed to open %s.\n", my_progname, filepath);
+			goto err;
+		}
+
+		file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL);
+		if (file == NULL) {
+			goto err;
+		}
+
+		if (opt_verbose) {
+			msg("%s\n", filepath);
+		}
+
+		if (stream_one_file(src_file, file) ||
+		    xb_stream_write_close(file) ||
+		    my_close(src_file, MYF(MY_WME))) {
+			goto err;
+		}
+	}
+
+	xb_stream_write_done(stream);
+
+	return 0;
+err:
+	xb_stream_write_done(stream);
+
+	return 1;
+}
+
+static
+file_entry_t *
+file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
+{
+	file_entry_t	*entry;
+	ds_file_t	*file;
+
+	entry = (file_entry_t *) my_malloc(sizeof(file_entry_t),
+					   MYF(MY_WME | MY_ZEROFILL));
+	if (entry == NULL) {
+		return NULL;
+	}
+
+	entry->path = my_strndup(path, pathlen, MYF(MY_WME));
+	if (entry->path == NULL) {
+		goto err;
+	}
+	entry->pathlen = pathlen;
+
+	file = ds_open(ds_ctxt, path, NULL);
+	if (file == NULL) {
+		msg("%s: failed to create file.\n", my_progname);
+		goto err;
+	}
+
+	if (opt_verbose) {
+		msg("%s\n", entry->path);
+	}
+
+	entry->file = file;
+	entry->ds_ctxt = ds_ctxt;
+
+	return entry;
+
+err:
+	if (entry->path != NULL) {
+		my_free(entry->path);
+	}
+	my_free(entry);
+
+	return NULL;
+}
+
+static
+uchar *
+get_file_entry_key(file_entry_t *entry, size_t *length,
+		   my_bool not_used __attribute__((unused)))
+{
+	*length = entry->pathlen;
+	return (uchar *) entry->path;
+}
+
+static
+void
+file_entry_free(file_entry_t *entry)
+{
+	ds_close(entry->file);
+	my_free(entry->path);
+	my_free(entry);
+}
+
+static
+int
+mode_extract(int argc __attribute__((unused)),
+	     char **argv __attribute__((unused)))
+{
+	xb_rstream_t		*stream;
+	xb_rstream_result_t	res;
+	xb_rstream_chunk_t	chunk;
+	HASH			filehash;
+	file_entry_t		*entry;
+	ds_ctxt_t		*ds_ctxt;
+
+	stream = xb_stream_read_new();
+	if (stream == NULL) {
+		msg("%s: xb_stream_read_new() failed.\n", my_progname);
+		return 1;
+	}
+
+	/* If --directory is specified, it is already set as CWD by now. */
+	ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
+
+	if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE,
+			  0, 0, (my_hash_get_key) get_file_entry_key,
+			  (my_hash_free_key) file_entry_free, MYF(0))) {
+		msg("%s: failed to initialize file hash.\n", my_progname);
+		goto err;
+	}
+
+	while ((res = xb_stream_read_chunk(stream, &chunk)) ==
+	       XB_STREAM_READ_CHUNK) {
+		/* If unknown type and ignorable flag is set, skip this chunk */
+		if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \
+		    !(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) {
+			continue;
+		}
+
+		/* See if we already have this file open */
+		entry = (file_entry_t *) my_hash_search(&filehash,
+							(uchar *) chunk.path,
+							chunk.pathlen);
+
+		if (entry == NULL) {
+			entry = file_entry_new(ds_ctxt, chunk.path,
+					       chunk.pathlen);
+			if (entry == NULL) {
+				goto err;
+			}
+			if (my_hash_insert(&filehash, (uchar *) entry)) {
+				msg("%s: my_hash_insert() failed.\n",
+				    my_progname);
+				goto err;
+			}
+		}
+
+		if (chunk.type == XB_CHUNK_TYPE_EOF) {
+			my_hash_delete(&filehash, (uchar *) entry);
+
+			continue;
+		}
+
+		if (entry->offset != chunk.offset) {
+			msg("%s: out-of-order chunk: real offset = 0x%llx, "
+			    "expected offset = 0x%llx\n", my_progname,
+			    chunk.offset, entry->offset);
+			goto err;
+		}
+
+		if (ds_write(entry->file, chunk.data, chunk.length)) {
+			msg("%s: my_write() failed.\n", my_progname);
+			goto err;
+		}
+
+		entry->offset += chunk.length;
+	};
+
+	if (res == XB_STREAM_READ_ERROR) {
+		goto err;
+	}
+
+	my_hash_free(&filehash);
+	ds_destroy(ds_ctxt);
+	xb_stream_read_done(stream);
+
+	return 0;
+err:
+	my_hash_free(&filehash);
+	ds_destroy(ds_ctxt);
+	xb_stream_read_done(stream);
+
+	return 1;
+}

extra/mariabackup/xbstream.h

@@ -0,0 +1,103 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+The xbstream format interface.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XBSTREAM_H
+#define XBSTREAM_H
+
+#include <my_base.h>
+
+/* Magic value in a chunk header */
+#define XB_STREAM_CHUNK_MAGIC "XBSTCK01"
+
+/* Chunk flags */
+/* Chunk can be ignored if unknown version/format */
+#define XB_STREAM_FLAG_IGNORABLE 0x01
+
+/* Magic + flags + type + path len */
+#define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \
+				   1 + 1 + 4)
+#define CHUNK_TYPE_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1)
+#define PATH_LENGTH_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1)
+
+typedef struct xb_wstream_struct xb_wstream_t;
+
+typedef struct xb_wstream_file_struct xb_wstream_file_t;
+
+typedef enum {
+	XB_STREAM_FMT_NONE,
+	XB_STREAM_FMT_TAR,
+	XB_STREAM_FMT_XBSTREAM
+} xb_stream_fmt_t;
+
+/************************************************************************
+Write interface. */
+
+typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file,
+					 void *userdata,
+					 const void *buf, size_t len);
+
+xb_wstream_t *xb_stream_write_new(void);
+
+xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path,
+					MY_STAT *mystat, void *userdata,
+					xb_stream_write_callback *onwrite);
+
+int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len);
+
+int xb_stream_write_close(xb_wstream_file_t *file);
+
+int xb_stream_write_done(xb_wstream_t *stream);
+
+/************************************************************************
+Read interface. */
+
+typedef enum {
+	XB_STREAM_READ_CHUNK,
+	XB_STREAM_READ_EOF,
+	XB_STREAM_READ_ERROR
+} xb_rstream_result_t;
+
+typedef enum {
+	XB_CHUNK_TYPE_UNKNOWN = '\0',
+	XB_CHUNK_TYPE_PAYLOAD = 'P',
+	XB_CHUNK_TYPE_EOF = 'E'
+} xb_chunk_type_t;
+
+typedef struct xb_rstream_struct xb_rstream_t;
+
+typedef struct {
+	uchar           flags;
+	xb_chunk_type_t type;
+	uint		pathlen;
+	char		path[FN_REFLEN];
+	size_t		length;
+	my_off_t	offset;
+	void		*data;
+	ulong		checksum;
+} xb_rstream_chunk_t;
+
+xb_rstream_t *xb_stream_read_new(void);
+
+xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream,
+					 xb_rstream_chunk_t *chunk);
+
+int xb_stream_read_done(xb_rstream_t *stream);
+
+#endif

extra/mariabackup/xbstream_read.c

@@ -0,0 +1,227 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+The xbstream format reader implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include <zlib.h>
+#include "common.h"
+#include "xbstream.h"
+
+/* Allocate 1 MB for the payload buffer initially */
+#define INIT_BUFFER_LEN (1024 * 1024)
+
+#ifndef MY_OFF_T_MAX
+#define MY_OFF_T_MAX (~(my_off_t)0UL)
+#endif
+
+struct xb_rstream_struct {
+	my_off_t	offset;
+	File 		fd;
+	void 		*buffer;
+	size_t		buflen;
+};
+
+xb_rstream_t *
+xb_stream_read_new(void)
+{
+	xb_rstream_t *stream;
+
+	stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE));
+
+	stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE));
+	stream->buflen = INIT_BUFFER_LEN;
+
+	stream->fd = fileno(stdin);
+	stream->offset = 0;
+
+#ifdef __WIN__
+	setmode(stream->fd, _O_BINARY);
+#endif
+
+	return stream;
+}
+
+static inline
+xb_chunk_type_t
+validate_chunk_type(uchar code)
+{
+	switch ((xb_chunk_type_t) code) {
+	case XB_CHUNK_TYPE_PAYLOAD:
+	case XB_CHUNK_TYPE_EOF:
+		return (xb_chunk_type_t) code;
+	default:
+		return XB_CHUNK_TYPE_UNKNOWN;
+	}
+}
+
+#define F_READ(buf,len)                                       	\
+	do {                                                      	\
+		if (xb_read_full(fd, buf, len) < len) {	\
+			msg("xb_stream_read_chunk(): my_read() failed.\n"); \
+			goto err;                                 	\
+		}							\
+	} while (0)
+
+xb_rstream_result_t
+xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
+{
+	uchar		tmpbuf[16];
+	uchar		*ptr = tmpbuf;
+	uint		pathlen;
+	size_t		tbytes;
+	ulonglong	ullval;
+	ulong		checksum_exp;
+	ulong		checksum;
+	File		fd = stream->fd;
+
+	xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN);
+
+	/* This is the only place where we expect EOF, so read with
+	xb_read_full() rather than F_READ() */
+	tbytes = xb_read_full(fd, ptr, CHUNK_HEADER_CONSTANT_LEN);
+	if (tbytes == 0) {
+		return XB_STREAM_READ_EOF;
+	} else if (tbytes < CHUNK_HEADER_CONSTANT_LEN) {
+		msg("xb_stream_read_chunk(): unexpected end of stream at "
+		    "offset 0x%llx.\n", stream->offset);
+		goto err;
+	}
+
+	ptr = tmpbuf;
+
+	/* Chunk magic value */
+	if (memcmp(tmpbuf, XB_STREAM_CHUNK_MAGIC, 8)) {
+		msg("xb_stream_read_chunk(): wrong chunk magic at offset "
+		    "0x%llx.\n", (ulonglong) stream->offset);
+		goto err;
+	}
+	ptr += 8;
+	stream->offset += 8;
+
+	/* Chunk flags */
+	chunk->flags = *ptr++;
+	stream->offset++;
+
+	/* Chunk type, ignore unknown ones if ignorable flag is set */
+	chunk->type = validate_chunk_type(*ptr);
+	if (chunk->type == XB_CHUNK_TYPE_UNKNOWN &&
+	    !(chunk->flags & XB_STREAM_FLAG_IGNORABLE)) {
+		msg("xb_stream_read_chunk(): unknown chunk type 0x%lu at "
+		    "offset 0x%llx.\n", (ulong) *ptr,
+		    (ulonglong) stream->offset);
+		goto err;
+	}
+	ptr++;
+	stream->offset++;
+
+	/* Path length */
+	pathlen = uint4korr(ptr);
+	if (pathlen >= FN_REFLEN) {
+		msg("xb_stream_read_chunk(): path length (%lu) is too large at "
+		    "offset 0x%llx.\n", (ulong) pathlen, stream->offset);
+		goto err;
+	}
+	chunk->pathlen = pathlen;
+	stream->offset +=4;
+
+	xb_ad((ptr + 4 - tmpbuf) == CHUNK_HEADER_CONSTANT_LEN);
+
+	/* Path */
+	if (chunk->pathlen > 0) {
+		F_READ((uchar *) chunk->path, pathlen);
+		stream->offset += pathlen;
+	}
+	chunk->path[pathlen] = '\0';
+
+	if (chunk->type == XB_CHUNK_TYPE_EOF) {
+		return XB_STREAM_READ_CHUNK;
+	}
+
+	/* Payload length */
+	F_READ(tmpbuf, 16);
+	ullval = uint8korr(tmpbuf);
+	if (ullval > (ulonglong) SIZE_T_MAX) {
+		msg("xb_stream_read_chunk(): chunk length is too large at "
+		    "offset 0x%llx: 0x%llx.\n", (ulonglong) stream->offset,
+		    ullval);
+		goto err;
+	}
+	chunk->length = (size_t) ullval;
+	stream->offset += 8;
+
+	/* Payload offset */
+	ullval = uint8korr(tmpbuf + 8);
+	if (ullval > (ulonglong) MY_OFF_T_MAX) {
+		msg("xb_stream_read_chunk(): chunk offset is too large at "
+		    "offset 0x%llx: 0x%llx.\n", (ulonglong) stream->offset,
+		    ullval);
+		goto err;
+	}
+	chunk->offset = (my_off_t) ullval;
+	stream->offset += 8;
+
+	/* Reallocate the buffer if needed */
+	if (chunk->length > stream->buflen) {
+		stream->buffer = my_realloc(stream->buffer, chunk->length,
+					    MYF(MY_WME));
+		if (stream->buffer == NULL) {
+			msg("xb_stream_read_chunk(): failed to increase buffer "
+			    "to %lu bytes.\n", (ulong) chunk->length);
+			goto err;
+		}
+		stream->buflen = chunk->length;
+	}
+
+	/* Checksum */
+	F_READ(tmpbuf, 4);
+	checksum_exp = uint4korr(tmpbuf);
+
+	/* Payload */
+	if (chunk->length > 0) {
+		F_READ(stream->buffer, chunk->length);
+		stream->offset += chunk->length;
+	}
+
+	checksum = crc32(0, stream->buffer, chunk->length);
+	if (checksum != checksum_exp) {
+		msg("xb_stream_read_chunk(): invalid checksum at offset "
+		    "0x%llx: expected 0x%lx, read 0x%lx.\n",
+		    (ulonglong) stream->offset, checksum_exp, checksum);
+		goto err;
+	}
+	stream->offset += 4;
+
+	chunk->data = stream->buffer;
+	chunk->checksum = checksum;
+
+	return XB_STREAM_READ_CHUNK;
+
+err:
+	return XB_STREAM_READ_ERROR;
+}
+
+int
+xb_stream_read_done(xb_rstream_t *stream)
+{
+	my_free(stream->buffer);
+	my_free(stream);
+
+	return 0;
+}

extra/mariabackup/xbstream_write.c

@@ -0,0 +1,280 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+The xbstream format writer implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <mysql_version.h>
+#include <my_base.h>
+#include <zlib.h>
+#include "common.h"
+#include "xbstream.h"
+
+/* Group writes smaller than this into a single chunk */
+#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024)
+
+struct xb_wstream_struct {
+	pthread_mutex_t	mutex;
+};
+
+struct xb_wstream_file_struct {
+	xb_wstream_t	*stream;
+	char		*path;
+	ulong		path_len;
+	char		chunk[XB_STREAM_MIN_CHUNK_SIZE];
+	char		*chunk_ptr;
+	size_t		chunk_free;
+	my_off_t	offset;
+	void		*userdata;
+	xb_stream_write_callback *write;
+};
+
+static int xb_stream_flush(xb_wstream_file_t *file);
+static int xb_stream_write_chunk(xb_wstream_file_t *file,
+				 const void *buf, size_t len);
+static int xb_stream_write_eof(xb_wstream_file_t *file);
+
+static
+ssize_t
+xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)),
+				 void *userdata __attribute__((unused)),
+				 const void *buf, size_t len)
+{
+	if (my_write(fileno(stdout), buf, len, MYF(MY_WME | MY_NABP)))
+		return -1;
+	return len;
+}
+
+xb_wstream_t *
+xb_stream_write_new(void)
+{
+	xb_wstream_t	*stream;
+
+	stream = (xb_wstream_t *) my_malloc(sizeof(xb_wstream_t), MYF(MY_FAE));
+	pthread_mutex_init(&stream->mutex, NULL);
+
+	return stream;;
+}
+
+xb_wstream_file_t *
+xb_stream_write_open(xb_wstream_t *stream, const char *path,
+		     MY_STAT *mystat __attribute__((unused)),
+		     void *userdata,
+		     xb_stream_write_callback *onwrite)
+{
+	xb_wstream_file_t	*file;
+	ulong			path_len;
+
+	path_len = strlen(path);
+
+	if (path_len > FN_REFLEN) {
+		msg("xb_stream_write_open(): file path is too long.\n");
+		return NULL;
+	}
+
+	file = (xb_wstream_file_t *) my_malloc(sizeof(xb_wstream_file_t) +
+					       path_len + 1, MYF(MY_FAE));
+
+	file->path = (char *) (file + 1);
+	memcpy(file->path, path, path_len + 1);
+	file->path_len = path_len;
+
+	file->stream = stream;
+	file->offset = 0;
+	file->chunk_ptr = file->chunk;
+	file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE;
+	if (onwrite) {
+#ifdef __WIN__
+		setmode(fileno(stdout), _O_BINARY);
+#endif
+		file->userdata = userdata;
+		file->write = onwrite;
+	} else {
+		file->userdata = NULL;
+		file->write = xb_stream_default_write_callback;
+	}
+
+	return file;
+}
+
+int
+xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len)
+{
+	if (len < file->chunk_free) {
+		memcpy(file->chunk_ptr, buf, len);
+		file->chunk_ptr += len;
+		file->chunk_free -= len;
+
+		return 0;
+	}
+
+	if (xb_stream_flush(file))
+		return 1;
+
+	return xb_stream_write_chunk(file, buf, len);
+}
+
+int
+xb_stream_write_close(xb_wstream_file_t *file)
+{
+	if (xb_stream_flush(file) ||
+	    xb_stream_write_eof(file)) {
+		my_free(file);
+		return 1;
+	}
+
+	my_free(file);
+
+	return 0;
+}
+
+int
+xb_stream_write_done(xb_wstream_t *stream)
+{
+	pthread_mutex_destroy(&stream->mutex);
+
+	my_free(stream);
+
+	return 0;
+}
+
+static
+int
+xb_stream_flush(xb_wstream_file_t *file)
+{
+	if (file->chunk_ptr == file->chunk) {
+		return 0;
+	}
+
+	if (xb_stream_write_chunk(file, file->chunk,
+				  file->chunk_ptr - file->chunk)) {
+		return 1;
+	}
+
+	file->chunk_ptr = file->chunk;
+	file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE;
+
+	return 0;
+}
+
+static
+int
+xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
+{
+	/* Chunk magic + flags + chunk type + path_len + path + len + offset +
+	checksum */
+	uchar		tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 +
+			       FN_REFLEN + 8 + 8 + 4];
+	uchar		*ptr;
+	xb_wstream_t	*stream = file->stream;
+	ulong		checksum;
+
+	/* Write xbstream header */
+	ptr = tmpbuf;
+
+	/* Chunk magic */
+	memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1);
+	ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1;
+
+	*ptr++ = 0;                              /* Chunk flags */
+
+	*ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD;  /* Chunk type */
+
+	int4store(ptr, file->path_len);          /* Path length */
+	ptr += 4;
+
+	memcpy(ptr, file->path, file->path_len); /* Path */
+	ptr += file->path_len;
+
+	int8store(ptr, len);                     /* Payload length */
+	ptr += 8;
+
+	pthread_mutex_lock(&stream->mutex);
+
+	int8store(ptr, file->offset);            /* Payload offset */
+	ptr += 8;
+
+	checksum = crc32(0, buf, len);           /* checksum */
+	int4store(ptr, checksum);
+	ptr += 4;
+
+	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
+
+	if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1)
+		goto err;
+
+
+	if (file->write(file, file->userdata, buf, len) == -1) /* Payload */
+		goto err;
+
+	file->offset+= len;
+
+	pthread_mutex_unlock(&stream->mutex);
+
+	return 0;
+
+err:
+
+	pthread_mutex_unlock(&stream->mutex);
+
+	return 1;
+}
+
+static
+int
+xb_stream_write_eof(xb_wstream_file_t *file)
+{
+	/* Chunk magic + flags + chunk type + path_len + path */
+	uchar		tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 +
+			       FN_REFLEN];
+	uchar		*ptr;
+	xb_wstream_t	*stream = file->stream;
+
+	pthread_mutex_lock(&stream->mutex);
+
+	/* Write xbstream header */
+	ptr = tmpbuf;
+
+	/* Chunk magic */
+	memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1);
+	ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1;
+
+	*ptr++ = 0;                              /* Chunk flags */
+
+	*ptr++ = (uchar) XB_CHUNK_TYPE_EOF;      /* Chunk type */
+
+	int4store(ptr, file->path_len);          /* Path length */
+	ptr += 4;
+
+	memcpy(ptr, file->path, file->path_len); /* Path */
+	ptr += file->path_len;
+
+	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
+
+	if (file->write(file, file->userdata, tmpbuf,
+			(ulonglong) (ptr - tmpbuf)) == -1)
+		goto err;
+
+	pthread_mutex_unlock(&stream->mutex);
+
+	return 0;
+err:
+
+	pthread_mutex_unlock(&stream->mutex);
+
+	return 1;
+}

extra/mariabackup/xtrabackup.h

@@ -0,0 +1,232 @@
+/******************************************************
+Copyright (c) 2011-2015 Percona LLC and/or its affiliates.
+
+Declarations for xtrabackup.cc
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XB_XTRABACKUP_H
+#define XB_XTRABACKUP_H
+
+#include <my_getopt.h>
+#include "datasink.h"
+#include "xbstream.h"
+#include "changed_page_bitmap.h"
+
+#ifdef __WIN__
+#define XB_FILE_UNDEFINED NULL
+#else
+#define XB_FILE_UNDEFINED (-1)
+#endif
+
+typedef struct {
+	ulint	page_size;
+	ulint	zip_size;
+	ulint	space_id;
+} xb_delta_info_t;
+
+/* ======== Datafiles iterator ======== */
+typedef struct {
+	fil_system_t	*system;
+	fil_space_t	*space;
+	fil_node_t	*node;
+	ibool		started;
+	os_ib_mutex_t	mutex;
+} datafiles_iter_t;
+
+/* value of the --incremental option */
+extern lsn_t incremental_lsn;
+
+extern char		*xtrabackup_target_dir;
+extern char		*xtrabackup_incremental_dir;
+extern char		*xtrabackup_incremental_basedir;
+extern char		*innobase_data_home_dir;
+extern char		*innobase_buffer_pool_filename;
+extern ds_ctxt_t	*ds_meta;
+extern ds_ctxt_t	*ds_data;
+
+/* The last checkpoint LSN at the backup startup time */
+extern lsn_t checkpoint_lsn_start;
+
+extern xb_page_bitmap *changed_page_bitmap;
+
+extern ulint	xtrabackup_rebuild_threads;
+
+extern char		*xtrabackup_incremental;
+extern my_bool		xtrabackup_incremental_force_scan;
+
+extern lsn_t		metadata_from_lsn;
+extern lsn_t		metadata_to_lsn;
+extern lsn_t		metadata_last_lsn;
+
+extern xb_stream_fmt_t	xtrabackup_stream_fmt;
+extern ibool		xtrabackup_stream;
+
+extern char		*xtrabackup_tables;
+extern char		*xtrabackup_tables_file;
+extern char		*xtrabackup_databases;
+extern char		*xtrabackup_databases_file;
+
+extern my_bool		xtrabackup_compact;
+extern ibool		xtrabackup_compress;
+extern ibool		xtrabackup_encrypt;
+
+extern my_bool		xtrabackup_backup;
+extern my_bool		xtrabackup_prepare;
+extern my_bool		xtrabackup_apply_log_only;
+extern my_bool		xtrabackup_copy_back;
+extern my_bool		xtrabackup_move_back;
+extern my_bool		xtrabackup_decrypt_decompress;
+
+extern char		*innobase_data_file_path;
+extern char		*innobase_doublewrite_file;
+extern char		*xtrabackup_encrypt_key;
+extern char		*xtrabackup_encrypt_key_file;
+extern longlong		innobase_log_file_size;
+extern long		innobase_log_files_in_group;
+extern longlong		innobase_page_size;
+
+extern const char	*xtrabackup_encrypt_algo_names[];
+extern TYPELIB		xtrabackup_encrypt_algo_typelib;
+
+extern int		xtrabackup_parallel;
+
+extern my_bool		xb_close_files;
+extern const char	*xtrabackup_compress_alg;
+extern uint		xtrabackup_compress_threads;
+extern ulonglong	xtrabackup_compress_chunk_size;
+extern ulong		xtrabackup_encrypt_algo;
+extern uint		xtrabackup_encrypt_threads;
+extern ulonglong	xtrabackup_encrypt_chunk_size;
+extern my_bool		xtrabackup_export;
+extern char		*xtrabackup_incremental_basedir;
+extern char		*xtrabackup_extra_lsndir;
+extern char		*xtrabackup_incremental_dir;
+extern ulint		xtrabackup_log_copy_interval;
+extern my_bool		xtrabackup_rebuild_indexes;
+extern char		*xtrabackup_stream_str;
+extern long		xtrabackup_throttle;
+extern longlong		xtrabackup_use_memory;
+
+extern my_bool		opt_galera_info;
+extern my_bool		opt_slave_info;
+extern my_bool		opt_no_lock;
+extern my_bool		opt_safe_slave_backup;
+extern my_bool		opt_rsync;
+extern my_bool		opt_force_non_empty_dirs;
+extern my_bool		opt_noversioncheck;
+extern my_bool		opt_no_backup_locks;
+extern my_bool		opt_decompress;
+extern my_bool		opt_remove_original;
+
+extern char		*opt_incremental_history_name;
+extern char		*opt_incremental_history_uuid;
+
+extern char		*opt_user;
+extern char		*opt_password;
+extern char		*opt_host;
+extern char		*opt_defaults_group;
+extern char		*opt_socket;
+extern uint		opt_port;
+extern char		*opt_login_path;
+extern char		*opt_log_bin;
+
+extern const char 	*query_type_names[];
+
+enum query_type_t {QUERY_TYPE_ALL, QUERY_TYPE_UPDATE,
+			QUERY_TYPE_SELECT};
+
+extern TYPELIB		query_type_typelib;
+
+extern ulong		opt_lock_wait_query_type;
+extern ulong		opt_kill_long_query_type;
+
+extern ulong		opt_decrypt_algo;
+
+extern uint		opt_kill_long_queries_timeout;
+extern uint		opt_lock_wait_timeout;
+extern uint		opt_lock_wait_threshold;
+extern uint		opt_debug_sleep_before_unlock;
+extern uint		opt_safe_slave_backup_timeout;
+
+extern const char	*opt_history;
+extern my_bool		opt_decrypt;
+
+#if defined(HAVE_OPENSSL)
+extern my_bool opt_use_ssl;
+extern my_bool opt_ssl_verify_server_cert;
+#if !defined(HAVE_YASSL)
+extern char *opt_server_public_key;
+#endif
+#endif
+
+enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON,
+			BINLOG_INFO_AUTO};
+
+extern ulong opt_binlog_info;
+
+void xtrabackup_io_throttling(void);
+my_bool xb_write_delta_metadata(const char *filename,
+				const xb_delta_info_t *info);
+
+datafiles_iter_t *datafiles_iter_new(fil_system_t *f_system);
+fil_node_t *datafiles_iter_next(datafiles_iter_t *it);
+void datafiles_iter_free(datafiles_iter_t *it);
+
+/************************************************************************
+Initialize the tablespace memory cache and populate it by scanning for and
+opening data files */
+ulint xb_data_files_init(void);
+
+/************************************************************************
+Destroy the tablespace memory cache. */
+void xb_data_files_close(void);
+
+/***********************************************************************
+Reads the space flags from a given data file and returns the compressed
+page size, or 0 if the space is not compressed. */
+ulint xb_get_zip_size(os_file_t file);
+
+/************************************************************************
+Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
+or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on
+the --tables or --tables-file options.
+
+@return TRUE if the table should be skipped. */
+my_bool
+check_if_skip_table(
+/******************/
+	const char*	name);	/*!< in: path to the table */
+
+/************************************************************************
+Check if parameter is set in defaults file or via command line argument
+@return true if parameter is set. */
+bool
+check_if_param_set(const char *param);
+
+
+void
+xtrabackup_backup_func(void);
+
+my_bool
+xb_get_one_option(int optid,
+		  const struct my_option *opt __attribute__((unused)),
+		  char *argument);
+
+const char*
+xb_get_copy_action(const char *dflt = "Copying");
+
+#endif /* XB_XTRABACKUP_H */

extra/mariabackup/xtrabackup_version.h.in

@@ -0,0 +1,27 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Version numbers definitions.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#ifndef XB_VERSION_H
+#define XB_VERSION_H
+
+#define XTRABACKUP_VERSION		"@XB_VERSION@"
+#define XTRABACKUP_REVISION		"@XB_REVISION@"
+
+#endif /* XB_VERSION_H */