Merge 10.1 into 10.2

cmake/os/Windows.cmake

@@ -139,8 +139,8 @@ IF(MSVC)
   ENDIF()
   
   #TODO: update the code and remove the disabled warnings
-  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996")
+  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996 /we4700")
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099 /we4700")
 
 ENDIF()
 

extra/mariabackup/backup_mysql.cc

@@ -113,6 +113,11 @@ xb_mysql_connect()
 			      (char *) &opt_secure_auth);
 	}
 
+	if (xb_plugin_dir && *xb_plugin_dir){
+		mysql_options(connection, MYSQL_PLUGIN_DIR, xb_plugin_dir);
+	}
+	mysql_options(connection, MYSQL_OPT_PROTOCOL, &opt_protocol);
+
 	msg_ts("Connecting to MySQL server host: %s, user: %s, password: %s, "
 	       "port: %s, socket: %s\n", opt_host ? opt_host : "localhost",
 	       opt_user ? opt_user : "not set",

extra/mariabackup/xtrabackup.cc

@@ -119,6 +119,7 @@ my_bool xtrabackup_export;
 
 longlong xtrabackup_use_memory;
 
+uint opt_protocol;
 long xtrabackup_throttle; /* 0:unlimited */
 static lint io_ticket;
 static os_event_t wait_throttle;
@@ -527,6 +528,7 @@ enum options_xtrabackup
 
   OPT_XTRA_TABLES_EXCLUDE,
   OPT_XTRA_DATABASES_EXCLUDE,
+  OPT_PROTOCOL
 };
 
 struct my_option xb_client_options[] =
@@ -759,6 +761,9 @@ struct my_option xb_client_options[] =
    0, 0, 0, GET_STR,
    REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 
+  {"protocol", OPT_PROTOCOL, "The protocol to use for connection (tcp, socket, pipe, memory).",
+   0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
   {"socket", 'S', "This option specifies the socket to use when "
    "connecting to the local database server with a UNIX domain socket.  "
    "The option accepts a string argument. See mysql --help for details.",
@@ -1290,8 +1295,13 @@ xb_get_one_option(int optid,
         start[1]=0 ;
     }
     break;
-
+  case OPT_PROTOCOL:
-
+    if (argument)
+    {
+       opt_protocol= find_type_or_exit(argument, &sql_protocol_typelib,
+                                    opt->name);
+    }
+    break;
 #include "sslopt-case.h"
 
   case '?':

extra/mariabackup/xtrabackup.h

@@ -43,6 +43,8 @@ extern char		*xtrabackup_incremental_dir;
 extern char		*xtrabackup_incremental_basedir;
 extern char		*innobase_data_home_dir;
 extern char		*innobase_buffer_pool_filename;
+extern char		*xb_plugin_dir;
+extern uint		opt_protocol;
 extern ds_ctxt_t	*ds_meta;
 extern ds_ctxt_t	*ds_data;
 

mysql-test/include/innodb_undo_tablespaces.combinations

@@ -0,0 +1,5 @@
+[undo0]
+innodb-undo-tablespaces=0
+
+[undo3]
+innodb-undo-tablespaces=3

mysql-test/include/innodb_undo_tablespaces.inc

@@ -0,0 +1,3 @@
+# The goal of including this file is to enable innodb_undo_tablespaces combinations
+# (see include/innodb_undo_tablespaces.combinations)
+

mysql-test/r/alter_table.result

@@ -2125,6 +2125,58 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8
 DROP TABLE t1;
 #
+# MDEV-8960 Can't refer the same column twice in one ALTER TABLE
+#
+CREATE TABLE t1 (
+`a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL,
+ALTER COLUMN `consultant_id` DROP DEFAULT;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `consultant_id` int(11) NOT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8
+DROP TABLE t1;
+CREATE TABLE t1 (
+`a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL,
+ALTER COLUMN `consultant_id` SET DEFAULT 2;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `consultant_id` int(11) NOT NULL DEFAULT 2
+) ENGINE=MyISAM DEFAULT CHARSET=utf8
+DROP TABLE t1;
+CREATE TABLE t1 (
+`a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2,
+ALTER COLUMN `consultant_id` DROP DEFAULT;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `consultant_id` int(11) NOT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8
+DROP TABLE t1;
+CREATE TABLE t1 (
+`a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2,
+ALTER COLUMN `consultant_id` DROP DEFAULT,
+MODIFY COLUMN `consultant_id` BIGINT;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `consultant_id` bigint(20) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8
+DROP TABLE t1;
+#
 # Start of 10.1 tests
 #
 #

mysql-test/suite/encryption/r/innodb_encryption.result

@@ -1,3 +1,5 @@
+call mtr.add_suppression("InnoDB: New log files created");
+call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables.");
 SET @start_global_value = @@global.innodb_encryption_threads;
 SHOW VARIABLES LIKE 'innodb_encrypt%';
 Variable_name	Value

mysql-test/suite/encryption/t/innodb_encryption.test

@@ -3,10 +3,14 @@
 #
 -- source include/have_innodb.inc
 -- source include/have_example_key_management_plugin.inc
+-- source include/innodb_undo_tablespaces.inc
 
 # embedded does not support restart
 -- source include/not_embedded.inc
 
+call mtr.add_suppression("InnoDB: New log files created");
+call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables.");
+
 SET @start_global_value = @@global.innodb_encryption_threads;
 
 SHOW VARIABLES LIKE 'innodb_encrypt%';

mysql-test/suite/innodb/t/xa_recovery.test

@@ -15,8 +15,14 @@ connect (con1,localhost,root);
 XA START 'x'; UPDATE t1 set a=2; XA END 'x'; XA PREPARE 'x';
 connection default;
 
+# innodb_force_recovery=2 prevents the purge and tests that the fix of
+# MDEV-13606 XA PREPARE transactions should survive innodb_force_recovery=1 or 2
+# is present.
+--let $restart_parameters= --innodb-force-recovery=2
 --let $shutdown_timeout=0
 --source include/restart_mysqld.inc
+--let $restart_parameters=
+--let $shutdown_timeout=
 
 disconnect con1;
 connect (con1,localhost,root);

mysql-test/suite/mariabackup/auth_plugin_win.opt

@@ -0,0 +1 @@
+--loose-enable-named-pipe

mysql-test/suite/mariabackup/auth_plugin_win.result

@@ -0,0 +1,5 @@
+INSTALL SONAME 'auth_named_pipe';
+CREATE USER 'USERNAME' IDENTIFIED WITH named_pipe;
+GRANT ALL PRIVILEGES ON *.* to USERNAME;
+DROP USER 'USERNAME';
+UNINSTALL SONAME 'auth_named_pipe';

mysql-test/suite/mariabackup/auth_plugin_win.test

@@ -0,0 +1,31 @@
+--source include/windows.inc
+--source include/not_embedded.inc
+
+if (!$AUTH_NAMED_PIPE_SO) {
+  skip No named pipe plugin;
+}
+
+if (!$USERNAME) {
+  skip USERNAME variable is undefined;
+}
+
+if (`SELECT count(*) <> 0 FROM mysql.user WHERE user = '$USERNAME'`) {
+  skip \$USER=$USER which exists in mysql.user;
+}
+
+INSTALL SONAME 'auth_named_pipe';
+
+--replace_result $USERNAME USERNAME
+eval CREATE USER '$USERNAME' IDENTIFIED WITH named_pipe;
+--replace_result $USERNAME USERNAME
+eval GRANT ALL PRIVILEGES ON *.* to $USERNAME;
+
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf -u $USERNAME --backup --protocol=pipe --target-dir=$targetdir;
+--enable_result_log
+--replace_result $USERNAME USERNAME
+eval DROP USER '$USERNAME';
+rmdir $targetdir;
+UNINSTALL SONAME 'auth_named_pipe';
+

mysql-test/t/alter_table.test

@@ -1765,6 +1765,48 @@ ALTER TABLE t1 CONVERT TO CHARACTER SET utf8;
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-8960 Can't refer the same column twice in one ALTER TABLE
+--echo #
+
+CREATE TABLE t1 (
+  `a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL,
+ALTER COLUMN `consultant_id` DROP DEFAULT;
+
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (
+  `a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL,
+ALTER COLUMN `consultant_id` SET DEFAULT 2;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (
+  `a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2,
+ALTER COLUMN `consultant_id` DROP DEFAULT;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (
+  `a` int(11) DEFAULT NULL
+) DEFAULT CHARSET=utf8;
+
+ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2,
+ALTER COLUMN `consultant_id` DROP DEFAULT,
+MODIFY COLUMN `consultant_id` BIGINT;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
 --echo #
 --echo # Start of 10.1 tests
 --echo #

mysys/my_init.c

@@ -200,7 +200,6 @@ Voluntary context switches %ld, Involuntary context switches %ld\n",
    _CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE );
    _CrtSetReportFile( _CRT_ASSERT, _CRTDBG_FILE_STDERR );
    _CrtCheckMemory();
-   _CrtDumpMemoryLeaks();
 #endif
   }
 

sql/sql_table.cc

@@ -7702,9 +7702,25 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
   {
     if (def->change && ! def->field)
     {
-      my_error(ER_BAD_FIELD_ERROR, MYF(0), def->change,
+      /*
-               table->s->table_name.str);
+        Check if there is modify for newly added field.
-      goto err;
+      */
+      Create_field *find;
+      find_it.rewind();
+      while((find=find_it++))
+      {
+        if (!my_strcasecmp(system_charset_info,find->field_name, def->field_name))
+          break;
+      }
+
+      if (find && !find->field)
+	find_it.remove();
+      else
+      {
+        my_error(ER_BAD_FIELD_ERROR, MYF(0), def->change,
+                 table->s->table_name.str);
+        goto err;
+      }
     }
     /*
       Check that the DATE/DATETIME not null field we are going to add is
@@ -7770,6 +7786,29 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
         find_it.after(def);			// Put column after this
       }
     }
+    /*
+      Check if there is alter for newly added field.
+    */
+    alter_it.rewind();
+    Alter_column *alter;
+    while ((alter=alter_it++))
+    {
+      if (!my_strcasecmp(system_charset_info,def->field_name, alter->name))
+        break;
+    }
+    if (alter)
+    {
+      if (def->sql_type == MYSQL_TYPE_BLOB)
+      {
+        my_error(ER_BLOB_CANT_HAVE_DEFAULT, MYF(0), def->change);
+        goto err;
+      }
+      if ((def->default_value= alter->default_value)) // Use new default
+        def->flags&= ~NO_DEFAULT_VALUE_FLAG;
+      else
+        def->flags|= NO_DEFAULT_VALUE_FLAG;
+      alter_it.remove();
+    }
   }
   if (alter_info->alter_list.elements)
   {

storage/connect/CMakeLists.txt

@@ -264,13 +264,6 @@ IF(CONNECT_WITH_JDBC)
     JdbcInterface.java ApacheInterface.java MariadbInterface.java
     MysqlInterface.java OracleInterface.java PostgresqlInterface.java
     JavaWrappers.jar)
-    # TODO: Find how to compile and install the java wrapper classes
-    # Find required libraries and include directories
-    SET (JAVA_SOURCES JdbcInterface.java)
-    add_jar(JdbcInterface ${JAVA_SOURCES})
-    INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar
-      ${CMAKE_CURRENT_BINARY_DIR}/JdbcInterface.jar
-      DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine)
     add_definitions(-DJDBC_SUPPORT)
 		IF(CONNECT_WITH_MONGO)
       SET(CONNECT_SOURCES ${CONNECT_SOURCES}
@@ -348,4 +341,31 @@ MYSQL_ADD_PLUGIN(connect ${CONNECT_SOURCES}
   LINK_LIBRARIES ${ZLIB_LIBRARY} ${XML_LIBRARY} ${ICONV_LIBRARY}
                  ${ODBC_LIBRARY} ${JDBC_LIBRARY} ${IPHLPAPI_LIBRARY})
 
+IF(NOT TARGET connect)
+  RETURN()
+ENDIF()
+
+# Install some extra files that belong to connect engine
+IF(WIN32)
+  # install ha_connect.lib
+  GET_TARGET_PROPERTY(CONNECT_LOCATION connect LOCATION)
+  STRING(REPLACE "dll" "lib" CONNECT_LIB ${CONNECT_LOCATION})
+  IF(CMAKE_CONFIGURATION_TYPES)
+    STRING(REPLACE "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}"
+      CONNECT_LIB ${CONNECT_LIB})
+  ENDIF()
+  INSTALL(FILES ${CONNECT_LIB}
+    DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine)
+ENDIF(WIN32)
+
+IF(CONNECT_WITH_JDBC AND JAVA_FOUND AND JNI_FOUND)
+  # TODO: Find how to compile and install the java wrapper classes
+  # Find required libraries and include directories
+  SET (JAVA_SOURCES JdbcInterface.java)
+  add_jar(JdbcInterface ${JAVA_SOURCES})
+  INSTALL(FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar
+    ${CMAKE_CURRENT_BINARY_DIR}/JdbcInterface.jar
+    DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine)
+ENDIF()
 

storage/innobase/buf/buf0rea.cc

@@ -478,13 +478,15 @@ buf_read_page_background(
 			<< " in the background"
 			" in a non-existing or being-dropped tablespace";
 		break;
+	case DB_PAGE_CORRUPTED:
 	case DB_DECRYPTION_FAILED:
 		ib::error()
-			<< "Background Page read failed to decrypt page "
+			<< "Background Page read failed to "
-			<< page_id;
+			"read or decrypt " << page_id;
 		break;
 	default:
-		ut_error;
+		ib::fatal() << "Error " << err << " in background read of "
+			<< page_id;
 	}
 
 	srv_stats.buf_pool_reads.add(count);
@@ -755,9 +757,10 @@ buf_read_ahead_linear(
 			case DB_TABLESPACE_DELETED:
 			case DB_ERROR:
 				break;
+			case DB_PAGE_CORRUPTED:
 			case DB_DECRYPTION_FAILED:
 				ib::error() << "linear readahead failed to"
-					" decrypt page "
+					" read or decrypt "
 					<< page_id_t(page_id.space(), i);
 				break;
 			default:
@@ -853,8 +856,9 @@ tablespace_deleted:
 			break;
 		case DB_TABLESPACE_DELETED:
 			goto tablespace_deleted;
+		case DB_PAGE_CORRUPTED:
 		case DB_DECRYPTION_FAILED:
-			ib::error() << "Failed to decrypt page " << page_id
+			ib::error() << "Failed to read or decrypt " << page_id
 				<< " for change buffer merge";
 			break;
 		default:
@@ -936,8 +940,8 @@ buf_read_recv_pages(
 				cur_page_id, page_size, true);
 		}
 
-		if (err == DB_DECRYPTION_FAILED) {
+		if (err == DB_DECRYPTION_FAILED || err == DB_PAGE_CORRUPTED) {
-			ib::error() << "Recovery failed to decrypt page "
+			ib::error() << "Recovery failed to read or decrypt "
 				<< cur_page_id;
 		}
 	}

storage/innobase/fil/fil0crypt.cc

@@ -168,7 +168,12 @@ fil_crypt_get_latest_key_version(
 				crypt_data->min_key_version,
 				key_version,
 				srv_fil_crypt_rotate_key_age)) {
-			os_event_set(fil_crypt_threads_event);
+			/* Below event seen as NULL-pointer at startup
+			when new database was created and we create a
+			checkpoint. Only seen when debugging. */
+			if (fil_crypt_threads_inited) {
+				os_event_set(fil_crypt_threads_event);
+			}
 		}
 	}
 

storage/innobase/row/row0umod.cc

@@ -34,6 +34,7 @@ Created 2/27/1997 Heikki Tuuri
 #include "trx0roll.h"
 #include "btr0btr.h"
 #include "mach0data.h"
+#include "ibuf0ibuf.h"
 #include "row0undo.h"
 #include "row0vers.h"
 #include "row0log.h"

storage/innobase/trx/trx0trx.cc

@@ -843,18 +843,9 @@ trx_resurrect_insert(
 				<< trx_get_id_for_print(trx)
 				<< " was in the XA prepared state.";
 
-			if (srv_force_recovery == 0) {
+			trx->state = TRX_STATE_PREPARED;
-
+			trx_sys->n_prepared_trx++;
-				trx->state = TRX_STATE_PREPARED;
+			trx_sys->n_prepared_recovered_trx++;
-				++trx_sys->n_prepared_trx;
-				++trx_sys->n_prepared_recovered_trx;
-			} else {
-
-				ib::info() << "Since innodb_force_recovery"
-					" > 0, we will force a rollback.";
-
-				trx->state = TRX_STATE_ACTIVE;
-			}
 		} else {
 			trx->state = TRX_STATE_COMMITTED_IN_MEMORY;
 		}
@@ -914,24 +905,14 @@ trx_resurrect_update_in_prepared_state(
 		ib::info() << "Transaction " << trx_get_id_for_print(trx)
 			<< " was in the XA prepared state.";
 
-		if (srv_force_recovery == 0) {
+		if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) {
-
+			trx_sys->n_prepared_trx++;
-			ut_ad(trx->state != TRX_STATE_FORCED_ROLLBACK);
+			trx_sys->n_prepared_recovered_trx++;
-
-			if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) {
-				++trx_sys->n_prepared_trx;
-				++trx_sys->n_prepared_recovered_trx;
-			} else {
-				ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED));
-			}
-
-			trx->state = TRX_STATE_PREPARED;
 		} else {
-			ib::info() << "Since innodb_force_recovery > 0, we"
+			ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED));
-				" will rollback it anyway.";
-
-			trx->state = TRX_STATE_ACTIVE;
 		}
+
+		trx->state = TRX_STATE_PREPARED;
 	} else {
 		trx->state = TRX_STATE_COMMITTED_IN_MEMORY;
 	}

storage/xtradb/buf/buf0rea.cc

@@ -428,13 +428,14 @@ read_ahead:
 					space, i);
 				break;
 			case DB_DECRYPTION_FAILED:
+			case DB_PAGE_CORRUPTED:
 				ib_logf(IB_LOG_LEVEL_ERROR,
-					"Random readahead failed to decrypt page "
+					"Random readahead failed to decrypt page or page corrupted "
 					ULINTPF ":" ULINTPF ".",
 					i, space);
 				break;
 			default:
-				ut_error;
+				ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in random readahead", err, ut_strerr(err));
 			}
 		}
 	}
@@ -570,13 +571,14 @@ buf_read_page_async(
 		break;
 
 	case DB_DECRYPTION_FAILED:
+	case DB_PAGE_CORRUPTED:
 		ib_logf(IB_LOG_LEVEL_ERROR,
-			"Async page read failed to decrypt page "
+			"Async page read failed to decrypt page or page corrupted "
 			ULINTPF ":" ULINTPF ".",
 			space, offset);
 		break;
 	default:
-		ut_error;
+		ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in async page read", err, ut_strerr(err));
 	}
 
 	srv_stats.buf_pool_reads.add(count);
@@ -863,13 +865,14 @@ buf_read_ahead_linear(
 				break;
 
 			case DB_DECRYPTION_FAILED:
+			case DB_PAGE_CORRUPTED:
 				ib_logf(IB_LOG_LEVEL_ERROR,
-					"Linear readahead failed to decrypt page "
+					"Linear readahead failed to decrypt page or page corrupted"
 					ULINTPF ":" ULINTPF ".",
 					i, space);
 				break;
 			default:
-				ut_error;
+				ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in linear readahead", err, ut_strerr(err));
 			}
 		}
 	}
@@ -963,13 +966,14 @@ tablespace_deleted:
 			ibuf_delete_for_discarded_space(space_ids[i]);
 			break;
 		case DB_DECRYPTION_FAILED:
+		case DB_PAGE_CORRUPTED:
 			ib_logf(IB_LOG_LEVEL_ERROR,
-				"Failed to decrypt insert buffer page "
+				"Failed to decrypt insert buffer page or page corrupted "
 				ULINTPF ":" ULINTPF ".",
 				space_ids[i], page_nos[i]);
 			break;
 		default:
-			ut_error;
+			ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in insert buffer read", err, ut_strerr(err));
 		}
 	}
 

storage/xtradb/fil/fil0crypt.cc

@@ -191,7 +191,12 @@ fil_crypt_get_latest_key_version(
 				crypt_data->min_key_version,
 				key_version,
 				srv_fil_crypt_rotate_key_age)) {
-			os_event_set(fil_crypt_threads_event);
+			/* Below event seen as NULL-pointer at startup
+			when new database was created and we create a
+			checkpoint. Only seen when debugging. */
+			if (fil_crypt_threads_inited) {
+				os_event_set(fil_crypt_threads_event);
+			}
 		}
 	}
 
@@ -1649,20 +1654,6 @@ fil_crypt_find_page_to_rotate(
 	return found;
 }
 
-/***********************************************************************
-Check if a page is uninitialized (doesn't need to be rotated)
-@param[in]	frame		Page to check
-@param[in]	zip_size	zip_size or 0
-@return true if page is uninitialized, false if not. */
-static inline
-bool
-fil_crypt_is_page_uninitialized(
-	const byte	*frame,
-	uint		zip_size)
-{
-	return (buf_page_is_zeroes(frame, zip_size));
-}
-
 #define fil_crypt_get_page_throttle(state,offset,mtr,sleeptime_ms) \
 	fil_crypt_get_page_throttle_func(state, offset, mtr, \
 					 sleeptime_ms, __FILE__, __LINE__)
@@ -1823,6 +1814,7 @@ fil_crypt_rotate_page(
 	fil_space_crypt_t *crypt_data = space->crypt_data;
 
 	ut_ad(space->n_pending_ops > 0);
+	ut_ad(offset > 0);
 
 	/* In fil_crypt_thread where key rotation is done we have
 	acquired space and checked that this space is not yet
@@ -1851,31 +1843,40 @@ fil_crypt_rotate_page(
 		byte* frame = buf_block_get_frame(block);
 		uint kv =  mach_read_from_4(frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
 
-		/* check if tablespace is closing after reading page */
+		if (space->is_stopping()) {
-		if (!space->is_stopping()) {
+			/* The tablespace is closing (in DROP TABLE or
+			TRUNCATE TABLE or similar): avoid further access */
+		} else if (!*reinterpret_cast<uint32_t*>(FIL_PAGE_OFFSET
+							 + frame)) {
+			/* It looks like this page was never
+			allocated. Because key rotation is accessing
+			pages in a pattern that is unlike the normal
+			B-tree and undo log access pattern, we cannot
+			invoke fseg_page_is_free() here, because that
+			could result in a deadlock. If we invoked
+			fseg_page_is_free() and released the
+			tablespace latch before acquiring block->lock,
+			then the fseg_page_is_free() information
+			could be stale already. */
+			ut_ad(kv == 0);
+			ut_ad(page_get_space_id(frame) == 0);
+		} else if (fil_crypt_needs_rotation(
+				   crypt_data->encryption,
+				   kv, key_state->key_version,
+				   key_state->rotate_key_age)) {
 
-			if (kv == 0 &&
+			modified = true;
-				fil_crypt_is_page_uninitialized(frame, zip_size)) {
-				;
-			} else if (fil_crypt_needs_rotation(
-					crypt_data->encryption,
-					kv, key_state->key_version,
-					key_state->rotate_key_age)) {
 
-				modified = true;
+			/* force rotation by dummy updating page */
+			mlog_write_ulint(frame + FIL_PAGE_SPACE_ID,
+					 space_id, MLOG_4BYTES, &mtr);
 
-				/* force rotation by dummy updating page */
+			/* statistics */
-				mlog_write_ulint(frame +
+			state->crypt_stat.pages_modified++;
-					FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
+		} else {
-					space_id, MLOG_4BYTES, &mtr);
+			if (crypt_data->is_encrypted()) {
-
+				if (kv < state->min_key_version_found) {
-				/* statistics */
+					state->min_key_version_found = kv;
-				state->crypt_stat.pages_modified++;
-			} else {
-				if (crypt_data->is_encrypted()) {
-					if (kv < state->min_key_version_found) {
-						state->min_key_version_found = kv;
-					}
 				}
 			}
 

storage/xtradb/fil/fil0fil.cc

@@ -2372,8 +2372,10 @@ the first page of a first data file at database startup.
 @param[out]	space_id		tablepspace ID
 @param[out]	flushed_lsn		flushed lsn value
 @param[out]	crypt_data		encryption crypt data
-@retval NULL on success, or if innodb_force_recovery is set
+@param[in]	check_first_page	true if first page contents
-@return pointer to an error message string */
+					should be checked
+@return NULL on success, or if innodb_force_recovery is set
+@retval pointer to an error message string */
 UNIV_INTERN
 const char*
 fil_read_first_page(
@@ -2382,7 +2384,8 @@ fil_read_first_page(
 	ulint*		flags,
 	ulint*		space_id,
 	lsn_t*		flushed_lsn,
-	fil_space_crypt_t**   crypt_data)
+	fil_space_crypt_t**   crypt_data,
+	bool		check_first_page)
 {
 	byte*		buf;
 	byte*		page;
@@ -2418,28 +2421,32 @@ fil_read_first_page(
 	*flags and *space_id as they were read from the first file and
 	do not validate the first page. */
 	if (!one_read_already) {
-		*space_id = fsp_header_get_space_id(page);
+		/* Undo tablespace does not contain correct FSP_HEADER,
-		*flags = fsp_header_get_flags(page);
+		and actually we really need to read only crypt_data. */
+		if (check_first_page) {
+			*space_id = fsp_header_get_space_id(page);
+			*flags = fsp_header_get_flags(page);
 
-		if (flushed_lsn) {
+			if (flushed_lsn) {
-			*flushed_lsn = mach_read_from_8(page +
+				*flushed_lsn = mach_read_from_8(page +
 				       FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
-		}
-
-		if (!fsp_flags_is_valid(*flags)) {
-			ulint cflags = fsp_flags_convert_from_101(*flags);
-			if (cflags == ULINT_UNDEFINED) {
-				ib_logf(IB_LOG_LEVEL_ERROR,
-					"Invalid flags 0x%x in tablespace %u",
-					unsigned(*flags), unsigned(*space_id));
-				return "invalid tablespace flags";
-			} else {
-				*flags = cflags;
 			}
-		}
 
-		if (!(IS_XTRABACKUP() && srv_backup_mode)) {
+			if (!fsp_flags_is_valid(*flags)) {
-			check_msg = fil_check_first_page(page, *space_id, *flags);
+				ulint cflags = fsp_flags_convert_from_101(*flags);
+				if (cflags == ULINT_UNDEFINED) {
+					ib_logf(IB_LOG_LEVEL_ERROR,
+						"Invalid flags 0x%x in tablespace %u",
+						unsigned(*flags), unsigned(*space_id));
+					return "invalid tablespace flags";
+				} else {
+					*flags = cflags;
+				}
+			}
+
+			if (!(IS_XTRABACKUP() && srv_backup_mode)) {
+				check_msg = fil_check_first_page(page, *space_id, *flags);
+			}
 		}
 
 		/* Possible encryption crypt data is also stored only to first page

storage/xtradb/fsp/fsp0fsp.cc

@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2044,15 +2045,6 @@ fseg_create_general(
 
 	mtr_x_lock(latch, mtr);
 
-	if (rw_lock_get_x_lock_count(latch) == 1) {
-		/* This thread did not own the latch before this call: free
-		excess pages from the insert buffer free list */
-
-		if (space == IBUF_SPACE_ID) {
-			ibuf_free_excess_pages();
-		}
-	}
-
 	if (!has_done_reservation) {
 		success = fsp_reserve_free_extents(&n_reserved, space, 2,
 						   FSP_NORMAL, mtr);
@@ -2623,15 +2615,6 @@ fseg_alloc_free_page_general(
 
 	mtr_x_lock(latch, mtr);
 
-	if (rw_lock_get_x_lock_count(latch) == 1) {
-		/* This thread did not own the latch before this call: free
-		excess pages from the insert buffer free list */
-
-		if (space == IBUF_SPACE_ID) {
-			ibuf_free_excess_pages();
-		}
-	}
-
 	inode = fseg_inode_get(seg_header, space, zip_size, mtr);
 
 	if (!has_done_reservation

storage/xtradb/ibuf/ibuf0ibuf.cc

@@ -2216,6 +2216,8 @@ ibuf_remove_free_page(void)
 	page_t*	root;
 	page_t*	bitmap_page;
 
+	log_free_check();
+
 	mtr_start(&mtr);
 
 	/* Acquire the fsp latch before the ibuf header, obeying the latching
@@ -2327,22 +2329,7 @@ ibuf_free_excess_pages(void)
 {
 	ulint		i;
 
-#ifdef UNIV_SYNC_DEBUG
+	if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) {
-	ut_ad(rw_lock_own(fil_space_get_latch(IBUF_SPACE_ID, NULL),
-			  RW_LOCK_EX));
-#endif /* UNIV_SYNC_DEBUG */
-
-	ut_ad(rw_lock_get_x_lock_count(
-		fil_space_get_latch(IBUF_SPACE_ID, NULL)) == 1);
-
-	/* NOTE: We require that the thread did not own the latch before,
-	because then we know that we can obey the correct latching order
-	for ibuf latches */
-
-	if (!ibuf) {
-		/* Not yet initialized; not sure if this is possible, but
-		does no harm to check for it. */
-
 		return;
 	}
 

storage/xtradb/include/fil0fil.h

@@ -804,8 +804,10 @@ the first page of a first data file at database startup.
 @param[out]	space_id		tablepspace ID
 @param[out]	flushed_lsn		flushed lsn value
 @param[out]	crypt_data		encryption crypt data
-@retval NULL on success, or if innodb_force_recovery is set
+@param[in]	check_first_page	true if first page contents
-@return pointer to an error message string */
+					should be checked
+@return NULL on success, or if innodb_force_recovery is set
+@retval pointer to an error message string */
 UNIV_INTERN
 const char*
 fil_read_first_page(
@@ -814,7 +816,8 @@ fil_read_first_page(
 	ulint*		flags,
 	ulint*		space_id,
 	lsn_t*		flushed_lsn,
-	fil_space_crypt_t**   crypt_data)
+	fil_space_crypt_t**   crypt_data,
+	bool		check_first_page=true)
 	MY_ATTRIBUTE((warn_unused_result));
 
 #endif /* !UNIV_HOTBACKUP */

storage/xtradb/row/row0ins.cc

@@ -38,6 +38,7 @@ Created 4/20/1996 Heikki Tuuri
 #include "btr0btr.h"
 #include "btr0cur.h"
 #include "mach0data.h"
+#include "ibuf0ibuf.h"
 #include "que0que.h"
 #include "row0upd.h"
 #include "row0sel.h"
@@ -3064,6 +3065,11 @@ row_ins_sec_index_entry(
 	if (err == DB_FAIL) {
 		mem_heap_empty(heap);
 
+		if (index->space == IBUF_SPACE_ID
+		    && !dict_index_is_unique(index)) {
+			ibuf_free_excess_pages();
+		}
+
 		/* Try then pessimistic descent to the B-tree */
 
 		log_free_check();

storage/xtradb/row/row0uins.cc

@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -201,6 +202,10 @@ row_undo_ins_remove_sec_low(
 		mtr_s_lock(dict_index_get_lock(index), &mtr);
 	} else {
 		ut_ad(mode == BTR_MODIFY_TREE);
+		if (index->space == IBUF_SPACE_ID
+		    && !dict_index_is_unique(index)) {
+			ibuf_free_excess_pages();
+		}
 		mtr_x_lock(dict_index_get_lock(index), &mtr);
 	}
 

storage/xtradb/row/row0umod.cc

@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -35,6 +36,7 @@ Created 2/27/1997 Heikki Tuuri
 #include "trx0roll.h"
 #include "btr0btr.h"
 #include "mach0data.h"
+#include "ibuf0ibuf.h"
 #include "row0undo.h"
 #include "row0vers.h"
 #include "row0log.h"
@@ -409,6 +411,11 @@ row_undo_mod_del_mark_or_remove_sec_low(
 
 	log_free_check();
 	mtr_start_trx(&mtr, thr_get_trx(thr));
+	if (mode == BTR_MODIFY_TREE
+	    && index->space == IBUF_SPACE_ID
+	    && !dict_index_is_unique(index)) {
+		ibuf_free_excess_pages();
+	}
 
 	if (*index->name == TEMP_INDEX_PREFIX) {
 		/* The index->online_status may change if the
@@ -581,6 +588,11 @@ row_undo_mod_del_unmark_sec_and_undo_update(
 
 	log_free_check();
 	mtr_start_trx(&mtr, thr_get_trx(thr));
+	if (mode == BTR_MODIFY_TREE
+	    && index->space == IBUF_SPACE_ID
+	    && !dict_index_is_unique(index)) {
+		ibuf_free_excess_pages();
+	}
 
 	if (*index->name == TEMP_INDEX_PREFIX) {
 		/* The index->online_status may change if the

storage/xtradb/srv/srv0start.cc

@@ -891,7 +891,7 @@ open_or_create_data_files(
 	bool		one_created	= false;
 	os_offset_t	size;
 	ulint		flags;
-	ulint		space;
+	ulint		space = 0;
 	ulint		rounded_size_pages;
 	char		name[10000];
 	fil_space_crypt_t*    crypt_data=NULL;
@@ -1369,11 +1369,30 @@ srv_undo_tablespace_open(
 		size = os_file_get_size(fh);
 		ut_a(size != (os_offset_t) -1);
 
+		/* Load the tablespace into InnoDB's internal
+		data structures. */
+
+		const char* check_msg;
+		fil_space_crypt_t* crypt_data = NULL;
+
+		/* Set the compressed page size to 0 (non-compressed) */
+		flags = FSP_FLAGS_PAGE_SSIZE();
+
+		/* Read first page to find out does the crypt_info
+		exists on undo tablespace. */
+		check_msg = fil_read_first_page(
+				fh, FALSE, &flags, &space,
+				NULL, &crypt_data, false);
+
 		ret = os_file_close(fh);
 		ut_a(ret);
 
-		/* Load the tablespace into InnoDB's internal
+		if (check_msg) {
-		data structures. */
+			ib_logf(IB_LOG_LEVEL_ERROR,
+				"%s in data file %s",
+				check_msg, name);
+			return (err);
+		}
 
 		/* We set the biggest space id to the undo tablespace
 		because InnoDB hasn't opened any other tablespace apart
@@ -1381,10 +1400,8 @@ srv_undo_tablespace_open(
 
 		fil_set_max_space_id_if_bigger(space);
 
-		/* Set the compressed page size to 0 (non-compressed) */
-		flags = FSP_FLAGS_PAGE_SSIZE();
 		fil_space_create(name, space, flags, FIL_TABLESPACE,
-				NULL /* no encryption */,
+				crypt_data,
 				true /* create */);
 
 		ut_a(fil_validate());
@@ -1491,6 +1508,18 @@ srv_undo_tablespaces_init(
 		n_undo_tablespaces = n_conf_tablespaces;
 
 		undo_tablespace_ids[n_conf_tablespaces] = ULINT_UNDEFINED;
+
+		if (backup_mode) {
+			ut_ad(!create_new_db);
+			/* MDEV-13561 FIXME: Determine srv_undo_space_id_start
+			from the undo001 file. */
+			srv_undo_space_id_start = 1;
+
+			for (i = 0; i < n_undo_tablespaces; i++) {
+				undo_tablespace_ids[i]
+					= i + srv_undo_space_id_start;
+			}
+		}
 	}
 
 	/* Open all the undo tablespaces that are currently in use. If we

storage/xtradb/trx/trx0trx.cc

@@ -720,25 +720,9 @@ trx_resurrect_insert(
 				"InnoDB: Transaction " TRX_ID_FMT " was in the"
 				" XA prepared state.\n", trx->id);
 
-			if (srv_force_recovery == 0) {
+			trx->state = TRX_STATE_PREPARED;
-
+			trx_sys->n_prepared_trx++;
-				/* XtraBackup should rollback prepared XA
+			trx_sys->n_prepared_recovered_trx++;
-				transactions */
-				if (IS_XTRABACKUP()) {
-					trx->state = TRX_STATE_ACTIVE;
-				}
-				else {
-					trx->state = TRX_STATE_PREPARED;
-					trx_sys->n_prepared_trx++;
-					trx_sys->n_prepared_recovered_trx++;
-				}
-			} else {
-				fprintf(stderr,
-					"InnoDB: Since innodb_force_recovery"
-					" > 0, we will rollback it anyway.\n");
-
-				trx->state = TRX_STATE_ACTIVE;
-			}
 		} else {
 			trx->state = TRX_STATE_COMMITTED_IN_MEMORY;
 		}
@@ -796,25 +780,14 @@ trx_resurrect_update_in_prepared_state(
 			"InnoDB: Transaction " TRX_ID_FMT
 			" was in the XA prepared state.\n", trx->id);
 
-		if (srv_force_recovery == 0) {
+		if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) {
-			if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) {
+			trx_sys->n_prepared_trx++;
-				if (!IS_XTRABACKUP()) {
+			trx_sys->n_prepared_recovered_trx++;
-					trx_sys->n_prepared_trx++;
-					trx_sys->n_prepared_recovered_trx++;
-				}
-			} else {
-				ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED));
-			}
-			/* XtraBackup should rollback prepared XA
-			transactions */
-			trx->state = IS_XTRABACKUP()?TRX_STATE_ACTIVE: TRX_STATE_PREPARED;
 		} else {
-			fprintf(stderr,
+			ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED));
-				"InnoDB: Since innodb_force_recovery"
-				" > 0, we will rollback it anyway.\n");
-
-			trx->state = TRX_STATE_ACTIVE;
 		}
+
+		trx->state = TRX_STATE_PREPARED;
 	} else {
 		trx->state = TRX_STATE_COMMITTED_IN_MEMORY;
 	}

win/packaging/create_msi.cmake.in

@@ -78,13 +78,6 @@ ELSE()
 ENDIF()
 
 SET(ENV{VS_UNICODE_OUTPUT})
-# Workaround for CMake bug#11452
-# Switch off the monolithic install
-EXECUTE_PROCESS(
-  COMMAND ${CMAKE_COMMAND} -DCPACK_MONOLITHIC_INSTALL=0 ${CMAKE_BINARY_DIR}
-  OUTPUT_QUIET
-)
-
 
 INCLUDE(${CMAKE_BINARY_DIR}/CPackConfig.cmake)
 
@@ -441,11 +434,4 @@ ENDIF()
 CONFIGURE_FILE(${CPACK_PACKAGE_FILE_NAME}.msi 
  ${CMAKE_BINARY_DIR}/${CPACK_PACKAGE_FILE_NAME}.msi 
   COPYONLY)
-  
-# Workaround for CMake bug#11452
-# Switch monolithic install on again
-EXECUTE_PROCESS(
-  COMMAND ${CMAKE_COMMAND} -DCPACK_MONOLITHIC_INSTALL=1 ${CMAKE_BINARY_DIR}
-  OUTPUT_QUIET
-)