Merging upstream version 1:10.11.7.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

18 files changed, 171 insertions, 134 deletions

diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h
index 5a0401fa..b42c543c 100644
--- a/storage/innobase/include/btr0btr.h
+++ b/storage/innobase/include/btr0btr.h
@@ -89,10 +89,12 @@ ATTRIBUTE_COLD void btr_decryption_failed(const dict_index_t &index);
 @param[in]	merge	whether change buffer merge should be attempted
 @param[in,out]	mtr	mini-transaction
 @param[out]	err	error code
+@param[out]	first	set if this is a first-time access to the page
 @return block */
 buf_block_t *btr_block_get(const dict_index_t &index,
                            uint32_t page, rw_lock_type_t mode, bool merge,
-                           mtr_t *mtr, dberr_t *err= nullptr);
+                           mtr_t *mtr, dberr_t *err= nullptr,
+                           bool *first= nullptr);
 
 /**************************************************************//**
 Gets the index id field of a page.
diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h
index 332b2039..cd7cc294 100644
--- a/storage/innobase/include/buf0buf.h
+++ b/storage/innobase/include/buf0buf.h
@@ -262,8 +262,6 @@ buf_block_t*
 buf_page_create_deferred(uint32_t space_id, ulint zip_size, mtr_t *mtr,
                          buf_block_t *free_block);
 
-/** Move a block to the start of the LRU list. */
-void buf_page_make_young(buf_page_t *bpage);
 /** Mark the page status as FREED for the given tablespace and page number.
 @param[in,out]	space	tablespace
 @param[in]	page	page number
@@ -285,15 +283,6 @@ there is danger of dropping from the buffer pool.
 @return true if bpage should be made younger */
 inline bool buf_page_peek_if_too_old(const buf_page_t *bpage);
 
-/** Move a page to the start of the buffer pool LRU list if it is too old.
-@param[in,out]	bpage		buffer pool page */
-inline void buf_page_make_young_if_needed(buf_page_t *bpage)
-{
-	if (UNIV_UNLIKELY(buf_page_peek_if_too_old(bpage))) {
-		buf_page_make_young(bpage);
-	}
-}
-
 /********************************************************************//**
 Increments the modify clock of a frame by 1. The caller must (1) own the
 buf_pool.mutex and block bufferfix count has to be zero, (2) or own an x-lock
@@ -656,12 +645,9 @@ public:
     access_time= 0;
   }
 
-  void set_os_unused()
+  void set_os_unused() const
   {
     MEM_NOACCESS(frame, srv_page_size);
-#ifdef MADV_FREE
-    madvise(frame, srv_page_size, MADV_FREE);
-#endif
   }
 
   void set_os_used() const
@@ -1301,6 +1287,11 @@ public:
   /** Resize from srv_buf_pool_old_size to srv_buf_pool_size. */
   inline void resize();
 
+#ifdef __linux__
+  /** Collect garbage (release pages from the LRU list) */
+  inline void garbage_collect();
+#endif
+
   /** @return whether resize() is in progress */
   bool resize_in_progress() const
   {
@@ -1507,10 +1498,8 @@ public:
         n_chunks_new / 4 * chunks->size;
   }
 
-  /** @return whether the buffer pool has run out */
-  TPOOL_SUPPRESS_TSAN
-  bool ran_out() const
-  { return UNIV_UNLIKELY(!try_LRU_scan || !UT_LIST_GET_LEN(free)); }
+  /** @return whether the buffer pool is running low */
+  bool need_LRU_eviction() const;
 
   /** @return whether the buffer pool is shrinking */
   inline bool is_shrinking() const
@@ -1836,6 +1825,9 @@ public:
   Set whenever the free list grows, along with a broadcast of done_free.
   Protected by buf_pool.mutex. */
   Atomic_relaxed<bool> try_LRU_scan;
+  /** Whether we have warned to be running out of buffer pool */
+  std::atomic_flag LRU_warned;
+
 	/* @} */
 
 	/** @name LRU replacement algorithm fields */
@@ -1898,7 +1890,8 @@ public:
   a delete-buffering operation is pending. Protected by mutex. */
   buf_page_t watch[innodb_purge_threads_MAX + 1];
   /** Reserve a buffer. */
-  buf_tmp_buffer_t *io_buf_reserve() { return io_buf.reserve(); }
+  buf_tmp_buffer_t *io_buf_reserve(bool wait_for_reads)
+  { return io_buf.reserve(wait_for_reads); }
 
   /** Remove a block from flush_list.
   @param bpage   buffer pool page */
@@ -1933,7 +1926,7 @@ private:
     void close();
 
     /** Reserve a buffer */
-    buf_tmp_buffer_t *reserve();
+    buf_tmp_buffer_t *reserve(bool wait_for_reads);
   } io_buf;
 
   /** whether resize() is in the critical path */
diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h
index 9932b0e5..6e7662d9 100644
--- a/storage/innobase/include/buf0dblwr.h
+++ b/storage/innobase/include/buf0dblwr.h
@@ -105,7 +105,8 @@ public:
   If we are upgrading from a version before MySQL 4.1, then this
   function performs the necessary update operations to support
   innodb_file_per_table. If we are in a crash recovery, this function
-  loads the pages from double write buffer into memory.
+  loads the pages from double write buffer which are not older than
+  the checkpoint into memory.
   @param file File handle
   @param path Path name of file
   @return DB_SUCCESS or error code */
diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h
index aec08e77..28410276 100644
--- a/storage/innobase/include/buf0lru.h
+++ b/storage/innobase/include/buf0lru.h
@@ -108,6 +108,16 @@ buf_LRU_add_block(
 				blocks in the LRU list, else put to the
 				start; if the LRU list is very short, added to
 				the start regardless of this parameter */
+
+/** Move a block to the start of the buf_pool.LRU list.
+@param bpage  buffer pool page */
+void buf_page_make_young(buf_page_t *bpage);
+/** Flag a page accessed in buf_pool and move it to the start of buf_pool.LRU
+if it is too old.
+@param bpage  buffer pool page
+@return whether this is not the first access */
+bool buf_page_make_young_if_needed(buf_page_t *bpage);
+
 /******************************************************************//**
 Adds a block to the LRU list of decompressed zip pages. */
 void
diff --git a/storage/innobase/include/dict0load.h b/storage/innobase/include/dict0load.h
index f7d33d5b..3143aafd 100644
--- a/storage/innobase/include/dict0load.h
+++ b/storage/innobase/include/dict0load.h
@@ -35,22 +35,16 @@ Created 4/24/1996 Heikki Tuuri
 #include "btr0types.h"
 
 #include <deque>
+#include <set>
 
 /** A stack of table names related through foreign key constraints */
 typedef std::deque<const char*, ut_allocator<const char*> >	dict_names_t;
 
-/** Check each tablespace found in the data dictionary.
-Then look at each table defined in SYS_TABLES that has a space_id > 0
-to find all the file-per-table tablespaces.
+/** Check MAX(SPACE) FROM SYS_TABLES and store it in fil_system.
+Open each data file if an encryption plugin has been loaded.
 
-In a crash recovery we already have some tablespace objects created from
-processing the REDO log. We will compare the
-space_id information in the data dictionary to what we find in the
-tablespace file. In addition, more validation will be done if recovery
-was needed and force_recovery is not set.
-
-We also scan the biggest space id, and store it to fil_system. */
-void dict_check_tablespaces_and_store_max_id();
+@param spaces  set of tablespace files to open */
+void dict_check_tablespaces_and_store_max_id(const std::set<uint32_t> *spaces);
 
 /** Make sure the data_file_name is saved in dict_table_t if needed.
 @param[in,out]	table		Table object */
diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h
index 0dc1b984..3b006daf 100644
--- a/storage/innobase/include/dict0stats.h
+++ b/storage/innobase/include/dict0stats.h
@@ -235,4 +235,13 @@ dict_stats_report_error(dict_table_t* table, bool defragment = false)
 void test_dict_stats_all();
 #endif /* UNIV_ENABLE_UNIT_TEST_DICT_STATS */
 
+/** Write all zeros (or 1 where it makes sense) into a table
+and its indexes'statistics members. The resulting stats
+correspond to an empty table.
+@param	table			table stats to be emptied
+@param  empty_defrag_stats	empty the defrag stats */
+void
+dict_stats_empty_table(
+	dict_table_t*	table,
+	bool		empty_defrag_stats);
 #endif /* dict0stats_h */
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index 6f58e3c1..cdc32515 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -359,8 +359,6 @@ struct fil_space_t final
   lsn_t max_lsn;
   /** tablespace identifier */
   uint32_t id;
-	/** whether undo tablespace truncation is in progress */
-	bool		is_being_truncated;
 	fil_type_t	purpose;/*!< purpose */
 	UT_LIST_BASE_NODE_T(fil_node_t) chain;
 				/*!< base node for the file chain */
@@ -440,6 +438,8 @@ private:
   /** LSN of freeing last page; protected by freed_range_mutex */
   lsn_t last_freed_lsn;
 
+  /** LSN of undo tablespace creation or 0; protected by latch */
+  lsn_t create_lsn;
 public:
   /** @return whether doublewrite buffering is needed */
   inline bool use_doublewrite() const;
@@ -447,6 +447,12 @@ public:
   /** @return whether a page has been freed */
   inline bool is_freed(uint32_t page);
 
+  /** Set create_lsn. */
+  inline void set_create_lsn(lsn_t lsn);
+
+  /** @return the latest tablespace rebuild LSN, or 0 */
+  lsn_t get_create_lsn() const { return create_lsn; }
+
   /** Apply freed_ranges to the file.
   @param writable whether the file is writable
   @return number of pages written or hole-punched */
@@ -524,9 +530,6 @@ public:
   /** Note that operations on the tablespace must stop. */
   inline void set_stopping();
 
-  /** Note that operations on the tablespace can resume after truncation */
-  inline void clear_stopping();
-
   /** Drop the tablespace and wait for any pending operations to cease
   @param id               tablespace identifier
   @param detached_handle  pointer to file to be closed later, or nullptr
@@ -1555,14 +1558,6 @@ inline void fil_space_t::set_stopping()
 #endif
 }
 
-inline void fil_space_t::clear_stopping()
-{
-  mysql_mutex_assert_owner(&fil_system.mutex);
-  static_assert(STOPPING_WRITES == 1U << 30, "compatibility");
-  ut_d(auto n=) n_pending.fetch_sub(STOPPING_WRITES, std::memory_order_relaxed);
-  ut_ad((n & STOPPING) == STOPPING_WRITES);
-}
-
 /** Flush pending writes from the file system cache to the file. */
 template<bool have_reference> inline void fil_space_t::flush()
 {
@@ -1802,7 +1797,7 @@ bool fil_comp_algo_loaded(ulint comp_algo);
 and write out FILE_MODIFY if needed, and write FILE_CHECKPOINT.
 @param lsn  checkpoint LSN
 @return current LSN */
-lsn_t fil_names_clear(lsn_t lsn);
+ATTRIBUTE_COLD lsn_t fil_names_clear(lsn_t lsn);
 
 #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH
 void test_make_filepath();
diff --git a/storage/innobase/include/fts0priv.inl b/storage/innobase/include/fts0priv.inl
index da14cfcb..3cb09c92 100644
--- a/storage/innobase/include/fts0priv.inl
+++ b/storage/innobase/include/fts0priv.inl
@@ -34,29 +34,6 @@ fts_write_object_id(
 	ib_id_t		id,		/* in: a table/index id */
 	char*		str)		/* in: buffer to write the id to */
 {
-
-#ifdef _WIN32
-
-	DBUG_EXECUTE_IF("innodb_test_wrong_non_windows_fts_aux_table_name",
-			return(sprintf(str, UINT64PFx, id)););
-
-	/* Use this to construct old(5.6.14 and 5.7.3) windows
-	ambiguous aux table names */
-	DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name",
-			return(sprintf(str, "%016llu", (ulonglong) id)););
-
-#else /* _WIN32 */
-
-	/* Use this to construct old(5.6.14 and 5.7.3) windows
-	ambiguous aux table names */
-	DBUG_EXECUTE_IF("innodb_test_wrong_windows_fts_aux_table_name",
-			return(sprintf(str, "%016llu", (ulonglong) id)););
-
-	DBUG_EXECUTE_IF("innodb_test_wrong_fts_aux_table_name",
-			return(sprintf(str, "%016llx", (ulonglong) id)););
-
-#endif /* _WIN32 */
-
 	return(sprintf(str, "%016llx", (ulonglong) id));
 }
 
diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h
index f873eabf..54851ca0 100644
--- a/storage/innobase/include/log0log.h
+++ b/storage/innobase/include/log0log.h
@@ -79,13 +79,6 @@ ATTRIBUTE_COLD void log_make_checkpoint();
 /** Make a checkpoint at the latest lsn on shutdown. */
 ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown();
 
-/**
-Checks that there is enough free space in the log to start a new query step.
-Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this
-function may only be called if the calling thread owns no synchronization
-objects! */
-ATTRIBUTE_COLD void log_check_margins();
-
 /******************************************************//**
 Prints info of the log. */
 void
@@ -179,24 +172,33 @@ private:
   std::atomic<lsn_t> flushed_to_disk_lsn;
   /** log sequence number when log resizing was initiated, or 0 */
   std::atomic<lsn_t> resize_lsn;
-  /** set when there may be need to flush the log buffer, or
-  preflush buffer pool pages, or initiate a log checkpoint.
+  /** set when there may be need to initiate a log checkpoint.
   This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */
-  std::atomic<bool> check_flush_or_checkpoint_;
-
+  std::atomic<bool> need_checkpoint;
 
 #if defined(__aarch64__)
-/* On ARM, we do more spinning */
-typedef srw_spin_lock log_rwlock_t;
-#define LSN_LOCK_ATTR MY_MUTEX_INIT_FAST
+  /* On ARM, we do more spinning */
+  typedef srw_spin_lock log_rwlock;
+  typedef pthread_mutex_wrapper<true> log_lsn_lock;
 #else
-typedef srw_lock log_rwlock_t;
-#define LSN_LOCK_ATTR nullptr
+  typedef srw_lock log_rwlock;
+  typedef srw_mutex log_lsn_lock;
 #endif
 
 public:
-  /** rw-lock protecting buf */
-  alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock_t latch;
+  /** rw-lock protecting writes to buf; normal mtr_t::commit()
+  outside any log checkpoint is covered by a shared latch */
+  alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock latch;
+private:
+  /** mutex protecting buf_free et al, together with latch */
+  log_lsn_lock lsn_lock;
+public:
+  /** first free offset within buf use; protected by lsn_lock */
+  Atomic_relaxed<size_t> buf_free;
+  /** number of write requests (to buf); protected by lsn_lock */
+  size_t write_to_buf;
+  /** number of append_prepare_wait(); protected by lsn_lock */
+  size_t waits;
 private:
   /** Last written LSN */
   lsn_t write_lsn;
@@ -227,20 +229,12 @@ private:
   /** Buffer for writing to resize_log; @see flush_buf */
   byte *resize_flush_buf;
 
-  /** spin lock protecting lsn, buf_free in append_prepare() */
-  alignas(CPU_LEVEL1_DCACHE_LINESIZE) pthread_mutex_t lsn_lock;
-  void init_lsn_lock() { pthread_mutex_init(&lsn_lock, LSN_LOCK_ATTR); }
-  void lock_lsn() { pthread_mutex_lock(&lsn_lock); }
-  void unlock_lsn() { pthread_mutex_unlock(&lsn_lock); }
-  void destroy_lsn_lock() { pthread_mutex_destroy(&lsn_lock); }
+  void init_lsn_lock() {lsn_lock.init(); }
+  void lock_lsn() { lsn_lock.wr_lock(); }
+  void unlock_lsn() {lsn_lock.wr_unlock(); }
+  void destroy_lsn_lock() { lsn_lock.destroy(); }
 
 public:
-  /** first free offset within buf use; protected by lsn_lock */
-  Atomic_relaxed<size_t> buf_free;
-  /** number of write requests (to buf); protected by exclusive lsn_lock */
-  ulint write_to_buf;
-  /** number of waits in append_prepare(); protected by lsn_lock */
-  ulint waits;
   /** recommended maximum size of buf, after which the buffer is flushed */
   size_t max_buf_free;
 
@@ -308,6 +302,9 @@ public:
 
   bool is_opened() const noexcept { return log.is_opened(); }
 
+  /** @return target write LSN to react on buf_free >= max_buf_free */
+  inline lsn_t get_write_target() const;
+
   /** @return LSN at which log resizing was started and is still in progress
       @retval 0 if no log resizing is in progress */
   lsn_t resize_in_progress() const noexcept
@@ -419,13 +416,14 @@ public:
   inline void persist(lsn_t lsn) noexcept;
 #endif
 
-  bool check_flush_or_checkpoint() const
+  bool check_for_checkpoint() const
+  {
+    return UNIV_UNLIKELY(need_checkpoint.load(std::memory_order_relaxed));
+  }
+  void set_check_for_checkpoint(bool need= true)
   {
-    return UNIV_UNLIKELY
-      (check_flush_or_checkpoint_.load(std::memory_order_relaxed));
+    need_checkpoint.store(need, std::memory_order_relaxed);
   }
-  void set_check_flush_or_checkpoint(bool flag= true)
-  { check_flush_or_checkpoint_.store(flag, std::memory_order_relaxed); }
 
   /** Make previous write_buf() durable and update flushed_to_disk_lsn. */
   bool flush(lsn_t lsn) noexcept;
@@ -446,8 +444,9 @@ public:
 
 private:
   /** Wait in append_prepare() for buffer to become available
+  @param lsn  log sequence number to write up to
   @param ex   whether log_sys.latch is exclusively locked */
-  ATTRIBUTE_COLD static void append_prepare_wait(bool ex) noexcept;
+  ATTRIBUTE_COLD void append_prepare_wait(lsn_t lsn, bool ex) noexcept;
 public:
   /** Reserve space in the log buffer for appending data.
   @tparam pmem  log_sys.is_pmem()
diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h
index 6d75e15a..a73b7279 100644
--- a/storage/innobase/include/log0recv.h
+++ b/storage/innobase/include/log0recv.h
@@ -44,6 +44,11 @@ ATTRIBUTE_COLD MY_ATTRIBUTE((nonnull, warn_unused_result))
 @return whether the page was recovered correctly */
 bool recv_recover_page(fil_space_t* space, buf_page_t* bpage);
 
+/** Read the latest checkpoint information from log file
+and store it in log_sys.next_checkpoint and recv_sys.file_checkpoint
+@return error code or DB_SUCCESS */
+dberr_t recv_recovery_read_checkpoint();
+
 /** Start recovering from a redo log checkpoint.
 of first system tablespace page
 @return error code or DB_SUCCESS */
@@ -114,7 +119,19 @@ struct recv_dblwr_t
   @param name      tablespace filepath
   @param file      tablespace file handle
   @return whether the operation failed */
-  bool restore_first_page(uint32_t space_id, const char *name, os_file_t file);
+  bool restore_first_page(uint32_t space_id, const char *name,
+                          pfs_os_file_t file);
+
+  /** Restore the first page of the given tablespace from
+  doublewrite buffer.
+  1) Find the page which has page_no as 0
+  2) Read first 3 pages from tablespace file
+  3) Compare the space_ids from the pages with page0 which
+  was retrieved from doublewrite buffer
+  @param name tablespace filepath
+  @param file tablespace file handle
+  @return space_id or 0 in case of error */
+  uint32_t find_first_page(const char *name, pfs_os_file_t file);
 
   typedef std::deque<byte*, ut_allocator<byte*> > list;
 
diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h
index 841cfab1..c916edc9 100644
--- a/storage/innobase/include/mtr0mtr.h
+++ b/storage/innobase/include/mtr0mtr.h
@@ -89,8 +89,9 @@ struct mtr_t {
   { auto s= m_memo.size(); rollback_to_savepoint(s - 1, s); }
 
   /** Commit a mini-transaction that is shrinking a tablespace.
-  @param space   tablespace that is being shrunk */
-  ATTRIBUTE_COLD void commit_shrink(fil_space_t &space);
+  @param space   tablespace that is being shrunk
+  @param size    new size in pages */
+  ATTRIBUTE_COLD void commit_shrink(fil_space_t &space, uint32_t size);
 
   /** Commit a mini-transaction that is deleting or renaming a file.
   @param space           tablespace that is being renamed or deleted
@@ -105,7 +106,7 @@ struct mtr_t {
   This is to be used at log_checkpoint().
   @param checkpoint_lsn   the log sequence number of a checkpoint, or 0
   @return current LSN */
-  lsn_t commit_files(lsn_t checkpoint_lsn= 0);
+  ATTRIBUTE_COLD lsn_t commit_files(lsn_t checkpoint_lsn= 0);
 
   /** @return mini-transaction savepoint (current size of m_memo) */
   ulint get_savepoint() const
diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h
index c9db6a1f..c8374515 100644
--- a/storage/innobase/include/os0file.h
+++ b/storage/innobase/include/os0file.h
@@ -142,9 +142,11 @@ static const ulint OS_FILE_NORMAL = 62;
 /* @} */
 
 /** Types for file create @{ */
-static const ulint OS_DATA_FILE = 100;
-static const ulint OS_LOG_FILE = 101;
-static const ulint OS_DATA_FILE_NO_O_DIRECT = 103;
+static constexpr ulint OS_DATA_FILE = 100;
+static constexpr ulint OS_LOG_FILE = 101;
+#if defined _WIN32 || defined HAVE_FCNTL_DIRECT
+static constexpr ulint OS_DATA_FILE_NO_O_DIRECT = 103;
+#endif
 /* @} */
 
 /** Error codes from os_file_get_last_error @{ */
@@ -373,7 +375,7 @@ os_file_create_simple_no_error_handling_func(
 	bool*		success)
 	MY_ATTRIBUTE((warn_unused_result));
 
-#ifdef  _WIN32
+#ifndef HAVE_FCNTL_DIRECT
 #define os_file_set_nocache(fd, file_name, operation_name) do{}while(0)
 #else
 /** Tries to disable OS caching on an opened file descriptor.
diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h
index db846795..457d9ab5 100644
--- a/storage/innobase/include/srv0srv.h
+++ b/storage/innobase/include/srv0srv.h
@@ -210,14 +210,11 @@ extern unsigned long long	srv_max_undo_log_size;
 extern uint	srv_n_fil_crypt_threads;
 extern uint	srv_n_fil_crypt_threads_started;
 
-/** Rate at which UNDO records should be purged. */
-extern ulong	srv_purge_rseg_truncate_frequency;
-
 /** Enable or Disable Truncate of UNDO tablespace. */
 extern my_bool	srv_undo_log_truncate;
 
 /** Default size of UNDO tablespace (10MiB for innodb_page_size=16k) */
-constexpr ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES= (10U << 20) /
+constexpr uint32_t SRV_UNDO_TABLESPACE_SIZE_IN_PAGES= (10U << 20) /
   UNIV_PAGE_SIZE_DEF;
 
 extern char*	srv_log_group_home_dir;
diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h
index 1dca0cc1..01067322 100644
--- a/storage/innobase/include/srw_lock.h
+++ b/storage/innobase/include/srw_lock.h
@@ -34,7 +34,6 @@ this program; if not, write to the Free Software Foundation, Inc.,
 # define SUX_LOCK_GENERIC /* Use dummy implementation for debugging purposes */
 #endif
 
-#ifdef SUX_LOCK_GENERIC
 /** An exclusive-only variant of srw_lock */
 template<bool spinloop>
 class pthread_mutex_wrapper final
@@ -70,7 +69,6 @@ template<>
 inline void pthread_mutex_wrapper<true>::wr_lock()
 { if (!wr_lock_try()) wr_wait(); }
 # endif
-#endif
 
 /** Futex-based mutex */
 template<bool spinloop>
@@ -541,7 +539,7 @@ public:
   /** @return whether any lock may be held by any thread */
   bool is_locked_or_waiting() const noexcept
   { return lock.is_locked_or_waiting(); }
-  /** @return whether an exclusive lock may be held by any thread */
+  /** @return whether a shared or exclusive lock may be held by any thread */
   bool is_locked() const noexcept { return lock.is_locked(); }
   /** @return whether an exclusive lock may be held by any thread */
   bool is_write_locked() const noexcept { return lock.is_write_locked(); }
diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h
index 3ddd2e98..0f4f8afa 100644
--- a/storage/innobase/include/trx0purge.h
+++ b/storage/innobase/include/trx0purge.h
@@ -140,6 +140,15 @@ private:
   bool m_initialized{false};
   /** whether purge is enabled; protected by latch and std::atomic */
   std::atomic<bool> m_enabled{false};
+  /** The primary candidate for iterator::free_history() is
+  rseg=trx_sys.rseg_array[skipped_rseg]. This field may be changed
+  after invoking rseg.set_skip_allocation() and rseg.clear_skip_allocation()
+  and while holding the exclusive rseg.latch.
+
+  This may only be 0 if innodb_undo_tablespaces=0, because rollback segment
+  0 always resides in the system tablespace and would never be used when
+  dedicated undo tablespaces are in use. */
+  Atomic_relaxed<uint8_t> skipped_rseg;
 public:
   /** whether purge is active (may hold table handles) */
   std::atomic<bool> m_active{false};
@@ -197,6 +206,11 @@ public:
 			return undo_no <= other.undo_no;
 		}
 
+		/** Remove unnecessary history data from a rollback segment.
+		@param rseg   rollback segment
+		@return error code */
+		inline dberr_t free_history_rseg(trx_rseg_t &rseg) const;
+
 		/** Free the undo pages up to this. */
 		dberr_t free_history() const;
 
@@ -240,14 +254,15 @@ public:
 					by the pq_mutex */
 	mysql_mutex_t	pq_mutex;	/*!< Mutex protecting purge_queue */
 
-	/** Undo tablespace file truncation (only accessed by the
-	srv_purge_coordinator_thread) */
-	struct {
-		/** The undo tablespace that is currently being truncated */
-		fil_space_t*	current;
-		/** The undo tablespace that was last truncated */
-		fil_space_t*	last;
-	} truncate;
+  /** innodb_undo_log_truncate=ON state;
+  only modified by purge_coordinator_callback() */
+  struct {
+    /** The undo tablespace that is currently being truncated */
+    Atomic_relaxed<fil_space_t*> current;
+    /** The number of the undo tablespace that was last truncated,
+    relative from srv_undo_space_id_start */
+    uint32_t last;
+  } truncate_undo_space;
 
   /** Create the instance */
   void create();
@@ -357,6 +372,26 @@ public:
     typically via purge_sys_t::view_guard. */
     return view.sees(id);
   }
+
+private:
+  /** Enable the use of a rollback segment and advance skipped_rseg,
+  after iterator::free_history_rseg() had invoked
+  rseg.set_skip_allocation(). */
+  inline void rseg_enable(trx_rseg_t &rseg);
+
+  /** Try to start truncating a tablespace.
+  @param id        undo tablespace identifier
+  @param size      the maximum desired undo tablespace size, in pages
+  @return undo tablespace whose truncation was started
+  @retval nullptr  if truncation is not currently possible */
+  inline fil_space_t *undo_truncate_try(uint32_t id, uint32_t size);
+public:
+  /** Check if innodb_undo_log_truncate=ON needs to be handled.
+  This is only to be called by purge_coordinator_callback().
+  @return undo tablespace chosen by innodb_undo_log_truncate=ON
+  @retval nullptr if truncation is not currently possible */
+  fil_space_t *truncating_tablespace();
+
   /** A wrapper around trx_sys_t::clone_oldest_view(). */
   template<bool also_end_view= false>
   void clone_oldest_view()
diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h
index 43e0c290..7fa43047 100644
--- a/storage/innobase/include/trx0rseg.h
+++ b/storage/innobase/include/trx0rseg.h
@@ -73,14 +73,15 @@ private:
   /** Reference counter to track is_persistent() transactions,
   with SKIP flag. */
   std::atomic<uint32_t> ref;
-
+public:
   /** Whether undo tablespace truncation is pending */
   static constexpr uint32_t SKIP= 1;
   /** Transaction reference count multiplier */
   static constexpr uint32_t REF= 2;
 
+  /** @return the reference count and flags */
   uint32_t ref_load() const { return ref.load(std::memory_order_relaxed); }
-
+private:
   /** Set the SKIP bit */
   void ref_set_skip()
   {
diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h
index 5dd0169f..3fa41fdf 100644
--- a/storage/innobase/include/trx0sys.h
+++ b/storage/innobase/include/trx0sys.h
@@ -902,8 +902,8 @@ public:
   uint64_t recovered_binlog_offset;
   /** Latest recovered binlog file name */
   char recovered_binlog_filename[TRX_SYS_MYSQL_LOG_NAME_LEN];
-  /** FIL_PAGE_LSN of the page with the latest recovered binlog metadata */
-  lsn_t recovered_binlog_lsn;
+  /** Set when latest position is from pre-version 10.3.5 TRX_SYS. */
+  bool recovered_binlog_is_legacy_pos;
 
 
   /**
@@ -1191,6 +1191,11 @@ public:
     return count;
   }
 
+  /** Disable further allocation of transactions in a rollback segment
+  that are subject to innodb_undo_log_truncate=ON
+  @param space   undo tablespace that will be truncated */
+  inline void undo_truncate_start(fil_space_t &space);
+
   /** Set the undo log empty value */
   void set_undo_non_empty(bool val)
   {
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index 3cfbe331..0a3e0d62 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -1108,6 +1108,7 @@ public:
   {
     ut_ad(state == TRX_STATE_NOT_STARTED);
     ut_ad(!id);
+    ut_ad(!*detailed_error);
     ut_ad(!mutex_is_owner());
     ut_ad(!has_logged());
     ut_ad(!is_referenced());