diff options
Diffstat (limited to '')
576 files changed, 6236 insertions, 5111 deletions
diff --git a/storage/archive/CMakeLists.txt b/storage/archive/CMakeLists.txt index 5b6818fc..5c7b6aa4 100644 --- a/storage/archive/CMakeLists.txt +++ b/storage/archive/CMakeLists.txt @@ -14,5 +14,5 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA SET(ARCHIVE_SOURCES azio.c ha_archive.cc ha_archive.h) -MYSQL_ADD_PLUGIN(archive ${ARCHIVE_SOURCES} STORAGE_ENGINE LINK_LIBRARIES ${ZLIB_LIBRARY}) +MYSQL_ADD_PLUGIN(archive ${ARCHIVE_SOURCES} STORAGE_ENGINE LINK_LIBRARIES ${ZLIB_LIBRARIES}) diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index 19a0ffe0..1ee1f071 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -267,6 +267,9 @@ ha_archive::ha_archive(handlerton *hton, TABLE_SHARE *table_arg) archive_reader_open= FALSE; } +/* Stack size 50264 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + int archive_discover(handlerton *hton, THD* thd, TABLE_SHARE *share) { DBUG_ENTER("archive_discover"); @@ -308,6 +311,7 @@ ret: my_free(frm_ptr); DBUG_RETURN(my_errno); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /** @brief Read version 1 meta file (5.0 compatibility routine). @@ -478,6 +482,10 @@ int ha_archive::read_data_header(azio_stream *file_to_read) See ha_example.cc for a longer description. */ + +/* Stack size 49608 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + Archive_share *ha_archive::get_share(const char *table_name, int *rc) { Archive_share *tmp_share; @@ -540,6 +548,7 @@ err: DBUG_RETURN(tmp_share); } +PRAGMA_REENABLE_CHECK_STACK_FRAME int Archive_share::init_archive_writer() @@ -761,6 +770,9 @@ int ha_archive::frm_compare(azio_stream *s) of creation. */ +/* Stack size 49608 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + int ha_archive::create(const char *name, TABLE *table_arg, HA_CREATE_INFO *create_info) { @@ -878,6 +890,7 @@ error: /* Return error number, if we got one */ DBUG_RETURN(error ? error : -1); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /* This is where the actual row is written out. @@ -1496,6 +1509,10 @@ int ha_archive::repair(THD* thd, HA_CHECK_OPT* check_opt) The table can become fragmented if data was inserted, read, and then inserted again. What we do is open up the file and recompress it completely. */ + +/* Stack size 50152 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + int ha_archive::optimize(THD* thd, HA_CHECK_OPT* check_opt) { int rc= 0; @@ -1621,6 +1638,7 @@ error: DBUG_RETURN(rc); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /* Below is an example of how to setup row level locking. diff --git a/storage/columnstore/CMakeLists.txt b/storage/columnstore/CMakeLists.txt index ebb138c7..3767d5da 100644 --- a/storage/columnstore/CMakeLists.txt +++ b/storage/columnstore/CMakeLists.txt @@ -1,6 +1,7 @@ #set(PLUGIN_COLUMNSTORE "NO" CACHE STRING "Enable ColumnStore engine") -if("NO" STREQUAL "${PLUGIN_COLUMNSTORE}") +if("NO" STREQUAL "${PLUGIN_COLUMNSTORE}" OR + (NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/columnstore/CMakeLists.txt)) return() endif() diff --git a/storage/columnstore/columnstore/CMakeLists.txt b/storage/columnstore/columnstore/CMakeLists.txt index 5d43aef5..0e6ad413 100644 --- a/storage/columnstore/columnstore/CMakeLists.txt +++ b/storage/columnstore/columnstore/CMakeLists.txt @@ -223,6 +223,7 @@ IF (MASK_LONGDOUBLE) MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE") ENDIF() + SET (CMAKE_REQUIRED_FLAGS "-Werror -Wall") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") SET (ENGINE_DT_LIB datatypes) diff --git a/storage/columnstore/columnstore/VERSION b/storage/columnstore/columnstore/VERSION index 35b4ad7f..fb113a00 100644 --- a/storage/columnstore/columnstore/VERSION +++ b/storage/columnstore/columnstore/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=6 COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=8 -COLUMNSTORE_VERSION_RELEASE=1 +COLUMNSTORE_VERSION_RELEASE=2 diff --git a/storage/columnstore/columnstore/datatypes/mcs_datatype.cpp b/storage/columnstore/columnstore/datatypes/mcs_datatype.cpp index 59bc2bf2..2ca95ba2 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_datatype.cpp +++ b/storage/columnstore/columnstore/datatypes/mcs_datatype.cpp @@ -1915,4 +1915,3 @@ const uint8_t* TypeHandlerUDecimal128::getEmptyValueForType( } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/datatypes/mcs_datatype.h b/storage/columnstore/columnstore/datatypes/mcs_datatype.h index a25c23e1..11b0d989 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_datatype.h +++ b/storage/columnstore/columnstore/datatypes/mcs_datatype.h @@ -2522,6 +2522,4 @@ class TypeHandlerTimestamp : public TypeHandlerTemporal } // end of namespace datatypes -#endif // MCS_DATATYPE_H_INCLUDED - -// vim:ts=2 sw=2: +#endif // MCS_DATATYPE_H_INCLUDED
\ No newline at end of file diff --git a/storage/columnstore/columnstore/datatypes/mcs_datatype_basic.h b/storage/columnstore/columnstore/datatypes/mcs_datatype_basic.h index 5ba6cfc1..16179f71 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_datatype_basic.h +++ b/storage/columnstore/columnstore/datatypes/mcs_datatype_basic.h @@ -80,4 +80,3 @@ uint64_t xFloatToMCSUInt64Round(SRC value) } // end of namespace datatypes #endif // MCS_DATATYPE_BASIC_H_INCLUDED -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/datatypes/mcs_double.h b/storage/columnstore/columnstore/datatypes/mcs_double.h index e34bf7aa..7e420f2f 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_double.h +++ b/storage/columnstore/columnstore/datatypes/mcs_double.h @@ -55,5 +55,4 @@ class TDouble } // end of namespace datatypes -#endif // MCS_DOUBLE_H_INCLUDED -// vim:ts=2 sw=2: +#endif // MCS_DOUBLE_H_INCLUDED
\ No newline at end of file diff --git a/storage/columnstore/columnstore/datatypes/mcs_float128.h b/storage/columnstore/columnstore/datatypes/mcs_float128.h index 058ad1ed..c25cfc90 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_float128.h +++ b/storage/columnstore/columnstore/datatypes/mcs_float128.h @@ -725,5 +725,4 @@ class TFloat128 } // namespace datatypes -#endif // MCS_TSFLOAT128_H_INCLUDED -// vim:ts=2 sw=2: +#endif // MCS_TSFLOAT128_H_INCLUDED
\ No newline at end of file diff --git a/storage/columnstore/columnstore/datatypes/mcs_int128.cpp b/storage/columnstore/columnstore/datatypes/mcs_int128.cpp index 13329fc4..58dee474 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_int128.cpp +++ b/storage/columnstore/columnstore/datatypes/mcs_int128.cpp @@ -113,4 +113,3 @@ std::ostream& operator<<(std::ostream& os, const TSInt128& x) } } // end of namespace datatypes -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/datatypes/mcs_int128.h b/storage/columnstore/columnstore/datatypes/mcs_int128.h index 02df0c7e..ddc37bdf 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_int128.h +++ b/storage/columnstore/columnstore/datatypes/mcs_int128.h @@ -324,4 +324,3 @@ class TSInt128 } // end of namespace datatypes #endif // MCS_TSINT128_H_INCLUDED -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/datatypes/mcs_int64.h b/storage/columnstore/columnstore/datatypes/mcs_int64.h index c44f1169..be379a62 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_int64.h +++ b/storage/columnstore/columnstore/datatypes/mcs_int64.h @@ -180,5 +180,4 @@ class TSInt64Null : public TSInt64, public TNullFlag } // end of namespace datatypes -#endif // MCS_INT64_H_INCLUDED -// vim:ts=2 sw=2: +#endif // MCS_INT64_H_INCLUDED
\ No newline at end of file diff --git a/storage/columnstore/columnstore/datatypes/mcs_longdouble.h b/storage/columnstore/columnstore/datatypes/mcs_longdouble.h index 55215642..f33d7658 100644 --- a/storage/columnstore/columnstore/datatypes/mcs_longdouble.h +++ b/storage/columnstore/columnstore/datatypes/mcs_longdouble.h @@ -55,5 +55,4 @@ class TLongDouble } // end of namespace datatypes -#endif // MCS_LONGDOUBLE_H_INCLUDED -// vim:ts=2 sw=2: +#endif // MCS_LONGDOUBLE_H_INCLUDED
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/ddlpackage/CMakeLists.txt b/storage/columnstore/columnstore/dbcon/ddlpackage/CMakeLists.txt index ed211403..5557ed3f 100644 --- a/storage/columnstore/columnstore/dbcon/ddlpackage/CMakeLists.txt +++ b/storage/columnstore/columnstore/dbcon/ddlpackage/CMakeLists.txt @@ -9,7 +9,7 @@ FIND_PACKAGE(FLEX REQUIRED) FLEX_TARGET(ddl_scan ddl.l ${CMAKE_CURRENT_BINARY_DIR}/ddl-scan.cpp COMPILE_FLAGS "-i -L -Pddl") ADD_FLEX_BISON_DEPENDENCY(ddl_scan ddl_gram) -set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) +set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-register -Wno-deprecated-register -Wno-sign-compare -DYY_NO_INPUT") ########### next target ############### diff --git a/storage/columnstore/columnstore/dbcon/ddlpackageproc/altertableprocessor.cpp b/storage/columnstore/columnstore/dbcon/ddlpackageproc/altertableprocessor.cpp index aca76bf4..c09c79b4 100644 --- a/storage/columnstore/columnstore/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/storage/columnstore/columnstore/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -2539,7 +2539,6 @@ void AlterTableProcessor::renameColumn(uint32_t sessionID, execplan::CalpontSyst } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/storage/columnstore/columnstore/dbcon/ddlpackageproc/createtableprocessor.cpp b/storage/columnstore/columnstore/dbcon/ddlpackageproc/createtableprocessor.cpp index f51d0395..f0ff32be 100644 --- a/storage/columnstore/columnstore/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/storage/columnstore/columnstore/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -830,4 +830,3 @@ void CreateTableProcessor::rollBackCreateTable(const string& error, BRM::TxnID t } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 3740035d..bf27af32 100644 --- a/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -1491,4 +1491,3 @@ int DDLPackageProcessor::commitTransaction(uint64_t uniqueId, BRM::TxnID txnID) } } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.h b/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.h index d2dd84c7..c2458528 100644 --- a/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/storage/columnstore/columnstore/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -54,7 +54,7 @@ #define EXPORT #endif -//#define IDB_DDL_DEBUG +// #define IDB_DDL_DEBUG namespace ddlpackageprocessor { #define SUMMARY_INFO(message) \ @@ -891,5 +891,4 @@ bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&) #undef EXPORT -#endif // DDLPACKAGEPROCESSOR_H -// vim:ts=4 sw=4: +#endif // DDLPACKAGEPROCESSOR_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/ddlpackageproc/droptableprocessor.cpp b/storage/columnstore/columnstore/dbcon/ddlpackageproc/droptableprocessor.cpp index e1ca785d..4cfc7946 100644 --- a/storage/columnstore/columnstore/dbcon/ddlpackageproc/droptableprocessor.cpp +++ b/storage/columnstore/columnstore/dbcon/ddlpackageproc/droptableprocessor.cpp @@ -1367,4 +1367,3 @@ TruncTableProcessor::DDLResult TruncTableProcessor::processPackage( } // namespace ddlpackageprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/dmlpackage/CMakeLists.txt b/storage/columnstore/columnstore/dbcon/dmlpackage/CMakeLists.txt index d7b5f235..e999cdae 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackage/CMakeLists.txt +++ b/storage/columnstore/columnstore/dbcon/dmlpackage/CMakeLists.txt @@ -10,7 +10,7 @@ FIND_PACKAGE(FLEX REQUIRED) FLEX_TARGET(dml_scan dml.l ${CMAKE_CURRENT_BINARY_DIR}/dml-scan.cpp COMPILE_FLAGS "-i -L -Pdml") ADD_FLEX_BISON_DEPENDENCY(dml_scan dml_gram) -set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) +set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS "-Wno-register -Wno-deprecated-register -Wno-sign-compare -DYY_NO_INPUT") ########### next target ############### diff --git a/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.cpp b/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.cpp index 3a71efaf..875e90d7 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.cpp +++ b/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.cpp @@ -97,4 +97,3 @@ AutoincrementData::OIDNextValue& AutoincrementData::getOidNextValueMap() return fOidNextValueMap; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.h b/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.h index 17be0648..81614f40 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.h +++ b/storage/columnstore/columnstore/dbcon/dmlpackageproc/autoincrementdata.h @@ -50,5 +50,4 @@ class AutoincrementData boost::mutex fOIDnextvalLock; }; -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/storage/columnstore/columnstore/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index b5217b0e..78b19b39 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/storage/columnstore/columnstore/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -913,4 +913,3 @@ int DMLPackageProcessor::endTransaction(uint64_t uniqueId, BRM::TxnID txnID, boo return rc; } } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/dmlpackageproc/insertpackageprocessor.cpp b/storage/columnstore/columnstore/dbcon/dmlpackageproc/insertpackageprocessor.cpp index fb844b69..3fd6b7ae 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackageproc/insertpackageprocessor.cpp +++ b/storage/columnstore/columnstore/dbcon/dmlpackageproc/insertpackageprocessor.cpp @@ -432,4 +432,3 @@ DMLPackageProcessor::DMLResult InsertPackageProcessor::processPackage(dmlpackage } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.cpp b/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.cpp index d5f94ec6..1c169917 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.cpp +++ b/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.cpp @@ -101,4 +101,3 @@ TablelockData::OIDTablelock& TablelockData::getOidTablelockMap() } } // namespace dmlpackageprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.h b/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.h index a5910d84..9c52d1cf 100644 --- a/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.h +++ b/storage/columnstore/columnstore/dbcon/dmlpackageproc/tablelockdata.h @@ -62,5 +62,4 @@ class TablelockData #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/execplan/calpontselectexecutionplan.h b/storage/columnstore/columnstore/dbcon/execplan/calpontselectexecutionplan.h index 10217d36..04b83689 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/calpontselectexecutionplan.h +++ b/storage/columnstore/columnstore/dbcon/execplan/calpontselectexecutionplan.h @@ -940,5 +940,4 @@ inline std::ostream& operator<<(std::ostream& os, const CalpontSelectExecutionPl } } // namespace execplan -#endif // CALPONTSELECTEXECUTIONPLAN_H -// vim:ts=4 sw=4: +#endif // CALPONTSELECTEXECUTIONPLAN_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/execplan/calpontsystemcatalog.h b/storage/columnstore/columnstore/dbcon/execplan/calpontsystemcatalog.h index 5be4b023..56322f3d 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/calpontsystemcatalog.h +++ b/storage/columnstore/columnstore/dbcon/execplan/calpontsystemcatalog.h @@ -1284,5 +1284,4 @@ bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatal } // namespace execplan -#endif // EXECPLAN_CALPONTSYSTEMCATALOG_H -// vim:ts=4 sw=4: +#endif // EXECPLAN_CALPONTSYSTEMCATALOG_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/execplan/clientrotator.cpp b/storage/columnstore/columnstore/dbcon/execplan/clientrotator.cpp index 7616f90e..395d7648 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/clientrotator.cpp +++ b/storage/columnstore/columnstore/dbcon/execplan/clientrotator.cpp @@ -399,4 +399,3 @@ void ClientRotator::writeToLog(int line, const string& msg, bool critical) const } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/execplan/clientrotator.h b/storage/columnstore/columnstore/dbcon/execplan/clientrotator.h index f0be4ff7..3ce8e97e 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/clientrotator.h +++ b/storage/columnstore/columnstore/dbcon/execplan/clientrotator.h @@ -167,5 +167,4 @@ class ClientRotator }; } // namespace execplan -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/execplan/constantcolumn.cpp b/storage/columnstore/columnstore/dbcon/execplan/constantcolumn.cpp index a3094285..bcdc6767 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/constantcolumn.cpp +++ b/storage/columnstore/columnstore/dbcon/execplan/constantcolumn.cpp @@ -347,4 +347,3 @@ bool ConstantColumn::operator!=(const TreeNode* t) const } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/execplan/constantfilter.cpp b/storage/columnstore/columnstore/dbcon/execplan/constantfilter.cpp index 4f837d0a..18f872e0 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/constantfilter.cpp +++ b/storage/columnstore/columnstore/dbcon/execplan/constantfilter.cpp @@ -308,4 +308,3 @@ void ConstantFilter::setSimpleColumnList() } } // namespace execplan -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/execplan/sessionmanager.h b/storage/columnstore/columnstore/dbcon/execplan/sessionmanager.h index 71076c26..9014a703 100644 --- a/storage/columnstore/columnstore/dbcon/execplan/sessionmanager.h +++ b/storage/columnstore/columnstore/dbcon/execplan/sessionmanager.h @@ -69,7 +69,7 @@ namespace execplan * immediately, causing all subsequent references to fail. This only affects * 'leakcheck'. */ -//#define DESTROYSHMSEG +// #define DESTROYSHMSEG class SessionManager { @@ -214,5 +214,4 @@ class SessionManager } // namespace execplan -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/anydatalist.cpp b/storage/columnstore/columnstore/dbcon/joblist/anydatalist.cpp index 4e5df471..da1ae05a 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/anydatalist.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/anydatalist.cpp @@ -170,4 +170,3 @@ std::ostream& omitOidInDL(std::ostream& strm) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/batchprimitiveprocessor-jl.h b/storage/columnstore/columnstore/dbcon/joblist/batchprimitiveprocessor-jl.h index 46c8a583..577a16d3 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/storage/columnstore/columnstore/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -44,7 +44,7 @@ #include "brm.h" #include "command-jl.h" #include "resourcemanager.h" -//#include "tableband.h" +// #include "tableband.h" namespace joblist { @@ -367,5 +367,4 @@ class BatchPrimitiveProcessorJL } // namespace joblist -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/columncommand-jl.h b/storage/columnstore/columnstore/dbcon/joblist/columncommand-jl.h index e0001902..2ac982cc 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/columncommand-jl.h +++ b/storage/columnstore/columnstore/dbcon/joblist/columncommand-jl.h @@ -130,5 +130,4 @@ class ColumnCommandJL : public CommandJL } // namespace joblist -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.cpp b/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.cpp index e0475c11..d03f21e0 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.cpp @@ -828,4 +828,3 @@ void CrossEngineStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.h b/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.h index 71c7386f..23f3315c 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/crossenginestep.h @@ -250,6 +250,4 @@ class CrossEngineStep : public BatchPrimitive, public TupleDeliveryStep } // namespace joblist -#endif // JOBLIST_CROSSENGINESTEP_H - -// vim:ts=4 sw=4: +#endif // JOBLIST_CROSSENGINESTEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/diskjoinstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/diskjoinstep.cpp index 87be2166..2f69f962 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/diskjoinstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/diskjoinstep.cpp @@ -163,7 +163,7 @@ void DiskJoinStep::smallReader() RGData rgData; bool more = true; int64_t memUsage = 0, combinedMemUsage = 0; - int rowCount = 0; + [[maybe_unused]] int rowCount = 0; RowGroup l_smallRG = smallRG; try @@ -224,7 +224,7 @@ void DiskJoinStep::largeReader() RGData rgData; bool more = true; int64_t largeSize = 0; - int rowCount = 0; + [[maybe_unused]] int rowCount = 0; RowGroup l_largeRG = largeRG; largeIterationCount++; diff --git a/storage/columnstore/columnstore/dbcon/joblist/distributedenginecomm.cpp b/storage/columnstore/columnstore/dbcon/joblist/distributedenginecomm.cpp index ced9bf81..4e735e58 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/distributedenginecomm.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/distributedenginecomm.cpp @@ -1150,4 +1150,3 @@ uint32_t DistributedEngineComm::MQE::getNextConnectionId(const size_t pmIndex, } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/elementtype.h b/storage/columnstore/columnstore/dbcon/joblist/elementtype.h index 0d1f6ed7..0b38d93e 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/elementtype.h +++ b/storage/columnstore/columnstore/dbcon/joblist/elementtype.h @@ -245,14 +245,14 @@ extern std::ostream& operator<<(std::ostream& out, const TupleType& rhs); #ifndef NO_DATALISTS -//#include "bandeddl.h" -//#include "wsdl.h" +// #include "bandeddl.h" +// #include "wsdl.h" #include "fifo.h" -//#include "bucketdl.h" -//#include "constantdatalist.h" -//#include "swsdl.h" -//#include "zdl.h" -//#include "deliverywsdl.h" +// #include "bucketdl.h" +// #include "constantdatalist.h" +// #include "swsdl.h" +// #include "zdl.h" +// #include "deliverywsdl.h" namespace joblist { @@ -639,5 +639,4 @@ extern std::ostream& omitOidInDL(std::ostream& strm); #endif -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/errorinfo.h b/storage/columnstore/columnstore/dbcon/joblist/errorinfo.h index 710cb89c..a709a938 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/errorinfo.h +++ b/storage/columnstore/columnstore/dbcon/joblist/errorinfo.h @@ -53,5 +53,4 @@ typedef boost::shared_ptr<ErrorInfo> SErrorInfo; } // namespace joblist -#endif // JOBLIST_ERROR_INFO_H_ -// vim:ts=4 sw=4: +#endif // JOBLIST_ERROR_INFO_H_
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/expressionstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/expressionstep.cpp index 2f5b96af..b182c7b2 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/expressionstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/expressionstep.cpp @@ -790,4 +790,3 @@ const string ExpressionStep::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/fifo.h b/storage/columnstore/columnstore/dbcon/joblist/fifo.h index 62bf8935..025e9f63 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/fifo.h +++ b/storage/columnstore/columnstore/dbcon/joblist/fifo.h @@ -530,5 +530,4 @@ void FIFO<element_t>::totalFileCounts(uint64_t& numFiles, uint64_t& numBytes) co } // namespace joblist -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/groupconcat.cpp b/storage/columnstore/columnstore/dbcon/joblist/groupconcat.cpp index 5eb0e1b1..f9a64645 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/groupconcat.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/groupconcat.cpp @@ -1050,4 +1050,3 @@ const string GroupConcatNoOrder::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/jlf_common.cpp b/storage/columnstore/columnstore/dbcon/joblist/jlf_common.cpp index 6ae8724e..47a1ecc6 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jlf_common.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/jlf_common.cpp @@ -822,4 +822,3 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/jlf_execplantojoblist.cpp b/storage/columnstore/columnstore/dbcon/joblist/jlf_execplantojoblist.cpp index 4ba29aa5..441ee1ea 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/jlf_execplantojoblist.cpp @@ -3446,7 +3446,6 @@ void JLF_ExecPlanToJobList::addJobSteps(JobStepVector& nsv, JobInfo& jobInfo, bo } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/storage/columnstore/columnstore/dbcon/joblist/jlf_graphics.cpp b/storage/columnstore/columnstore/dbcon/joblist/jlf_graphics.cpp index 7365e1c4..348fb85d 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jlf_graphics.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/jlf_graphics.cpp @@ -418,7 +418,6 @@ ostream& writeDotCmds(ostream& dotFile, const JobStepVector& query, const JobSte } // end namespace jlf_graphics -// vim:ts=4 sw=4 syntax=cpp: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/storage/columnstore/columnstore/dbcon/joblist/jlf_subquery.cpp b/storage/columnstore/columnstore/dbcon/joblist/jlf_subquery.cpp index 91d8a5de..0ea66ac0 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jlf_subquery.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/jlf_subquery.cpp @@ -875,4 +875,3 @@ SJSTEP doUnionSub(CalpontExecutionPlan* ep, JobInfo& jobInfo) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/jlf_tuplejoblist.cpp b/storage/columnstore/columnstore/dbcon/joblist/jlf_tuplejoblist.cpp index 7678d278..d388ce9f 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/jlf_tuplejoblist.cpp @@ -4559,7 +4559,6 @@ SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/storage/columnstore/columnstore/dbcon/joblist/joblist.cpp b/storage/columnstore/columnstore/dbcon/joblist/joblist.cpp index fd58bde0..7adc6a98 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/joblist.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/joblist.cpp @@ -1233,4 +1233,3 @@ void TupleJobList::abort() #pragma clang diagnostic pop #endif -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/joblist.h b/storage/columnstore/columnstore/dbcon/joblist/joblist.h index 61f498a2..4c6eb19e 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/joblist.h +++ b/storage/columnstore/columnstore/dbcon/joblist/joblist.h @@ -261,5 +261,4 @@ typedef boost::shared_ptr<TupleJobList> STJLP; #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/joblistfactory.cpp b/storage/columnstore/columnstore/dbcon/joblist/joblistfactory.cpp index 648b6f1c..26809d39 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/joblistfactory.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/joblistfactory.cpp @@ -2314,7 +2314,6 @@ SJLP JobListFactory::makeJobList(CalpontExecutionPlan* cplan, ResourceManager* r } } // namespace joblist -// vim:ts=4 sw=4: #ifdef __clang__ #pragma clang diagnostic pop diff --git a/storage/columnstore/columnstore/dbcon/joblist/jobstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/jobstep.cpp index 36d13f35..8f42dff1 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jobstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/jobstep.cpp @@ -244,4 +244,3 @@ void JobStep::handleException(std::exception_ptr e, const int errorCode, const u } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/jobstep.h b/storage/columnstore/columnstore/dbcon/joblist/jobstep.h index e3c669cf..47371eb1 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/jobstep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/jobstep.h @@ -567,5 +567,4 @@ typedef boost::shared_ptr<JobStep> SJSTEP; } // namespace joblist -#endif // JOBLIST_JOBSTEP_H_ -// vim:ts=4 sw=4: +#endif // JOBLIST_JOBSTEP_H_
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/lbidlist.cpp b/storage/columnstore/columnstore/dbcon/joblist/lbidlist.cpp index 26080f0c..e784eb26 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/lbidlist.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/lbidlist.cpp @@ -920,4 +920,3 @@ template bool LBIDList::checkRangeOverlap<int64_t>(int64_t min, int64_t max, int } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/limitedorderby.cpp b/storage/columnstore/columnstore/dbcon/joblist/limitedorderby.cpp index 3aee6fc0..f4e573bd 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/limitedorderby.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/limitedorderby.cpp @@ -303,4 +303,3 @@ const string LimitedOrderBy::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/passthrucommand-jl.cpp b/storage/columnstore/columnstore/dbcon/joblist/passthrucommand-jl.cpp index b512e37b..84e63f9c 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/passthrucommand-jl.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/passthrucommand-jl.cpp @@ -110,4 +110,3 @@ uint16_t PassThruCommandJL::getWidth() } }; // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/pcolscan.cpp b/storage/columnstore/columnstore/dbcon/joblist/pcolscan.cpp index aa6fd57b..2a3e64fb 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/pcolscan.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/pcolscan.cpp @@ -472,4 +472,3 @@ void pColScanStep::appendFilter(const std::vector<const execplan::Filter*>& fs) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/pcolstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/pcolstep.cpp index fe9a6d5c..3a314a7d 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/pcolstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/pcolstep.cpp @@ -616,4 +616,3 @@ void pColStep::appendFilter(const std::vector<const execplan::Filter*>& fs) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/pdictionaryscan.cpp b/storage/columnstore/columnstore/dbcon/joblist/pdictionaryscan.cpp index 08b65a70..910f0a8b 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/pdictionaryscan.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/pdictionaryscan.cpp @@ -929,4 +929,3 @@ uint16_t pDictionaryScan::planFlagsToPrimFlags(uint32_t planFlags) } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/primitivemsg.h b/storage/columnstore/columnstore/dbcon/joblist/primitivemsg.h index 3d467a8e..216182c9 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/primitivemsg.h +++ b/storage/columnstore/columnstore/dbcon/joblist/primitivemsg.h @@ -885,5 +885,4 @@ struct LbidAtVer #pragma pack(pop) -#endif // JOBLIST_PRIMITIVE_H -// vim:ts=4 sw=4: +#endif // JOBLIST_PRIMITIVE_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/primitivestep.h b/storage/columnstore/columnstore/dbcon/joblist/primitivestep.h index d20ef5a1..e25b3899 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/primitivestep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/primitivestep.h @@ -111,18 +111,24 @@ class pColStep : public JobStep pColStep(const PassThruStep& rhs); - virtual ~pColStep(){} + virtual ~pColStep() + { + } /** @brief Starts processing. Set at least the RID list before calling. * * Starts processing. Set at least the RID list before calling this. */ - virtual void run(){} + virtual void run() + { + } /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(){} + virtual void join() + { + } virtual const std::string toString() const; @@ -347,19 +353,25 @@ class pColScanStep : public JobStep const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); pColScanStep(const pColStep& rhs); - ~pColScanStep(){} + ~pColScanStep() + { + } /** @brief Starts processing. * * Starts processing. */ - virtual void run(){} + virtual void run() + { + } /** @brief Sync's the caller with the end of execution. * * Does nothing. Returns when this instance is finished. */ - virtual void join(){} + virtual void join() + { + } virtual bool isDictCol() const { @@ -549,12 +561,18 @@ class pDictionaryStep : public JobStep pDictionaryStep(execplan::CalpontSystemCatalog::OID oid, execplan::CalpontSystemCatalog::OID tabelOid, const execplan::CalpontSystemCatalog::ColType& ct, const JobInfo& jobInfo); - virtual ~pDictionaryStep(){} + virtual ~pDictionaryStep() + { + } /** @brief virtual void Run method */ - virtual void run(){} - virtual void join(){} + virtual void run() + { + } + virtual void join() + { + } // void setOutList(StringDataList* rids); void setInputList(DataList_t* rids) { @@ -1506,7 +1524,7 @@ class FilterStep : public JobStep protected: // void unblockDataLists(FifoDataList* fifo, StringFifoDataList* strFifo, StrDataList* strResult, - //DataList_t* result); + // DataList_t* result); private: // This i/f is not meaningful in this step @@ -1676,5 +1694,4 @@ class PseudoColStep : public pColStep } // namespace joblist -#endif // JOBLIST_PRIMITIVESTEP_H -// vim:ts=4 sw=4: +#endif // JOBLIST_PRIMITIVESTEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/rowestimator.cpp b/storage/columnstore/columnstore/dbcon/joblist/rowestimator.cpp index 9babae7c..fa7e8920 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/rowestimator.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/rowestimator.cpp @@ -269,8 +269,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const m float tempFactor = 1.0; uint64_t adjustedMin = 0, adjustedMax = 0; - uint128_t adjustedBigMin, adjustedBigMax; - uint32_t distinctValuesEstimate; + uint128_t adjustedBigMin = 0, adjustedBigMax = 0; + uint32_t distinctValuesEstimate = 0; // Adjust values based on column type and estimate the if (!ct.isWideDecimalType()) diff --git a/storage/columnstore/columnstore/dbcon/joblist/subquerystep.cpp b/storage/columnstore/columnstore/dbcon/joblist/subquerystep.cpp index a019c24f..fbc46912 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/subquerystep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/subquerystep.cpp @@ -533,4 +533,3 @@ void SubAdapterStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/subquerystep.h b/storage/columnstore/columnstore/dbcon/joblist/subquerystep.h index fb0eca13..e7a10f57 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/subquerystep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/subquerystep.h @@ -274,5 +274,4 @@ class SubAdapterStep : public JobStep, public TupleDeliveryStep } // namespace joblist -#endif // SUBQUERY_STEP_H -// vim:ts=4 sw=4: +#endif // SUBQUERY_STEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/subquerytransformer.cpp b/storage/columnstore/columnstore/dbcon/joblist/subquerytransformer.cpp index 816b6aa7..48f1cc2d 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/subquerytransformer.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/subquerytransformer.cpp @@ -622,4 +622,3 @@ void SimpleScalarTransformer::getScalarResult() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tuple-bps.cpp b/storage/columnstore/columnstore/dbcon/joblist/tuple-bps.cpp index f9a6d736..c730e494 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tuple-bps.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/tuple-bps.cpp @@ -3334,4 +3334,3 @@ template bool TupleBPS::compareSingleValue<int64_t>(uint8_t COP, int64_t val1, i template bool TupleBPS::compareSingleValue<int128_t>(uint8_t COP, int128_t val1, int128_t val2) const; } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.cpp b/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.cpp index 3f06ed1c..16ce5611 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.cpp @@ -5962,4 +5962,3 @@ void TupleAggregateStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.h b/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.h index 95d6ee42..f15c87bc 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/tupleaggregatestep.h @@ -225,6 +225,4 @@ class TupleAggregateStep : public JobStep, public TupleDeliveryStep } // namespace joblist -#endif // JOBLIST_TUPLEAGGREGATESTEP_H - -// vim:ts=4 sw=4: +#endif // JOBLIST_TUPLEAGGREGATESTEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.cpp index f6a7d70c..b93e4137 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.cpp @@ -1258,4 +1258,3 @@ void TupleAnnexStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.h b/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.h index 96cc0064..fa761262 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/tupleannexstep.h @@ -194,6 +194,4 @@ class reservablePQ : private std::priority_queue<T> } // namespace joblist -#endif // JOBLIST_TUPLEANNEXSTEP_H - -// vim:ts=4 sw=4: +#endif // JOBLIST_TUPLEANNEXSTEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.cpp index ee252ec0..065b087d 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.cpp @@ -842,4 +842,3 @@ const string TupleConstantBooleanStep::toString() const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.h b/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.h index 0aca68ff..e1bd3b93 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/tupleconstantstep.h @@ -190,6 +190,4 @@ class TupleConstantBooleanStep : public TupleConstantStep } // namespace joblist -#endif // JOBLIST_TUPLECONSTANTSTEP_H - -// vim:ts=4 sw=4: +#endif // JOBLIST_TUPLECONSTANTSTEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.cpp b/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.cpp index 8625723e..0b646b46 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.cpp @@ -2017,4 +2017,3 @@ void TupleHashJoinStep::abort() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.h b/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.h index 8531cfd7..45ee8729 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.h +++ b/storage/columnstore/columnstore/dbcon/joblist/tuplehashjoin.h @@ -653,5 +653,4 @@ class TupleHashJoinStep : public JobStep, public TupleDeliveryStep } // namespace joblist -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.cpp index 701e77b3..60496895 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.cpp @@ -408,4 +408,3 @@ void TupleHavingStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.h b/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.h index 068f3d35..eae7ae0c 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/tuplehavingstep.h @@ -117,5 +117,3 @@ class TupleHavingStep : public ExpressionStep, public TupleDeliveryStep } // namespace joblist #endif // JOBLIST_TUPLEHAVINGSTEP_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/virtualtable.cpp b/storage/columnstore/columnstore/dbcon/joblist/virtualtable.cpp index dc9bf275..8fe1ec67 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/virtualtable.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/virtualtable.cpp @@ -158,4 +158,3 @@ const CalpontSystemCatalog::ColType& VirtualTable::columnType(uint32_t i) const } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.cpp b/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.cpp index 1ab8d398..ab7b8ca8 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.cpp +++ b/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.cpp @@ -1599,4 +1599,3 @@ void WindowFunctionStep::formatMiniStats() } } // namespace joblist -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.h b/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.h index e2f03eb4..fb5597e7 100644 --- a/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.h +++ b/storage/columnstore/columnstore/dbcon/joblist/windowfunctionstep.h @@ -226,6 +226,4 @@ class WindowFunctionStep : public JobStep, public TupleDeliveryStep } // namespace joblist -#endif // JOBLIST_WINDOWFUNCTIONSTEP_H - -// vim:ts=4 sw=4: +#endif // JOBLIST_WINDOWFUNCTIONSTEP_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_autoi.cpp b/storage/columnstore/columnstore/dbcon/mysql/ha_autoi.cpp index ddfbcdc2..4b9a2ae3 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_autoi.cpp +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_autoi.cpp @@ -176,4 +176,3 @@ bool parseAutoincrementColumnComment(std::string comment, uint64_t& startValue) return autoincrement; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_datatype.h b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_datatype.h index 2f9e625d..b0fd54a6 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_datatype.h +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_datatype.h @@ -842,6 +842,4 @@ class WriteBatchFieldMariaDB : public WriteBatchField } // end of namespace datatypes -#endif - -// vim:ts=2 sw=2: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_ddl.cpp b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_ddl.cpp index 2d18688f..305b8e07 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_ddl.cpp +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_ddl.cpp @@ -2810,4 +2810,3 @@ extern "C" } } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_dml.cpp b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_dml.cpp index bd142382..a3f97db7 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_dml.cpp +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_dml.cpp @@ -1001,4 +1001,3 @@ int ha_mcs_impl_close_connection_(handlerton* hton, THD* thd, cal_connection_inf return rc; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_execplan.cpp b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_execplan.cpp index 55f6de40..798e5af8 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_execplan.cpp +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_mcs_execplan.cpp @@ -10481,4 +10481,3 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } } // namespace cal_impl_if -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/mysql/ha_pseudocolumn.cpp b/storage/columnstore/columnstore/dbcon/mysql/ha_pseudocolumn.cpp index 5b1010e8..48811328 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/ha_pseudocolumn.cpp +++ b/storage/columnstore/columnstore/dbcon/mysql/ha_pseudocolumn.cpp @@ -584,4 +584,3 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& } } // namespace cal_impl_if -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dbcon/mysql/idb_mysql.h b/storage/columnstore/columnstore/dbcon/mysql/idb_mysql.h index bc966ffc..06911c6c 100644 --- a/storage/columnstore/columnstore/dbcon/mysql/idb_mysql.h +++ b/storage/columnstore/columnstore/dbcon/mysql/idb_mysql.h @@ -33,8 +33,8 @@ bool isnan(T); #endif #endif -//#define INFINIDB_DEBUG -//#define DEBUG_WALK_COND +// #define INFINIDB_DEBUG +// #define DEBUG_WALK_COND #define MYSQL_SERVER 1 // needed for definition of struct THD in mysql_priv.h #define USE_CALPONT_REGEX @@ -119,5 +119,4 @@ inline char* idb_mysql_query_str(THD* thd) } } // namespace -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/ddlproc/ddlproc.cpp b/storage/columnstore/columnstore/ddlproc/ddlproc.cpp index e479fbf4..f203f85d 100644 --- a/storage/columnstore/columnstore/ddlproc/ddlproc.cpp +++ b/storage/columnstore/columnstore/ddlproc/ddlproc.cpp @@ -248,4 +248,3 @@ int main(int argc, char** argv) return ServiceDDLProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/ddlproc/ddlprocessor.cpp b/storage/columnstore/columnstore/ddlproc/ddlprocessor.cpp index eb6f3d09..cca42af1 100644 --- a/storage/columnstore/columnstore/ddlproc/ddlprocessor.cpp +++ b/storage/columnstore/columnstore/ddlproc/ddlprocessor.cpp @@ -876,4 +876,3 @@ int DDLProcessor::commitTransaction(uint32_t txnID, std::string& errorMsg) return rc; } } // namespace ddlprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.cpp b/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.cpp index 1a61e1eb..48307417 100644 --- a/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.cpp +++ b/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.cpp @@ -547,4 +547,3 @@ void BatchInsertProc::getError(int& errorCode, std::string& errMsg) errMsg = fErrMsg; } } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.h b/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.h index c7cccb1d..f889c89c 100644 --- a/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.h +++ b/storage/columnstore/columnstore/dmlproc/batchinsertprocessor.h @@ -94,5 +94,4 @@ class BatchInsertProc }; } // namespace dmlprocessor -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/dmlproc/dmlproc.cpp b/storage/columnstore/columnstore/dmlproc/dmlproc.cpp index f1fa9cc4..fb79ae59 100644 --- a/storage/columnstore/columnstore/dmlproc/dmlproc.cpp +++ b/storage/columnstore/columnstore/dmlproc/dmlproc.cpp @@ -639,14 +639,7 @@ int ServiceDMLProc::Child() // Couldn't check the return code b/c // fuser returns 1 for unused port. -#if defined(__GNUC__) && __GNUC__ >= 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - (void)::system(cmd.c_str()); -#pragma GCC diagnostic pop -#else - (void)::system(cmd.c_str()); -#endif + std::ignore = ::system(cmd.c_str()); } catch (...) { @@ -698,4 +691,3 @@ int main(int argc, char** argv) return ServiceDMLProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dmlproc/dmlprocessor.cpp b/storage/columnstore/columnstore/dmlproc/dmlprocessor.cpp index 33e9b3d1..0cc5337a 100644 --- a/storage/columnstore/columnstore/dmlproc/dmlprocessor.cpp +++ b/storage/columnstore/columnstore/dmlproc/dmlprocessor.cpp @@ -2001,4 +2001,3 @@ void DMLProcessor::log(const std::string& msg, logging::LOG_TYPE level) } } // namespace dmlprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/dmlproc/dmlprocessor.h b/storage/columnstore/columnstore/dmlproc/dmlprocessor.h index b1863600..35465cda 100644 --- a/storage/columnstore/columnstore/dmlproc/dmlprocessor.h +++ b/storage/columnstore/columnstore/dmlproc/dmlprocessor.h @@ -331,5 +331,4 @@ class RollbackTransactionProcessor : public dmlpackageprocessor::DMLPackageProce } // namespace dmlprocessor -#endif // DMLPROCESSOR_H -// vim:ts=4 sw=4: +#endif // DMLPROCESSOR_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/exemgr/activestatementcounter.cpp b/storage/columnstore/columnstore/exemgr/activestatementcounter.cpp index 27a3df1d..b25e6154 100644 --- a/storage/columnstore/columnstore/exemgr/activestatementcounter.cpp +++ b/storage/columnstore/columnstore/exemgr/activestatementcounter.cpp @@ -57,4 +57,3 @@ void ActiveStatementCounter::decr(bool& counted) --fStatementCount; condvar.notify_one(); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/exemgr/activestatementcounter.h b/storage/columnstore/columnstore/exemgr/activestatementcounter.h index e8e75c58..fbbf953c 100644 --- a/storage/columnstore/columnstore/exemgr/activestatementcounter.h +++ b/storage/columnstore/columnstore/exemgr/activestatementcounter.h @@ -63,5 +63,4 @@ class ActiveStatementCounter BRM::VSS fVss; }; -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/exemgr/main.cpp b/storage/columnstore/columnstore/exemgr/main.cpp index 513cc075..abb0991c 100644 --- a/storage/columnstore/columnstore/exemgr/main.cpp +++ b/storage/columnstore/columnstore/exemgr/main.cpp @@ -1825,4 +1825,3 @@ int main(int argc, char* argv[]) return ServiceExeMgr(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.cpp b/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.cpp index d33ac883..47153584 100644 --- a/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.cpp +++ b/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.cpp @@ -1063,4 +1063,3 @@ std::string Oam::itoa(const int i) return ss.str(); } } // namespace oam -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.h b/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.h index 504a537d..7e762ff0 100644 --- a/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.h +++ b/storage/columnstore/columnstore/oam/oamcpp/liboamcpp.h @@ -425,5 +425,4 @@ class Oam #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/oamapps/columnstoreDB/columnstoreDB.cpp b/storage/columnstore/columnstore/oamapps/columnstoreDB/columnstoreDB.cpp index f716f9bd..3bdca8b6 100644 --- a/storage/columnstore/columnstore/oamapps/columnstoreDB/columnstoreDB.cpp +++ b/storage/columnstore/columnstore/oamapps/columnstoreDB/columnstoreDB.cpp @@ -271,7 +271,7 @@ int main(int argc, char** argv) Oam oam; BRM::DBRM dbrm; - char c; + signed char c; // Invokes member function `int operator ()(void);' while ((c = getopt(argc, argv, "c:h")) != -1) diff --git a/storage/columnstore/columnstore/oamapps/postConfigure/mycnfUpgrade.cpp b/storage/columnstore/columnstore/oamapps/postConfigure/mycnfUpgrade.cpp index d46bd773..a5b51dec 100644 --- a/storage/columnstore/columnstore/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/storage/columnstore/columnstore/oamapps/postConfigure/mycnfUpgrade.cpp @@ -284,4 +284,3 @@ int main(int argc, char* argv[]) exit(0); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/oamapps/sessionWalker/sessionwalker.cpp b/storage/columnstore/columnstore/oamapps/sessionWalker/sessionwalker.cpp index 5f096012..06cf08bd 100644 --- a/storage/columnstore/columnstore/oamapps/sessionWalker/sessionwalker.cpp +++ b/storage/columnstore/columnstore/oamapps/sessionWalker/sessionwalker.cpp @@ -133,4 +133,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/blockcache/blockcacheclient.h b/storage/columnstore/columnstore/primitives/blockcache/blockcacheclient.h index ccdb5b16..c4b06c7b 100644 --- a/storage/columnstore/columnstore/primitives/blockcache/blockcacheclient.h +++ b/storage/columnstore/columnstore/primitives/blockcache/blockcacheclient.h @@ -177,5 +177,4 @@ class blockCacheClient } // namespace dbbc -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/blockcache/blockrequestprocessor.h b/storage/columnstore/columnstore/primitives/blockcache/blockrequestprocessor.h index edb1479f..3a1de568 100644 --- a/storage/columnstore/columnstore/primitives/blockcache/blockrequestprocessor.h +++ b/storage/columnstore/columnstore/primitives/blockcache/blockrequestprocessor.h @@ -191,5 +191,4 @@ class BlockRequestProcessor }; } // namespace dbbc -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/blockcache/filerequest.h b/storage/columnstore/columnstore/primitives/blockcache/filerequest.h index 19bd9256..0ede2eef 100644 --- a/storage/columnstore/columnstore/primitives/blockcache/filerequest.h +++ b/storage/columnstore/columnstore/primitives/blockcache/filerequest.h @@ -321,5 +321,4 @@ class fileRequest } // namespace dbbc -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/blockcache/iomanager.cpp b/storage/columnstore/columnstore/primitives/blockcache/iomanager.cpp index 81eb787c..d8cae6f3 100644 --- a/storage/columnstore/columnstore/primitives/blockcache/iomanager.cpp +++ b/storage/columnstore/columnstore/primitives/blockcache/iomanager.cpp @@ -1442,4 +1442,3 @@ void ioManager::handleBlockReadError(fileRequest* fr, const string& errMsg, bool } } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/blockcache/iomanager.h b/storage/columnstore/columnstore/primitives/blockcache/iomanager.h index 6cf61569..546675d7 100644 --- a/storage/columnstore/columnstore/primitives/blockcache/iomanager.h +++ b/storage/columnstore/columnstore/primitives/blockcache/iomanager.h @@ -43,7 +43,7 @@ #include "fileblockrequestqueue.h" #include "filebuffermgr.h" -//#define SHARED_NOTHING_DEMO_2 +// #define SHARED_NOTHING_DEMO_2 namespace dbbc { @@ -148,5 +148,4 @@ void dropFDCache(); void purgeFDCache(std::vector<BRM::FileInfo>& files); } // namespace dbbc -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/linux-port/column.cpp b/storage/columnstore/columnstore/primitives/linux-port/column.cpp index fb13ffba..f07416a2 100644 --- a/storage/columnstore/columnstore/primitives/linux-port/column.cpp +++ b/storage/columnstore/columnstore/primitives/linux-port/column.cpp @@ -18,7 +18,7 @@ #include <iostream> #include <sstream> -//#define NDEBUG +// #define NDEBUG #include <cassert> #include <cmath> #include <functional> @@ -1636,7 +1636,7 @@ template <typename T, typename std::enable_if<sizeof(T) == sizeof(int8_t) || siz T>::type* = nullptr> // gcc >= 5 #else sizeof(T) == sizeof(int128_t), - T>::type*> // gcc 4.8.5 + T>::type*> // gcc 4.8.5 #endif #else sizeof(T) == sizeof(int128_t), @@ -1672,7 +1672,7 @@ template <typename T, #if ___GNUC__ >= 5 typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type* = nullptr> // gcc >= 5 #else - typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type*> // gcc 4.8.5 + typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type*> // gcc 4.8.5 #endif #else typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type* = nullptr> @@ -1763,5 +1763,4 @@ template void primitives::PrimitiveProcessor::columnScanAndFilter<int64_t>(NewCo template void primitives::PrimitiveProcessor::columnScanAndFilter<int128_t>(NewColRequestHeader*, ColResultHeader*); -} // namespace primitives -// vim:ts=4 sw=4: +} // namespace primitives
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/linux-port/dictionary.cpp b/storage/columnstore/columnstore/primitives/linux-port/dictionary.cpp index 2942d4fd..360d8b60 100644 --- a/storage/columnstore/columnstore/primitives/linux-port/dictionary.cpp +++ b/storage/columnstore/columnstore/primitives/linux-port/dictionary.cpp @@ -620,4 +620,3 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector<uint8_t>* out, } } // namespace primitives -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/linux-port/index.cpp b/storage/columnstore/columnstore/primitives/linux-port/index.cpp index bbfc949e..2d86d35e 100644 --- a/storage/columnstore/columnstore/primitives/linux-port/index.cpp +++ b/storage/columnstore/columnstore/primitives/linux-port/index.cpp @@ -1130,4 +1130,3 @@ void PrimitiveProcessor::p_IdxList(const IndexListHeader* rqst, IndexListHeader* } } // namespace primitives -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/linux-port/primitiveprocessor.h b/storage/columnstore/columnstore/primitives/linux-port/primitiveprocessor.h index baaf054f..453d41d8 100644 --- a/storage/columnstore/columnstore/primitives/linux-port/primitiveprocessor.h +++ b/storage/columnstore/columnstore/primitives/linux-port/primitiveprocessor.h @@ -563,5 +563,4 @@ boost::shared_ptr<ParsedColumnFilter> _parseColumnFilter( } // namespace primitives -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/primproc/batchprimitiveprocessor.cpp b/storage/columnstore/columnstore/primitives/primproc/batchprimitiveprocessor.cpp index e16b7ff1..5d898d05 100644 --- a/storage/columnstore/columnstore/primitives/primproc/batchprimitiveprocessor.cpp +++ b/storage/columnstore/columnstore/primitives/primproc/batchprimitiveprocessor.cpp @@ -2806,4 +2806,3 @@ void BatchPrimitiveProcessor::buildVSSCache(uint32_t loopCount) } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/primproc/columncommand.cpp b/storage/columnstore/columnstore/primitives/primproc/columncommand.cpp index 16426f49..a7069359 100644 --- a/storage/columnstore/columnstore/primitives/primproc/columncommand.cpp +++ b/storage/columnstore/columnstore/primitives/primproc/columncommand.cpp @@ -1226,4 +1226,3 @@ void ColumnCommandInt128::issuePrimitive() } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/primproc/columncommand.h b/storage/columnstore/columnstore/primitives/primproc/columncommand.h index b3ec49d7..c260f23d 100644 --- a/storage/columnstore/columnstore/primitives/primproc/columncommand.h +++ b/storage/columnstore/columnstore/primitives/primproc/columncommand.h @@ -307,5 +307,4 @@ inline void ColumnCommand::fillEmptyBlock<messageqcpp::ByteStream::hexbyte>(uint } // namespace primitiveprocessor -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/primitives/primproc/primitiveserver.cpp b/storage/columnstore/columnstore/primitives/primproc/primitiveserver.cpp index 99aa7a31..19c1f6a4 100644 --- a/storage/columnstore/columnstore/primitives/primproc/primitiveserver.cpp +++ b/storage/columnstore/columnstore/primitives/primproc/primitiveserver.cpp @@ -2575,4 +2575,3 @@ bool BPPV::aborted() // end workaround } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/primproc/primproc.cpp b/storage/columnstore/columnstore/primitives/primproc/primproc.cpp index 96ab2088..ce757c32 100644 --- a/storage/columnstore/columnstore/primitives/primproc/primproc.cpp +++ b/storage/columnstore/columnstore/primitives/primproc/primproc.cpp @@ -786,4 +786,3 @@ int main(int argc, char** argv) return ServicePrimProc(opt).Run(); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/primitives/primproc/udf.cpp b/storage/columnstore/columnstore/primitives/primproc/udf.cpp index 7a836590..188e5128 100644 --- a/storage/columnstore/columnstore/primitives/primproc/udf.cpp +++ b/storage/columnstore/columnstore/primitives/primproc/udf.cpp @@ -69,4 +69,3 @@ void loadUDFs() } } // namespace primitiveprocessor -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/storage-manager/src/smcat.cpp b/storage/columnstore/columnstore/storage-manager/src/smcat.cpp index e931ec8a..48dc5077 100644 --- a/storage/columnstore/columnstore/storage-manager/src/smcat.cpp +++ b/storage/columnstore/columnstore/storage-manager/src/smcat.cpp @@ -94,7 +94,7 @@ void catFileOffline(const char* filename, int prefixlen) void catFileOnline(const char* filename, int prefixlen) { uint8_t data[8192]; - off_t offset = 0; + [[maybe_unused]] off_t offset = 0; int read_err, write_err, count; idbdatafile::SMDataFile df(filename, O_RDONLY, 0); diff --git a/storage/columnstore/columnstore/storage-manager/src/unit_tests.cpp b/storage/columnstore/columnstore/storage-manager/src/unit_tests.cpp index 3d2e7ec2..dc37007a 100644 --- a/storage/columnstore/columnstore/storage-manager/src/unit_tests.cpp +++ b/storage/columnstore/columnstore/storage-manager/src/unit_tests.cpp @@ -1063,10 +1063,10 @@ bool copytask(bool connectionTest = false) copy_cmd* cmd = (copy_cmd*)buf; cmd->opcode = COPY; cmd->file1.flen = strlen(source); - strncpy(cmd->file1.filename, source, cmd->file1.flen); + memcpy(cmd->file1.filename, source, cmd->file1.flen); f_name* file2 = (f_name*)&cmd->file1.filename[cmd->file1.flen]; file2->flen = strlen(dest); - strncpy(file2->filename, dest, file2->flen); + memcpy(file2->filename, dest, file2->flen); uint len = (uint64_t)&file2->filename[file2->flen] - (uint64_t)buf; diff --git a/storage/columnstore/columnstore/tests/primitives_column_scan_and_filter.cpp b/storage/columnstore/columnstore/tests/primitives_column_scan_and_filter.cpp index 70f921b3..eeed651e 100644 --- a/storage/columnstore/columnstore/tests/primitives_column_scan_and_filter.cpp +++ b/storage/columnstore/columnstore/tests/primitives_column_scan_and_filter.cpp @@ -965,4 +965,3 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters) EXPECT_EQ(out->Max, __col16block_cdf_umax); EXPECT_EQ(out->Min, __col16block_cdf_umin); } -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/tests/primitives_scan_bench.cpp b/storage/columnstore/columnstore/tests/primitives_scan_bench.cpp index 2d6b77c9..c2431b8d 100644 --- a/storage/columnstore/columnstore/tests/primitives_scan_bench.cpp +++ b/storage/columnstore/columnstore/tests/primitives_scan_bench.cpp @@ -427,4 +427,3 @@ BENCHMARK_DEFINE_F(FilterBenchFixture, BM_ColumnScan8Byte1FilterVectorizedCode)( BENCHMARK_REGISTER_F(FilterBenchFixture, BM_ColumnScan8Byte1FilterVectorizedCode); BENCHMARK_MAIN(); -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/tools/bincvt/li2bin.cpp b/storage/columnstore/columnstore/tools/bincvt/li2bin.cpp index 51b1d8b6..91bf9c4c 100644 --- a/storage/columnstore/columnstore/tools/bincvt/li2bin.cpp +++ b/storage/columnstore/columnstore/tools/bincvt/li2bin.cpp @@ -278,4 +278,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/clearShm/main.cpp b/storage/columnstore/columnstore/tools/clearShm/main.cpp index e37e27db..d31f9a75 100644 --- a/storage/columnstore/columnstore/tools/clearShm/main.cpp +++ b/storage/columnstore/columnstore/tools/clearShm/main.cpp @@ -221,4 +221,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/dbbuilder/dbbuilder.cpp b/storage/columnstore/columnstore/tools/dbbuilder/dbbuilder.cpp index 38b9ffd5..79d2a4be 100644 --- a/storage/columnstore/columnstore/tools/dbbuilder/dbbuilder.cpp +++ b/storage/columnstore/columnstore/tools/dbbuilder/dbbuilder.cpp @@ -324,4 +324,3 @@ int main(int argc, char* argv[]) return 1; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/ddlcleanup/ddlcleanup.cpp b/storage/columnstore/columnstore/tools/ddlcleanup/ddlcleanup.cpp index bbbbac43..6d1ca008 100644 --- a/storage/columnstore/columnstore/tools/ddlcleanup/ddlcleanup.cpp +++ b/storage/columnstore/columnstore/tools/ddlcleanup/ddlcleanup.cpp @@ -80,4 +80,3 @@ int main(int argc, char** argv) return ddlcleanuputil::ddl_cleanup(); } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/editem/editem.cpp b/storage/columnstore/columnstore/tools/editem/editem.cpp index 2dcb6ce8..bce63ed0 100644 --- a/storage/columnstore/columnstore/tools/editem/editem.cpp +++ b/storage/columnstore/columnstore/tools/editem/editem.cpp @@ -1040,4 +1040,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/qfe/server.cpp b/storage/columnstore/columnstore/tools/qfe/server.cpp index 018cf470..7b3276de 100644 --- a/storage/columnstore/columnstore/tools/qfe/server.cpp +++ b/storage/columnstore/columnstore/tools/qfe/server.cpp @@ -513,4 +513,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/sendPlan/sendplan.cpp b/storage/columnstore/columnstore/tools/sendPlan/sendplan.cpp index 5374d867..bd8f4bb4 100644 --- a/storage/columnstore/columnstore/tools/sendPlan/sendplan.cpp +++ b/storage/columnstore/columnstore/tools/sendPlan/sendplan.cpp @@ -390,4 +390,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/tools/setConfig/main.cpp b/storage/columnstore/columnstore/tools/setConfig/main.cpp index 1c536167..7d7fe396 100644 --- a/storage/columnstore/columnstore/tools/setConfig/main.cpp +++ b/storage/columnstore/columnstore/tools/setConfig/main.cpp @@ -148,4 +148,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/cacheutils/cacheutils.cpp b/storage/columnstore/columnstore/utils/cacheutils/cacheutils.cpp index ead2483a..a0042e15 100644 --- a/storage/columnstore/columnstore/utils/cacheutils/cacheutils.cpp +++ b/storage/columnstore/columnstore/utils/cacheutils/cacheutils.cpp @@ -354,4 +354,3 @@ int purgePrimProcFdCache(const std::vector<BRM::FileInfo> files, const int pmId) } } // namespace cacheutils -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/cacheutils/cacheutils.h b/storage/columnstore/columnstore/utils/cacheutils/cacheutils.h index 13d01ef6..fdc01da7 100644 --- a/storage/columnstore/columnstore/utils/cacheutils/cacheutils.h +++ b/storage/columnstore/columnstore/utils/cacheutils/cacheutils.h @@ -73,6 +73,4 @@ int dropPrimProcFdCache(); int purgePrimProcFdCache(const std::vector<BRM::FileInfo> files, const int pmId); } // namespace cacheutils -// vim:ts=4 sw=4: - #endif diff --git a/storage/columnstore/columnstore/utils/common/hashfamily.h b/storage/columnstore/columnstore/utils/common/hashfamily.h index 6733beaf..eb40a4df 100644 --- a/storage/columnstore/columnstore/utils/common/hashfamily.h +++ b/storage/columnstore/columnstore/utils/common/hashfamily.h @@ -51,5 +51,4 @@ class HashFamily }; } // namespace utils -#endif -// vim:ts=2 sw=2: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/common/mcs_basic_types.h b/storage/columnstore/columnstore/utils/common/mcs_basic_types.h index 2e1caf32..e19f78a8 100644 --- a/storage/columnstore/columnstore/utils/common/mcs_basic_types.h +++ b/storage/columnstore/columnstore/utils/common/mcs_basic_types.h @@ -22,5 +22,4 @@ using int128_t = __int128; using uint128_t = unsigned __int128; -#endif // MCS_BASIC_TYPES_H_INCLUDED -// vim:ts=2 sw=2: +#endif // MCS_BASIC_TYPES_H_INCLUDED
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/common/simd_sse.h b/storage/columnstore/columnstore/utils/common/simd_sse.h index c7da2ec0..d9873414 100644 --- a/storage/columnstore/columnstore/utils/common/simd_sse.h +++ b/storage/columnstore/columnstore/utils/common/simd_sse.h @@ -892,4 +892,3 @@ class SimdFilterProcessor< } // namespace simd #endif // if defined(__x86_64__ ) -// vim:ts=2 sw=2: diff --git a/storage/columnstore/columnstore/utils/compress/idbcompress.cpp b/storage/columnstore/columnstore/utils/compress/idbcompress.cpp index a0a98387..adc7a9cd 100644 --- a/storage/columnstore/columnstore/utils/compress/idbcompress.cpp +++ b/storage/columnstore/columnstore/utils/compress/idbcompress.cpp @@ -672,4 +672,3 @@ std::shared_ptr<CompressInterface> getCompressorByType( #endif } // namespace compress -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/configcpp/configcpp.cpp b/storage/columnstore/columnstore/utils/configcpp/configcpp.cpp index a7194ef7..e102a0cb 100644 --- a/storage/columnstore/columnstore/utils/configcpp/configcpp.cpp +++ b/storage/columnstore/columnstore/utils/configcpp/configcpp.cpp @@ -669,4 +669,3 @@ std::string Config::getTempFileDir(Config::TempDirPurpose what) } } // namespace config -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/configcpp/configcpp.h b/storage/columnstore/columnstore/utils/configcpp/configcpp.h index f573f292..35f290e8 100644 --- a/storage/columnstore/columnstore/utils/configcpp/configcpp.h +++ b/storage/columnstore/columnstore/utils/configcpp/configcpp.h @@ -244,5 +244,4 @@ class Config #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/configcpp/configstream.cpp b/storage/columnstore/columnstore/utils/configcpp/configstream.cpp index 059f690e..cfb1a4ac 100644 --- a/storage/columnstore/columnstore/utils/configcpp/configstream.cpp +++ b/storage/columnstore/columnstore/utils/configcpp/configstream.cpp @@ -64,4 +64,3 @@ void ConfigStream::init(const xmlChar* xp) } } // namespace config -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/configcpp/configstream.h b/storage/columnstore/columnstore/utils/configcpp/configstream.h index b42c6506..8c32e31d 100644 --- a/storage/columnstore/columnstore/utils/configcpp/configstream.h +++ b/storage/columnstore/columnstore/utils/configcpp/configstream.h @@ -62,5 +62,4 @@ class ConfigStream } // namespace config -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/configcpp/xmlparser.cpp b/storage/columnstore/columnstore/utils/configcpp/xmlparser.cpp index 03da1f95..37b6bdde 100644 --- a/storage/columnstore/columnstore/utils/configcpp/xmlparser.cpp +++ b/storage/columnstore/columnstore/utils/configcpp/xmlparser.cpp @@ -285,4 +285,3 @@ const vector<string> XMLParser::enumSection(const xmlDocPtr doc, const string& s } } // namespace config -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/configcpp/xmlparser.h b/storage/columnstore/columnstore/utils/configcpp/xmlparser.h index 48467d24..c1804992 100644 --- a/storage/columnstore/columnstore/utils/configcpp/xmlparser.h +++ b/storage/columnstore/columnstore/utils/configcpp/xmlparser.h @@ -64,5 +64,4 @@ class XMLParser } // namespace config -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/dataconvert/dataconvert.cpp b/storage/columnstore/columnstore/utils/dataconvert/dataconvert.cpp index 14195433..37ff38c4 100644 --- a/storage/columnstore/columnstore/utils/dataconvert/dataconvert.cpp +++ b/storage/columnstore/columnstore/utils/dataconvert/dataconvert.cpp @@ -1418,7 +1418,7 @@ boost::any DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAtt { int64_t val64; number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); - char ival = (char)val64; + signed char ival = (signed char)val64; if (ival < 0 && ival != static_cast<int8_t>(joblist::TINYINTEMPTYROW) && ival != static_cast<int8_t>(joblist::TINYINTNULL)) @@ -3388,4 +3388,3 @@ void DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd& u } } // namespace dataconvert -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.cpp b/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.cpp index a9defdf4..ca196c29 100644 --- a/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.cpp +++ b/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.cpp @@ -270,4 +270,3 @@ int ddl_cleanup() } } // namespace ddlcleanuputil -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.h b/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.h index 7eebdf15..b76052fa 100644 --- a/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.h +++ b/storage/columnstore/columnstore/utils/ddlcleanup/ddlcleanuputil.h @@ -29,6 +29,4 @@ namespace ddlcleanuputil { int ddl_cleanup(); } -// vim:ts=4 sw=4: - #endif diff --git a/storage/columnstore/columnstore/utils/funcexp/func_abs.cpp b/storage/columnstore/columnstore/utils/funcexp/func_abs.cpp index 64ac5d01..cd05ade7 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_abs.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_abs.cpp @@ -79,4 +79,3 @@ long double Func_abs::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_add_time.cpp b/storage/columnstore/columnstore/utils/funcexp/func_add_time.cpp index 5a2506b3..82c173ed 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_add_time.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_add_time.cpp @@ -303,4 +303,3 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, boo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_ascii.cpp b/storage/columnstore/columnstore/utils/funcexp/func_ascii.cpp index eb93f601..5cf8653d 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_ascii.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_ascii.cpp @@ -56,4 +56,3 @@ int64_t Func_ascii::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_between.cpp b/storage/columnstore/columnstore/utils/funcexp/func_between.cpp index 3ce66eb6..65712363 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_between.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_between.cpp @@ -376,4 +376,3 @@ bool Func_notbetween::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_bitwise.cpp b/storage/columnstore/columnstore/utils/funcexp/func_bitwise.cpp index c10030b9..998696ae 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_bitwise.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_bitwise.cpp @@ -494,4 +494,3 @@ bool Func_bit_count::fix(execplan::FunctionColumn& col) const } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_case.cpp b/storage/columnstore/columnstore/utils/funcexp/func_case.cpp index e8bfbd0f..fba7691e 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_case.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_case.cpp @@ -723,4 +723,3 @@ int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_cast.cpp b/storage/columnstore/columnstore/utils/funcexp/func_cast.cpp index 4b224c6d..ef0e2f51 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_cast.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_cast.cpp @@ -1549,4 +1549,3 @@ double Func_cast_double::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_ceil.cpp b/storage/columnstore/columnstore/utils/funcexp/func_ceil.cpp index dc559a5d..e5c1f872 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_ceil.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_ceil.cpp @@ -598,4 +598,3 @@ IDB_Decimal Func_ceil::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_char.cpp b/storage/columnstore/columnstore/utils/funcexp/func_char.cpp index 7ead2b94..0c4adeaf 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_char.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_char.cpp @@ -178,4 +178,3 @@ string Func_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontS } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_char_length.cpp b/storage/columnstore/columnstore/utils/funcexp/func_char_length.cpp index 553d5439..5270d1d8 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_char_length.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_char_length.cpp @@ -121,4 +121,3 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_coalesce.cpp b/storage/columnstore/columnstore/utils/funcexp/func_coalesce.cpp index ebabde05..bbb5a279 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_coalesce.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_coalesce.cpp @@ -244,4 +244,3 @@ execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row, FunctionP } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_concat.cpp b/storage/columnstore/columnstore/utils/funcexp/func_concat.cpp index 73b2a2b2..05835d23 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_concat.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_concat.cpp @@ -65,4 +65,3 @@ string Func_concat::getStrVal(Row& row, FunctionParm& parm, bool& isNull, Calpon } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_concat_oracle.cpp b/storage/columnstore/columnstore/utils/funcexp/func_concat_oracle.cpp index 28713a10..46631e7a 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_concat_oracle.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_concat_oracle.cpp @@ -71,4 +71,3 @@ string Func_concat_oracle::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_concat_ws.cpp b/storage/columnstore/columnstore/utils/funcexp/func_concat_ws.cpp index 2c07c5b0..ea0cf22d 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_concat_ws.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_concat_ws.cpp @@ -121,4 +121,3 @@ string Func_concat_ws::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_conv.cpp b/storage/columnstore/columnstore/utils/funcexp/func_conv.cpp index 5f73f5ea..037ce83e 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_conv.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_conv.cpp @@ -304,4 +304,3 @@ string Func_conv::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_convert_tz.cpp b/storage/columnstore/columnstore/utils/funcexp/func_convert_tz.cpp index 03efdad8..36339279 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_convert_tz.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_convert_tz.cpp @@ -220,4 +220,3 @@ string Func_convert_tz::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_crc32.cpp b/storage/columnstore/columnstore/utils/funcexp/func_crc32.cpp index 8d728c11..e573db87 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_crc32.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_crc32.cpp @@ -64,4 +64,3 @@ int64_t Func_crc32::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_date.cpp b/storage/columnstore/columnstore/utils/funcexp/func_date.cpp index 5544e8b3..b6e554e5 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_date.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_date.cpp @@ -149,4 +149,3 @@ string Func_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_date_add.cpp b/storage/columnstore/columnstore/utils/funcexp/func_date_add.cpp index 2e715acc..6cc47fda 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_date_add.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_date_add.cpp @@ -816,4 +816,3 @@ string Func_date_add::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_date_format.cpp b/storage/columnstore/columnstore/utils/funcexp/func_date_format.cpp index 962fc811..8a3ba974 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_date_format.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_date_format.cpp @@ -416,4 +416,3 @@ int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, FunctionParm& p } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_day.cpp b/storage/columnstore/columnstore/utils/funcexp/func_day.cpp index bd4a484d..e7954e3d 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_day.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_day.cpp @@ -143,4 +143,3 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_dayname.cpp b/storage/columnstore/columnstore/utils/funcexp/func_dayname.cpp index 07078a25..e8e069db 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_dayname.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_dayname.cpp @@ -179,4 +179,3 @@ string Func_dayname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_dayofweek.cpp b/storage/columnstore/columnstore/utils/funcexp/func_dayofweek.cpp index 441bfeeb..c8fb40ab 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_dayofweek.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_dayofweek.cpp @@ -166,4 +166,3 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_dayofyear.cpp b/storage/columnstore/columnstore/utils/funcexp/func_dayofyear.cpp index fb9d9742..215ec215 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_dayofyear.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_dayofyear.cpp @@ -160,4 +160,3 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_div.cpp b/storage/columnstore/columnstore/utils/funcexp/func_div.cpp index 7169992b..d36e75c9 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_div.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_div.cpp @@ -108,4 +108,3 @@ string Func_div::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_elt.cpp b/storage/columnstore/columnstore/utils/funcexp/func_elt.cpp index 22446095..860dd1ae 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_elt.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_elt.cpp @@ -96,4 +96,3 @@ string Func_elt::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_exp.cpp b/storage/columnstore/columnstore/utils/funcexp/func_exp.cpp index bab2fc92..d57b2855 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_exp.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_exp.cpp @@ -110,4 +110,3 @@ long double Func_exp::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_extract.cpp b/storage/columnstore/columnstore/utils/funcexp/func_extract.cpp index a6f8d635..b051f267 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_extract.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_extract.cpp @@ -254,4 +254,3 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_find_in_set.cpp b/storage/columnstore/columnstore/utils/funcexp/func_find_in_set.cpp index 94c1243d..85b0c7e2 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_find_in_set.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_find_in_set.cpp @@ -127,4 +127,3 @@ execplan::IDB_Decimal Func_find_in_set::getDecimalVal(rowgroup::Row& row, Functi } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_floor.cpp b/storage/columnstore/columnstore/utils/funcexp/func_floor.cpp index 7c0dacc8..f497400b 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_floor.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_floor.cpp @@ -555,4 +555,3 @@ IDB_Decimal Func_floor::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_from_days.cpp b/storage/columnstore/columnstore/utils/funcexp/func_from_days.cpp index e9f45b42..a3e964ee 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_from_days.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_from_days.cpp @@ -80,4 +80,3 @@ int64_t Func_from_days::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_from_unixtime.cpp b/storage/columnstore/columnstore/utils/funcexp/func_from_unixtime.cpp index cca4083b..b01bfab3 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_from_unixtime.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_from_unixtime.cpp @@ -212,4 +212,3 @@ long double Func_from_unixtime::getLongDoubleVal(rowgroup::Row& row, FunctionPar } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_get_format.cpp b/storage/columnstore/columnstore/utils/funcexp/func_get_format.cpp index d902c087..00f51364 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_get_format.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_get_format.cpp @@ -107,4 +107,3 @@ string Func_get_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_greatest.cpp b/storage/columnstore/columnstore/utils/funcexp/func_greatest.cpp index e8562be2..b3dea5be 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_greatest.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_greatest.cpp @@ -240,4 +240,3 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_hex.cpp b/storage/columnstore/columnstore/utils/funcexp/func_hex.cpp index 9e2fa114..c1a111cb 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_hex.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_hex.cpp @@ -140,4 +140,3 @@ string Func_hex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_hour.cpp b/storage/columnstore/columnstore/utils/funcexp/func_hour.cpp index 7d2da741..0550ef3e 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_hour.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_hour.cpp @@ -156,4 +156,3 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_idbpartition.cpp b/storage/columnstore/columnstore/utils/funcexp/func_idbpartition.cpp index 3b71cdd9..b7b5e697 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_idbpartition.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_idbpartition.cpp @@ -63,4 +63,3 @@ string Func_idbpartition::getStrVal(Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_if.cpp b/storage/columnstore/columnstore/utils/funcexp/func_if.cpp index 98fe8a33..d5941df8 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_if.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_if.cpp @@ -234,4 +234,3 @@ int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, Calpo } } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_ifnull.cpp b/storage/columnstore/columnstore/utils/funcexp/func_ifnull.cpp index 52dae255..a2450a5a 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_ifnull.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_ifnull.cpp @@ -205,4 +205,3 @@ bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_in.cpp b/storage/columnstore/columnstore/utils/funcexp/func_in.cpp index 89ed8959..d452fc24 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_in.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_in.cpp @@ -386,4 +386,3 @@ bool Func_notin::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_insert.cpp b/storage/columnstore/columnstore/utils/funcexp/func_insert.cpp index 236752e9..72fd06e4 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_insert.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_insert.cpp @@ -102,4 +102,3 @@ std::string Func_insert::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_instr.cpp b/storage/columnstore/columnstore/utils/funcexp/func_instr.cpp index ab0379cc..71eb81d0 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_instr.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_instr.cpp @@ -85,4 +85,3 @@ int64_t Func_instr::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_isnull.cpp b/storage/columnstore/columnstore/utils/funcexp/func_isnull.cpp index 48337b3e..4ff7e643 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_isnull.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_isnull.cpp @@ -79,4 +79,3 @@ bool Func_isnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, Calpont } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_last_day.cpp b/storage/columnstore/columnstore/utils/funcexp/func_last_day.cpp index 89f173a7..e74a2d51 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_last_day.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_last_day.cpp @@ -195,4 +195,3 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_lcase.cpp b/storage/columnstore/columnstore/utils/funcexp/func_lcase.cpp index 817e82ae..eabdfd40 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_lcase.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_lcase.cpp @@ -68,4 +68,3 @@ std::string Func_lcase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_least.cpp b/storage/columnstore/columnstore/utils/funcexp/func_least.cpp index add78cfa..d488b751 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_least.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_least.cpp @@ -218,4 +218,3 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_left.cpp b/storage/columnstore/columnstore/utils/funcexp/func_left.cpp index a90d5f46..de31d177 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_left.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_left.cpp @@ -73,4 +73,3 @@ std::string Func_left::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_length.cpp b/storage/columnstore/columnstore/utils/funcexp/func_length.cpp index f63e2f1b..a6f7236f 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_length.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_length.cpp @@ -55,4 +55,3 @@ int64_t Func_length::getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_lpad.cpp b/storage/columnstore/columnstore/utils/funcexp/func_lpad.cpp index 0ad0bdc2..8a5544a6 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_lpad.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_lpad.cpp @@ -127,4 +127,3 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_ltrim.cpp b/storage/columnstore/columnstore/utils/funcexp/func_ltrim.cpp index 39fd7165..3d7e9691 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_ltrim.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_ltrim.cpp @@ -93,4 +93,3 @@ std::string Func_ltrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_ltrim_oracle.cpp b/storage/columnstore/columnstore/utils/funcexp/func_ltrim_oracle.cpp index 234808e7..70fc1dc8 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_ltrim_oracle.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_ltrim_oracle.cpp @@ -97,4 +97,3 @@ std::string Func_ltrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_makedate.cpp b/storage/columnstore/columnstore/utils/funcexp/func_makedate.cpp index 0418508d..fb49f54c 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_makedate.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_makedate.cpp @@ -190,4 +190,3 @@ string Func_makedate::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_maketime.cpp b/storage/columnstore/columnstore/utils/funcexp/func_maketime.cpp index 6d0a9d90..ad918ccc 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_maketime.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_maketime.cpp @@ -171,4 +171,3 @@ string Func_maketime::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_math.cpp b/storage/columnstore/columnstore/utils/funcexp/func_math.cpp index 7d60ae50..74a12aa7 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_math.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_math.cpp @@ -2276,4 +2276,3 @@ double Func_degrees::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, Ca } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_md5.cpp b/storage/columnstore/columnstore/utils/funcexp/func_md5.cpp index 6ef058e8..0ce6b0a0 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_md5.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_md5.cpp @@ -512,4 +512,3 @@ string Func_md5::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_microsecond.cpp b/storage/columnstore/columnstore/utils/funcexp/func_microsecond.cpp index aca1e3ae..8c94647b 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_microsecond.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_microsecond.cpp @@ -137,4 +137,3 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_minute.cpp b/storage/columnstore/columnstore/utils/funcexp/func_minute.cpp index af310c87..8e572454 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_minute.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_minute.cpp @@ -137,4 +137,3 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_mod.cpp b/storage/columnstore/columnstore/utils/funcexp/func_mod.cpp index bf33a649..dff1a4e0 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_mod.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_mod.cpp @@ -531,4 +531,3 @@ std::string Func_mod::getStrVal(Row& row, FunctionParm& fp, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_month.cpp b/storage/columnstore/columnstore/utils/funcexp/func_month.cpp index 50ee3600..ea66abf3 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_month.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_month.cpp @@ -142,4 +142,3 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNu } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_monthname.cpp b/storage/columnstore/columnstore/utils/funcexp/func_monthname.cpp index 8ea7c3e3..c674f454 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_monthname.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_monthname.cpp @@ -191,4 +191,3 @@ execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, Function } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_nullif.cpp b/storage/columnstore/columnstore/utils/funcexp/func_nullif.cpp index ba4ab740..3821151e 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_nullif.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_nullif.cpp @@ -964,7 +964,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -1022,4 +1022,3 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, FunctionPar } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_period_add.cpp b/storage/columnstore/columnstore/utils/funcexp/func_period_add.cpp index 23b96415..4038651e 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_period_add.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_period_add.cpp @@ -94,4 +94,3 @@ int64_t Func_period_add::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_period_diff.cpp b/storage/columnstore/columnstore/utils/funcexp/func_period_diff.cpp index 470ab60c..080d15a4 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_period_diff.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_period_diff.cpp @@ -113,4 +113,3 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_pow.cpp b/storage/columnstore/columnstore/utils/funcexp/func_pow.cpp index 734b50a2..236e7e2c 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_pow.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_pow.cpp @@ -121,4 +121,3 @@ long double Func_pow::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_quarter.cpp b/storage/columnstore/columnstore/utils/funcexp/func_quarter.cpp index 04c4babb..5f1fd003 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_quarter.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_quarter.cpp @@ -139,4 +139,3 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_quote.cpp b/storage/columnstore/columnstore/utils/funcexp/func_quote.cpp index 05ce4cec..b82cf7ba 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_quote.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_quote.cpp @@ -80,4 +80,3 @@ std::string Func_quote::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_rand.cpp b/storage/columnstore/columnstore/utils/funcexp/func_rand.cpp index 223949bb..b7da263d 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_rand.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_rand.cpp @@ -98,4 +98,3 @@ double Func_rand::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_regexp.cpp b/storage/columnstore/columnstore/utils/funcexp/func_regexp.cpp index cb495c8c..60e1459d 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_regexp.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_regexp.cpp @@ -248,4 +248,3 @@ bool Func_regexp::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_repeat.cpp b/storage/columnstore/columnstore/utils/funcexp/func_repeat.cpp index 6b0e6ace..ebbfc355 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_repeat.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_repeat.cpp @@ -90,4 +90,3 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_replace.cpp b/storage/columnstore/columnstore/utils/funcexp/func_replace.cpp index 05e0fe35..da18fb3f 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_replace.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_replace.cpp @@ -173,4 +173,3 @@ std::string Func_replace::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_replace_oracle.cpp b/storage/columnstore/columnstore/utils/funcexp/func_replace_oracle.cpp index b8eb7f6e..113b49d9 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_replace_oracle.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_replace_oracle.cpp @@ -167,4 +167,3 @@ std::string Func_replace_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_reverse.cpp b/storage/columnstore/columnstore/utils/funcexp/func_reverse.cpp index 2b383ee7..db57d095 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_reverse.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_reverse.cpp @@ -93,4 +93,3 @@ std::string Func_reverse::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_right.cpp b/storage/columnstore/columnstore/utils/funcexp/func_right.cpp index 4615256e..2c1ec612 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_right.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_right.cpp @@ -73,4 +73,3 @@ std::string Func_right::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_round.cpp b/storage/columnstore/columnstore/utils/funcexp/func_round.cpp index 16855324..c6e67336 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_round.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_round.cpp @@ -645,7 +645,7 @@ string Func_round::getStrVal(Row& row, FunctionParm& parm, bool& isNull, Calpont { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); int64_t e = (x.scale < 0) ? (-x.scale) : x.scale; - int64_t p = 1; + [[maybe_unused]] int64_t p = 1; while (e-- > 0) p *= 10; @@ -716,4 +716,3 @@ int64_t Func_round::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_rpad.cpp b/storage/columnstore/columnstore/utils/funcexp/func_rpad.cpp index fba47bfa..a2a538ff 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_rpad.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_rpad.cpp @@ -127,4 +127,3 @@ std::string Func_rpad::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_rtrim.cpp b/storage/columnstore/columnstore/utils/funcexp/func_rtrim.cpp index 426ec871..ee61906b 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_rtrim.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_rtrim.cpp @@ -157,4 +157,3 @@ std::string Func_rtrim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_rtrim_oracle.cpp b/storage/columnstore/columnstore/utils/funcexp/func_rtrim_oracle.cpp index 1b47d7dd..93713e11 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_rtrim_oracle.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_rtrim_oracle.cpp @@ -161,4 +161,3 @@ std::string Func_rtrim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, b } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_sec_to_time.cpp b/storage/columnstore/columnstore/utils/funcexp/func_sec_to_time.cpp index d3b4af67..9a2fef4c 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_sec_to_time.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_sec_to_time.cpp @@ -231,4 +231,3 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, Functi } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_second.cpp b/storage/columnstore/columnstore/utils/funcexp/func_second.cpp index 7731ea66..6136c3ed 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_second.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_second.cpp @@ -144,4 +144,3 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_sha.cpp b/storage/columnstore/columnstore/utils/funcexp/func_sha.cpp index bc74a0a2..ec3fac1f 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_sha.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_sha.cpp @@ -655,4 +655,3 @@ string Func_sha::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_sign.cpp b/storage/columnstore/columnstore/utils/funcexp/func_sign.cpp index a0cece67..426fbb0c 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_sign.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_sign.cpp @@ -71,4 +71,3 @@ string Func_sign::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_space.cpp b/storage/columnstore/columnstore/utils/funcexp/func_space.cpp index 93e765a5..9a9a8578 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_space.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_space.cpp @@ -57,4 +57,3 @@ std::string Func_space::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_str_to_date.cpp b/storage/columnstore/columnstore/utils/funcexp/func_str_to_date.cpp index 0580021f..ab0fb4f6 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_str_to_date.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_str_to_date.cpp @@ -264,4 +264,3 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_strcmp.cpp b/storage/columnstore/columnstore/utils/funcexp/func_strcmp.cpp index a16c68cc..63b3b0b2 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_strcmp.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_strcmp.cpp @@ -74,4 +74,3 @@ std::string Func_strcmp::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_substr.cpp b/storage/columnstore/columnstore/utils/funcexp/func_substr.cpp index cb45acc0..9a39a4f1 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_substr.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_substr.cpp @@ -98,4 +98,3 @@ std::string Func_substr::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_substring_index.cpp b/storage/columnstore/columnstore/utils/funcexp/func_substring_index.cpp index ffae25e5..82f94a7e 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_substring_index.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_substring_index.cpp @@ -205,4 +205,3 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row, FunctionParm& fp } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_sysdate.cpp b/storage/columnstore/columnstore/utils/funcexp/func_sysdate.cpp index b30e66c5..780492f3 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_sysdate.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_sysdate.cpp @@ -105,4 +105,3 @@ int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_time.cpp b/storage/columnstore/columnstore/utils/funcexp/func_time.cpp index 14d2246a..e898f782 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_time.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_time.cpp @@ -169,4 +169,3 @@ double Func_time::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_time_format.cpp b/storage/columnstore/columnstore/utils/funcexp/func_time_format.cpp index d075658b..1205e21b 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_time_format.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_time_format.cpp @@ -229,4 +229,3 @@ string Func_time_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_time_to_sec.cpp b/storage/columnstore/columnstore/utils/funcexp/func_time_to_sec.cpp index 388ef2ca..3cf718aa 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_time_to_sec.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_time_to_sec.cpp @@ -195,4 +195,3 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_timediff.cpp b/storage/columnstore/columnstore/utils/funcexp/func_timediff.cpp index d3f89139..03000476 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_timediff.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_timediff.cpp @@ -349,4 +349,3 @@ double Func_timediff::getDoubleVal(rowgroup::Row& row, FunctionParm& fp, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_timestampdiff.cpp b/storage/columnstore/columnstore/utils/funcexp/func_timestampdiff.cpp index 07e7eda7..6ad2f395 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_timestampdiff.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_timestampdiff.cpp @@ -196,4 +196,3 @@ int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_to_days.cpp b/storage/columnstore/columnstore/utils/funcexp/func_to_days.cpp index f597c5fd..d6f831be 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_to_days.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_to_days.cpp @@ -151,4 +151,3 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_trim.cpp b/storage/columnstore/columnstore/utils/funcexp/func_trim.cpp index 636afe44..d4782356 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_trim.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_trim.cpp @@ -168,4 +168,3 @@ std::string Func_trim::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& isN } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_trim_oracle.cpp b/storage/columnstore/columnstore/utils/funcexp/func_trim_oracle.cpp index 5e1226c5..4daec213 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_trim_oracle.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_trim_oracle.cpp @@ -166,4 +166,3 @@ std::string Func_trim_oracle::getStrVal(rowgroup::Row& row, FunctionParm& fp, bo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_truncate.cpp b/storage/columnstore/columnstore/utils/funcexp/func_truncate.cpp index 87f045af..7a529c00 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_truncate.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_truncate.cpp @@ -522,7 +522,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -613,7 +613,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN { // strip off micro seconds value = value.substr(0, 14); - int64_t x = atoll(value.c_str()); + x = atoll(value.c_str()); if (s > 5) s = 0; @@ -721,4 +721,3 @@ int64_t Func_truncate::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_ucase.cpp b/storage/columnstore/columnstore/utils/funcexp/func_ucase.cpp index 9fa00e67..1cbbd6a2 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_ucase.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_ucase.cpp @@ -77,4 +77,3 @@ std::string Func_ucase::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_unhex.cpp b/storage/columnstore/columnstore/utils/funcexp/func_unhex.cpp index 38da2cb2..92ef5fa3 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_unhex.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_unhex.cpp @@ -112,4 +112,3 @@ string Func_unhex::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_unix_timestamp.cpp b/storage/columnstore/columnstore/utils/funcexp/func_unix_timestamp.cpp index d5b65fe2..f9c81e6b 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_unix_timestamp.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_unix_timestamp.cpp @@ -223,4 +223,3 @@ string Func_unix_timestamp::getStrVal(rowgroup::Row& row, FunctionParm& parm, bo } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_week.cpp b/storage/columnstore/columnstore/utils/funcexp/func_week.cpp index 8f7a3af0..e6ab52f3 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_week.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_week.cpp @@ -169,4 +169,3 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_weekday.cpp b/storage/columnstore/columnstore/utils/funcexp/func_weekday.cpp index 302293b8..e78d12c6 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_weekday.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_weekday.cpp @@ -164,4 +164,3 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_year.cpp b/storage/columnstore/columnstore/utils/funcexp/func_year.cpp index 2f4fc827..9ea12fbd 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_year.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_year.cpp @@ -140,4 +140,3 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNul } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/func_yearweek.cpp b/storage/columnstore/columnstore/utils/funcexp/func_yearweek.cpp index 3b85692b..2ad94139 100644 --- a/storage/columnstore/columnstore/utils/funcexp/func_yearweek.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/func_yearweek.cpp @@ -173,4 +173,3 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& i } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/funcexp/functor.cpp b/storage/columnstore/columnstore/utils/funcexp/functor.cpp index 4eed1688..3da34944 100644 --- a/storage/columnstore/columnstore/utils/funcexp/functor.cpp +++ b/storage/columnstore/columnstore/utils/funcexp/functor.cpp @@ -361,4 +361,3 @@ string Func::longDoubleToString(long double ld) } } // namespace funcexp -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/libmysql_client/libmysql_client.h b/storage/columnstore/columnstore/utils/libmysql_client/libmysql_client.h index 4207a705..931886b7 100644 --- a/storage/columnstore/columnstore/utils/libmysql_client/libmysql_client.h +++ b/storage/columnstore/columnstore/utils/libmysql_client/libmysql_client.h @@ -91,4 +91,3 @@ class LibMySQL #endif // UTILS_LIBMYSQL_CL_H -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/loggingcpp/errorcodes.cpp b/storage/columnstore/columnstore/utils/loggingcpp/errorcodes.cpp index e52a5e32..c8e879c0 100644 --- a/storage/columnstore/columnstore/utils/loggingcpp/errorcodes.cpp +++ b/storage/columnstore/columnstore/utils/loggingcpp/errorcodes.cpp @@ -103,4 +103,3 @@ string ErrorCodes::errorString(uint16_t code) const return (fPreamble + msg); } } // namespace logging -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/loggingcpp/exceptclasses.h b/storage/columnstore/columnstore/utils/loggingcpp/exceptclasses.h index 92aebbbb..7bd6b625 100644 --- a/storage/columnstore/columnstore/utils/loggingcpp/exceptclasses.h +++ b/storage/columnstore/columnstore/utils/loggingcpp/exceptclasses.h @@ -307,5 +307,4 @@ class ProtocolError : public std::logic_error } // namespace logging -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/messageqcpp/inetstreamsocket.cpp b/storage/columnstore/columnstore/utils/messageqcpp/inetstreamsocket.cpp index 81ed82f6..45f605bc 100644 --- a/storage/columnstore/columnstore/utils/messageqcpp/inetstreamsocket.cpp +++ b/storage/columnstore/columnstore/utils/messageqcpp/inetstreamsocket.cpp @@ -101,8 +101,6 @@ using boost::scoped_array; // some static functions namespace { -using messageqcpp::ByteStream; - // @bug 2441 - Retry after 512 read() error. // ERESTARTSYS (512) is a kernal I/O errno that is similar to a EINTR, except // that it is not supposed to "leak" out into the user space. But we are @@ -937,16 +935,8 @@ void InetStreamSocket::connect(const sockaddr* serv_addr) char buf = '\0'; (void)::recv(socketParms().sd(), &buf, 1, 0); #else -#if defined(__GNUC__) && __GNUC__ >= 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#pragma GCC diagnostic pop -#else char buf = '\0'; - ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer -#endif // pragma + std::ignore = ::read(socketParms().sd(), &buf, 1); // we know 1 byte is in the recv buffer #endif return; } diff --git a/storage/columnstore/columnstore/utils/messageqcpp/messagequeue.h b/storage/columnstore/columnstore/utils/messageqcpp/messagequeue.h index 7d9cba82..30dd20ff 100644 --- a/storage/columnstore/columnstore/utils/messageqcpp/messagequeue.h +++ b/storage/columnstore/columnstore/utils/messageqcpp/messagequeue.h @@ -337,5 +337,4 @@ inline void MessageQueueClient::syncProto(bool use) #undef EXPORT -#endif // MESSAGEQCPP_MESSAGEQUEUE_H -// vim:ts=4 sw=4: +#endif // MESSAGEQCPP_MESSAGEQUEUE_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/multicast/multicast.cpp b/storage/columnstore/columnstore/utils/multicast/multicast.cpp index 116f0382..b471c170 100644 --- a/storage/columnstore/columnstore/utils/multicast/multicast.cpp +++ b/storage/columnstore/columnstore/utils/multicast/multicast.cpp @@ -93,4 +93,3 @@ void MulticastSender::send(const ByteStream& msg) } // namespace multicast -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/regr/regrmysql.cpp b/storage/columnstore/columnstore/utils/regr/regrmysql.cpp index 1b480aa8..d9e9a51b 100644 --- a/storage/columnstore/columnstore/utils/regr/regrmysql.cpp +++ b/storage/columnstore/columnstore/utils/regr/regrmysql.cpp @@ -1405,4 +1405,3 @@ extern "C" return valOut; } } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/rowgroup/rowgroup.cpp b/storage/columnstore/columnstore/utils/rowgroup/rowgroup.cpp index 4e767e1d..7c297f8f 100644 --- a/storage/columnstore/columnstore/utils/rowgroup/rowgroup.cpp +++ b/storage/columnstore/columnstore/utils/rowgroup/rowgroup.cpp @@ -1751,4 +1751,3 @@ RowGroup RowGroup::truncate(uint32_t cols) } // namespace rowgroup -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/rowgroup/rowgroup.h b/storage/columnstore/columnstore/utils/rowgroup/rowgroup.h index a17449d0..6dfebfec 100644 --- a/storage/columnstore/columnstore/utils/rowgroup/rowgroup.h +++ b/storage/columnstore/columnstore/utils/rowgroup/rowgroup.h @@ -31,7 +31,7 @@ #include <vector> #include <string> #include <stdexcept> -//#define NDEBUG +// #define NDEBUG #include <cassert> #include <boost/shared_ptr.hpp> #include <boost/shared_array.hpp> @@ -1525,8 +1525,8 @@ class RowGroup : public messageqcpp::Serializeable inline void setUseStringTable(bool); // RGData *convertToInlineData(uint64_t *size = NULL) const; // caller manages the memory returned by - //this void convertToInlineDataInPlace(); RGData *convertToStringTable(uint64_t *size = NULL) const; void - //convertToStringTableInPlace(); + // this void convertToInlineDataInPlace(); RGData *convertToStringTable(uint64_t *size = NULL) + // const; void convertToStringTableInPlace(); void serializeRGData(messageqcpp::ByteStream&) const; inline uint32_t getStringTableThreshold() const; @@ -2184,5 +2184,4 @@ inline void RGData::getRow(uint32_t num, Row* row) } // namespace rowgroup -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/rwlock/rwlock.cpp b/storage/columnstore/columnstore/utils/rwlock/rwlock.cpp index 73a9c5db..ca7b82e9 100644 --- a/storage/columnstore/columnstore/utils/rwlock/rwlock.cpp +++ b/storage/columnstore/columnstore/utils/rwlock/rwlock.cpp @@ -746,4 +746,3 @@ LockState RWLock::getLockState() } } // namespace rwlock -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/rwlock/rwlock.h b/storage/columnstore/columnstore/utils/rwlock/rwlock.h index 33702570..d18640a4 100644 --- a/storage/columnstore/columnstore/utils/rwlock/rwlock.h +++ b/storage/columnstore/columnstore/utils/rwlock/rwlock.h @@ -46,9 +46,15 @@ namespace rwlock { -const std::array<const std::string, 7> RWLockNames = { { - "all", "VSS", "ExtentMap", "FreeList", "VBBM", "CopyLocks", "ExtentMapIndex", -} }; +const std::array<const std::string, 7> RWLockNames = {{ + "all", + "VSS", + "ExtentMap", + "FreeList", + "VBBM", + "CopyLocks", + "ExtentMapIndex", +}}; /// the layout of the shmseg struct State @@ -283,5 +289,4 @@ class RWLock #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/startup/installdir.cpp b/storage/columnstore/columnstore/utils/startup/installdir.cpp index 65437a9c..2db2128d 100644 --- a/storage/columnstore/columnstore/utils/startup/installdir.cpp +++ b/storage/columnstore/columnstore/utils/startup/installdir.cpp @@ -112,4 +112,3 @@ const string StartUp::tmpDir() } } // namespace startup -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/startup/installdir.h b/storage/columnstore/columnstore/utils/startup/installdir.h index 4d9b8989..c902d4bf 100644 --- a/storage/columnstore/columnstore/utils/startup/installdir.h +++ b/storage/columnstore/columnstore/utils/startup/installdir.h @@ -53,5 +53,4 @@ class StartUp } // namespace startup -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/testbc/iomanager.cpp b/storage/columnstore/columnstore/utils/testbc/iomanager.cpp index 7c3cc4d6..b2944e5b 100644 --- a/storage/columnstore/columnstore/utils/testbc/iomanager.cpp +++ b/storage/columnstore/columnstore/utils/testbc/iomanager.cpp @@ -325,4 +325,3 @@ fileRequest* ioManager::getNextRequest() } } // namespace dbbc -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/testbc/iomanager.h b/storage/columnstore/columnstore/utils/testbc/iomanager.h index 20704c58..aab251dd 100644 --- a/storage/columnstore/columnstore/utils/testbc/iomanager.h +++ b/storage/columnstore/columnstore/utils/testbc/iomanager.h @@ -99,5 +99,4 @@ class ioManager }; } // namespace dbbc -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/threadpool/prioritythreadpool.cpp b/storage/columnstore/columnstore/utils/threadpool/prioritythreadpool.cpp index b26144b6..908c0ba9 100644 --- a/storage/columnstore/columnstore/utils/threadpool/prioritythreadpool.cpp +++ b/storage/columnstore/columnstore/utils/threadpool/prioritythreadpool.cpp @@ -322,4 +322,3 @@ void PriorityThreadPool::stop() } } // namespace threadpool -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/udfsdk/udfmysql.cpp b/storage/columnstore/columnstore/utils/udfsdk/udfmysql.cpp index 55272942..f32fc6b3 100644 --- a/storage/columnstore/columnstore/utils/udfsdk/udfmysql.cpp +++ b/storage/columnstore/columnstore/utils/udfsdk/udfmysql.cpp @@ -535,4 +535,3 @@ extern "C" return 0; } } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/udfsdk/udfsdk.cpp b/storage/columnstore/columnstore/utils/udfsdk/udfsdk.cpp index 0cf80618..dc5e4fa8 100644 --- a/storage/columnstore/columnstore/utils/udfsdk/udfsdk.cpp +++ b/storage/columnstore/columnstore/utils/udfsdk/udfsdk.cpp @@ -375,4 +375,3 @@ int64_t MCS_isnull::getDatetimeIntVal(Row& row, FunctionParm& parm, bool& isNull } } // namespace udfsdk -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/udfsdk/udfsdk.h b/storage/columnstore/columnstore/utils/udfsdk/udfsdk.h index 9e53601b..6c0ee013 100644 --- a/storage/columnstore/columnstore/utils/udfsdk/udfsdk.h +++ b/storage/columnstore/columnstore/utils/udfsdk/udfsdk.h @@ -314,4 +314,3 @@ class MCS_isnull : public funcexp::Func #undef EXPORT -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/framebound.cpp b/storage/columnstore/columnstore/utils/windowfunction/framebound.cpp index 043620f5..56b50d2e 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/framebound.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/framebound.cpp @@ -72,4 +72,3 @@ const string FrameBound::toString() const } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/framebound.h b/storage/columnstore/columnstore/utils/windowfunction/framebound.h index 27038ae1..22203c73 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/framebound.h +++ b/storage/columnstore/columnstore/utils/windowfunction/framebound.h @@ -146,6 +146,4 @@ extern std::map<int, std::string> colType2String; } // namespace windowfunction -#endif // UTILS_FRAMEBOUND_H - -// vim:ts=4 sw=4: +#endif // UTILS_FRAMEBOUND_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.cpp b/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.cpp index 0b356bce..e884d930 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.cpp @@ -411,4 +411,3 @@ template class FrameBoundExpressionRange<float>; template class FrameBoundExpressionRange<uint64_t>; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.h b/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.h index 0a36a543..0688e7da 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.h +++ b/storage/columnstore/columnstore/utils/windowfunction/frameboundrange.h @@ -214,6 +214,4 @@ class FrameBoundExpressionRange : public FrameBoundConstantRange<T> } // namespace windowfunction -#endif // UTILS_FRAMEBOUNDRANGE_H - -// vim:ts=4 sw=4: +#endif // UTILS_FRAMEBOUNDRANGE_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.cpp b/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.cpp index 616b1e0e..3d442741 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.cpp @@ -139,4 +139,3 @@ template class FrameBoundExpressionRow<float>; template class FrameBoundExpressionRow<uint64_t>; } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.h b/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.h index ac0b6c0a..bc7f450a 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.h +++ b/storage/columnstore/columnstore/utils/windowfunction/frameboundrow.h @@ -148,5 +148,3 @@ class FrameBoundExpressionRow : public FrameBoundConstantRow } // namespace windowfunction #endif // UTILS_FRAMEBOUNDROW_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/idborderby.cpp b/storage/columnstore/columnstore/utils/windowfunction/idborderby.cpp index 30d442e7..5160eed2 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/idborderby.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/idborderby.cpp @@ -898,4 +898,3 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) } } // namespace ordering -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_count.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_count.cpp index c45cd297..bc469476 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_count.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_count.cpp @@ -184,4 +184,3 @@ template boost::shared_ptr<WindowFunctionType> WF_count<int64_t>::makeFunction(i WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_count.h b/storage/columnstore/columnstore/utils/windowfunction/wf_count.h index 2dcbd963..1c4da82e 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_count.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_count.h @@ -49,5 +49,3 @@ class WF_count : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_COUNT_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.cpp index 4e7de53f..b3babe8b 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.cpp @@ -306,4 +306,3 @@ template void WF_lead_lag<double>::parseParms(const std::vector<execplan::SRCP>& template void WF_lead_lag<string>::parseParms(const std::vector<execplan::SRCP>&); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.h b/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.h index 9aaea0c8..e6290eb8 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_lead_lag.h @@ -54,5 +54,3 @@ class WF_lead_lag : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_LEAD_LAG_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.cpp index 4120fe32..b5b63bde 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.cpp @@ -185,4 +185,3 @@ template boost::shared_ptr<WindowFunctionType> WF_min_max<int64_t>::makeFunction WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.h b/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.h index bd8091ed..9305ebcb 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_min_max.h @@ -48,5 +48,3 @@ class WF_min_max : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_MIN_MAX_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.cpp index 338984a0..f3c9e2a1 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.cpp @@ -271,4 +271,3 @@ template boost::shared_ptr<WindowFunctionType> WF_nth_value<int64_t>::makeFuncti WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.h b/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.h index d5d98b4b..e78709f2 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_nth_value.h @@ -52,5 +52,3 @@ class WF_nth_value : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_NTH_VALUE_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.cpp index 375fd81d..4b663460 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.cpp @@ -154,4 +154,3 @@ void WF_ntile::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.h b/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.h index 154f725b..db3de2cf 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_ntile.h @@ -49,5 +49,3 @@ class WF_ntile : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_NTILE_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.cpp index 4d0dfce0..24d6b9f2 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.cpp @@ -382,4 +382,3 @@ template boost::shared_ptr<WindowFunctionType> WF_percentile<int64_t>::makeFunct WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.h b/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.h index 952f29af..713a2698 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_percentile.h @@ -50,5 +50,3 @@ class WF_percentile : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_PERCENTILE_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.cpp index f0459b7f..540ca808 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.cpp @@ -145,4 +145,3 @@ void WF_ranking::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.h b/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.h index df2e7471..dcfb9448 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_ranking.h @@ -48,5 +48,3 @@ class WF_ranking : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_RANKING_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.cpp index 6453c0a0..6dd58ce3 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.cpp @@ -86,4 +86,3 @@ void WF_row_number::operator()(int64_t b, int64_t e, int64_t c) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.h b/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.h index d0976c25..bcb8d996 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_row_number.h @@ -47,5 +47,3 @@ class WF_row_number : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_ROW_NUMBER_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_stats.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_stats.cpp index a4b6f4f5..73a668ac 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_stats.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_stats.cpp @@ -234,4 +234,3 @@ template boost::shared_ptr<WindowFunctionType> WF_stats<int64_t>::makeFunction(i WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_stats.h b/storage/columnstore/columnstore/utils/windowfunction/wf_stats.h index 725d1c6c..6a50df11 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_stats.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_stats.h @@ -50,5 +50,3 @@ class WF_stats : public WindowFunctionType } // namespace windowfunction #endif // UTILS_WF_STATS_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.cpp index d988f0a1..573b6e09 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.cpp @@ -306,4 +306,3 @@ template boost::shared_ptr<WindowFunctionType> WF_sum_avg<int64_t, long double>: int, const string&, int, WindowFunctionColumn*); } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.h b/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.h index 14cbd356..bdb5bc38 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_sum_avg.h @@ -61,6 +61,4 @@ class WF_sum_avg : public WindowFunctionType } // namespace windowfunction -#endif // UTILS_WF_SUM_AVG_H - -// vim:ts=4 sw=4: +#endif // UTILS_WF_SUM_AVG_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.cpp b/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.cpp index 274c9701..0894c6ad 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.cpp @@ -1191,4 +1191,3 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) fPrev = c; } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.h b/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.h index ab777761..22fae5a6 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.h +++ b/storage/columnstore/columnstore/utils/windowfunction/wf_udaf.h @@ -113,6 +113,4 @@ class WF_udaf : public WindowFunctionType } // namespace windowfunction -#endif // UTILS_WF_UDAF_H - -// vim:ts=4 sw=4: +#endif // UTILS_WF_UDAF_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/utils/windowfunction/windowframe.cpp b/storage/columnstore/columnstore/utils/windowfunction/windowframe.cpp index cbc9352c..491e0a0f 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/windowframe.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/windowframe.cpp @@ -81,4 +81,3 @@ const string WindowFrame::toString() const } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/windowframe.h b/storage/columnstore/columnstore/utils/windowfunction/windowframe.h index 159ee84d..cb20c5fe 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/windowframe.h +++ b/storage/columnstore/columnstore/utils/windowfunction/windowframe.h @@ -121,5 +121,3 @@ class WindowFrame } // namespace windowfunction #endif // UTILS_WINDOWFRAME_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/windowfunction.cpp b/storage/columnstore/columnstore/utils/windowfunction/windowfunction.cpp index a0e84493..941fff91 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/windowfunction.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/windowfunction.cpp @@ -259,4 +259,3 @@ void WindowFunction::sort(std::vector<RowPosition>::iterator v, uint64_t n) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/windowfunction.h b/storage/columnstore/columnstore/utils/windowfunction/windowfunction.h index 53f5b9a4..2d65f8fc 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/windowfunction.h +++ b/storage/columnstore/columnstore/utils/windowfunction/windowfunction.h @@ -113,5 +113,3 @@ class WindowFunction } // namespace windowfunction #endif // UTILS_WINDOWFUNCTION_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.cpp b/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.cpp index ccf74609..43f05f4b 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.cpp +++ b/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.cpp @@ -797,4 +797,3 @@ void WindowFunctionType::constParms(const std::vector<SRCP>& functionParms) } } // namespace windowfunction -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.h b/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.h index c1af9bc7..e64ac30c 100644 --- a/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.h +++ b/storage/columnstore/columnstore/utils/windowfunction/windowfunctiontype.h @@ -305,5 +305,3 @@ extern std::map<int, std::string> colType2String; } // namespace windowfunction #endif // UTILS_WINDOWFUNCTIONTYPE_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/versioning/BRM/brmshmimpl.cpp b/storage/columnstore/columnstore/versioning/BRM/brmshmimpl.cpp index 74028241..44750c63 100644 --- a/storage/columnstore/columnstore/versioning/BRM/brmshmimpl.cpp +++ b/storage/columnstore/columnstore/versioning/BRM/brmshmimpl.cpp @@ -410,4 +410,3 @@ void BRMManagedShmImpl::remap(const bool readOnly) } // namespace BRM -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/versioning/BRM/load_brm.cpp b/storage/columnstore/columnstore/versioning/BRM/load_brm.cpp index 4378fb76..b72bd3c2 100644 --- a/storage/columnstore/columnstore/versioning/BRM/load_brm.cpp +++ b/storage/columnstore/columnstore/versioning/BRM/load_brm.cpp @@ -127,4 +127,3 @@ int main(int argc, char** argv) return 0; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/versioning/BRM/slavecomm.cpp b/storage/columnstore/columnstore/versioning/BRM/slavecomm.cpp index 7b665a1a..1f490dd8 100644 --- a/storage/columnstore/columnstore/versioning/BRM/slavecomm.cpp +++ b/storage/columnstore/columnstore/versioning/BRM/slavecomm.cpp @@ -2432,4 +2432,3 @@ void SlaveComm::do_dmlReleaseLBIDRanges(ByteStream& msg) } // namespace BRM -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/versioning/BRM/vbbm.cpp b/storage/columnstore/columnstore/versioning/BRM/vbbm.cpp index 253bd051..2ed2454d 100644 --- a/storage/columnstore/columnstore/versioning/BRM/vbbm.cpp +++ b/storage/columnstore/columnstore/versioning/BRM/vbbm.cpp @@ -40,7 +40,6 @@ #include <boost/interprocess/shared_memory_object.hpp> #include <boost/interprocess/mapped_region.hpp> -namespace bi = boost::interprocess; #include <boost/scoped_array.hpp> #include <boost/scoped_ptr.hpp> @@ -1076,8 +1075,8 @@ void VBBM::save(string filename) } var = VBBM_MAGIC_V2; - int bytesWritten = 0; - int bytesToWrite = 12; + [[maybe_unused]] int bytesWritten = 0; + [[maybe_unused]] int bytesToWrite = 12; bytesWritten += out->write((char*)&var, 4); bytesWritten += out->write((char*)&vbbm->vbCurrentSize, 4); bytesWritten += out->write((char*)&vbbm->nFiles, 4); diff --git a/storage/columnstore/columnstore/writeengine/client/we_clients.cpp b/storage/columnstore/columnstore/writeengine/client/we_clients.cpp index c696dbb6..b6d66e2e 100644 --- a/storage/columnstore/columnstore/writeengine/client/we_clients.cpp +++ b/storage/columnstore/columnstore/writeengine/client/we_clients.cpp @@ -545,4 +545,3 @@ void WEClients::addDataToOutput(SBS sbs, uint32_t connIndex) } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/client/we_clients.h b/storage/columnstore/columnstore/writeengine/client/we_clients.h index 8028181a..a059b739 100644 --- a/storage/columnstore/columnstore/writeengine/client/we_clients.h +++ b/storage/columnstore/columnstore/writeengine/client/we_clients.h @@ -32,7 +32,7 @@ #include <boost/scoped_array.hpp> #include "bytestream.h" -//#include "we_message.h" +// #include "we_message.h" #include "threadsafequeue.h" #include "rwlock_local.h" #include "resourcemanager.h" @@ -181,5 +181,4 @@ class WEClients #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.cpp b/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.cpp index d155715b..9f910193 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.cpp +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.cpp @@ -52,4 +52,3 @@ void Redistribute::handleRedistributeMessage(ByteStream& bs, IOSocket& ios) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.h b/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.h index 62c33b83..8e01ed01 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.h +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistribute.h @@ -45,6 +45,4 @@ class Redistribute } // namespace redistribute -#endif // WE_REDISTRIBUTE_H - -// vim:ts=4 sw=4: +#endif // WE_REDISTRIBUTE_H
\ No newline at end of file diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.cpp b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.cpp index a54e47ff..9cb5e139 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.cpp +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.cpp @@ -698,4 +698,3 @@ void RedistributeControl::logMessage(const string& msg) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.h b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.h index 65535051..6b466508 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.h +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrol.h @@ -126,5 +126,3 @@ class RedistributeControl } // namespace redistribute #endif // WE_REDISTRIBUTECONTROL_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.cpp b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.cpp index ce808eb2..774f4011 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -838,4 +838,3 @@ void RedistributeControlThread::doStopAction() } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.h b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.h index 8f3116fa..92581ebd 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.h +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributecontrolthread.h @@ -127,5 +127,3 @@ class RedistributeControlThread } // namespace redistribute #endif // WE_REDISTRIBUTECONTROLTHREAD_H - -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.cpp b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.cpp index 67f1f586..5825ea53 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.cpp +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.cpp @@ -1520,4 +1520,3 @@ void RedistributeWorkerThread::logMessage(const string& msg, int line) } // namespace redistribute -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.h b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.h index cdd3a62a..c1933bf7 100644 --- a/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.h +++ b/storage/columnstore/columnstore/writeengine/redistribute/we_redistributeworkerthread.h @@ -142,6 +142,4 @@ class RedistributeWorkerThread } // namespace redistribute -#endif // WE_REDISTRIBUTEWORKERTHREAD_H - -// vim:ts=4 sw=4: +#endif
\ No newline at end of file diff --git a/storage/columnstore/columnstore/writeengine/server/we_ddlcommandproc.cpp b/storage/columnstore/columnstore/writeengine/server/we_ddlcommandproc.cpp index 1d832f9e..3970e6c1 100644 --- a/storage/columnstore/columnstore/writeengine/server/we_ddlcommandproc.cpp +++ b/storage/columnstore/columnstore/writeengine/server/we_ddlcommandproc.cpp @@ -4952,4 +4952,3 @@ void WE_DDLCommandProc::purgeFDCache() } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/shared/we_brm.cpp b/storage/columnstore/columnstore/writeengine/shared/we_brm.cpp index a049806b..fb5d88fa 100644 --- a/storage/columnstore/columnstore/writeengine/shared/we_brm.cpp +++ b/storage/columnstore/columnstore/writeengine/shared/we_brm.cpp @@ -1741,4 +1741,3 @@ int BRMWrapper::getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMi } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/shared/we_chunkmanager.cpp b/storage/columnstore/columnstore/writeengine/shared/we_chunkmanager.cpp index b34535c7..784f9990 100644 --- a/storage/columnstore/columnstore/writeengine/shared/we_chunkmanager.cpp +++ b/storage/columnstore/columnstore/writeengine/shared/we_chunkmanager.cpp @@ -2667,4 +2667,3 @@ int ChunkManager::checkFixLastDictChunk(const FID& fid, uint16_t root, uint32_t } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/splitter/we_splitterapp.cpp b/storage/columnstore/columnstore/writeengine/splitter/we_splitterapp.cpp index 366c64ad..22a29100 100644 --- a/storage/columnstore/columnstore/writeengine/splitter/we_splitterapp.cpp +++ b/storage/columnstore/columnstore/writeengine/splitter/we_splitterapp.cpp @@ -572,4 +572,3 @@ int main(int argc, char** argv) return SPLTR_EXIT_STATUS; } -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/wrapper/we_colop.cpp b/storage/columnstore/columnstore/writeengine/wrapper/we_colop.cpp index 25bed1eb..bbe3dc42 100644 --- a/storage/columnstore/columnstore/writeengine/wrapper/we_colop.cpp +++ b/storage/columnstore/columnstore/writeengine/wrapper/we_colop.cpp @@ -1910,4 +1910,3 @@ int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.cpp b/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.cpp index 64bfff06..421ecbcb 100644 --- a/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.cpp +++ b/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.cpp @@ -109,4 +109,3 @@ ColsExtsInfoMap& TableMetaData::getColsExtsInfoMap() return fColsExtsInfoMap; } } // namespace WriteEngine -// vim:ts=4 sw=4: diff --git a/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.h b/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.h index 2e4f114a..830fecba 100644 --- a/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.h +++ b/storage/columnstore/columnstore/writeengine/wrapper/we_tablemetadata.h @@ -99,5 +99,4 @@ class TableMetaData #undef EXPORT -#endif -// vim:ts=4 sw=4: +#endif // WE_TABLEMETADATA_H__
\ No newline at end of file diff --git a/storage/columnstore/columnstore/writeengine/wrapper/writeengine.cpp b/storage/columnstore/columnstore/writeengine/wrapper/writeengine.cpp index 81520412..41471472 100644 --- a/storage/columnstore/columnstore/writeengine/wrapper/writeengine.cpp +++ b/storage/columnstore/columnstore/writeengine/wrapper/writeengine.cpp @@ -6628,4 +6628,3 @@ int WriteEngineWrapper::RemoveTxnFromLBIDMap(const TxnID txnid) } } // end of namespace -// vim:ts=4 sw=4: diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 51d1cf2f..31f7d2eb 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -406,7 +406,7 @@ MYSQL_ADD_PLUGIN(connect ${CONNECT_SOURCES} STORAGE_ENGINE COMPONENT connect-engine RECOMPILE_FOR_EMBEDDED - LINK_LIBRARIES ${ZLIB_LIBRARY} ${XML_LIBRARY} ${ICONV_LIBRARY} + LINK_LIBRARIES ${ZLIB_LIBRARIES} ${XML_LIBRARY} ${ICONV_LIBRARY} ${ODBC_LIBRARY} ${JDBC_LIBRARY} ${MONGOC_LIBRARY} ${IPHLPAPI_LIBRARY} ${REST_LIBRARY}) IF(NOT TARGET connect) diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 54ecde32..e9aa3db6 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -6454,6 +6454,9 @@ char *ha_connect::GetDBfromName(const char *name) ha_create_table() in handle.cc */ +/* Stack size 25608 in clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + int ha_connect::create(const char *name, TABLE *table_arg, HA_CREATE_INFO *create_info) { @@ -6998,6 +7001,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, table= st; DBUG_RETURN(rc); } // end of create +PRAGMA_REENABLE_CHECK_STACK_FRAME /** Used to check whether a file based outward table can be populated by @@ -7005,6 +7009,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, - file does not exist or is void - user has file privilege */ + +/* Stack size 16664 in clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + bool ha_connect::FileExists(const char *fn, bool bf) { if (!fn || !*fn) @@ -7055,6 +7063,7 @@ bool ha_connect::FileExists(const char *fn, bool bf) return true; } // end of FileExists +PRAGMA_REENABLE_CHECK_STACK_FRAME // Called by SameString and NoFieldOptionChange bool ha_connect::CheckString(PCSZ str1, PCSZ str2) diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h index 44430e56..94b292ed 100644 --- a/storage/connect/ioapi.h +++ b/storage/connect/ioapi.h @@ -144,7 +144,7 @@ typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stre typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); -/* here is the "old" 32 bits structure structure */ +/* here is the "old" 32 bits structure */ typedef struct zlib_filefunc_def_s { open_file_func zopen_file; diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index 67f22ce2..ab588dd4 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -93,7 +93,6 @@ class LIBXMLDOC : public XMLDOCUMENT { xmlXPathContextPtr Ctxp; xmlXPathObjectPtr Xop; xmlXPathObjectPtr NlXop; - xmlErrorPtr Xerr; char *Buf; // Temporary bool Nofreelist; }; // end of class LIBXMLDOC @@ -327,7 +326,6 @@ LIBXMLDOC::LIBXMLDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) Ctxp = NULL; Xop = NULL; NlXop = NULL; - Xerr = NULL; Buf = NULL; Nofreelist = false; } // end of LIBXMLDOC constructor @@ -365,8 +363,8 @@ bool LIBXMLDOC::ParseFile(PGLOBAL g, char *fn) Encoding = (char*)Docp->encoding; return false; - } else if ((Xerr = xmlGetLastError())) - xmlResetError(Xerr); + } else if (xmlGetLastError()) + xmlResetLastError(); return true; } // end of ParseFile @@ -505,9 +503,9 @@ int LIBXMLDOC::DumpDoc(PGLOBAL g, char *ofn) #if 1 // This function does not crash ( if (xmlSaveFormatFileEnc((const char *)ofn, Docp, Encoding, 0) < 0) { - xmlErrorPtr err = xmlGetLastError(); + const xmlError *err = xmlGetLastError(); strcpy(g->Message, (err) ? err->message : "Error saving XML doc"); - xmlResetError(Xerr); + xmlResetLastError(); rc = -1; } // endif Save // rc = xmlDocDump(of, Docp); @@ -546,8 +544,8 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) if (Nlist) { xmlXPathFreeNodeSet(Nlist); - if ((Xerr = xmlGetLastError())) - xmlResetError(Xerr); + if (xmlGetLastError()) + xmlResetLastError(); Nlist = NULL; } // endif Nlist @@ -555,8 +553,8 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) if (Xop) { xmlXPathFreeObject(Xop); - if ((Xerr = xmlGetLastError())) - xmlResetError(Xerr); + if (xmlGetLastError()) + xmlResetLastError(); Xop = NULL; } // endif Xop @@ -564,8 +562,8 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) if (NlXop) { xmlXPathFreeObject(NlXop); - if ((Xerr = xmlGetLastError())) - xmlResetError(Xerr); + if (xmlGetLastError()) + xmlResetLastError(); NlXop = NULL; } // endif NlXop @@ -573,8 +571,8 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) if (Ctxp) { xmlXPathFreeContext(Ctxp); - if ((Xerr = xmlGetLastError())) - xmlResetError(Xerr); + if (xmlGetLastError()) + xmlResetLastError(); Ctxp = NULL; } // endif Ctxp @@ -590,6 +588,7 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) /******************************************************************/ xmlNodeSetPtr LIBXMLDOC::GetNodeList(PGLOBAL g, xmlNodePtr np, char *xp) { + const xmlError *xerr; xmlNodeSetPtr nl; if (trace(1)) @@ -649,11 +648,11 @@ xmlNodeSetPtr LIBXMLDOC::GetNodeList(PGLOBAL g, xmlNodePtr np, char *xp) } else xmlXPathFreeObject(Xop); // Caused node not found - if ((Xerr = xmlGetLastError())) { - strcpy(g->Message, Xerr->message); - xmlResetError(Xerr); + if ((xerr = xmlGetLastError())) { + strcpy(g->Message, xerr->message); + xmlResetLastError(); return NULL; - } // endif Xerr + } // endif xerr } // endif Xop @@ -1079,7 +1078,7 @@ void XML2NODE::AddText(PGLOBAL g, PCSZ txtp) /******************************************************************/ void XML2NODE::DeleteChild(PGLOBAL g, PXNODE dnp) { - xmlErrorPtr xerr; + const xmlError *xerr; if (trace(1)) htrc("DeleteChild: node=%p\n", dnp); @@ -1122,7 +1121,7 @@ err: if (trace(1)) htrc("DeleteChild: errmsg=%-.256s\n", xerr->message); - xmlResetError(xerr); + xmlResetLastError(); } // end of DeleteChild /* -------------------- class XML2NODELIST ---------------------- */ diff --git a/storage/connect/mysql-test/connect/r/drop-open-error.result b/storage/connect/mysql-test/connect/r/drop-open-error.result index a5d1e893..f9b9b7e8 100644 --- a/storage/connect/mysql-test/connect/r/drop-open-error.result +++ b/storage/connect/mysql-test/connect/r/drop-open-error.result @@ -3,7 +3,7 @@ create table tcon engine=connect table_type=mysql CONNECTION='mysql://root@local ERROR HY000: Too long value for 'SRCDEF' drop table mdev9949; Warnings: -Warning 1017 Can't find file: 'MYSQLD_DATADIR/test/mdev9949.dos' (errno: 2 "No such file or directory") +Warning 1017 Can't find file: 'DATADIR/test/mdev9949.dos' (errno: 2 "No such file or directory") drop table t1; select @@secure_file_priv 'must be NULL'; must be NULL diff --git a/storage/connect/mysql-test/connect/r/mysql.result b/storage/connect/mysql-test/connect/r/mysql.result index 1dcbca88..d1c68dcc 100644 --- a/storage/connect/mysql-test/connect/r/mysql.result +++ b/storage/connect/mysql-test/connect/r/mysql.result @@ -229,6 +229,7 @@ a 20 30 # Start of mysqldump ------ +/*!999999\- enable the sandbox mode */ /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t2` ( diff --git a/storage/connect/mysql-test/connect/t/drop-open-error.test b/storage/connect/mysql-test/connect/t/drop-open-error.test index 69e634e8..dd286c96 100644 --- a/storage/connect/mysql-test/connect/t/drop-open-error.test +++ b/storage/connect/mysql-test/connect/t/drop-open-error.test @@ -13,8 +13,9 @@ error ER_VALUE_TOO_LONG; create table tcon engine=connect table_type=mysql CONNECTION='mysql://root@localhost/test/t1' SRCDEF='select c from t1 where c in ("foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar", "foo", "bar", "qux", "foobar")'; # copy the invalid frm (as created by the statement above before the MDEV-9949 fix) +let $MARIADB_DATADIR=`select @@datadir`; copy_file $MTR_SUITE_DIR/std_data/mdev9949.frm $datadir/test/mdev9949.frm; ---replace_result $datadir MYSQLD_DATADIR/ ./ MYSQLD_DATADIR/ +--replace_result $MARIADB_DATADIR DATADIR/ './' 'DATADIR/' drop table mdev9949; drop table t1; diff --git a/storage/connect/tabmul.cpp b/storage/connect/tabmul.cpp index d0b7b316..7e2857f1 100644 --- a/storage/connect/tabmul.cpp +++ b/storage/connect/tabmul.cpp @@ -125,8 +125,11 @@ PTDB TDBMUL::Duplicate(PGLOBAL g) /* have a LRECL that is the sum of the lengths of all components. */ /* This is why we use a big filename array to take care of that. */ /***********************************************************************/ + +PRAGMA_DISABLE_CHECK_STACK_FRAME + bool TDBMUL::InitFileNames(PGLOBAL g) - { +{ #define PFNZ 4096 #define FNSZ (_MAX_DRIVE+_MAX_DIR+_MAX_FNAME+_MAX_EXT) PTDBDIR dirp; @@ -234,6 +237,7 @@ bool TDBMUL::InitFileNames(PGLOBAL g) NumFiles = n; return false; } // end of InitFileNames +PRAGMA_REENABLE_CHECK_STACK_FRAME /***********************************************************************/ /* The table column list is the sub-table column list. */ diff --git a/storage/connect/unzip.c b/storage/connect/unzip.c index 90935043..de69f4b6 100644 --- a/storage/connect/unzip.c +++ b/storage/connect/unzip.c @@ -122,7 +122,7 @@ const char unz_copyright[] = " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; -/* unz_file_info_interntal contain internal info about a file in zipfile*/ +/* unz_file_info64_internal contain internal info about a file in zipfile*/ typedef struct unz_file_info64_internal_s { ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ @@ -542,7 +542,7 @@ local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) return 0; - /* number of the disk with the start of the zip64 end of central directory */ + /* number of the disk with the start of the zip64 end of central directory */ if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) return 0; if (uL != 0) @@ -590,10 +590,10 @@ local unzFile unzOpenInternal (const void *path, ZPOS64_T central_pos; uLong uL; - uLong number_disk; /* number of the current dist, used for - spaning ZIP, unsupported, always 0*/ - uLong number_disk_with_CD; /* number the the disk with central dir, used - for spaning ZIP, unsupported, always 0*/ + uLong number_disk; /* number of the current disk, used for + spanning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the disk with central dir, used + for spanning ZIP, unsupported, always 0*/ ZPOS64_T number_entry_CD; /* total number of entries in the central dir (same than number_entry on nospan) */ diff --git a/storage/connect/unzip.h b/storage/connect/unzip.h index 2104e391..ef045109 100644 --- a/storage/connect/unzip.h +++ b/storage/connect/unzip.h @@ -306,7 +306,7 @@ extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, Get Info about the current file if pfile_info!=NULL, the *pfile_info structure will contain somes info about the current file - if szFileName!=NULL, the filemane string will be copied in szFileName + if szFileName!=NULL, the filename string will be copied in szFileName (fileNameBufferSize is the size of the buffer) if extraField!=NULL, the extra field information will be copied in extraField (extraFieldBufferSize is the size of the buffer). diff --git a/storage/connect/zip.c b/storage/connect/zip.c index 3d3d4cad..523ea222 100644 --- a/storage/connect/zip.c +++ b/storage/connect/zip.c @@ -575,7 +575,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) return 0; - /* number of the disk with the start of the zip64 end of central directory */ + /* number of the disk with the start of the zip64 end of central directory */ if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) return 0; if (uL != 0) @@ -614,9 +614,9 @@ local int LoadCentralDirectoryRecord(zip64_internal* pziinit) { ZPOS64_T central_pos; uLong uL; - uLong number_disk; /* number of the current dist, used for + uLong number_disk; /* number of the current disk, used for spanning ZIP, unsupported, always 0*/ - uLong number_disk_with_CD; /* number the the disk with central dir, used + uLong number_disk_with_CD; /* number of the disk with central dir, used for spanning ZIP, unsupported, always 0*/ ZPOS64_T number_entry; ZPOS64_T number_entry_CD; /* total number of entries in @@ -814,71 +814,66 @@ local int LoadCentralDirectoryRecord(zip64_internal* pziinit) { /************************************************************/ extern zipFile ZEXPORT zipOpen3(const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) { - zip64_internal ziinit; zip64_internal* zi; int err=ZIP_OK; - ziinit.z_filefunc.zseek32_file = NULL; - ziinit.z_filefunc.ztell32_file = NULL; + if (!(zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)))) + return NULL; + + zi->z_filefunc.zseek32_file = NULL; + zi->z_filefunc.ztell32_file = NULL; if (pzlib_filefunc64_32_def==NULL) - fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + fill_fopen64_filefunc(&zi->z_filefunc.zfile_func64); else - ziinit.z_filefunc = *pzlib_filefunc64_32_def; + zi->z_filefunc = *pzlib_filefunc64_32_def; - ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + zi->filestream = ZOPEN64(zi->z_filefunc, pathname, (append == APPEND_STATUS_CREATE) ? (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); - if (ziinit.filestream == NULL) + if (zi->filestream == NULL) + { + free(zi); return NULL; + } if (append == APPEND_STATUS_CREATEAFTER) - ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + ZSEEK64(zi->z_filefunc,zi->filestream,0,SEEK_END); - ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); - ziinit.in_opened_file_inzip = 0; - ziinit.ci.stream_initialised = 0; - ziinit.number_entry = 0; - ziinit.add_position_when_writing_offset = 0; - init_linkedlist(&(ziinit.central_dir)); - - - - zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); - if (zi==NULL) - { - ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); - return NULL; - } + zi->begin_pos = ZTELL64(zi->z_filefunc,zi->filestream); + zi->in_opened_file_inzip = 0; + zi->ci.stream_initialised = 0; + zi->number_entry = 0; + zi->add_position_when_writing_offset = 0; + init_linkedlist(&(zi->central_dir)); /* now we add file in a zipfile */ # ifndef NO_ADDFILEINEXISTINGZIP - ziinit.globalcomment = NULL; + zi->globalcomment = NULL; if (append == APPEND_STATUS_ADDINZIP) { // Read and Cache Central Directory Records - err = LoadCentralDirectoryRecord(&ziinit); + err = LoadCentralDirectoryRecord(zi); } if (globalcomment) { - *globalcomment = ziinit.globalcomment; + *globalcomment = zi->globalcomment; } # endif /* !NO_ADDFILEINEXISTINGZIP*/ if (err != ZIP_OK) { # ifndef NO_ADDFILEINEXISTINGZIP - free(ziinit.globalcomment); + free(zi->globalcomment); # endif /* !NO_ADDFILEINEXISTINGZIP*/ free(zi); return NULL; } else { - *zi = ziinit; return (zipFile)zi; } } @@ -1027,7 +1022,6 @@ extern int ZEXPORT zipOpenNewFileInZip4_64(zipFile file, const char* filename, c int err = ZIP_OK; # ifdef NOCRYPT - (crcForCrypting); if (password != NULL) return ZIP_PARAMERROR; # endif @@ -1043,6 +1037,17 @@ extern int ZEXPORT zipOpenNewFileInZip4_64(zipFile file, const char* filename, c return ZIP_PARAMERROR; #endif + // The filename and comment length must fit in 16 bits. + if ((filename!=NULL) && (strlen(filename)>0xffff)) + return ZIP_PARAMERROR; + if ((comment!=NULL) && (strlen(comment)>0xffff)) + return ZIP_PARAMERROR; + // The extra field length must fit in 16 bits. If the member also requires + // a Zip64 extra block, that will also need to fit within that 16-bit + // length, but that will be checked for later. + if ((size_extrafield_local>0xffff) || (size_extrafield_global>0xffff)) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; if (zi->in_opened_file_inzip == 1) @@ -1597,7 +1602,7 @@ extern int ZEXPORT zipCloseFileInZipRaw64(zipFile file, ZPOS64_T uncompressed_si if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) { - // we can not write more data to the buffer that we have room for. + // we cannot write more data to the buffer that we have room for. return ZIP_BADZIPFILE; } @@ -1861,7 +1866,7 @@ extern int ZEXPORT zipClose(zipFile file, const char* global_comment) { free_linkedlist(&(zi->central_dir)); pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; - if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + if(pos >= 0xffffffff || zi->number_entry >= 0xFFFF) { ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); diff --git a/storage/connect/zip.h b/storage/connect/zip.h index 5fc08413..3e230d34 100644 --- a/storage/connect/zip.h +++ b/storage/connect/zip.h @@ -177,9 +177,9 @@ extern int ZEXPORT zipOpenNewFileInZip64(zipFile file, filename : the filename in zip (if NULL, '-' without quote will be used *zipfi contain supplemental information if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local - contains the extrafield data the the local header + contains the extrafield data for the local header if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global - contains the extrafield data the the local header + contains the extrafield data for the global header if comment != NULL, comment contain the comment string method contain the compression method (0 for store, Z_DEFLATED for deflate) level contain the level of compression (can be Z_DEFAULT_COMPRESSION) diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 5f7f0c1e..404ad22a 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -440,31 +440,22 @@ int ha_heap::external_lock(THD *thd, int lock_type) SYNOPSIS disable_indexes() - mode mode of operation: - HA_KEY_SWITCH_NONUNIQ disable all non-unique keys - HA_KEY_SWITCH_ALL disable all keys - HA_KEY_SWITCH_NONUNIQ_SAVE dis. non-uni. and make persistent - HA_KEY_SWITCH_ALL_SAVE dis. all keys and make persistent DESCRIPTION - Disable indexes and clear keys to use for scanning. - - IMPLEMENTATION - HA_KEY_SWITCH_NONUNIQ is not implemented. - HA_KEY_SWITCH_NONUNIQ_SAVE is not implemented with HEAP. - HA_KEY_SWITCH_ALL_SAVE is not implemented with HEAP. + See handler::ha_disable_indexes() RETURN 0 ok HA_ERR_WRONG_COMMAND mode not implemented. */ -int ha_heap::disable_indexes(uint mode) +int ha_heap::disable_indexes(key_map map, bool persist) { int error; - if (mode == HA_KEY_SWITCH_ALL) + if (!persist) { + DBUG_ASSERT(map.is_clear_all()); if (!(error= heap_disable_indexes(file))) set_keys_for_scanning(); } @@ -482,11 +473,6 @@ int ha_heap::disable_indexes(uint mode) SYNOPSIS enable_indexes() - mode mode of operation: - HA_KEY_SWITCH_NONUNIQ enable all non-unique keys - HA_KEY_SWITCH_ALL enable all keys - HA_KEY_SWITCH_NONUNIQ_SAVE en. non-uni. and make persistent - HA_KEY_SWITCH_ALL_SAVE en. all keys and make persistent DESCRIPTION Enable indexes and set keys to use for scanning. @@ -495,10 +481,7 @@ int ha_heap::disable_indexes(uint mode) since the heap storage engine cannot repair the indexes. To be sure, call handler::delete_all_rows() before. - IMPLEMENTATION - HA_KEY_SWITCH_NONUNIQ is not implemented. - HA_KEY_SWITCH_NONUNIQ_SAVE is not implemented with HEAP. - HA_KEY_SWITCH_ALL_SAVE is not implemented with HEAP. + See also handler::ha_enable_indexes() RETURN 0 ok @@ -506,12 +489,13 @@ int ha_heap::disable_indexes(uint mode) HA_ERR_WRONG_COMMAND mode not implemented. */ -int ha_heap::enable_indexes(uint mode) +int ha_heap::enable_indexes(key_map map, bool persist) { int error; - if (mode == HA_KEY_SWITCH_ALL) + if (!persist) { + DBUG_ASSERT(map.is_prefix(table->s->keys)); if (!(error= heap_enable_indexes(file))) set_keys_for_scanning(); } diff --git a/storage/heap/ha_heap.h b/storage/heap/ha_heap.h index 18e0d1a9..d8636f7f 100644 --- a/storage/heap/ha_heap.h +++ b/storage/heap/ha_heap.h @@ -101,8 +101,8 @@ public: int external_lock(THD *thd, int lock_type); int delete_all_rows(void); int reset_auto_increment(ulonglong value); - int disable_indexes(uint mode); - int enable_indexes(uint mode); + int disable_indexes(key_map map, bool persist); + int enable_indexes(key_map map, bool persist); int indexes_are_disabled(void); ha_rows records_in_range(uint inx, const key_range *start_key, const key_range *end_key, page_range *pages); diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 32c0a437..b3125ca9 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -48,6 +48,13 @@ IF(UNIX) IF(HAVE_LIBNUMA) LINK_LIBRARIES(numa) ENDIF() + IF(CMAKE_SIZEOF_VOID_P EQUAL 8) + IF(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch|AARCH|p(ower)?pc|x86_|amd)64") + OPTION(WITH_INNODB_PMEM "Support memory-mapped InnoDB redo log" ON) + ELSE() # Disable by default on ISA that are not covered by our CI + OPTION(WITH_INNODB_PMEM "Support memory-mapped InnoDB redo log" OFF) + ENDIF() + ENDIF() ENDIF() ENDIF() @@ -71,7 +78,7 @@ ADD_FEATURE_INFO(INNODB_ROOT_GUESS WITH_INNODB_ROOT_GUESS OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF) IF(WITH_INNODB_EXTRA_DEBUG) - ADD_DEFINITIONS(-DUNIV_ZIP_DEBUG) + ADD_DEFINITIONS(-DUNIV_ZIP_DEBUG -DLOG_LATCH_DEBUG) ENDIF() ADD_FEATURE_INFO(INNODB_EXTRA_DEBUG WITH_INNODB_EXTRA_DEBUG "Extra InnoDB debug checks") @@ -133,7 +140,6 @@ SET(INNOBASE_SOURCES btr/btr0pcur.cc btr/btr0sea.cc btr/btr0defragment.cc - buf/buf0block_hint.cc buf/buf0buddy.cc buf/buf0buf.cc buf/buf0dblwr.cc @@ -428,26 +434,16 @@ SET(INNOBASE_SOURCES ut/ut0vec.cc ut/ut0wqueue.cc) -OPTION(WITH_PMEM "Support redo log in persistent memory" OFF) -FIND_PACKAGE(PMEM) -IF(PMEM_FOUND) - INCLUDE_DIRECTORIES(${PMEM_INCLUDES}) - ADD_COMPILE_FLAGS(log/log0log.cc log/log0recv.cc - buf/buf0flu.cc mtr/mtr0mtr.cc trx/trx0trx.cc srv/srv0start.cc - COMPILE_FLAGS "-DHAVE_PMEM") - SET(PMEM_LIBRARY ${PMEM_LIBRARIES}) -ELSE() - IF(WITH_PMEM) - MESSAGE(FATAL_ERROR "WITH_PMEM=ON cannot be satisfied") - ENDIF() +IF(WITH_INNODB_PMEM) + ADD_DEFINITIONS(-DHAVE_PMEM) + SET(INNOBASE_SOURCES ${INNOBASE_SOURCES} include/cache.h sync/cache.cc) ENDIF() MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE MODULE_OUTPUT_NAME ha_innodb DEFAULT RECOMPILE_FOR_EMBEDDED LINK_LIBRARIES - ${ZLIB_LIBRARY} - ${PMEM_LIBRARY} + ${ZLIB_LIBRARIES} ${NUMA_LIBRARY} ${LIBSYSTEMD} ${LINKER_SCRIPT}) diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 705ff035..6b3a3733 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -264,6 +264,8 @@ btr_root_block_get( mtr_t* mtr, /*!< in: mtr */ dberr_t* err) /*!< out: error code */ { + ut_ad(mode != RW_NO_LATCH); + if (!index->table || !index->table->space) { *err= DB_TABLESPACE_NOT_FOUND; @@ -285,13 +287,12 @@ btr_root_block_get( if (UNIV_LIKELY(block != nullptr)) { - if (UNIV_UNLIKELY(mode == RW_NO_LATCH)); - else if (!!page_is_comp(block->page.frame) != - index->table->not_redundant() || - btr_page_get_index_id(block->page.frame) != index->id || - !fil_page_index_page_check(block->page.frame) || - index->is_spatial() != - (fil_page_get_type(block->page.frame) == FIL_PAGE_RTREE)) + if (!!page_is_comp(block->page.frame) != + index->table->not_redundant() || + btr_page_get_index_id(block->page.frame) != index->id || + !fil_page_index_page_check(block->page.frame) || + index->is_spatial() != + (fil_page_get_type(block->page.frame) == FIL_PAGE_RTREE)) { *err= DB_PAGE_CORRUPTED; block= nullptr; @@ -561,13 +562,39 @@ btr_page_alloc_for_ibuf( { buf_page_make_young_if_needed(&new_block->page); *err= flst_remove(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, new_block, - PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, mtr); + PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, + fil_system.sys_space->free_limit, mtr); } ut_d(if (*err == DB_SUCCESS) flst_validate(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, mtr)); return new_block; } +static MY_ATTRIBUTE((nonnull, warn_unused_result)) +/** Acquire a latch on the index root page for allocating or freeing pages. +@param index index tree +@param mtr mini-transaction +@param err error code +@return root page +@retval nullptr if an error occurred */ +buf_block_t *btr_root_block_sx(dict_index_t *index, mtr_t *mtr, dberr_t *err) +{ + buf_block_t *root= + mtr->get_already_latched(page_id_t{index->table->space_id, index->page}, + MTR_MEMO_PAGE_SX_FIX); + if (!root) + { + root= btr_root_block_get(index, RW_SX_LATCH, mtr, err); + if (UNIV_UNLIKELY(!root)) + return root; + } +#ifdef BTR_CUR_HASH_ADAPT + else + ut_ad(!root->index || !root->index->freed()); +#endif + return root; +} + /**************************************************************//** Allocates a new file page to be used in an index tree. NOTE: we assume that the caller has made the reservation for free extents! @@ -589,21 +616,9 @@ btr_page_alloc_low( page should be initialized. */ dberr_t* err) /*!< out: error code */ { - const auto savepoint= mtr->get_savepoint(); - buf_block_t *root= btr_root_block_get(index, RW_NO_LATCH, mtr, err); + buf_block_t *root= btr_root_block_sx(index, mtr, err); if (UNIV_UNLIKELY(!root)) return root; - - const bool have_latch= mtr->have_u_or_x_latch(*root); -#ifdef BTR_CUR_HASH_ADAPT - ut_ad(!have_latch || !root->index || !root->index->freed()); -#endif - mtr->rollback_to_savepoint(savepoint); - - if (!have_latch && - UNIV_UNLIKELY(!(root= btr_root_block_get(index, RW_SX_LATCH, mtr, err)))) - return root; - fseg_header_t *seg_header= root->page.frame + (level ? PAGE_HEADER + PAGE_BTR_SEG_TOP : PAGE_HEADER + PAGE_BTR_SEG_LEAF); return fseg_alloc_free_page_general(seg_header, hint_page_no, file_direction, @@ -652,7 +667,8 @@ btr_page_free_for_ibuf( buf_block_t *root= btr_get_latched_root(*index, mtr); dberr_t err= flst_add_first(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, - block, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, mtr); + block, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, + fil_system.sys_space->free_limit, mtr); ut_d(if (err == DB_SUCCESS) flst_validate(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, mtr)); return err; @@ -696,24 +712,16 @@ dberr_t btr_page_free(dict_index_t* index, buf_block_t* block, mtr_t* mtr, fil_space_t *space= index->table->space; dberr_t err; - const auto savepoint= mtr->get_savepoint(); - if (buf_block_t *root= btr_root_block_get(index, RW_NO_LATCH, mtr, &err)) + if (buf_block_t *root= btr_root_block_sx(index, mtr, &err)) { - const bool have_latch= mtr->have_u_or_x_latch(*root); -#ifdef BTR_CUR_HASH_ADAPT - ut_ad(!have_latch || !root->index || !root->index->freed()); -#endif - mtr->rollback_to_savepoint(savepoint); - if (have_latch || - (root= btr_root_block_get(index, RW_SX_LATCH, mtr, &err))) - err= fseg_free_page(&root->page.frame[blob || - page_is_leaf(block->page.frame) - ? PAGE_HEADER + PAGE_BTR_SEG_LEAF - : PAGE_HEADER + PAGE_BTR_SEG_TOP], - space, page, mtr, space_latched); + err= fseg_free_page(&root->page.frame[blob || + page_is_leaf(block->page.frame) + ? PAGE_HEADER + PAGE_BTR_SEG_LEAF + : PAGE_HEADER + PAGE_BTR_SEG_TOP], + space, page, mtr, space_latched); + if (err == DB_SUCCESS) + buf_page_free(space, page, mtr); } - if (err == DB_SUCCESS) - buf_page_free(space, page, mtr); /* The page was marked free in the allocation bitmap, but it should remain exclusively latched until mtr_t::commit() or until it @@ -1291,54 +1299,71 @@ btr_read_autoinc(dict_index_t* index) return autoinc; } -/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, -or fall back to MAX(auto_increment_column). -@param[in] table table containing an AUTO_INCREMENT column -@param[in] col_no index of the AUTO_INCREMENT column -@return the AUTO_INCREMENT value -@retval 0 on error or if no AUTO_INCREMENT value was used yet */ -ib_uint64_t -btr_read_autoinc_with_fallback(const dict_table_t* table, unsigned col_no) +dict_index_t *dict_table_t::get_index(const dict_col_t &col) const { - ut_ad(table->persistent_autoinc); - ut_ad(!table->is_temporary()); + dict_index_t *index= dict_table_get_first_index(this); - dict_index_t* index = dict_table_get_first_index(table); + while (index && (index->fields[0].col != &col || index->is_corrupted())) + index= dict_table_get_next_index(index); - if (index == NULL) { - return 0; - } + return index; +} - mtr_t mtr; - mtr.start(); - buf_block_t* block = buf_page_get( - page_id_t(index->table->space_id, index->page), - index->table->space->zip_size(), - RW_S_LATCH, &mtr); - - ib_uint64_t autoinc = block - ? page_get_autoinc(block->page.frame) : 0; - const bool retry = block && autoinc == 0 - && !page_is_empty(block->page.frame); - mtr.commit(); +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, +or fall back to MAX(auto_increment_column). +@param table table containing an AUTO_INCREMENT column +@param col_no index of the AUTO_INCREMENT column +@param mysql_version TABLE_SHARE::mysql_version +@param max the maximum value of the AUTO_INCREMENT column +@return the AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +uint64_t btr_read_autoinc_with_fallback(const dict_table_t *table, + unsigned col_no, ulong mysql_version, + uint64_t max) +{ + ut_ad(table->persistent_autoinc); + ut_ad(!table->is_temporary()); - if (retry) { - /* This should be an old data file where - PAGE_ROOT_AUTO_INC was initialized to 0. - Fall back to reading MAX(autoinc_col). - There should be an index on it. */ - const dict_col_t* autoinc_col - = dict_table_get_nth_col(table, col_no); - while (index && index->fields[0].col != autoinc_col) { - index = dict_table_get_next_index(index); - } + uint64_t autoinc= 0; + mtr_t mtr; + mtr.start(); - if (index) { - autoinc = row_search_max_autoinc(index); - } - } + if (buf_block_t *block= + buf_page_get(page_id_t(table->space_id, + dict_table_get_first_index(table)->page), + table->space->zip_size(), RW_SX_LATCH, &mtr)) + { + autoinc= page_get_autoinc(block->page.frame); - return autoinc; + if (autoinc > 0 && autoinc <= max && mysql_version >= 100210); + else if (dict_index_t *index= + table->get_index(*dict_table_get_nth_col(table, col_no))) + { + /* Read MAX(autoinc_col), in case this table had originally been + created before MariaDB 10.2.4 introduced persistent AUTO_INCREMENT + and MariaDB 10.2.10 fixed MDEV-12123, and there could be a garbage + value in the PAGE_ROOT_AUTO_INC field. */ + const uint64_t max_autoinc= row_search_max_autoinc(index); + const bool need_adjust{autoinc > max || autoinc < max_autoinc}; + ut_ad(max_autoinc <= max); + + if (UNIV_UNLIKELY(need_adjust) && !high_level_read_only && !opt_readonly) + { + sql_print_information("InnoDB: Resetting PAGE_ROOT_AUTO_INC from " + UINT64PF " to " UINT64PF + " on table %`.*s.%`s (created with version %lu)", + autoinc, max_autoinc, + int(table->name.dblen()), table->name.m_name, + table->name.basename(), mysql_version); + autoinc= max_autoinc; + index->set_modified(mtr); + page_set_autoinc(block, max_autoinc, &mtr, true); + } + } + } + + mtr.commit(); + return autoinc; } /** Write the next available AUTO_INCREMENT value to PAGE_ROOT_AUTO_INC. diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc index 5bf68c58..e2513ad6 100644 --- a/storage/innobase/btr/btr0bulk.cc +++ b/storage/innobase/btr/btr0bulk.cc @@ -837,7 +837,7 @@ PageBulk::release() m_block->page.fix(); /* No other threads can modify this block. */ - m_modify_clock = buf_block_get_modify_clock(m_block); + m_modify_clock = m_block->modify_clock; m_mtr.commit(); } diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 46afb73b..2fc05b06 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -817,7 +817,7 @@ static ulint btr_node_ptr_max_size(const dict_index_t* index) /* Determine the maximum length of the index field. */ field_max_size = dict_col_get_fixed_size(col, comp); - if (field_max_size) { + if (field_max_size && field->fixed_len) { /* dict_index_add_col() should guarantee this */ ut_ad(!field->prefix_len || field->fixed_len == field->prefix_len); @@ -935,7 +935,7 @@ static inline page_cur_mode_t btr_cur_nonleaf_mode(page_cur_mode_t mode) return PAGE_CUR_LE; } -static MY_ATTRIBUTE((nonnull)) +MY_ATTRIBUTE((nonnull,warn_unused_result)) /** Acquire a latch on the previous page without violating the latching order. @param block index page @param page_id page identifier with valid space identifier @@ -946,8 +946,9 @@ static MY_ATTRIBUTE((nonnull)) @retval 0 if an error occurred @retval 1 if the page could be latched in the wrong order @retval -1 if the latch on block was temporarily released */ -int btr_latch_prev(buf_block_t *block, page_id_t page_id, ulint zip_size, - rw_lock_type_t rw_latch, mtr_t *mtr, dberr_t *err) +static int btr_latch_prev(buf_block_t *block, page_id_t page_id, + ulint zip_size, + rw_lock_type_t rw_latch, mtr_t *mtr, dberr_t *err) { ut_ad(rw_latch == RW_S_LATCH || rw_latch == RW_X_LATCH); ut_ad(page_id.space() == block->page.id().space()); @@ -955,47 +956,80 @@ int btr_latch_prev(buf_block_t *block, page_id_t page_id, ulint zip_size, const auto prev_savepoint= mtr->get_savepoint(); ut_ad(block == mtr->at_savepoint(prev_savepoint - 1)); - page_id.set_page_no(btr_page_get_prev(block->page.frame)); + const page_t *const page= block->page.frame; + page_id.set_page_no(btr_page_get_prev(page)); + /* We are holding a latch on the current page. + + We will start by buffer-fixing the left sibling. Waiting for a latch + on it while holding a latch on the current page could lead to a + deadlock, because another thread could hold that latch and wait for + a right sibling page latch (the current page). + + If there is a conflict, we will temporarily release our latch on the + current block while waiting for a latch on the left sibling. The + buffer-fixes on both blocks will prevent eviction. */ + + retry: buf_block_t *prev= buf_page_get_gen(page_id, zip_size, RW_NO_LATCH, nullptr, BUF_GET, mtr, err, false); if (UNIV_UNLIKELY(!prev)) return 0; int ret= 1; - if (UNIV_UNLIKELY(rw_latch == RW_S_LATCH)) + static_assert(MTR_MEMO_PAGE_S_FIX == mtr_memo_type_t(BTR_SEARCH_LEAF), ""); + static_assert(MTR_MEMO_PAGE_X_FIX == mtr_memo_type_t(BTR_MODIFY_LEAF), ""); + + if (rw_latch == RW_S_LATCH + ? prev->page.lock.s_lock_try() : prev->page.lock.x_lock_try()) { - if (UNIV_LIKELY(prev->page.lock.s_lock_try())) + mtr->lock_register(prev_savepoint, mtr_memo_type_t(rw_latch)); + if (UNIV_UNLIKELY(prev->page.id() != page_id)) { - mtr->lock_register(prev_savepoint, MTR_MEMO_PAGE_S_FIX); - goto prev_latched; + fail: + /* the page was just read and found to be corrupted */ + mtr->rollback_to_savepoint(prev_savepoint); + return 0; } - block->page.lock.s_unlock(); } else { - if (UNIV_LIKELY(prev->page.lock.x_lock_try())) + ut_ad(mtr->at_savepoint(mtr->get_savepoint() - 1)->page.id() == page_id); + mtr->release_last_page(); + if (rw_latch == RW_S_LATCH) + block->page.lock.s_unlock(); + else + block->page.lock.x_unlock(); + + prev= buf_page_get_gen(page_id, zip_size, rw_latch, prev, + BUF_GET, mtr, err); + if (rw_latch == RW_S_LATCH) + block->page.lock.s_lock(); + else + block->page.lock.x_lock(); + + const page_id_t prev_page_id= page_id; + page_id.set_page_no(btr_page_get_prev(page)); + + if (UNIV_UNLIKELY(page_id != prev_page_id)) { - mtr->lock_register(prev_savepoint, MTR_MEMO_PAGE_X_FIX); - goto prev_latched; + mtr->release_last_page(); + if (page_id.page_no() == FIL_NULL) + return -1; + goto retry; } - block->page.lock.x_unlock(); + + if (UNIV_UNLIKELY(!prev)) + goto fail; + + ret= -1; } - ret= -1; - mtr->lock_register(prev_savepoint - 1, MTR_MEMO_BUF_FIX); - mtr->rollback_to_savepoint(prev_savepoint); - prev= buf_page_get_gen(page_id, zip_size, rw_latch, prev, - BUF_GET, mtr, err, false); - if (UNIV_UNLIKELY(!prev)) - return 0; - mtr->upgrade_buffer_fix(prev_savepoint - 1, rw_latch); - - prev_latched: - if (memcmp_aligned<2>(FIL_PAGE_TYPE + prev->page.frame, - FIL_PAGE_TYPE + block->page.frame, 2) || - memcmp_aligned<2>(PAGE_HEADER + PAGE_INDEX_ID + prev->page.frame, - PAGE_HEADER + PAGE_INDEX_ID + block->page.frame, 8) || - page_is_comp(prev->page.frame) != page_is_comp(block->page.frame)) + const page_t *const p= prev->page.frame; + if (memcmp_aligned<4>(FIL_PAGE_NEXT + p, FIL_PAGE_OFFSET + page, 4) || + memcmp_aligned<2>(FIL_PAGE_TYPE + p, FIL_PAGE_TYPE + page, 2) || + memcmp_aligned<2>(PAGE_HEADER + PAGE_INDEX_ID + p, + PAGE_HEADER + PAGE_INDEX_ID + page, 8) || + page_is_comp(p) != page_is_comp(page)) { ut_ad("corrupted" == 0); // FIXME: remove this *err= DB_CORRUPTION; @@ -6092,7 +6126,6 @@ btr_store_big_rec_extern_fields( for (ulint blob_npages = 0;; ++blob_npages) { buf_block_t* block; const ulint commit_freq = 4; - uint32_t r_extents; ut_ad(page_align(field_ref) == page_align(rec)); @@ -6127,22 +6160,14 @@ btr_store_big_rec_extern_fields( hint_prev = rec_block->page.id().page_no(); } - error = fsp_reserve_free_extents( - &r_extents, index->table->space, 1, - FSP_BLOB, &mtr, 1); - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { -alloc_fail: - mtr.commit(); - goto func_exit; - } - block = btr_page_alloc(index, hint_prev + 1, FSP_NO_DIR, 0, &mtr, &mtr, &error); - index->table->space->release_free_extents(r_extents); if (!block) { - goto alloc_fail; +alloc_fail: + mtr.commit(); + goto func_exit; } const uint32_t space_id = block->page.id().space(); diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 2131fb94..de0f9e93 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -179,10 +179,8 @@ before_first: cursor->old_n_fields, &cursor->old_rec_buf, &cursor->buf_size); - cursor->block_when_stored.store(block); - - /* Function try to check if block is S/X latch. */ - cursor->modify_clock = buf_block_get_modify_clock(block); + cursor->old_page_id = block->page.id(); + cursor->modify_clock = block->modify_clock; } /**************************************************************//** @@ -214,101 +212,80 @@ btr_pcur_copy_stored_position( } /** Optimistically latches the leaf page or pages requested. -@param[in] block guessed buffer block -@param[in,out] pcur cursor -@param[in,out] latch_mode BTR_SEARCH_LEAF, ... -@param[in,out] mtr mini-transaction -@return true if success */ +@param pcur persistent cursor +@param latch_mode BTR_SEARCH_LEAF, ... +@param mtr mini-transaction +@return true on success */ TRANSACTIONAL_TARGET -static bool btr_pcur_optimistic_latch_leaves(buf_block_t *block, - btr_pcur_t *pcur, +static bool btr_pcur_optimistic_latch_leaves(btr_pcur_t *pcur, btr_latch_mode *latch_mode, mtr_t *mtr) { - ut_ad(block->page.buf_fix_count()); - ut_ad(block->page.in_file()); - ut_ad(block->page.frame); - static_assert(BTR_SEARCH_PREV & BTR_SEARCH_LEAF, ""); static_assert(BTR_MODIFY_PREV & BTR_MODIFY_LEAF, ""); static_assert((BTR_SEARCH_PREV ^ BTR_MODIFY_PREV) == (RW_S_LATCH ^ RW_X_LATCH), ""); + buf_block_t *const block= + buf_page_optimistic_fix(pcur->btr_cur.page_cur.block, pcur->old_page_id); + + if (!block) + return false; + + if (*latch_mode == BTR_SEARCH_LEAF || *latch_mode == BTR_MODIFY_LEAF) + return buf_page_optimistic_get(block, rw_lock_type_t(*latch_mode), + pcur->modify_clock, mtr); + + ut_ad(*latch_mode == BTR_SEARCH_PREV || *latch_mode == BTR_MODIFY_PREV); const rw_lock_type_t mode= rw_lock_type_t(*latch_mode & (RW_X_LATCH | RW_S_LATCH)); - switch (*latch_mode) { - default: - ut_ad(*latch_mode == BTR_SEARCH_LEAF || *latch_mode == BTR_MODIFY_LEAF); - return buf_page_optimistic_get(mode, block, pcur->modify_clock, mtr); - case BTR_SEARCH_PREV: - case BTR_MODIFY_PREV: - page_id_t id{0}; - uint32_t left_page_no; - ulint zip_size; - buf_block_t *left_block= nullptr; - { - transactional_shared_lock_guard<block_lock> g{block->page.lock}; - if (block->modify_clock != pcur->modify_clock) - return false; - id= block->page.id(); - zip_size= block->zip_size(); - left_page_no= btr_page_get_prev(block->page.frame); - } - - if (left_page_no != FIL_NULL) - { - left_block= - buf_page_get_gen(page_id_t(id.space(), left_page_no), zip_size, - mode, nullptr, BUF_GET_POSSIBLY_FREED, mtr); - - if (!left_block); - else if (btr_page_get_next(left_block->page.frame) != id.page_no()) - { -release_left_block: - mtr->release_last_page(); - return false; - } - else - buf_page_make_young_if_needed(&left_block->page); - } - - if (buf_page_optimistic_get(mode, block, pcur->modify_clock, mtr)) - { - if (btr_page_get_prev(block->page.frame) == left_page_no) - { - /* block was already buffer-fixed while entering the function and - buf_page_optimistic_get() buffer-fixes it again. */ - ut_ad(2 <= block->page.buf_fix_count()); - *latch_mode= btr_latch_mode(mode); - return true; - } - - mtr->release_last_page(); - } - - ut_ad(block->page.buf_fix_count()); - if (left_block) - goto release_left_block; - return false; + uint64_t modify_clock; + uint32_t left_page_no; + const page_t *const page= block->page.frame; + { + transactional_shared_lock_guard<block_lock> g{block->page.lock}; + modify_clock= block->modify_clock; + left_page_no= btr_page_get_prev(page); } -} -/** Structure acts as functor to do the latching of leaf pages. -It returns true if latching of leaf pages succeeded and false -otherwise. */ -struct optimistic_latch_leaves -{ - btr_pcur_t *const cursor; - btr_latch_mode *const latch_mode; - mtr_t *const mtr; + const auto savepoint= mtr->get_savepoint(); + mtr->memo_push(block, MTR_MEMO_BUF_FIX); - bool operator()(buf_block_t *hint) const + if (UNIV_UNLIKELY(modify_clock != pcur->modify_clock)) { - return hint && - btr_pcur_optimistic_latch_leaves(hint, cursor, latch_mode, mtr); + fail: + mtr->rollback_to_savepoint(savepoint); + return false; + } + + buf_block_t *prev; + if (left_page_no != FIL_NULL) + { + prev= buf_page_get_gen(page_id_t(pcur->old_page_id.space(), + left_page_no), block->zip_size(), + mode, nullptr, BUF_GET_POSSIBLY_FREED, mtr); + if (!prev || + page_is_comp(prev->page.frame) != page_is_comp(block->page.frame) || + memcmp_aligned<2>(block->page.frame, prev->page.frame, 2) || + memcmp_aligned<2>(block->page.frame + PAGE_HEADER + PAGE_INDEX_ID, + prev->page.frame + PAGE_HEADER + PAGE_INDEX_ID, 8)) + goto fail; } -}; + else + prev= nullptr; + + mtr->upgrade_buffer_fix(savepoint, mode); + + if (UNIV_UNLIKELY(block->modify_clock != modify_clock) || + UNIV_UNLIKELY(block->page.is_freed()) || + (prev && + memcmp_aligned<4>(FIL_PAGE_NEXT + prev->page.frame, + FIL_PAGE_OFFSET + page, 4))) + goto fail; + + return true; +} /** Restores the stored position of a persistent cursor bufferfixing the page and obtaining the specified latches. If the cursor position @@ -331,6 +308,7 @@ btr_pcur_t::SAME_UNIQ cursor position is on user rec and points on the record with the same unique field values as in the stored record, btr_pcur_t::NOT_SAME cursor position is not on user rec or points on the record with not the samebuniq field values as in the stored */ +TRANSACTIONAL_TARGET btr_pcur_t::restore_status btr_pcur_t::restore_position(btr_latch_mode restore_latch_mode, mtr_t *mtr) { @@ -361,7 +339,6 @@ btr_pcur_t::restore_position(btr_latch_mode restore_latch_mode, mtr_t *mtr) latch_mode = BTR_LATCH_MODE_WITHOUT_INTENTION(restore_latch_mode); pos_state = BTR_PCUR_IS_POSITIONED; - block_when_stored.clear(); return restore_status::NOT_SAME; } @@ -378,9 +355,8 @@ btr_pcur_t::restore_position(btr_latch_mode restore_latch_mode, mtr_t *mtr) case BTR_SEARCH_PREV: case BTR_MODIFY_PREV: /* Try optimistic restoration. */ - if (block_when_stored.run_with_hint( - optimistic_latch_leaves{this, &restore_latch_mode, - mtr})) { + if (btr_pcur_optimistic_latch_leaves(this, &restore_latch_mode, + mtr)) { pos_state = BTR_PCUR_IS_POSITIONED; latch_mode = restore_latch_mode; @@ -485,16 +461,22 @@ btr_pcur_t::restore_position(btr_latch_mode restore_latch_mode, mtr_t *mtr) since the cursor can now be on a different page! But we can retain the value of old_rec */ - block_when_stored.store(btr_pcur_get_block(this)); - modify_clock= buf_block_get_modify_clock( - block_when_stored.block()); + old_page_id = btr_cur.page_cur.block->page.id(); + modify_clock = btr_cur.page_cur.block->modify_clock; mem_heap_free(heap); return restore_status::SAME_ALL; } - if (n_matched_fields >= index->n_uniq) - ret_val= restore_status::SAME_UNIQ; + if (n_matched_fields >= index->n_uniq + /* Unique indexes can contain "NULL" keys, and if all + unique fields are NULL and not all tuple + fields match to record fields, then treat it as if + restored cursor position points to the record with + not the same unique key. */ + && !(index->n_nullable + && dtuple_contains_null(tuple, index->n_uniq))) + ret_val= restore_status::SAME_UNIQ; } mem_heap_free(heap); @@ -612,40 +594,33 @@ btr_pcur_move_backward_from_page( return true; } - buf_block_t* block = btr_pcur_get_block(cursor); - - if (page_has_prev(block->page.frame)) { - buf_block_t* left_block - = mtr->at_savepoint(mtr->get_savepoint() - 1); - const page_t* const left = left_block->page.frame; - if (memcmp_aligned<4>(left + FIL_PAGE_NEXT, - block->page.frame - + FIL_PAGE_OFFSET, 4)) { - /* This should be the right sibling page, or - if there is none, the current block. */ - ut_ad(left_block == block - || !memcmp_aligned<4>(left + FIL_PAGE_PREV, - block->page.frame - + FIL_PAGE_OFFSET, 4)); - /* The previous one must be the left sibling. */ - left_block - = mtr->at_savepoint(mtr->get_savepoint() - 2); - ut_ad(!memcmp_aligned<4>(left_block->page.frame - + FIL_PAGE_NEXT, - block->page.frame - + FIL_PAGE_OFFSET, 4)); - } + buf_block_t* block = mtr->at_savepoint(0); + ut_ad(block == btr_pcur_get_block(cursor)); + const page_t* const page = block->page.frame; + /* btr_pcur_optimistic_latch_leaves() will acquire a latch on + the preceding page if one exists; + if that fails, btr_cur_t::search_leaf() invoked by + btr_pcur_open_with_no_init() will also acquire a latch on the + succeeding page. Our caller only needs one page latch. */ + ut_ad(mtr->get_savepoint() <= 3); + + if (page_has_prev(page)) { + buf_block_t* const left_block = mtr->at_savepoint(1); + ut_ad(!memcmp_aligned<4>(page + FIL_PAGE_OFFSET, + left_block->page.frame + + FIL_PAGE_NEXT, 4)); if (btr_pcur_is_before_first_on_page(cursor)) { + /* Reposition on the previous page. */ page_cur_set_after_last(left_block, &cursor->btr_cur.page_cur); /* Release the right sibling. */ - } else { - /* Release the left sibling. */ + mtr->rollback_to_savepoint(0, 1); block = left_block; } - mtr->release(*block); } + mtr->rollback_to_savepoint(1); + ut_ad(block == mtr->at_savepoint(0)); cursor->latch_mode = latch_mode; cursor->old_rec = nullptr; return false; diff --git a/storage/innobase/buf/buf0block_hint.cc b/storage/innobase/buf/buf0block_hint.cc deleted file mode 100644 index 6bd01faa..00000000 --- a/storage/innobase/buf/buf0block_hint.cc +++ /dev/null @@ -1,59 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2020, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License, version 2.0, as published by the -Free Software Foundation. - -This program is also distributed with certain software (including but not -limited to OpenSSL) that is licensed under separate terms, as designated in a -particular file or component or in included license documentation. The authors -of MySQL hereby grant you an additional permission to link the program and -your derivative works with the separately licensed software that they have -included with MySQL. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, -for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ - -#include "buf0block_hint.h" -namespace buf { - -TRANSACTIONAL_TARGET -void Block_hint::buffer_fix_block_if_still_valid() -{ - /* To check if m_block belongs to the current buf_pool, we must - prevent freeing memory while we check, and until we buffer-fix the - block. For this purpose it is enough to latch any of the many - latches taken by buf_pool_t::resize(). - - Similar to buf_page_optimistic_get(), we must validate - m_block->page.id() after acquiring the hash_lock, because the object - may have been freed and not actually attached to buf_pool.page_hash - at the moment. (The block could have been reused to store a - different page, and that slice of buf_pool.page_hash could be protected - by another hash_lock that we are not holding.) - - Finally, we must ensure that the block is not being freed. */ - if (m_block) - { - auto &cell= buf_pool.page_hash.cell_get(m_page_id.fold()); - transactional_shared_lock_guard<page_hash_latch> g - {buf_pool.page_hash.lock_get(cell)}; - if (buf_pool.is_uncompressed(m_block) && m_page_id == m_block->page.id() && - m_block->page.frame && m_block->page.in_file()) - m_block->page.fix(); - else - clear(); - } -} -} // namespace buf diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 23b5b776..49f73105 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -77,6 +77,8 @@ struct set_numa_interleave_t if (srv_numa_interleave) { struct bitmask *numa_mems_allowed = numa_get_mems_allowed(); + MEM_MAKE_DEFINED(numa_mems_allowed, + sizeof *numa_mems_allowed); ib::info() << "Setting NUMA memory policy to" " MPOL_INTERLEAVE"; if (set_mempolicy(MPOL_INTERLEAVE, @@ -1062,6 +1064,7 @@ inline bool buf_pool_t::chunk_t::create(size_t bytes) if (srv_numa_interleave) { struct bitmask *numa_mems_allowed= numa_get_mems_allowed(); + MEM_MAKE_DEFINED(numa_mems_allowed, sizeof *numa_mems_allowed); if (mbind(mem, mem_size(), MPOL_INTERLEAVE, numa_mems_allowed->maskp, numa_mems_allowed->size, MPOL_MF_MOVE)) @@ -1591,17 +1594,14 @@ inline bool buf_pool_t::withdraw_blocks() /* reserve free_list length */ if (UT_LIST_GET_LEN(withdraw) < withdraw_target) { - buf_flush_LRU( - std::max<ulint>(withdraw_target - - UT_LIST_GET_LEN(withdraw), - srv_LRU_scan_depth), - true); - mysql_mutex_unlock(&buf_pool.mutex); - buf_dblwr.flush_buffered_writes(); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - buf_flush_wait_LRU_batch_end(); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - mysql_mutex_lock(&buf_pool.mutex); + try_LRU_scan = false; + mysql_mutex_unlock(&mutex); + mysql_mutex_lock(&flush_list_mutex); + page_cleaner_wakeup(true); + my_cond_wait(&done_flush_list, + &flush_list_mutex.m_mutex); + mysql_mutex_unlock(&flush_list_mutex); + mysql_mutex_lock(&mutex); } /* relocate blocks/buddies in withdrawn area */ @@ -2298,7 +2298,10 @@ buf_page_t *buf_pool_t::watch_set(const page_id_t id, got_block: bpage->fix(); if (watch_is_sentinel(*bpage)) + { + ut_ad(!bpage->oldest_modification()); bpage= nullptr; + } page_hash.lock_get(chain).unlock(); return bpage; } @@ -2370,6 +2373,7 @@ void buf_pool_t::watch_unset(const page_id_t id, buf_pool_t::hash_chain &chain) } else { + ut_ad(!w->oldest_modification()); const auto state= w->state(); ut_ad(~buf_page_t::LRU_MASK & state); ut_ad(state >= buf_page_t::UNFIXED + 1); @@ -2856,9 +2860,10 @@ got_block_fixed: if (state > buf_page_t::READ_FIX && state < buf_page_t::WRITE_FIX) { if (mode == BUF_PEEK_IF_IN_POOL) { ignore_block: + block->unfix(); +ignore_unfixed: ut_ad(mode == BUF_GET_POSSIBLY_FREED || mode == BUF_PEEK_IF_IN_POOL); - block->unfix(); if (err) { *err = DB_CORRUPTION; } @@ -2872,16 +2877,32 @@ ignore_block: in buf_page_t::read_complete() or buf_pool_t::corrupted_evict(), or after buf_zip_decompress() in this function. */ - block->page.lock.s_lock(); + if (rw_latch != RW_NO_LATCH) { + block->page.lock.s_lock(); + } else if (!block->page.lock.s_lock_try()) { + /* For RW_NO_LATCH, we should not try to acquire S or X + latch directly as we could be violating the latching + order resulting in deadlock. Instead we try latching the + page and retry in case of a failure. */ + goto wait_for_read; + } state = block->page.state(); ut_ad(state < buf_page_t::READ_FIX || state >= buf_page_t::WRITE_FIX); const page_id_t id{block->page.id()}; block->page.lock.s_unlock(); - if (UNIV_UNLIKELY(id != page_id)) { + if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED)) { + if (UNIV_UNLIKELY(id == page_id)) { + /* The page read was completed, and + another thread marked the page as free + while we were waiting. */ + goto ignore_block; + } + ut_ad(id == page_id_t{~0ULL}); block->page.unfix(); + if (++retries < BUF_PAGE_READ_MAX_RETRIES) { goto loop; } @@ -2892,6 +2913,7 @@ ignore_block: return nullptr; } + ut_ad(id == page_id); } else if (mode != BUF_PEEK_IF_IN_POOL) { } else if (!mtr) { ut_ad(!block->page.oldest_modification()); @@ -2918,6 +2940,7 @@ free_unfixed_block: if (UNIV_UNLIKELY(!block->page.frame)) { if (!block->page.lock.x_lock_try()) { wait_for_unzip: +wait_for_read: /* The page is being read or written, or another thread is executing buf_zip_decompress() in buf_page_get_low() on it. */ @@ -3098,83 +3121,72 @@ re_evict_fail: #endif /* UNIV_DEBUG */ ut_ad(block->page.frame); + /* The state = block->page.state() may be stale at this point, + and in fact, at any point of time if we consider its + buffer-fix component. If the block is being read into the + buffer pool, it is possible that buf_page_t::read_complete() + will invoke buf_pool_t::corrupted_evict() and therefore + invalidate it (invoke buf_page_t::set_corrupt_id() and set the + state to FREED). Therefore, after acquiring the page latch we + must recheck the state. */ + if (state >= buf_page_t::UNFIXED && allow_ibuf_merge && fil_page_get_type(block->page.frame) == FIL_PAGE_INDEX && page_is_leaf(block->page.frame)) { block->page.lock.x_lock(); - ut_ad(block->page.id() == page_id - || (state >= buf_page_t::READ_FIX - && state < buf_page_t::WRITE_FIX)); - -#ifdef BTR_CUR_HASH_ADAPT - btr_search_drop_page_hash_index(block, true); -#endif /* BTR_CUR_HASH_ADAPT */ - - dberr_t e; - - if (UNIV_UNLIKELY(block->page.id() != page_id)) { -page_id_mismatch: - state = block->page.state(); - e = DB_CORRUPTION; -ibuf_merge_corrupted: - if (err) { - *err = e; - } - - if (block->page.id().is_corrupted()) { - buf_pool.corrupted_evict(&block->page, state); - } - return nullptr; - } - state = block->page.state(); ut_ad(state < buf_page_t::READ_FIX); if (state >= buf_page_t::IBUF_EXIST && state < buf_page_t::REINIT) { block->page.clear_ibuf_exist(); - e = ibuf_merge_or_delete_for_page(block, page_id, - block->zip_size()); - if (UNIV_UNLIKELY(e != DB_SUCCESS)) { - goto ibuf_merge_corrupted; + if (dberr_t local_err = + ibuf_merge_or_delete_for_page(block, page_id, + block->zip_size())) { + if (err) { + *err = local_err; + } + goto release_and_ignore_block; } + } else if (state < buf_page_t::UNFIXED) { +release_and_ignore_block: + block->page.lock.x_unlock(); + goto ignore_block; } - if (rw_latch == RW_X_LATCH) { - goto get_latch_valid; - } else { +#ifdef BTR_CUR_HASH_ADAPT + btr_search_drop_page_hash_index(block, true); +#endif /* BTR_CUR_HASH_ADAPT */ + + switch (rw_latch) { + case RW_NO_LATCH: + block->page.lock.x_unlock(); + break; + case RW_S_LATCH: block->page.lock.x_unlock(); - goto get_latch; + block->page.lock.s_lock(); + break; + case RW_SX_LATCH: + block->page.lock.x_u_downgrade(); + break; + default: + ut_ad(rw_latch == RW_X_LATCH); } + + mtr->memo_push(block, mtr_memo_type_t(rw_latch)); } else { -get_latch: switch (rw_latch) { case RW_NO_LATCH: mtr->memo_push(block, MTR_MEMO_BUF_FIX); return block; case RW_S_LATCH: block->page.lock.s_lock(); - ut_ad(!block->page.is_read_fixed()); - if (UNIV_UNLIKELY(block->page.id() != page_id)) { - block->page.lock.s_unlock(); - block->page.lock.x_lock(); - goto page_id_mismatch; - } -get_latch_valid: - mtr->memo_push(block, mtr_memo_type_t(rw_latch)); -#ifdef BTR_CUR_HASH_ADAPT - btr_search_drop_page_hash_index(block, true); -#endif /* BTR_CUR_HASH_ADAPT */ break; case RW_SX_LATCH: block->page.lock.u_lock(); ut_ad(!block->page.is_io_fixed()); - if (UNIV_UNLIKELY(block->page.id() != page_id)) { - block->page.lock.u_x_upgrade(); - goto page_id_mismatch; - } - goto get_latch_valid; + break; default: ut_ad(rw_latch == RW_X_LATCH); if (block->page.lock.x_lock_upgraded()) { @@ -3183,17 +3195,26 @@ get_latch_valid: mtr->page_lock_upgrade(*block); return block; } - if (UNIV_UNLIKELY(block->page.id() != page_id)) { - goto page_id_mismatch; - } - goto get_latch_valid; } - ut_ad(page_id_t(page_get_space_id(block->page.frame), - page_get_page_no(block->page.frame)) - == page_id); + mtr->memo_push(block, mtr_memo_type_t(rw_latch)); + state = block->page.state(); + + if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED)) { + mtr->release_last_page(); + goto ignore_unfixed; + } + + ut_ad(state < buf_page_t::READ_FIX + || state > buf_page_t::WRITE_FIX); + +#ifdef BTR_CUR_HASH_ADAPT + btr_search_drop_page_hash_index(block, true); +#endif /* BTR_CUR_HASH_ADAPT */ } + ut_ad(page_id_t(page_get_space_id(block->page.frame), + page_get_page_no(block->page.frame)) == page_id); return block; } @@ -3289,83 +3310,76 @@ buf_page_get_gen( return block; } -/********************************************************************//** -This is the general function used to get optimistic access to a database -page. -@return TRUE if success */ TRANSACTIONAL_TARGET -bool buf_page_optimistic_get(ulint rw_latch, buf_block_t *block, - uint64_t modify_clock, mtr_t *mtr) +buf_block_t *buf_page_optimistic_fix(buf_block_t *block, page_id_t id) +{ + buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(id.fold()); + transactional_shared_lock_guard<page_hash_latch> g + {buf_pool.page_hash.lock_get(chain)}; + if (UNIV_UNLIKELY(!buf_pool.is_uncompressed(block) || + id != block->page.id() || !block->page.frame)) + return nullptr; + const auto state= block->page.state(); + if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED || + state >= buf_page_t::READ_FIX)) + return nullptr; + block->page.fix(); + return block; +} + +buf_block_t *buf_page_optimistic_get(buf_block_t *block, + rw_lock_type_t rw_latch, + uint64_t modify_clock, mtr_t *mtr) { - ut_ad(block); - ut_ad(mtr); ut_ad(mtr->is_active()); ut_ad(rw_latch == RW_S_LATCH || rw_latch == RW_X_LATCH); + ut_ad(block->page.buf_fix_count()); - if (have_transactional_memory); - else if (UNIV_UNLIKELY(!block->page.frame)) - return false; - else + if (rw_latch == RW_S_LATCH) { - const auto state= block->page.state(); - if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED || - state >= buf_page_t::READ_FIX)) - return false; - } + if (!block->page.lock.s_lock_try()) + { + fail: + block->page.unfix(); + return nullptr; + } - bool success; - const page_id_t id{block->page.id()}; - buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(id.fold()); - bool have_u_not_x= false; + ut_ad(!ibuf_inside(mtr) || + ibuf_page(block->page.id(), block->zip_size(), nullptr)); - { - transactional_shared_lock_guard<page_hash_latch> g - {buf_pool.page_hash.lock_get(chain)}; - if (UNIV_UNLIKELY(id != block->page.id() || !block->page.frame)) - return false; - const auto state= block->page.state(); - if (UNIV_UNLIKELY(state < buf_page_t::UNFIXED || - state >= buf_page_t::READ_FIX)) - return false; - - if (rw_latch == RW_S_LATCH) - success= block->page.lock.s_lock_try(); - else + if (modify_clock != block->modify_clock || block->page.is_freed()) { - have_u_not_x= block->page.lock.have_u_not_x(); - success= have_u_not_x || block->page.lock.x_lock_try(); + block->page.lock.s_unlock(); + goto fail; } - } - if (!success) - return false; - - if (have_u_not_x) + ut_ad(!block->page.is_read_fixed()); + buf_page_make_young_if_needed(&block->page); + mtr->memo_push(block, MTR_MEMO_PAGE_S_FIX); + } + else if (block->page.lock.have_u_not_x()) { block->page.lock.u_x_upgrade(); + block->page.unfix(); mtr->page_lock_upgrade(*block); - ut_ad(id == block->page.id()); ut_ad(modify_clock == block->modify_clock); } + else if (!block->page.lock.x_lock_try()) + goto fail; else { - ut_ad(rw_latch == RW_S_LATCH || !block->page.is_io_fixed()); - ut_ad(id == block->page.id()); - ut_ad(!ibuf_inside(mtr) || ibuf_page(id, block->zip_size(), nullptr)); + ut_ad(!block->page.is_io_fixed()); + ut_ad(!ibuf_inside(mtr) || + ibuf_page(block->page.id(), block->zip_size(), nullptr)); if (modify_clock != block->modify_clock || block->page.is_freed()) { - if (rw_latch == RW_S_LATCH) - block->page.lock.s_unlock(); - else - block->page.lock.x_unlock(); - return false; + block->page.lock.x_unlock(); + goto fail; } - block->page.fix(); - ut_ad(!block->page.is_read_fixed()); buf_page_make_young_if_needed(&block->page); - mtr->memo_push(block, mtr_memo_type_t(rw_latch)); + mtr->memo_push(block, MTR_MEMO_PAGE_X_FIX); } ut_d(if (!(++buf_dbg_counter % 5771)) buf_pool.validate()); @@ -3375,7 +3389,7 @@ bool buf_page_optimistic_get(ulint rw_latch, buf_block_t *block, ut_ad(~buf_page_t::LRU_MASK & state); ut_ad(block->page.frame); - return true; + return block; } /** Try to S-latch a page. diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index d4628985..d364be31 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -274,30 +274,22 @@ buf_flush_relocate_on_flush_list( ut_d(buf_flush_validate_low()); } -/** Note that a block is no longer dirty, while not removing -it from buf_pool.flush_list -@param temporary whether the page belongs to the temporary tablespace -@param error whether an error may have occurred while writing */ -inline void buf_page_t::write_complete(bool temporary, bool error) +void buf_page_t::write_complete(bool persistent, bool error, uint32_t state) { - ut_ad(temporary == fsp_is_system_temporary(id().space())); - if (UNIV_UNLIKELY(error)); - else if (temporary) - { - ut_ad(oldest_modification() == 2); - oldest_modification_= 0; - } - else + ut_ad(!persistent == fsp_is_system_temporary(id().space())); + ut_ad(state >= WRITE_FIX); + + if (UNIV_LIKELY(!error)) { + ut_d(lsn_t om= oldest_modification()); + ut_ad(om >= 2); + ut_ad(persistent == (om > 2)); /* We use release memory order to guarantee that callers of oldest_modification_acquire() will observe the block as being detached from buf_pool.flush_list, after reading the value 0. */ - ut_ad(oldest_modification() > 2); - oldest_modification_.store(1, std::memory_order_release); + oldest_modification_.store(persistent, std::memory_order_release); } - const auto s= state(); - ut_ad(s >= WRITE_FIX); - zip.fix.fetch_sub((s >= WRITE_FIX_REINIT) + zip.fix.fetch_sub((state >= WRITE_FIX_REINIT) ? (WRITE_FIX_REINIT - UNFIXED) : (WRITE_FIX - UNFIXED)); lock.u_unlock(true); @@ -311,18 +303,10 @@ inline void buf_pool_t::n_flush_inc() inline void buf_pool_t::n_flush_dec() { - mysql_mutex_lock(&flush_list_mutex); + mysql_mutex_assert_owner(&flush_list_mutex); ut_ad(page_cleaner_status >= LRU_FLUSH); if ((page_cleaner_status-= LRU_FLUSH) < LRU_FLUSH) pthread_cond_broadcast(&done_flush_LRU); - mysql_mutex_unlock(&flush_list_mutex); -} - -inline void buf_pool_t::n_flush_dec_holding_mutex() -{ - mysql_mutex_assert_owner(&flush_list_mutex); - ut_ad(page_cleaner_status >= LRU_FLUSH); - page_cleaner_status-= LRU_FLUSH; } /** Complete write of a file page from buf_pool. @@ -352,28 +336,26 @@ void buf_page_write_complete(const IORequest &request, bool error) mysql_mutex_assert_not_owner(&buf_pool.mutex); mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); - if (request.is_LRU()) + const bool persistent= bpage->oldest_modification() != 2; + + if (UNIV_UNLIKELY(!persistent) && UNIV_LIKELY(!error)) { - const bool temp= bpage->oldest_modification() == 2; - if (!temp && state < buf_page_t::WRITE_FIX_REINIT && - request.node->space->use_doublewrite()) - buf_dblwr.write_completed(); /* We must hold buf_pool.mutex while releasing the block, so that no other thread can access it before we have freed it. */ mysql_mutex_lock(&buf_pool.mutex); - bpage->write_complete(temp, error); - if (!error) - buf_LRU_free_page(bpage, true); + bpage->write_complete(persistent, error, state); + buf_LRU_free_page(bpage, true); mysql_mutex_unlock(&buf_pool.mutex); - - buf_pool.n_flush_dec(); } else { + bpage->write_complete(persistent, error, state); if (state < buf_page_t::WRITE_FIX_REINIT && request.node->space->use_doublewrite()) + { + ut_ad(persistent); buf_dblwr.write_completed(); - bpage->write_complete(false, error); + } } } @@ -740,17 +722,15 @@ ATTRIBUTE_COLD void buf_pool_t::release_freed_page(buf_page_t *bpage) noexcept } /** Write a flushable page to a file or free a freeable block. -@param evict whether to evict the page on write completion @param space tablespace @return whether a page write was initiated and buf_pool.mutex released */ -bool buf_page_t::flush(bool evict, fil_space_t *space) +bool buf_page_t::flush(fil_space_t *space) { mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); ut_ad(in_file()); ut_ad(in_LRU_list); ut_ad((space->purpose == FIL_TYPE_TEMPORARY) == (space == fil_system.temp_space)); - ut_ad(evict || space != fil_system.temp_space); ut_ad(space->referenced()); const auto s= state(); @@ -797,22 +777,11 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) mysql_mutex_unlock(&buf_pool.mutex); IORequest::Type type= IORequest::WRITE_ASYNC; - if (UNIV_UNLIKELY(evict)) - { - type= IORequest::WRITE_LRU; - mysql_mutex_lock(&buf_pool.flush_list_mutex); - buf_pool.n_flush_inc(); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - } /* Apart from the U-lock, this block will also be protected by is_write_fixed() and oldest_modification()>1. Thus, it cannot be relocated or removed. */ - DBUG_PRINT("ib_buf", ("%s %u page %u:%u", - evict ? "LRU" : "flush_list", - id().space(), id().page_no())); - buf_block_t *block= reinterpret_cast<buf_block_t*>(this); page_t *write_frame= zip.data; @@ -864,10 +833,7 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) { switch (space->chain.start->punch_hole) { case 1: - static_assert(IORequest::PUNCH_LRU - IORequest::PUNCH == - IORequest::WRITE_LRU - IORequest::WRITE_ASYNC, ""); - type= - IORequest::Type(type + (IORequest::PUNCH - IORequest::WRITE_ASYNC)); + type= IORequest::PUNCH; break; case 2: size= orig_size; @@ -894,10 +860,8 @@ bool buf_page_t::flush(bool evict, fil_space_t *space) /** Check whether a page can be flushed from the buf_pool. @param id page identifier @param fold id.fold() -@param evict true=buf_pool.LRU; false=buf_pool.flush_list @return whether the page can be flushed */ -static bool buf_flush_check_neighbor(const page_id_t id, ulint fold, - bool evict) +static bool buf_flush_check_neighbor(const page_id_t id, ulint fold) { mysql_mutex_assert_owner(&buf_pool.mutex); ut_ad(fold == id.fold()); @@ -906,26 +870,16 @@ static bool buf_flush_check_neighbor(const page_id_t id, ulint fold, const buf_page_t *bpage= buf_pool.page_hash.get(id, buf_pool.page_hash.cell_get(fold)); - if (!bpage || buf_pool.watch_is_sentinel(*bpage)) - return false; - - /* We avoid flushing 'non-old' blocks in an eviction flush, because the - flushed blocks are soon freed */ - if (evict && !bpage->is_old()) - return false; - - return bpage->oldest_modification() > 1 && !bpage->is_io_fixed(); + return bpage && bpage->oldest_modification() > 1 && !bpage->is_io_fixed(); } /** Check which neighbors of a page can be flushed from the buf_pool. @param space tablespace @param id page identifier of a dirty page @param contiguous whether to consider contiguous areas of pages -@param evict true=buf_pool.LRU; false=buf_pool.flush_list @return last page number that can be flushed */ static page_id_t buf_flush_check_neighbors(const fil_space_t &space, - page_id_t &id, bool contiguous, - bool evict) + page_id_t &id, bool contiguous) { ut_ad(id.page_no() < space.size + (space.physical_size() == 2048 ? 1 @@ -958,7 +912,7 @@ static page_id_t buf_flush_check_neighbors(const fil_space_t &space, for (page_id_t i= id - 1;; --i) { fold--; - if (!buf_flush_check_neighbor(i, fold, evict)) + if (!buf_flush_check_neighbor(i, fold)) { low= i + 1; break; @@ -974,7 +928,7 @@ static page_id_t buf_flush_check_neighbors(const fil_space_t &space, while (++i < high) { ++fold; - if (!buf_flush_check_neighbor(i, fold, evict)) + if (!buf_flush_check_neighbor(i, fold)) break; } @@ -1051,14 +1005,13 @@ and also write zeroes or punch the hole for the freed ranges of pages. @param page_id page identifier @param bpage buffer page @param contiguous whether to consider contiguous areas of pages -@param evict true=buf_pool.LRU; false=buf_pool.flush_list @param n_flushed number of pages flushed so far in this batch @param n_to_flush maximum number of pages we are allowed to flush @return number of pages flushed */ static ulint buf_flush_try_neighbors(fil_space_t *space, const page_id_t page_id, buf_page_t *bpage, - bool contiguous, bool evict, + bool contiguous, ulint n_flushed, ulint n_to_flush) { ut_ad(space->id == page_id.space()); @@ -1072,7 +1025,7 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, ut_ad(lsn >= bpage->oldest_modification()); if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) { - ut_a(!bpage->flush(evict, space)); + ut_a(!bpage->flush(space)); mysql_mutex_unlock(&buf_pool.mutex); return 0; } @@ -1082,7 +1035,7 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, ulint count= 0; page_id_t id= page_id; - page_id_t high= buf_flush_check_neighbors(*space, id, contiguous, evict); + page_id_t high= buf_flush_check_neighbors(*space, id, contiguous); ut_ad(page_id >= id); ut_ad(page_id < high); @@ -1119,7 +1072,7 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, ut_ad(!buf_pool.watch_is_sentinel(*b)); ut_ad(b->oldest_modification() > 1); flush: - if (b->flush(evict, space)) + if (b->flush(space)) { ++count; continue; @@ -1127,9 +1080,10 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, } /* We avoid flushing 'non-old' blocks in an eviction flush, because the flushed blocks are soon freed */ - else if ((!evict || b->is_old()) && !buf_pool.watch_is_sentinel(*b) && - b->oldest_modification() > 1 && b->lock.u_lock_try(true)) + else if (b->oldest_modification() > 1 && b->lock.u_lock_try(true)) { + /* For the buf_pool.watch[] sentinels, oldest_modification() == 0 */ + ut_ad(!buf_pool.watch_is_sentinel(*b)); if (b->oldest_modification() < 2) b->lock.u_unlock(true); else @@ -1251,10 +1205,8 @@ static void buf_flush_discard_page(buf_page_t *bpage) /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush -@param evict whether dirty pages are to be evicted after flushing them @param n counts of flushed and evicted pages */ -static void buf_flush_LRU_list_batch(ulint max, bool evict, - flush_counters_t *n) +static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n) { ulint scanned= 0; ulint free_limit= srv_LRU_scan_depth; @@ -1302,8 +1254,12 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, if (state < buf_page_t::READ_FIX && bpage->lock.u_lock_try(true)) { ut_ad(!bpage->is_io_fixed()); - bool do_evict= evict; switch (bpage->oldest_modification()) { + case 2: + /* LRU flushing will always evict pages of the temporary tablespace, + in buf_page_write_complete(). */ + ++n->evicted; + break; case 1: mysql_mutex_lock(&buf_pool.flush_list_mutex); if (ut_d(lsn_t lsn=) bpage->oldest_modification()) @@ -1316,12 +1272,8 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, case 0: bpage->lock.u_unlock(true); goto evict; - case 2: - /* LRU flushing will always evict pages of the temporary tablespace. */ - do_evict= true; } - /* Block is ready for flush. Dispatch an IO request. - If do_evict, the page may be evicted by buf_page_write_complete(). */ + /* Block is ready for flush. Dispatch an IO request. */ const page_id_t page_id(bpage->id()); const uint32_t space_id= page_id.space(); if (!space || space->id != space_id) @@ -1356,6 +1308,7 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); + ++n->evicted; continue; } @@ -1368,8 +1321,8 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, if (neighbors && space->is_rotational()) n->flushed+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, - do_evict, n->flushed, max); - else if (bpage->flush(do_evict, space)) + n->flushed, max); + else if (bpage->flush(space)) ++n->flushed; else continue; @@ -1387,24 +1340,25 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, space->release(); if (scanned) + { MONITOR_INC_VALUE_CUMULATIVE(MONITOR_LRU_BATCH_SCANNED, MONITOR_LRU_BATCH_SCANNED_NUM_CALL, MONITOR_LRU_BATCH_SCANNED_PER_CALL, scanned); + } } /** Flush and move pages from LRU or unzip_LRU list to the free list. Whether LRU or unzip_LRU is used depends on the state of the system. @param max maximum number of blocks to flush -@param evict whether dirty pages are to be evicted after flushing them @param n counts of flushed and evicted pages */ -static void buf_do_LRU_batch(ulint max, bool evict, flush_counters_t *n) +static void buf_do_LRU_batch(ulint max, flush_counters_t *n) { if (buf_LRU_evict_from_unzip_LRU()) buf_free_from_unzip_LRU_list_batch(); n->evicted= 0; n->flushed= 0; - buf_flush_LRU_list_batch(max, evict, n); + buf_flush_LRU_list_batch(max, n); mysql_mutex_assert_owner(&buf_pool.mutex); buf_lru_freed_page_count+= n->evicted; @@ -1516,8 +1470,8 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) { if (neighbors && space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, - neighbors == 1, false, count, max_n); - else if (bpage->flush(false, space)) + neighbors == 1, count, max_n); + else if (bpage->flush(space)) ++count; else continue; @@ -1536,10 +1490,13 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) space->release(); if (scanned) + { MONITOR_INC_VALUE_CUMULATIVE(MONITOR_FLUSH_BATCH_SCANNED, MONITOR_FLUSH_BATCH_SCANNED_NUM_CALL, MONITOR_FLUSH_BATCH_SCANNED_PER_CALL, scanned); + } + return count; } @@ -1683,7 +1640,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) goto was_freed; } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (bpage->flush(false, space)) + if (bpage->flush(space)) { ++n_flush; if (!--max_n_flush) @@ -1741,27 +1698,22 @@ and move clean blocks to buf_pool.free. The caller must invoke buf_dblwr.flush_buffered_writes() after releasing buf_pool.mutex. @param max_n wished maximum mumber of blocks flushed -@param evict whether to evict pages after flushing -@return evict ? number of processed pages : number of pages written */ -ulint buf_flush_LRU(ulint max_n, bool evict) +@return number of pages written */ +static ulint buf_flush_LRU(ulint max_n) { mysql_mutex_assert_owner(&buf_pool.mutex); flush_counters_t n; - buf_do_LRU_batch(max_n, evict, &n); + buf_do_LRU_batch(max_n, &n); ulint pages= n.flushed; if (n.evicted) { - if (evict) - pages+= n.evicted; buf_pool.try_LRU_scan= true; pthread_cond_broadcast(&buf_pool.done_free); } - else if (!pages && !buf_pool.try_LRU_scan && - !buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) - { + else if (!pages && !buf_pool.try_LRU_scan) /* For example, with the minimum innodb_buffer_pool_size=5M and the default innodb_page_size=16k there are only a little over 316 pages in the buffer pool. The buffer pool can easily be exhausted @@ -1775,18 +1727,13 @@ ulint buf_flush_LRU(ulint max_n, bool evict) (3) This thread is the only one that could make progress, but we fail to do so because all the pages that we scanned are buffer-fixed or latched by some thread. */ - sql_print_warning("InnoDB: Could not free any blocks in the buffer pool!" - " %zu blocks are in use and %zu free." - " Consider increasing innodb_buffer_pool_size.", - UT_LIST_GET_LEN(buf_pool.LRU), - UT_LIST_GET_LEN(buf_pool.free)); - } + buf_pool.LRU_warn(); return pages; } #ifdef HAVE_PMEM -# include <libpmem.h> +# include "cache.h" #endif /** Write checkpoint information to the log header and release mutex. @@ -1900,8 +1847,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept ut_ad(!log.is_opened()); bool success; log.m_file= - os_file_create_func(get_log_file_path().c_str(), - OS_FILE_OPEN | OS_FILE_ON_ERROR_NO_EXIT, + os_file_create_func(get_log_file_path().c_str(), OS_FILE_OPEN, OS_FILE_NORMAL, OS_LOG_FILE, false, &success); ut_a(success); ut_a(log.is_opened()); @@ -1916,7 +1862,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept { my_munmap(buf, file_size); buf= resize_buf; - buf_free= START_OFFSET + (get_lsn() - resizing); + set_buf_free(START_OFFSET + (get_lsn() - resizing)); } else #endif @@ -1958,9 +1904,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept static bool log_checkpoint_low(lsn_t oldest_lsn, lsn_t end_lsn) { ut_ad(!srv_read_only_mode); -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); ut_ad(oldest_lsn <= end_lsn); ut_ad(end_lsn == log_sys.get_lsn()); @@ -2327,7 +2271,7 @@ func_exit: sum_pages += last_pages_in; - const ulint time_elapsed = std::max<ulint>(curr_time - prev_time, 1); + const ulint time_elapsed = std::max<ulint>(ulint(curr_time - prev_time), 1); /* We update our variables every innodb_flushing_avg_loops iterations to smooth out transition in workload. */ @@ -2541,26 +2485,16 @@ static void buf_flush_page_cleaner() { buf_pool.page_cleaner_set_idle(false); buf_pool.n_flush_inc(); - /* Remove clean blocks from buf_pool.flush_list before the LRU scan. */ - for (buf_page_t *p= UT_LIST_GET_FIRST(buf_pool.flush_list); p; ) - { - const lsn_t lsn{p->oldest_modification()}; - ut_ad(lsn > 2 || lsn == 1); - buf_page_t *n= UT_LIST_GET_NEXT(list, p); - if (lsn <= 1) - buf_pool.delete_from_flush_list(p); - p= n; - } mysql_mutex_unlock(&buf_pool.flush_list_mutex); n= srv_max_io_capacity; mysql_mutex_lock(&buf_pool.mutex); LRU_flush: - n= buf_flush_LRU(n, false); + n= buf_flush_LRU(n); mysql_mutex_unlock(&buf_pool.mutex); last_pages+= n; check_oldest_and_set_idle: mysql_mutex_lock(&buf_pool.flush_list_mutex); - buf_pool.n_flush_dec_holding_mutex(); + buf_pool.n_flush_dec(); oldest_lsn= buf_pool.get_oldest_modification(0); if (!oldest_lsn) goto fully_unemployed; @@ -2693,6 +2627,16 @@ static void buf_flush_page_cleaner() #endif } +ATTRIBUTE_COLD void buf_pool_t::LRU_warn() +{ + mysql_mutex_assert_owner(&mutex); + if (!LRU_warned.test_and_set(std::memory_order_acquire)) + sql_print_warning("InnoDB: Could not free any blocks in the buffer pool!" + " %zu blocks are in use and %zu free." + " Consider increasing innodb_buffer_pool_size.", + UT_LIST_GET_LEN(LRU), UT_LIST_GET_LEN(free)); +} + /** Initialize page_cleaner. */ ATTRIBUTE_COLD void buf_flush_page_cleaner_init() { diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 2a8d6ff2..33d01b6b 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -385,142 +385,76 @@ we put it to free list to be used. @return the free control block, in state BUF_BLOCK_MEMORY */ buf_block_t *buf_LRU_get_free_block(bool have_mutex) { - ulint n_iterations = 0; - ulint flush_failures = 0; - MONITOR_INC(MONITOR_LRU_GET_FREE_SEARCH); - if (have_mutex) { - mysql_mutex_assert_owner(&buf_pool.mutex); - goto got_mutex; - } - DBUG_EXECUTE_IF("recv_ran_out_of_buffer", - if (recv_recovery_is_on() - && recv_sys.apply_log_recs) { - mysql_mutex_lock(&buf_pool.mutex); - goto flush_lru; - }); -get_mutex: - mysql_mutex_lock(&buf_pool.mutex); -got_mutex: - buf_LRU_check_size_of_non_data_objects(); - buf_block_t* block; + bool waited= false; + MONITOR_INC(MONITOR_LRU_GET_FREE_SEARCH); + if (!have_mutex) + mysql_mutex_lock(&buf_pool.mutex); + + buf_LRU_check_size_of_non_data_objects(); - IF_DBUG(static bool buf_lru_free_blocks_error_printed,); - DBUG_EXECUTE_IF("ib_lru_force_no_free_page", - if (!buf_lru_free_blocks_error_printed) { - n_iterations = 21; - goto not_found;}); + buf_block_t *block; retry: - /* If there is a block in the free list, take it */ - if ((block = buf_LRU_get_free_only()) != nullptr) { + /* If there is a block in the free list, take it */ + block= buf_LRU_get_free_only(); + if (block) + { got_block: - const ulint LRU_size = UT_LIST_GET_LEN(buf_pool.LRU); - const ulint available = UT_LIST_GET_LEN(buf_pool.free); - const ulint scan_depth = srv_LRU_scan_depth / 2; - ut_ad(LRU_size <= BUF_LRU_MIN_LEN || available >= scan_depth - || buf_pool.need_LRU_eviction()); - - if (!have_mutex) { - mysql_mutex_unlock(&buf_pool.mutex); - } - - if (UNIV_UNLIKELY(available < scan_depth) - && LRU_size > BUF_LRU_MIN_LEN) { - mysql_mutex_lock(&buf_pool.flush_list_mutex); - if (!buf_pool.page_cleaner_active()) { - buf_pool.page_cleaner_wakeup(true); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - } - - block->page.zip.clear(); - return block; - } + const ulint LRU_size= UT_LIST_GET_LEN(buf_pool.LRU); + const ulint available= UT_LIST_GET_LEN(buf_pool.free); + const ulint scan_depth= srv_LRU_scan_depth / 2; + ut_ad(LRU_size <= BUF_LRU_MIN_LEN || + available >= scan_depth || buf_pool.need_LRU_eviction()); - MONITOR_INC( MONITOR_LRU_GET_FREE_LOOPS ); - if (n_iterations || buf_pool.try_LRU_scan) { - /* If no block was in the free list, search from the - end of the LRU list and try to free a block there. - If we are doing for the first time we'll scan only - tail of the LRU list otherwise we scan the whole LRU - list. */ - if (buf_LRU_scan_and_free_block(n_iterations - ? ULINT_UNDEFINED : 100)) { - goto retry; - } + if (UNIV_UNLIKELY(available < scan_depth) && LRU_size > BUF_LRU_MIN_LEN) + { + mysql_mutex_lock(&buf_pool.flush_list_mutex); + if (!buf_pool.page_cleaner_active()) + buf_pool.page_cleaner_wakeup(true); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + } - /* Tell other threads that there is no point - in scanning the LRU list. */ - buf_pool.try_LRU_scan = false; - } + if (!have_mutex) + mysql_mutex_unlock(&buf_pool.mutex); - for (;;) { - if ((block = buf_LRU_get_free_only()) != nullptr) { - goto got_block; - } - const bool wake = buf_pool.need_LRU_eviction(); - mysql_mutex_unlock(&buf_pool.mutex); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - const auto n_flush = buf_pool.n_flush(); - if (wake && !buf_pool.page_cleaner_active()) { - buf_pool.page_cleaner_wakeup(true); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - mysql_mutex_lock(&buf_pool.mutex); - if (!n_flush) { - goto not_found; - } - if (!buf_pool.try_LRU_scan) { - my_cond_wait(&buf_pool.done_free, - &buf_pool.mutex.m_mutex); - } - } - -not_found: - if (n_iterations > 1) { - MONITOR_INC( MONITOR_LRU_GET_FREE_WAITS ); - } + block->page.zip.clear(); + return block; + } - if (n_iterations == 21 - && srv_buf_pool_old_size == srv_buf_pool_size - && buf_pool.LRU_warned.test_and_set(std::memory_order_acquire)) { - IF_DBUG(buf_lru_free_blocks_error_printed = true,); - mysql_mutex_unlock(&buf_pool.mutex); - ib::warn() << "Difficult to find free blocks in the buffer pool" - " (" << n_iterations << " search iterations)! " - << flush_failures << " failed attempts to" - " flush a page!" - " Consider increasing innodb_buffer_pool_size." - " Pending flushes (fsync): " - << fil_n_pending_tablespace_flushes - << ". " << os_n_file_reads << " OS file reads, " - << os_n_file_writes << " OS file writes, " - << os_n_fsyncs - << " OS fsyncs."; - mysql_mutex_lock(&buf_pool.mutex); - } + MONITOR_INC(MONITOR_LRU_GET_FREE_LOOPS); + if (waited || buf_pool.try_LRU_scan) + { + /* If no block was in the free list, search from the end of the + LRU list and try to free a block there. If we are doing for the + first time we'll scan only tail of the LRU list otherwise we scan + the whole LRU list. */ + if (buf_LRU_scan_and_free_block(waited ? ULINT_UNDEFINED : 100)) + goto retry; + + /* Tell other threads that there is no point in scanning the LRU + list. */ + buf_pool.try_LRU_scan= false; + } - /* No free block was found: try to flush the LRU list. - The freed blocks will be up for grabs for all threads. + waited= true; - TODO: A more elegant way would have been to return one freed - up block to the caller here but the code that deals with - removing the block from buf_pool.page_hash and buf_pool.LRU is fairly - involved (particularly in case of ROW_FORMAT=COMPRESSED pages). We - can do that in a separate patch sometime in future. */ -#ifndef DBUG_OFF -flush_lru: -#endif - if (!buf_flush_LRU(innodb_lru_flush_size, true)) { - MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT); - ++flush_failures; - } + while (!(block= buf_LRU_get_free_only())) + { + buf_pool.stat.LRU_waits++; + + timespec abstime; + set_timespec(abstime, 1); + + mysql_mutex_lock(&buf_pool.flush_list_mutex); + if (!buf_pool.page_cleaner_active()) + buf_pool.page_cleaner_wakeup(true); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (my_cond_timedwait(&buf_pool.done_free, &buf_pool.mutex.m_mutex, + &abstime)) + buf_pool.LRU_warn(); + } - n_iterations++; - buf_pool.stat.LRU_waits++; - mysql_mutex_unlock(&buf_pool.mutex); - buf_dblwr.flush_buffered_writes(); - goto get_mutex; + goto got_block; } /** Move the LRU_old pointer so that the length of the old blocks list diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 9041c6a2..76a5e710 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -575,7 +575,7 @@ fail: hash_lock.lock_shared(); const buf_page_t* bpage= buf_pool.page_hash.get(i, chain); - if (!bpage) + if (!bpage || buf_pool.watch_is_sentinel(*bpage)) { hash_lock.unlock_shared(); if (i == page_id) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 5d3cab17..a1295c33 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -657,47 +657,22 @@ dict_table_t::parse_name<>(char(&)[NAME_LEN + 1], char(&)[NAME_LEN + 1], /** Acquire MDL shared for the table name. @tparam trylock whether to use non-blocking operation @param[in,out] table table object -@param[in,out] thd background thread -@param[out] mdl mdl ticket +@param[in,out] mdl_context MDL context +@param[out] mdl MDL ticket @param[in] table_op operation to perform when opening @return table object after locking MDL shared @retval nullptr if the table is not readable, or if trylock && MDL blocked */ template<bool trylock> +__attribute__((nonnull, warn_unused_result)) dict_table_t* dict_acquire_mdl_shared(dict_table_t *table, - THD *thd, - MDL_ticket **mdl, + MDL_context *mdl_context, MDL_ticket **mdl, dict_table_op_t table_op) { - if (!table || !mdl) - return table; - - MDL_context *mdl_context= static_cast<MDL_context*>(thd_mdl_context(thd)); - size_t db_len; - dict_table_t *not_found= nullptr; - - if (trylock) - { - dict_sys.freeze(SRW_LOCK_CALL); - db_len= dict_get_db_name_len(table->name.m_name); - dict_sys.unfreeze(); - } - else - { - ut_ad(dict_sys.frozen_not_locked()); - db_len= dict_get_db_name_len(table->name.m_name); - } - - if (db_len == 0) - return table; /* InnoDB system tables are not covered by MDL */ - - if (!mdl_context) - return nullptr; - table_id_t table_id= table->id; char db_buf[NAME_LEN + 1], db_buf1[NAME_LEN + 1]; char tbl_buf[NAME_LEN + 1], tbl_buf1[NAME_LEN + 1]; - size_t tbl_len; + size_t db_len, tbl_len; bool unaccessible= false; if (!table->parse_name<!trylock>(db_buf, tbl_buf, &db_len, &tbl_len)) @@ -768,7 +743,6 @@ retry: if (!table || !table->is_accessible()) { - table= nullptr; return_without_mdl: if (trylock) dict_sys.unfreeze(); @@ -777,7 +751,7 @@ return_without_mdl: mdl_context->release_lock(*mdl); *mdl= nullptr; } - return not_found; + return nullptr; } size_t db1_len, tbl1_len; @@ -815,6 +789,50 @@ return_without_mdl: } template dict_table_t* dict_acquire_mdl_shared<false> +(dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); + +/** Acquire MDL shared for the table name. +@tparam trylock whether to use non-blocking operation +@param[in,out] table table object +@param[in,out] thd background thread +@param[out] mdl mdl ticket +@param[in] table_op operation to perform when opening +@return table object after locking MDL shared +@retval nullptr if the table is not readable, or if trylock && MDL blocked */ +template<bool trylock> +dict_table_t* +dict_acquire_mdl_shared(dict_table_t *table, + THD *thd, + MDL_ticket **mdl, + dict_table_op_t table_op) +{ + if (!table || !mdl) + return table; + + MDL_context *mdl_context= static_cast<MDL_context*>(thd_mdl_context(thd)); + size_t db_len; + + if (trylock) + { + dict_sys.freeze(SRW_LOCK_CALL); + db_len= dict_get_db_name_len(table->name.m_name); + dict_sys.unfreeze(); + } + else + { + ut_ad(dict_sys.frozen_not_locked()); + db_len= dict_get_db_name_len(table->name.m_name); + } + + if (db_len == 0) + return table; /* InnoDB system tables are not covered by MDL */ + + return mdl_context + ? dict_acquire_mdl_shared<trylock>(table, mdl_context, mdl, table_op) + : nullptr; +} + +template dict_table_t* dict_acquire_mdl_shared<false> (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); template dict_table_t* dict_acquire_mdl_shared<true> (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); @@ -960,9 +978,6 @@ void dict_sys_t::lock_wait(SRW_LOCK_ARGS(const char *file, unsigned line)) { latch.wr_lock(SRW_LOCK_ARGS(file, line)); latch_ex_wait_start.store(0, std::memory_order_relaxed); - ut_ad(!latch_readers); - ut_ad(!latch_ex); - ut_d(latch_ex= pthread_self()); return; } @@ -978,35 +993,36 @@ void dict_sys_t::lock_wait(SRW_LOCK_ARGS(const char *file, unsigned line)) ib::warn() << "A long wait (" << waited << " seconds) was observed for dict_sys.latch"; latch.wr_lock(SRW_LOCK_ARGS(file, line)); - ut_ad(!latch_readers); - ut_ad(!latch_ex); - ut_d(latch_ex= pthread_self()); } #ifdef UNIV_PFS_RWLOCK ATTRIBUTE_NOINLINE void dict_sys_t::unlock() { - ut_ad(latch_ex == pthread_self()); - ut_ad(!latch_readers); - ut_d(latch_ex= 0); latch.wr_unlock(); } ATTRIBUTE_NOINLINE void dict_sys_t::freeze(const char *file, unsigned line) { latch.rd_lock(file, line); - ut_ad(!latch_ex); - ut_d(latch_readers++); } ATTRIBUTE_NOINLINE void dict_sys_t::unfreeze() { - ut_ad(!latch_ex); - ut_ad(latch_readers--); latch.rd_unlock(); } #endif /* UNIV_PFS_RWLOCK */ +/** Report an error about failing to open a table. +@param name table name */ +static void dict_table_open_failed(const table_name_t &name) +{ + my_printf_error(ER_TABLE_CORRUPT, + "Table %`.*s.%`s is corrupted." + " Please drop the table and recreate.", + MYF(ME_ERROR_LOG), + int(name.dblen()), name.m_name, name.basename()); +} + /**********************************************************************//** Returns a table object and increments its open handle count. NOTE! This is a high-level function to be used mainly from outside the @@ -1039,18 +1055,20 @@ dict_table_open_on_name( if (!(ignore_err & ~DICT_ERR_IGNORE_FK_NOKEY) && !table->is_readable() && table->corrupted) { - ulint algo = table->space->get_compression_algo(); - if (algo <= PAGE_ALGORITHM_LAST && !fil_comp_algo_loaded(algo)) { - my_printf_error(ER_PROVIDER_NOT_LOADED, - "Table %s is compressed with %s, which is not currently loaded. " - "Please load the %s provider plugin to open the table", - MYF(ME_ERROR_LOG), table->name, - page_compression_algorithms[algo], page_compression_algorithms[algo]); - } else { - my_printf_error(ER_TABLE_CORRUPT, - "Table %s is corrupted. Please drop the table and recreate.", - MYF(ME_ERROR_LOG), table->name); - } + ulint algo= table->space->get_compression_algo(); + if (algo <= PAGE_ALGORITHM_LAST && !fil_comp_algo_loaded(algo)) + my_printf_error(ER_PROVIDER_NOT_LOADED, + "Table %`.*s.%`s is compressed with %s," + " which is not currently loaded. " + "Please load the %s provider plugin" + " to open the table", + MYF(ME_ERROR_LOG), + int(table->name.dblen()), table->name.m_name, + table->name.basename(), + page_compression_algorithms[algo], + page_compression_algorithms[algo]); + else + dict_table_open_failed(table->name); dict_sys.unfreeze(); DBUG_RETURN(nullptr); } @@ -1070,8 +1088,7 @@ dict_table_open_on_name( if (!(ignore_err & ~DICT_ERR_IGNORE_FK_NOKEY) && !table->is_readable() && table->corrupted) { - ib::error() << "Table " << table->name - << " is corrupted. Please drop the table and recreate."; + dict_table_open_failed(table->name); if (!dict_locked) dict_sys.unlock(); DBUG_RETURN(nullptr); @@ -1992,7 +2009,6 @@ dict_index_add_to_cache( new_index->n_fields = new_index->n_def; new_index->trx_id = index->trx_id; new_index->set_committed(index->is_committed()); - new_index->nulls_equal = index->nulls_equal; n_ord = new_index->n_uniq; /* Flag the ordering columns and also set column max_prefix */ diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index f11187b9..f647278d 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -3867,6 +3867,10 @@ release_and_exit: goto release_and_exit; } +#ifdef ENABLED_DEBUG_SYNC + DEBUG_SYNC(thd, "dict_stats_mdl_acquired"); +#endif /* ENABLED_DEBUG_SYNC */ + trx = trx_create(); trx_start_internal_read_only(trx); diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 97cb3994..d4e6c6f3 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2290,7 +2290,7 @@ void fil_space_crypt_close_tablespace(const fil_space_t *space) << space->chain.start->name << " (" << space->id << ") active threads " << crypt_data->rotate_state.active_threads - << "flushing=" + << " flushing=" << crypt_data->rotate_state.flushing << "."; last = now; } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index bd0ace7c..e8d23657 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -342,7 +342,7 @@ static bool fil_node_open_file_low(fil_node_t *node) ut_ad(node->space->is_closing()); mysql_mutex_assert_owner(&fil_system.mutex); static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility"); -#if defined _WIN32 || defined HAVE_FCNTL_DIRECT +#if defined _WIN32 || defined O_DIRECT ulint type; switch (FSP_FLAGS_GET_ZIP_SSIZE(node->space->flags)) { case 1: @@ -361,8 +361,7 @@ static bool fil_node_open_file_low(fil_node_t *node) bool success; node->handle= os_file_create(innodb_data_file_key, node->name, node->is_raw_disk - ? OS_FILE_OPEN_RAW | OS_FILE_ON_ERROR_NO_EXIT - : OS_FILE_OPEN | OS_FILE_ON_ERROR_NO_EXIT, + ? OS_FILE_OPEN_RAW : OS_FILE_OPEN, OS_FILE_AIO, type, srv_read_only_mode, &success); @@ -928,9 +927,7 @@ bool fil_space_free(uint32_t id, bool x_latched) log_sys.latch.wr_unlock(); } else { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); if (space->max_lsn) { ut_d(space->max_lsn = 0); fil_system.named_spaces.remove(*space); @@ -1691,30 +1688,27 @@ pfs_os_file_t fil_delete_tablespace(uint32_t id) /*******************************************************************//** Allocates and builds a file name from a path, a table or tablespace name and a suffix. The string must be freed by caller with ut_free(). -@param[in] path NULL or the directory path or the full path and filename. +@param[in] path nullptr or the directory path or the full path and filename @param[in] name {} if path is full, or Table/Tablespace name -@param[in] ext the file extension to use -@param[in] trim_name true if the last name on the path should be trimmed. +@param[in] extension the file extension to use +@param[in] trim_name true if the last name on the path should be trimmed @return own: file name */ -char* fil_make_filepath(const char *path, const fil_space_t::name_type &name, - ib_extention ext, bool trim_name) +char* fil_make_filepath_low(const char *path, + const fil_space_t::name_type &name, + ib_extention extension, bool trim_name) { /* The path may contain the basename of the file, if so we do not need the name. If the path is NULL, we can use the default path, but there needs to be a name. */ ut_ad(path || name.data()); - /* If we are going to strip a name off the path, there better be a - path and a new name to put back on. */ - ut_ad(!trim_name || (path && name.data())); - if (path == NULL) { path = fil_path_to_mysql_datadir; } ulint len = 0; /* current length */ ulint path_len = strlen(path); - const char* suffix = dot_ext[ext]; + const char* suffix = dot_ext[extension]; ulint suffix_len = strlen(suffix); ulint full_len = path_len + 1 + name.size() + suffix_len + 1; @@ -1797,8 +1791,16 @@ char* fil_make_filepath(const char *path, const fil_space_t::name_type &name, char *fil_make_filepath(const char* path, const table_name_t name, ib_extention suffix, bool strip_name) { - return fil_make_filepath(path, {name.m_name, strlen(name.m_name)}, - suffix, strip_name); + return fil_make_filepath_low(path, {name.m_name, strlen(name.m_name)}, + suffix, strip_name); +} + +/** Wrapper function over fil_make_filepath_low() to build directory name. +@param path the directory path or the full path and filename +@return own: directory name */ +static inline char *fil_make_dirpath(const char *path) +{ + return fil_make_filepath_low(path, fil_space_t::name_type{}, NO_EXT, true); } dberr_t fil_space_t::rename(const char *path, bool log, bool replace) @@ -1839,14 +1841,32 @@ dberr_t fil_space_t::rename(const char *path, bool log, bool replace) return DB_TABLESPACE_NOT_FOUND; } - exists= false; - if (replace); - else if (!os_file_status(path, &exists, &ftype) || exists) + if (!replace) { - sql_print_error("InnoDB: Cannot rename '%s' to '%s'" - " because the target file exists.", - old_path, path); - return DB_TABLESPACE_EXISTS; + char *schema_path= fil_make_dirpath(path); + if (!schema_path) + return DB_ERROR; + + exists= false; + bool schema_fail= os_file_status(schema_path, &exists, &ftype) && !exists; + ut_free(schema_path); + + if (schema_fail) + { + sql_print_error("InnoDB: Cannot rename '%s' to '%s'" + " because the target schema directory doesn't exist.", + old_path, path); + return DB_ERROR; + } + + exists= false; + if (!os_file_status(path, &exists, &ftype) || exists) + { + sql_print_error("InnoDB: Cannot rename '%s' to '%s'" + " because the target file exists.", + old_path, path); + return DB_TABLESPACE_EXISTS; + } } mtr_t mtr; @@ -1906,7 +1926,7 @@ fil_ibd_create( static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096, "compatibility"); -#if defined _WIN32 || defined HAVE_FCNTL_DIRECT +#if defined _WIN32 || defined O_DIRECT ulint type; switch (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { case 1: @@ -1922,7 +1942,7 @@ fil_ibd_create( file = os_file_create( innodb_data_file_key, path, - OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT, + OS_FILE_CREATE, OS_FILE_AIO, type, srv_read_only_mode, &success); if (!success) { @@ -3037,9 +3057,7 @@ void fil_names_dirty( fil_space_t* space) { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); ut_ad(recv_recovery_is_on()); ut_ad(log_sys.get_lsn() != 0); ut_ad(space->max_lsn == 0); @@ -3053,9 +3071,7 @@ fil_names_dirty( tablespace was modified for the first time since fil_names_clear(). */ ATTRIBUTE_NOINLINE ATTRIBUTE_COLD void mtr_t::name_write() { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); ut_d(fil_space_validate_for_mtr_commit(m_user_space)); ut_ad(!m_user_space->max_lsn); m_user_space->max_lsn= log_sys.get_lsn(); @@ -3079,9 +3095,7 @@ ATTRIBUTE_COLD lsn_t fil_names_clear(lsn_t lsn) { mtr_t mtr; -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); ut_ad(lsn); ut_ad(log_sys.is_latest()); diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 1c20efcd..62f90f53 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -502,9 +502,10 @@ err_exit: return DB_SUCCESS; } - sql_print_error("InnoDB: %s in datafile: %s, Space ID: " - UINT32PF ", " "Flags: " UINT32PF, - error_txt, m_filepath, m_space_id, m_flags); + sql_print_information( + "InnoDB: %s in datafile: %s, Space ID: " UINT32PF + ", " "Flags: " UINT32PF, + error_txt, m_filepath, m_space_id, m_flags); m_is_valid = false; return DB_CORRUPTION; } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 87672a82..5f34fe93 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -259,6 +259,7 @@ inline void xdes_init(const buf_block_t &block, xdes_t *descr, mtr_t *mtr) } /** Mark a page used in an extent descriptor. +@param[in] space tablespace @param[in,out] seg_inode segment inode @param[in,out] iblock segment inode page @param[in] page page number @@ -268,8 +269,9 @@ inline void xdes_init(const buf_block_t &block, xdes_t *descr, mtr_t *mtr) @return error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fseg_mark_page_used(fseg_inode_t *seg_inode, buf_block_t *iblock, - ulint page, xdes_t *descr, buf_block_t *xdes, mtr_t *mtr) +fseg_mark_page_used(const fil_space_t *space, + fseg_inode_t *seg_inode, buf_block_t *iblock, + uint32_t page, xdes_t *descr, buf_block_t *xdes, mtr_t *mtr) { ut_ad(fil_page_get_type(iblock->page.frame) == FIL_PAGE_INODE); ut_ad(!((page_offset(seg_inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); @@ -278,15 +280,16 @@ fseg_mark_page_used(fseg_inode_t *seg_inode, buf_block_t *iblock, const uint16_t xoffset= uint16_t(descr - xdes->page.frame + XDES_FLST_NODE); const uint16_t ioffset= uint16_t(seg_inode - iblock->page.frame); + const uint32_t limit= space->free_limit; if (!xdes_get_n_used(descr)) { /* We move the extent from the free list to the NOT_FULL list */ if (dberr_t err= flst_remove(iblock, uint16_t(FSEG_FREE + ioffset), - xdes, xoffset, mtr)) + xdes, xoffset, limit, mtr)) return err; if (dberr_t err= flst_add_last(iblock, uint16_t(FSEG_NOT_FULL + ioffset), - xdes, xoffset, mtr)) + xdes, xoffset, limit, mtr)) return err; } @@ -303,10 +306,10 @@ fseg_mark_page_used(fseg_inode_t *seg_inode, buf_block_t *iblock, { /* We move the extent from the NOT_FULL list to the FULL list */ if (dberr_t err= flst_remove(iblock, uint16_t(FSEG_NOT_FULL + ioffset), - xdes, xoffset, mtr)) + xdes, xoffset, limit, mtr)) return err; if (dberr_t err= flst_add_last(iblock, uint16_t(FSEG_FULL + ioffset), - xdes, xoffset, mtr)) + xdes, xoffset, limit, mtr)) return err; mtr->write<4>(*iblock, seg_inode + FSEG_NOT_FULL_N_USED, not_full_n_used - FSP_EXTENT_SIZE); @@ -884,7 +887,7 @@ fsp_fill_free_list( xdes_set_free<false>(*xdes, descr, FSP_IBUF_BITMAP_OFFSET, mtr); xdes_set_state(*xdes, descr, XDES_FREE_FRAG, mtr); if (dberr_t err= flst_add_last(header, FSP_HEADER_OFFSET + FSP_FREE_FRAG, - xdes, xoffset, mtr)) + xdes, xoffset, space->free_limit, mtr)) return err; byte *n_used= FSP_HEADER_OFFSET + FSP_FRAG_N_USED + header->page.frame; mtr->write<4>(*header, n_used, 2U + mach_read_from_4(n_used)); @@ -893,7 +896,7 @@ fsp_fill_free_list( { if (dberr_t err= flst_add_last(header, FSP_HEADER_OFFSET + FSP_FREE, - xdes, xoffset, mtr)) + xdes, xoffset, space->free_limit, mtr)) return err; count++; } @@ -944,7 +947,11 @@ corrupted: first = flst_get_first(FSP_HEADER_OFFSET + FSP_FREE + header->page.frame); - if (first.page == FIL_NULL) { + if (first.page >= space->free_limit) { + if (first.page != FIL_NULL) { + goto flst_corrupted; + } + *err = fsp_fill_free_list(false, space, header, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { goto corrupted; @@ -955,6 +962,17 @@ corrupted: if (first.page == FIL_NULL) { return nullptr; /* No free extents left */ } + if (first.page >= space->free_limit) { + goto flst_corrupted; + } + } + + if (first.boffset < FSP_HEADER_OFFSET + FSP_HEADER_SIZE + || first.boffset >= space->physical_size() + - (XDES_SIZE + FIL_PAGE_DATA_END)) { + flst_corrupted: + *err = DB_CORRUPTION; + goto corrupted; } descr = xdes_lst_get_descriptor(*space, first, mtr, @@ -967,7 +985,7 @@ corrupted: *err = flst_remove(header, FSP_HEADER_OFFSET + FSP_FREE, desc_block, static_cast<uint16_t>(descr - desc_block->page.frame + XDES_FLST_NODE), - mtr); + space->free_limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { return nullptr; } @@ -984,11 +1002,12 @@ MY_ATTRIBUTE((nonnull, warn_unused_result)) @param[in,out] xdes extent descriptor page @param[in,out] descr extent descriptor @param[in] bit slot to allocate in the extent +@param[in] space tablespace @param[in,out] mtr mini-transaction @return error code */ static dberr_t fsp_alloc_from_free_frag(buf_block_t *header, buf_block_t *xdes, xdes_t *descr, - ulint bit, mtr_t *mtr) + uint32_t bit, fil_space_t *space, mtr_t *mtr) { if (UNIV_UNLIKELY(xdes_get_state(descr) != XDES_FREE_FRAG || !xdes_is_free(descr, bit))) @@ -1001,14 +1020,15 @@ fsp_alloc_from_free_frag(buf_block_t *header, buf_block_t *xdes, xdes_t *descr, if (xdes_is_full(descr)) { + const uint32_t limit= space->free_limit; /* The fragment is full: move it to another list */ const uint16_t xoffset= static_cast<uint16_t>(descr - xdes->page.frame + XDES_FLST_NODE); if (dberr_t err= flst_remove(header, FSP_HEADER_OFFSET + FSP_FREE_FRAG, - xdes, xoffset, mtr)) + xdes, xoffset, limit, mtr)) return err; if (dberr_t err= flst_add_last(header, FSP_HEADER_OFFSET + FSP_FULL_FRAG, - xdes, xoffset, mtr)) + xdes, xoffset, limit, mtr)) return err; xdes_set_state(*xdes, descr, XDES_FULL_FRAG, mtr); n_used-= FSP_EXTENT_SIZE; @@ -1070,8 +1090,11 @@ buf_block_t *fsp_alloc_free_page(fil_space_t *space, uint32_t hint, /* Else take the first extent in free_frag list */ fil_addr_t first = flst_get_first(FSP_HEADER_OFFSET + FSP_FREE_FRAG + block->page.frame); - if (first.page == FIL_NULL) + if (first.page >= space->free_limit) { + if (first.page != FIL_NULL) + goto flst_corrupted; + /* There are no partially full fragments: allocate a free extent and add it to the FREE_FRAG list. NOTE that the allocation may have as a side-effect that an extent containing a descriptor @@ -1082,13 +1105,23 @@ buf_block_t *fsp_alloc_free_page(fil_space_t *space, uint32_t hint, return nullptr; *err= flst_add_last(block, FSP_HEADER_OFFSET + FSP_FREE_FRAG, xdes, static_cast<uint16_t>(descr - xdes->page.frame + - XDES_FLST_NODE), mtr); + XDES_FLST_NODE), + space->free_limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) return nullptr; xdes_set_state(*xdes, descr, XDES_FREE_FRAG, mtr); } else { + if (first.boffset < FSP_HEADER_OFFSET + FSP_HEADER_SIZE || + first.boffset >= space->physical_size() - + (XDES_SIZE + FIL_PAGE_DATA_END)) + { + flst_corrupted: + *err= DB_CORRUPTION; + goto err_exit; + } + descr= xdes_lst_get_descriptor(*space, first, mtr, &xdes, err); if (!descr) return nullptr; @@ -1135,7 +1168,7 @@ buf_block_t *fsp_alloc_free_page(fil_space_t *space, uint32_t hint, } } - *err= fsp_alloc_from_free_frag(block, xdes, descr, free, mtr); + *err= fsp_alloc_from_free_frag(block, xdes, descr, free, space, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) goto corrupted; return fsp_page_create(space, page_no, init_mtr); @@ -1174,7 +1207,8 @@ static dberr_t fsp_free_extent(fil_space_t* space, uint32_t offset, space->free_len++; return flst_add_last(block, FSP_HEADER_OFFSET + FSP_FREE, xdes, static_cast<uint16_t>(descr - xdes->page.frame + - XDES_FLST_NODE), mtr); + XDES_FLST_NODE), + space->free_limit, mtr); } MY_ATTRIBUTE((nonnull)) @@ -1228,16 +1262,17 @@ static dberr_t fsp_free_page(fil_space_t *space, uint32_t offset, mtr_t *mtr) const uint16_t xoffset= static_cast<uint16_t>(descr - xdes->page.frame + XDES_FLST_NODE); + const uint32_t limit = space->free_limit; if (state == XDES_FULL_FRAG) { /* The fragment was full: move it to another list */ err = flst_remove(header, FSP_HEADER_OFFSET + FSP_FULL_FRAG, - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } err = flst_add_last(header, FSP_HEADER_OFFSET + FSP_FREE_FRAG, - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } @@ -1259,7 +1294,7 @@ static dberr_t fsp_free_page(fil_space_t *space, uint32_t offset, mtr_t *mtr) if (!xdes_get_n_used(descr)) { /* The extent has become free: move it to another list */ err = flst_remove(header, FSP_HEADER_OFFSET + FSP_FREE_FRAG, - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (err == DB_SUCCESS) { err = fsp_free_extent(space, offset, mtr); } @@ -1353,7 +1388,7 @@ static dberr_t fsp_alloc_seg_inode_page(fil_space_t *space, #endif return flst_add_last(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FREE, - block, FSEG_INODE_PAGE_NODE, mtr); + block, FSEG_INODE_PAGE_NODE, space->free_limit, mtr); } MY_ATTRIBUTE((nonnull, warn_unused_result)) @@ -1409,12 +1444,13 @@ fsp_alloc_seg_inode(fil_space_t *space, buf_block_t *header, { /* There are no other unused headers left on the page: move it to another list */ + const uint32_t limit= space->free_limit; *err= flst_remove(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FREE, - block, FSEG_INODE_PAGE_NODE, mtr); + block, FSEG_INODE_PAGE_NODE, limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) return nullptr; *err= flst_add_last(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FULL, - block, FSEG_INODE_PAGE_NODE, mtr); + block, FSEG_INODE_PAGE_NODE, limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) return nullptr; } @@ -1447,16 +1483,17 @@ static void fsp_free_seg_inode(fil_space_t *space, fseg_inode_t *inode, } const ulint physical_size= space->physical_size(); + const uint32_t limit= space->free_limit; if (ULINT_UNDEFINED == fsp_seg_inode_page_find_free(iblock->page.frame, 0, physical_size)) { /* Move the page to another list */ if (flst_remove(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FULL, - iblock, FSEG_INODE_PAGE_NODE, mtr) != DB_SUCCESS) + iblock, FSEG_INODE_PAGE_NODE, limit, mtr) != DB_SUCCESS) return; if (flst_add_last(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FREE, - iblock, FSEG_INODE_PAGE_NODE, mtr) != DB_SUCCESS) + iblock, FSEG_INODE_PAGE_NODE, limit, mtr) != DB_SUCCESS) return; } @@ -1468,7 +1505,7 @@ static void fsp_free_seg_inode(fil_space_t *space, fseg_inode_t *inode, /* There are no other used headers left on the page: free it */ if (flst_remove(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FREE, - iblock, FSEG_INODE_PAGE_NODE, mtr) == DB_SUCCESS) + iblock, FSEG_INODE_PAGE_NODE, limit, mtr) == DB_SUCCESS) fsp_free_page(space, iblock->page.id().page_no(), mtr); } @@ -1841,7 +1878,8 @@ static dberr_t fseg_fill_free_list(const fseg_inode_t *inode, static_cast<uint16_t>(inode - iblock->page.frame + FSEG_FREE), xdes, static_cast<uint16_t>(descr - xdes->page.frame + - XDES_FLST_NODE), mtr)) + XDES_FLST_NODE), + space->free_limit, mtr)) return err; xdes_set_state(*xdes, descr, XDES_FSEG, mtr); mtr->memcpy(*xdes, descr + XDES_ID, inode + FSEG_ID, 8); @@ -1876,11 +1914,25 @@ fseg_alloc_free_extent( ut_ad(!memcmp(FSEG_MAGIC_N_BYTES, FSEG_MAGIC_N + inode, 4)); ut_d(space->modify_check(*mtr)); + if (UNIV_UNLIKELY(page_offset(inode) < FSEG_ARR_OFFSET)) + { + corrupted: + *err= DB_CORRUPTION; + space->set_corrupted(); + return nullptr; + } + if (flst_get_len(inode + FSEG_FREE)) { + const fil_addr_t first= flst_get_first(inode + FSEG_FREE); + if (first.page >= space->free_limit || + first.boffset < FSP_HEADER_OFFSET + FSP_HEADER_SIZE || + first.boffset >= space->physical_size() - + (XDES_SIZE + FIL_PAGE_DATA_END)) + goto corrupted; + /* Segment free list is not empty, allocate from it */ - return xdes_lst_get_descriptor(*space, flst_get_first(inode + FSEG_FREE), - mtr, xdes, err); + return xdes_lst_get_descriptor(*space, first, mtr, xdes, err); } xdes_t* descr= fsp_alloc_free_extent(space, 0, xdes, mtr, err); @@ -1892,7 +1944,8 @@ fseg_alloc_free_extent( static_cast<uint16_t>(inode - iblock->page.frame + FSEG_FREE), *xdes, static_cast<uint16_t>(descr - (*xdes)->page.frame + - XDES_FLST_NODE), mtr); + XDES_FLST_NODE), + space->free_limit, mtr); if (UNIV_LIKELY(*err != DB_SUCCESS)) return nullptr; /* Try to fill the segment free list */ @@ -1978,29 +2031,42 @@ fseg_alloc_free_page_low( } } - /* In the big if-else below we look for ret_page and ret_descr */ - /*-------------------------------------------------------------*/ - if ((xdes_get_state(descr) == XDES_FSEG) - && mach_read_from_8(descr + XDES_ID) == seg_id - && xdes_is_free(descr, hint % FSP_EXTENT_SIZE)) { + const uint32_t extent_size = FSP_EXTENT_SIZE; + ret_descr = descr; + /* Try to get the page from extent which belongs to segment */ + if (xdes_get_state(descr) == XDES_FSEG + && mach_read_from_8(descr + XDES_ID) == seg_id) { + /* Get the page from the segment extent */ + if (xdes_is_free(descr, hint % extent_size)) { take_hinted_page: - /* 1. We can take the hinted page - =================================*/ - ret_descr = descr; - ret_page = hint; - /* Skip the check for extending the tablespace. If the - page hint were not within the size of the tablespace, - we would have got (descr == NULL) above and reset the hint. */ - goto got_hinted_page; - /*-----------------------------------------------------------*/ - } else if (xdes_get_state(descr) == XDES_FREE - && reserved - used < reserved / FSEG_FILLFACTOR - && used >= FSEG_FRAG_LIMIT) { - - /* 2. We allocate the free extent from space and can take - ========================================================= - the hinted page - ===============*/ + ret_page = hint; + goto got_hinted_page; + } else if (!xdes_is_full(descr)) { + /* Take the page from the same extent as the + hinted page (and the extent already belongs to + the segment) */ + ret_page = xdes_find_free(descr, hint % extent_size); + if (ret_page == FIL_NULL) { + ut_ad(!has_done_reservation); + return nullptr; + } + ret_page += xdes_get_offset(ret_descr); + goto alloc_done; + } + } + + /** If the number of unused but reserved pages in a segment is + esser than minimum value of 1/8 of reserved pages or + 4 * FSP_EXTENT_SIZE and there are at least half of extent size + used pages, then we allow a new empty extent to be added to + the segment in fseg_alloc_free_page_general(). Otherwise, we use + unused pages of the segment. */ + if (used < extent_size / 2 || + reserved - used >= reserved / 8 || + reserved - used >= extent_size * 4) { + } else if (xdes_get_state(descr) == XDES_FREE) { + /* Allocate the free extent from space and can + take the hinted page */ ret_descr = fsp_alloc_free_extent(space, hint, &xdes, mtr, err); @@ -2020,61 +2086,42 @@ take_hinted_page: + FSEG_FREE), xdes, static_cast<uint16_t>(ret_descr - xdes->page.frame - + XDES_FLST_NODE), mtr); + + XDES_FLST_NODE), + space->free_limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { return nullptr; } /* Try to fill the segment free list */ *err = fseg_fill_free_list(seg_inode, iblock, space, - hint + FSP_EXTENT_SIZE, mtr); + hint + extent_size, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { return nullptr; } goto take_hinted_page; - /*-----------------------------------------------------------*/ - } else if ((direction != FSP_NO_DIR) - && ((reserved - used) < reserved / FSEG_FILLFACTOR) - && (used >= FSEG_FRAG_LIMIT) - && (ret_descr = fseg_alloc_free_extent(seg_inode, iblock, - &xdes, space, - mtr, err))) { - /* 3. We take any free extent (which was already assigned above - =============================================================== - in the if-condition to ret_descr) and take the lowest or - ======================================================== - highest page in it, depending on the direction - ==============================================*/ + } else if (direction != FSP_NO_DIR) { + + ret_descr = fseg_alloc_free_extent(seg_inode, iblock, + &xdes, space, mtr, err); + + if (!ret_descr) { + ut_ad(*err != DB_SUCCESS); + return nullptr; + } + /* Take any free extent (which was already assigned + above in the if-condition to ret_descr) and take the + lowest or highest page in it, depending on the direction */ ret_page = xdes_get_offset(ret_descr); if (direction == FSP_DOWN) { - ret_page += FSP_EXTENT_SIZE - 1; - } - ut_ad(!has_done_reservation || ret_page != FIL_NULL); - /*-----------------------------------------------------------*/ - } else if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { - return nullptr; - } else if ((xdes_get_state(descr) == XDES_FSEG) - && mach_read_from_8(descr + XDES_ID) == seg_id - && (!xdes_is_full(descr))) { - - /* 4. We can take the page from the same extent as the - ====================================================== - hinted page (and the extent already belongs to the - ================================================== - segment) - ========*/ - ret_descr = descr; - ret_page = xdes_find_free(ret_descr, hint % FSP_EXTENT_SIZE); - if (ret_page == FIL_NULL) { - ut_ad(!has_done_reservation); - } else { - ret_page += xdes_get_offset(ret_descr); + ret_page += extent_size - 1; } - /*-----------------------------------------------------------*/ - } else if (reserved - used > 0) { - /* 5. We take any unused page from the segment - ==============================================*/ + goto alloc_done; + } + + /* Try to take individual page from the segment or tablespace */ + if (reserved - used > 0) { + /* Take any unused page from the segment */ fil_addr_t first; if (flst_get_len(seg_inode + FSEG_NOT_FULL) > 0) { @@ -2083,7 +2130,15 @@ take_hinted_page: first = flst_get_first(seg_inode + FSEG_FREE); } else { ut_ad(!has_done_reservation); - return(NULL); + return nullptr; + } + + if (first.page >= space->free_limit + || first.boffset < FSP_HEADER_OFFSET + FSP_HEADER_SIZE + || first.boffset >= space->physical_size() + - (XDES_SIZE + FIL_PAGE_DATA_END)) { + *err= DB_CORRUPTION; + return nullptr; } ret_descr = xdes_lst_get_descriptor(*space, first, mtr, &xdes); @@ -2097,10 +2152,9 @@ take_hinted_page: } else { ret_page += xdes_get_offset(ret_descr); } - /*-----------------------------------------------------------*/ - } else if (used < FSEG_FRAG_LIMIT) { - /* 6. We allocate an individual page from the space - ===================================================*/ + + } else if (used < extent_size / 2) { + /* Allocate an individual page from the space */ buf_block_t* block = fsp_alloc_free_page( space, hint, mtr, init_mtr, err); @@ -2123,13 +2177,11 @@ take_hinted_page: /* fsp_alloc_free_page() invoked fsp_init_file_page() already. */ return(block); - /*-----------------------------------------------------------*/ } else { - /* 7. We allocate a new extent and take its first page - ======================================================*/ + /* In worst case, try to allocate a new extent + and take its first page */ ret_descr = fseg_alloc_free_extent(seg_inode, iblock, &xdes, space, mtr, err); - if (!ret_descr) { ut_ad(!has_done_reservation || *err); return nullptr; @@ -2142,14 +2194,13 @@ take_hinted_page: /* Page could not be allocated */ ut_ad(!has_done_reservation); - return(NULL); + return nullptr; } - +alloc_done: if (space->size <= ret_page && !is_predefined_tablespace(space->id)) { /* It must be that we are extending a single-table tablespace whose size is still < 64 pages */ - - if (ret_page >= FSP_EXTENT_SIZE) { + if (ret_page >= extent_size) { sql_print_error("InnoDB: Trying to extend '%s'" " by single page(s) though the" " space size " UINT32PF "." @@ -2157,33 +2208,34 @@ take_hinted_page: space->chain.start->name, space->size, ret_page); ut_ad(!has_done_reservation); - return(NULL); + return nullptr; } if (!fsp_try_extend_data_file_with_pages( space, ret_page, header, mtr)) { /* No disk space left */ ut_ad(!has_done_reservation); - return(NULL); + return nullptr; } } -got_hinted_page: - /* ret_descr == NULL if the block was allocated from free_frag - (XDES_FREE_FRAG) */ + /* Skip the check for extending the tablespace. + If the page hint were not within the size of the tablespace, + descr set to nullptr above and reset the hint and the block + was allocated from free_frag (XDES_FREE_FRAG) */ if (ret_descr != NULL) { +got_hinted_page: /* At this point we know the extent and the page offset. The extent is still in the appropriate list (FSEG_NOT_FULL or FSEG_FREE), and the page is not yet marked as used. */ - ut_d(buf_block_t* xxdes); ut_ad(xdes_get_descriptor(space, ret_page, mtr, err, &xxdes) == ret_descr); ut_ad(xdes == xxdes); - ut_ad(xdes_is_free(ret_descr, ret_page % FSP_EXTENT_SIZE)); + ut_ad(xdes_is_free(ret_descr, ret_page % extent_size)); - *err = fseg_mark_page_used(seg_inode, iblock, ret_page, - ret_descr, xdes, mtr); + *err = fseg_mark_page_used(space, seg_inode, iblock, ret_page, + ret_descr, xdes, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { return nullptr; } @@ -2524,18 +2576,19 @@ corrupted: const uint16_t xoffset= uint16_t(descr - xdes->page.frame + XDES_FLST_NODE); const uint16_t ioffset= uint16_t(seg_inode - iblock->page.frame); + const uint32_t limit = space->free_limit; if (xdes_is_full(descr)) { /* The fragment is full: move it to another list */ err = flst_remove(iblock, static_cast<uint16_t>(FSEG_FULL + ioffset), - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } err = flst_add_last(iblock, static_cast<uint16_t>(FSEG_NOT_FULL + ioffset), - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } @@ -2553,7 +2606,7 @@ corrupted: if (!xdes_get_n_used(descr)) { err = flst_remove(iblock, static_cast<uint16_t>(FSEG_NOT_FULL + ioffset), - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } @@ -2698,11 +2751,12 @@ fseg_free_extent( #endif /* BTR_CUR_HASH_ADAPT */ uint16_t lst; + uint32_t limit = space->free_limit; if (xdes_is_full(descr)) { lst = static_cast<uint16_t>(FSEG_FULL + ioffset); remove: - err = flst_remove(iblock, lst, xdes, xoffset, mtr); + err = flst_remove(iblock, lst, xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } @@ -2712,7 +2766,7 @@ remove: } else { err = flst_remove( iblock, static_cast<uint16_t>(FSEG_NOT_FULL + ioffset), - xdes, xoffset, mtr); + xdes, xoffset, limit, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } @@ -2962,7 +3016,10 @@ fseg_get_first_extent( return nullptr; } - if (first.page == FIL_NULL) + if (first.page >= space->free_limit || + first.boffset < FSP_HEADER_OFFSET + FSP_HEADER_SIZE || + first.boffset >= space->physical_size() - + (XDES_SIZE + FIL_PAGE_DATA_END)) goto corrupted; return xdes_lst_get_descriptor(*space, first, mtr, nullptr, err); diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 0775d939..4d9a1d3a 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -2187,6 +2187,22 @@ fts_trx_row_get_new_state( return(result); } +/** Compare two doubly indirected pointers */ +static int fts_ptr2_cmp(const void *p1, const void *p2) +{ + const void *a= **static_cast<const void*const*const*>(p1); + const void *b= **static_cast<const void*const*const*>(p2); + return b > a ? -1 : a > b; +} + +/** Compare a singly indirected pointer to a doubly indirected one */ +static int fts_ptr1_ptr2_cmp(const void *p1, const void *p2) +{ + const void *a= *static_cast<const void*const*>(p1); + const void *b= **static_cast<const void*const*const*>(p2); + return b > a ? -1 : a > b; +} + /******************************************************************//** Create a savepoint instance. @return savepoint instance */ @@ -2209,8 +2225,8 @@ fts_savepoint_create( savepoint->name = mem_heap_strdup(heap, name); } - savepoint->tables = rbt_create( - sizeof(fts_trx_table_t*), fts_trx_table_cmp); + static_assert(!offsetof(fts_trx_table_t, table), "ABI"); + savepoint->tables = rbt_create(sizeof(fts_trx_table_t*), fts_ptr2_cmp); return(savepoint); } @@ -2258,6 +2274,19 @@ fts_trx_create( return(ftt); } +/** Compare two doc_id */ +static inline int doc_id_cmp(doc_id_t a, doc_id_t b) +{ + return b > a ? -1 : a > b; +} + +/** Compare two DOC_ID. */ +int fts_doc_id_cmp(const void *p1, const void *p2) +{ + return doc_id_cmp(*static_cast<const doc_id_t*>(p1), + *static_cast<const doc_id_t*>(p2)); +} + /******************************************************************//** Create an FTS trx table. @return FTS trx table */ @@ -2276,7 +2305,8 @@ fts_trx_table_create( ftt->table = table; ftt->fts_trx = fts_trx; - ftt->rows = rbt_create(sizeof(fts_trx_row_t), fts_trx_row_doc_id_cmp); + static_assert(!offsetof(fts_trx_row_t, doc_id), "ABI"); + ftt->rows = rbt_create(sizeof(fts_trx_row_t), fts_doc_id_cmp); return(ftt); } @@ -2300,7 +2330,8 @@ fts_trx_table_clone( ftt->table = ftt_src->table; ftt->fts_trx = ftt_src->fts_trx; - ftt->rows = rbt_create(sizeof(fts_trx_row_t), fts_trx_row_doc_id_cmp); + static_assert(!offsetof(fts_trx_row_t, doc_id), "ABI"); + ftt->rows = rbt_create(sizeof(fts_trx_row_t), fts_doc_id_cmp); /* Copy the rb tree values to the new savepoint. */ rbt_merge_uniq(ftt->rows, ftt_src->rows); @@ -2325,13 +2356,9 @@ fts_trx_init( { fts_trx_table_t* ftt; ib_rbt_bound_t parent; - ib_rbt_t* tables; - fts_savepoint_t* savepoint; - - savepoint = static_cast<fts_savepoint_t*>(ib_vector_last(savepoints)); - - tables = savepoint->tables; - rbt_search_cmp(tables, &parent, &table->id, fts_trx_table_id_cmp, NULL); + ib_rbt_t* tables = static_cast<fts_savepoint_t*>( + ib_vector_last(savepoints))->tables; + rbt_search_cmp(tables, &parent, &table, fts_ptr1_ptr2_cmp, nullptr); if (parent.result == 0) { fts_trx_table_t** fttp; @@ -3860,6 +3887,13 @@ fts_write_node( return(error); } +/** Sort an array of doc_id */ +void fts_doc_ids_sort(ib_vector_t *doc_ids) +{ + doc_id_t *const data= reinterpret_cast<doc_id_t*>(doc_ids->data); + std::sort(data, data + doc_ids->used); +} + /*********************************************************************//** Add rows to the DELETED_CACHE table. @return DB_SUCCESS if all went well else error code*/ @@ -3881,7 +3915,7 @@ fts_sync_add_deleted_cache( ut_a(ib_vector_size(doc_ids) > 0); - ib_vector_sort(doc_ids, fts_doc_id_cmp); + fts_doc_ids_sort(doc_ids); info = pars_info_create(); @@ -5575,8 +5609,8 @@ fts_savepoint_rollback_last_stmt( l_ftt = rbt_value(fts_trx_table_t*, node); rbt_search_cmp( - s_tables, &parent, &(*l_ftt)->table->id, - fts_trx_table_id_cmp, NULL); + s_tables, &parent, &(*l_ftt)->table, + fts_ptr1_ptr2_cmp, nullptr); if (parent.result == 0) { fts_trx_table_t** s_ftt; diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index fe31767d..30889e59 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1016,7 +1016,7 @@ fts_table_fetch_doc_ids( que_graph_free(graph); if (error == DB_SUCCESS) { - ib_vector_sort(doc_ids->doc_ids, fts_doc_id_cmp); + fts_doc_ids_sort(doc_ids->doc_ids); } if (alloc_bk_trx) { diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 9c92a117..b8f22076 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -385,22 +385,6 @@ fts_query_terms_in_document( ulint* total); /*!< out: total words in document */ #endif -/******************************************************************** -Compare two fts_doc_freq_t doc_ids. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int -fts_freq_doc_id_cmp( -/*================*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ -{ - const fts_doc_freq_t* fq1 = (const fts_doc_freq_t*) p1; - const fts_doc_freq_t* fq2 = (const fts_doc_freq_t*) p2; - - return((int) (fq1->doc_id - fq2->doc_id)); -} - #if 0 /*******************************************************************//** Print the table used for calculating LCS. */ @@ -506,14 +490,11 @@ fts_query_compare_rank( if (r2->rank < r1->rank) { return(-1); } else if (r2->rank == r1->rank) { - if (r1->doc_id < r2->doc_id) { - return(1); - } else if (r1->doc_id > r2->doc_id) { - return(1); + return -1; } - return(0); + return r1->doc_id > r2->doc_id; } return(1); @@ -674,8 +655,9 @@ fts_query_add_word_freq( word_freq.doc_count = 0; + static_assert(!offsetof(fts_doc_freq_t, doc_id), "ABI"); word_freq.doc_freqs = rbt_create( - sizeof(fts_doc_freq_t), fts_freq_doc_id_cmp); + sizeof(fts_doc_freq_t), fts_doc_id_cmp); parent.last = rbt_add_node( query->word_freqs, &parent, &word_freq); @@ -1253,8 +1235,9 @@ fts_query_intersect( /* Create the rb tree that will hold the doc ids of the intersection. */ + static_assert(!offsetof(fts_ranking_t, doc_id), "ABI"); query->intersection = rbt_create( - sizeof(fts_ranking_t), fts_ranking_doc_id_cmp); + sizeof(fts_ranking_t), fts_doc_id_cmp); query->total_size += SIZEOF_RBT_CREATE; @@ -1540,8 +1523,9 @@ fts_merge_doc_ids( to create a new result set for fts_query_intersect(). */ if (query->oper == FTS_EXIST) { + static_assert(!offsetof(fts_ranking_t, doc_id), "ABI"); query->intersection = rbt_create( - sizeof(fts_ranking_t), fts_ranking_doc_id_cmp); + sizeof(fts_ranking_t), fts_doc_id_cmp); query->total_size += SIZEOF_RBT_CREATE; } @@ -3012,8 +2996,9 @@ fts_query_visitor( if (query->oper == FTS_EXIST) { ut_ad(query->intersection == NULL); + static_assert(!offsetof(fts_ranking_t, doc_id), "ABI"); query->intersection = rbt_create( - sizeof(fts_ranking_t), fts_ranking_doc_id_cmp); + sizeof(fts_ranking_t), fts_doc_id_cmp); query->total_size += SIZEOF_RBT_CREATE; } @@ -3123,8 +3108,8 @@ fts_ast_visit_sub_exp( /* Create new result set to store the sub-expression result. We will merge this result set with the parent after processing. */ - query->doc_ids = rbt_create(sizeof(fts_ranking_t), - fts_ranking_doc_id_cmp); + static_assert(!offsetof(fts_ranking_t, doc_id), "ABI"); + query->doc_ids = rbt_create(sizeof(fts_ranking_t), fts_doc_id_cmp); query->total_size += SIZEOF_RBT_CREATE; @@ -3661,8 +3646,9 @@ fts_query_prepare_result( result = static_cast<fts_result_t*>( ut_zalloc_nokey(sizeof(*result))); + static_assert(!offsetof(fts_ranking_t, doc_id), "ABI"); result->rankings_by_id = rbt_create( - sizeof(fts_ranking_t), fts_ranking_doc_id_cmp); + sizeof(fts_ranking_t), fts_doc_id_cmp); query->total_size += sizeof(fts_result_t) + SIZEOF_RBT_CREATE; result_is_null = true; @@ -4038,7 +4024,7 @@ fts_query( DEBUG_SYNC_C("fts_deleted_doc_ids_append"); /* Sort the vector so that we can do a binary search over the ids. */ - ib_vector_sort(query.deleted->doc_ids, fts_doc_id_cmp); + fts_doc_ids_sort(query.deleted->doc_ids); /* Convert the query string to lower case before parsing. We own the ut_malloc'ed result and so remember to free it before return. */ @@ -4065,8 +4051,9 @@ fts_query( query.heap = mem_heap_create(128); /* Create the rb tree for the doc id (current) set. */ + static_assert(!offsetof(fts_ranking_t, doc_id), "ABI"); query.doc_ids = rbt_create( - sizeof(fts_ranking_t), fts_ranking_doc_id_cmp); + sizeof(fts_ranking_t), fts_doc_id_cmp); query.parser = index->parser; query.total_size += SIZEOF_RBT_CREATE; diff --git a/storage/innobase/fut/fut0lst.cc b/storage/innobase/fut/fut0lst.cc index a52027f2..48e2fbe3 100644 --- a/storage/innobase/fut/fut0lst.cc +++ b/storage/innobase/fut/fut0lst.cc @@ -113,17 +113,18 @@ static void flst_add_to_empty(buf_block_t *base, uint16_t boffset, } /** Insert a node after another one. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] cur insert position block -@param[in] coffset byte offset of the insert position -@param[in,out] add block to be added -@param[in] aoffset byte offset of the block to be added -@param[in,out] mtr mini-transaction */ +@param base base node block +@param boffset byte offset of the base node +@param cur insert position block +@param coffset byte offset of the insert position +@param add block to be added +@param aoffset byte offset of the block to be added +@param limit fil_space_t::free_limit +@param mtr mini-transaction */ static dberr_t flst_insert_after(buf_block_t *base, uint16_t boffset, buf_block_t *cur, uint16_t coffset, buf_block_t *add, uint16_t aoffset, - mtr_t *mtr) + uint32_t limit, mtr_t *mtr) { ut_ad(base != cur || boffset != coffset); ut_ad(base != add || boffset != aoffset); @@ -139,6 +140,15 @@ static dberr_t flst_insert_after(buf_block_t *base, uint16_t boffset, MTR_MEMO_PAGE_SX_FIX)); fil_addr_t next_addr= flst_get_next_addr(cur->page.frame + coffset); + if (next_addr.page >= limit) + { + if (UNIV_UNLIKELY(next_addr.page != FIL_NULL)) + return DB_CORRUPTION; + } + else if (UNIV_UNLIKELY(next_addr.boffset < FIL_PAGE_DATA || + next_addr.boffset >= base->physical_size() - + FIL_PAGE_DATA_END)) + return DB_CORRUPTION; flst_write_addr(*add, add->page.frame + aoffset + FLST_PREV, cur->page.id().page_no(), coffset, mtr); @@ -167,18 +177,19 @@ static dberr_t flst_insert_after(buf_block_t *base, uint16_t boffset, } /** Insert a node before another one. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] cur insert position block -@param[in] coffset byte offset of the insert position -@param[in,out] add block to be added -@param[in] aoffset byte offset of the block to be added -@param[in,out] mtr mini-transaction +@param base base node block +@param boffset byte offset of the base node +@param cur insert position block +@param coffset byte offset of the insert position +@param add block to be added +@param aoffset byte offset of the block to be added +@param limit fil_space_t::free_limit +@param mtr mini-transaction @return error code */ static dberr_t flst_insert_before(buf_block_t *base, uint16_t boffset, buf_block_t *cur, uint16_t coffset, buf_block_t *add, uint16_t aoffset, - mtr_t *mtr) + uint32_t limit, mtr_t *mtr) { ut_ad(base != cur || boffset != coffset); ut_ad(base != add || boffset != aoffset); @@ -194,6 +205,15 @@ static dberr_t flst_insert_before(buf_block_t *base, uint16_t boffset, MTR_MEMO_PAGE_SX_FIX)); fil_addr_t prev_addr= flst_get_prev_addr(cur->page.frame + coffset); + if (prev_addr.page >= limit) + { + if (UNIV_UNLIKELY(prev_addr.page != FIL_NULL)) + return DB_CORRUPTION; + } + else if (UNIV_UNLIKELY(prev_addr.boffset < FIL_PAGE_DATA || + prev_addr.boffset >= base->physical_size() - + FIL_PAGE_DATA_END)) + return DB_CORRUPTION; flst_write_addr(*add, add->page.frame + aoffset + FLST_PREV, prev_addr.page, prev_addr.boffset, mtr); @@ -234,14 +254,9 @@ void flst_init(const buf_block_t& block, byte *base, mtr_t *mtr) flst_zero_both(block, base + FLST_FIRST, mtr); } -/** Append a file list node to a list. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] add block to be added -@param[in] aoffset byte offset of the node to be added -@param[in,outr] mtr mini-transaction */ dberr_t flst_add_last(buf_block_t *base, uint16_t boffset, - buf_block_t *add, uint16_t aoffset, mtr_t *mtr) + buf_block_t *add, uint16_t aoffset, + uint32_t limit, mtr_t *mtr) { ut_ad(base != add || boffset != aoffset); ut_ad(boffset < base->physical_size()); @@ -258,6 +273,13 @@ dberr_t flst_add_last(buf_block_t *base, uint16_t boffset, else { fil_addr_t addr= flst_get_last(base->page.frame + boffset); + if (UNIV_UNLIKELY(addr.page >= limit)) + return DB_CORRUPTION; + else if (UNIV_UNLIKELY(addr.boffset < FIL_PAGE_DATA || + addr.boffset >= base->physical_size() - + FIL_PAGE_DATA_END)) + return DB_CORRUPTION; + buf_block_t *cur= add; dberr_t err; if (addr.page != add->page.id().page_no() && @@ -266,19 +288,13 @@ dberr_t flst_add_last(buf_block_t *base, uint16_t boffset, BUF_GET_POSSIBLY_FREED, mtr, &err))) return err; return flst_insert_after(base, boffset, cur, addr.boffset, - add, aoffset, mtr); + add, aoffset, limit, mtr); } } -/** Prepend a file list node to a list. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] add block to be added -@param[in] aoffset byte offset of the node to be added -@param[in,out] mtr mini-transaction -@return error code */ dberr_t flst_add_first(buf_block_t *base, uint16_t boffset, - buf_block_t *add, uint16_t aoffset, mtr_t *mtr) + buf_block_t *add, uint16_t aoffset, + uint32_t limit, mtr_t *mtr) { ut_ad(base != add || boffset != aoffset); ut_ad(boffset < base->physical_size()); @@ -296,6 +312,12 @@ dberr_t flst_add_first(buf_block_t *base, uint16_t boffset, else { fil_addr_t addr= flst_get_first(base->page.frame + boffset); + if (UNIV_UNLIKELY(addr.page >= limit)) + return DB_CORRUPTION; + else if (UNIV_UNLIKELY(addr.boffset < FIL_PAGE_DATA || + addr.boffset >= base->physical_size() - + FIL_PAGE_DATA_END)) + return DB_CORRUPTION; buf_block_t *cur= add; dberr_t err; if (addr.page != add->page.id().page_no() && @@ -304,19 +326,13 @@ dberr_t flst_add_first(buf_block_t *base, uint16_t boffset, BUF_GET_POSSIBLY_FREED, mtr, &err))) return err; return flst_insert_before(base, boffset, cur, addr.boffset, - add, aoffset, mtr); + add, aoffset, limit, mtr); } } -/** Remove a file list node. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] cur block to be removed -@param[in] coffset byte offset of the current record to be removed -@param[in,out] mtr mini-transaction -@return error code */ dberr_t flst_remove(buf_block_t *base, uint16_t boffset, - buf_block_t *cur, uint16_t coffset, mtr_t *mtr) + buf_block_t *cur, uint16_t coffset, + uint32_t limit, mtr_t *mtr) { ut_ad(boffset < base->physical_size()); ut_ad(coffset < cur->physical_size()); @@ -329,9 +345,27 @@ dberr_t flst_remove(buf_block_t *base, uint16_t boffset, const fil_addr_t next_addr= flst_get_next_addr(cur->page.frame + coffset); dberr_t err= DB_SUCCESS; - if (prev_addr.page == FIL_NULL) + if (next_addr.page >= limit) + { + if (next_addr.page != FIL_NULL) + return DB_CORRUPTION; + } + else if (UNIV_UNLIKELY(next_addr.boffset < FIL_PAGE_DATA || + next_addr.boffset >= base->physical_size() - + FIL_PAGE_DATA_END)) + return DB_CORRUPTION; + + if (prev_addr.page >= limit) + { + if (prev_addr.page != FIL_NULL) + return DB_CORRUPTION; flst_write_addr(*base, base->page.frame + boffset + FLST_FIRST, next_addr.page, next_addr.boffset, mtr); + } + else if (UNIV_UNLIKELY(prev_addr.boffset < FIL_PAGE_DATA || + prev_addr.boffset >= base->physical_size() - + FIL_PAGE_DATA_END)) + return DB_CORRUPTION; else { buf_block_t *b= cur; @@ -375,25 +409,19 @@ void flst_validate(const buf_block_t *base, uint16_t boffset, mtr_t *mtr) ut_ad(mtr->memo_contains_flagged(base, MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX)); - /* We use two mini-transaction handles: the first is used to lock - the base node, and prevent other threads from modifying the list. - The second is used to traverse the list. We cannot run the second - mtr without committing it at times, because if the list is long, - the x-locked pages could fill the buffer, resulting in a deadlock. */ - mtr_t mtr2; - const uint32_t len= flst_get_len(base->page.frame + boffset); fil_addr_t addr= flst_get_first(base->page.frame + boffset); for (uint32_t i= len; i--; ) { - mtr2.start(); + ut_ad(addr.boffset >= FIL_PAGE_DATA); + ut_ad(addr.boffset < base->physical_size() - FIL_PAGE_DATA_END); const buf_block_t *b= buf_page_get_gen(page_id_t(base->page.id().space(), addr.page), base->zip_size(), RW_SX_LATCH, nullptr, BUF_GET, mtr); ut_ad(b); addr= flst_get_next_addr(b->page.frame + addr.boffset); - mtr2.commit(); + mtr->release_last_page(); } ut_ad(addr.page == FIL_NULL); @@ -402,13 +430,14 @@ void flst_validate(const buf_block_t *base, uint16_t boffset, mtr_t *mtr) for (uint32_t i= len; i--; ) { - mtr2.start(); + ut_ad(addr.boffset >= FIL_PAGE_DATA); + ut_ad(addr.boffset < base->physical_size() - FIL_PAGE_DATA_END); const buf_block_t *b= buf_page_get_gen(page_id_t(base->page.id().space(), addr.page), base->zip_size(), RW_SX_LATCH, nullptr, BUF_GET, mtr); ut_ad(b); addr= flst_get_prev_addr(b->page.frame + addr.boffset); - mtr2.commit(); + mtr->release_last_page(); } ut_ad(addr.page == FIL_NULL); diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 0df9a7de..4aab68e9 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -289,10 +289,6 @@ rtr_pcur_getnext_from_path( mtr->rollback_to_savepoint(1); } - ut_ad((my_latch_mode | 4) == BTR_CONT_MODIFY_TREE - || !page_is_leaf(btr_cur_get_page(btr_cur)) - || !btr_cur->page_cur.block->page.lock.have_any()); - const auto block_savepoint = mtr->get_savepoint(); block = buf_page_get_gen( page_id_t(index->table->space_id, @@ -511,7 +507,7 @@ rtr_pcur_move_to_next( mysql_mutex_unlock(&rtr_info->matches->rtr_match_mutex); cursor->btr_cur.page_cur.rec = rec.r_rec; - cursor->btr_cur.page_cur.block = &rtr_info->matches->block; + cursor->btr_cur.page_cur.block = rtr_info->matches->block; DEBUG_SYNC_C("rtr_pcur_move_to_next_return"); return(true); @@ -672,8 +668,13 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, buf_mode, mtr, &err, false); if (!block) { - if (err == DB_DECRYPTION_FAILED) - btr_decryption_failed(*index); + if (err) + { + err_exit: + if (err == DB_DECRYPTION_FAILED) + btr_decryption_failed(*index); + mtr->rollback_to_savepoint(savepoint); + } func_exit: if (UNIV_LIKELY_NULL(heap)) mem_heap_free(heap); @@ -737,7 +738,8 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, #endif } - if (height == 0) { + if (height == 0) + { if (rw_latch == RW_NO_LATCH) { ut_ad(block == mtr->at_savepoint(block_savepoint)); @@ -821,7 +823,7 @@ dberr_t rtr_search_to_nth_level(ulint level, const dtuple_t *tuple, if (page_cur_search_with_match(tuple, page_mode, &up_match, &low_match, &cur->page_cur, nullptr)) { err= DB_CORRUPTION; - goto func_exit; + goto err_exit; } } @@ -1316,21 +1318,15 @@ rtr_create_rtr_info( rtr_info->index = index; if (init_matches) { - rtr_info->heap = mem_heap_create(sizeof(*(rtr_info->matches))); rtr_info->matches = static_cast<matched_rec_t*>( - mem_heap_zalloc( - rtr_info->heap, - sizeof(*rtr_info->matches))); + ut_zalloc_nokey(sizeof *rtr_info->matches)); rtr_info->matches->matched_recs = UT_NEW_NOKEY(rtr_rec_vector()); - rtr_info->matches->bufp = page_align(rtr_info->matches->rec_buf - + UNIV_PAGE_SIZE_MAX + 1); mysql_mutex_init(rtr_match_mutex_key, &rtr_info->matches->rtr_match_mutex, nullptr); - rtr_info->matches->block.page.lock.init(); } rtr_info->path = UT_NEW_NOKEY(rtr_node_path_t()); @@ -1449,18 +1445,16 @@ rtr_clean_rtr_info( if (free_all) { if (rtr_info->matches) { - if (rtr_info->matches->matched_recs != NULL) { - UT_DELETE(rtr_info->matches->matched_recs); + if (rtr_info->matches->block) { + buf_block_free(rtr_info->matches->block); + rtr_info->matches->block = nullptr; } - rtr_info->matches->block.page.lock.free(); + UT_DELETE(rtr_info->matches->matched_recs); mysql_mutex_destroy( &rtr_info->matches->rtr_match_mutex); - } - - if (rtr_info->heap) { - mem_heap_free(rtr_info->heap); + ut_free(rtr_info->matches); } if (initialized) { @@ -1570,7 +1564,7 @@ rtr_check_discard_page( if (auto matches = rtr_info->matches) { mysql_mutex_lock(&matches->rtr_match_mutex); - if (matches->block.page.id() == id) { + if (matches->block->page.id() == id) { matches->matched_recs->clear(); matches->valid = false; } @@ -1584,23 +1578,6 @@ rtr_check_discard_page( lock_sys.prdt_page_free_from_discard(id, true); } -/** Structure acts as functor to get the optimistic access of the page. -It returns true if it successfully gets the page. */ -struct optimistic_get -{ - btr_pcur_t *const r_cursor; - mtr_t *const mtr; - - optimistic_get(btr_pcur_t *r_cursor,mtr_t *mtr) - :r_cursor(r_cursor), mtr(mtr) {} - - bool operator()(buf_block_t *hint) const - { - return hint && buf_page_optimistic_get( - RW_X_LATCH, hint, r_cursor->modify_clock, mtr); - } -}; - /** Restore the stored position of a persistent cursor bufferfixing the page */ static bool @@ -1632,8 +1609,11 @@ rtr_cur_restore_position( r_cursor->modify_clock = 100; ); - if (r_cursor->block_when_stored.run_with_hint( - optimistic_get(r_cursor, mtr))) { + if (buf_page_optimistic_fix(r_cursor->btr_cur.page_cur.block, + r_cursor->old_page_id) + && buf_page_optimistic_get(r_cursor->btr_cur.page_cur.block, + RW_X_LATCH, r_cursor->modify_clock, + mtr)) { ut_ad(r_cursor->pos_state == BTR_PCUR_IS_POSITIONED); ut_ad(r_cursor->rel_pos == BTR_PCUR_ON); @@ -1778,7 +1758,7 @@ rtr_leaf_push_match_rec( ulint data_len; rtr_rec_t rtr_rec; - buf = match_rec->block.page.frame + match_rec->used; + buf = match_rec->block->page.frame + match_rec->used; ut_ad(page_rec_is_leaf(rec)); copy = rec_copy(buf, rec, offsets); @@ -1875,43 +1855,6 @@ rtr_non_leaf_insert_stack_push( new_seq, level, child_no, my_cursor, mbr_inc); } -/** Copy a buf_block_t, except "block->page.lock". -@param[in,out] matches copy to match->block -@param[in] block block to copy */ -static -void -rtr_copy_buf( - matched_rec_t* matches, - const buf_block_t* block) -{ - /* Copy all members of "block" to "matches->block" except "lock". - We skip "lock" because it is not used - from the dummy buf_block_t we create here and because memcpy()ing - it generates (valid) compiler warnings that the vtable pointer - will be copied. */ - matches->block.page.lock.free(); - new (&matches->block.page) buf_page_t(block->page); - matches->block.page.frame = block->page.frame; - matches->block.unzip_LRU = block->unzip_LRU; - - ut_d(matches->block.in_unzip_LRU_list = block->in_unzip_LRU_list); - ut_d(matches->block.in_withdraw_list = block->in_withdraw_list); - - /* Skip buf_block_t::lock */ - matches->block.modify_clock = block->modify_clock; -#ifdef BTR_CUR_HASH_ADAPT - matches->block.n_hash_helps = block->n_hash_helps; - matches->block.n_fields = block->n_fields; - matches->block.left_side = block->left_side; -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - matches->block.n_pointers = 0; -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ - matches->block.curr_n_fields = block->curr_n_fields; - matches->block.curr_left_side = block->curr_left_side; - matches->block.index = block->index; -#endif /* BTR_CUR_HASH_ADAPT */ -} - /****************************************************************//** Generate a shadow copy of the page block header to save the matched records */ @@ -1925,16 +1868,18 @@ rtr_init_match( { ut_ad(matches->matched_recs->empty()); matches->locked = false; - rtr_copy_buf(matches, block); - matches->block.page.frame = matches->bufp; matches->valid = false; + if (!matches->block) { + matches->block = buf_block_alloc(); + } + + matches->block->page.init(buf_page_t::MEMORY, block->page.id()); /* We have to copy PAGE_*_SUPREMUM_END bytes so that we can use infimum/supremum of this page as normal btr page for search. */ - memcpy(matches->block.page.frame, page, page_is_comp(page) - ? PAGE_NEW_SUPREMUM_END : PAGE_OLD_SUPREMUM_END); matches->used = page_is_comp(page) ? PAGE_NEW_SUPREMUM_END : PAGE_OLD_SUPREMUM_END; + memcpy(matches->block->page.frame, page, matches->used); #ifdef RTR_SEARCH_DIAGNOSTIC ulint pageno = page_get_page_no(page); fprintf(stderr, "INNODB_RTR: Searching leaf page %d\n", @@ -2361,7 +2306,7 @@ rtr_cur_search_with_match( #endif /* UNIV_DEBUG */ /* Pop the last match record and position on it */ match_rec->matched_recs->pop_back(); - page_cur_position(test_rec.r_rec, &match_rec->block, + page_cur_position(test_rec.r_rec, match_rec->block, cursor); } } else { diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 407834f2..dfe034ec 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -47,10 +47,13 @@ this program; if not, write to the Free Software Foundation, Inc., #include <my_bitmap.h> #include <mysql/service_thd_alloc.h> #include <mysql/service_thd_wait.h> +#include <mysql/service_print_check_msg.h> #include "sql_type_geom.h" #include "scope.h" #include "srv0srv.h" +extern my_bool opt_readonly; + // MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system; // MYSQL_PLUGIN_IMPORT extern char mysql_unpacked_real_data_home[]; @@ -116,6 +119,7 @@ thread_local ha_handler_stats *mariadb_stats= &mariadb_dummy_stats; #include "snappy-c.h" #include <limits> +#include <myisamchk.h> // TT_FOR_UPGRADE #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) @@ -874,6 +878,10 @@ static MYSQL_THDVAR_BOOL(table_locks, PLUGIN_VAR_OPCMDARG, /* check_func */ NULL, /* update_func */ NULL, /* default */ TRUE); +static MYSQL_THDVAR_BOOL(snapshot_isolation, PLUGIN_VAR_OPCMDARG, + "Use snapshot isolation (write-write conflict detection).", + NULL, NULL, FALSE); + static MYSQL_THDVAR_BOOL(strict_mode, PLUGIN_VAR_OPCMDARG, "Use strict mode when evaluating create options.", NULL, NULL, TRUE); @@ -2166,6 +2174,9 @@ convert_error_code_to_mysql( return(HA_ERR_LOCK_DEADLOCK); + case DB_RECORD_CHANGED: + return HA_ERR_RECORD_CHANGED; + case DB_LOCK_WAIT_TIMEOUT: /* Starting from 5.0.13, we let MySQL just roll back the latest SQL statement in a lock wait timeout. Previously, we @@ -2809,6 +2820,8 @@ innobase_trx_init( trx->check_unique_secondary = !thd_test_options( thd, OPTION_RELAXED_UNIQUE_CHECKS); + trx->snapshot_isolation = THDVAR(thd, snapshot_isolation) & 1; + #ifdef WITH_WSREP trx->wsrep = wsrep_on(thd); #endif @@ -4003,7 +4016,7 @@ static int innodb_init_params() data_mysql_default_charset_coll = (ulint) default_charset_info->number; -#ifdef HAVE_FCNTL_DIRECT +#if !defined _WIN32 && defined O_DIRECT if (srv_use_atomic_writes && my_may_have_atomic_write) { /* Force O_DIRECT on Unixes (on Windows writes are always @@ -4342,7 +4355,7 @@ innobase_start_trx_and_assign_read_view( Do this only if transaction is using REPEATABLE READ isolation level. */ trx->isolation_level = innobase_map_isolation_level( - thd_get_trx_isolation(thd)); + thd_get_trx_isolation(thd)) & 3; if (trx->isolation_level == TRX_ISO_REPEATABLE_READ) { trx->read_view.open(trx); @@ -5345,67 +5358,6 @@ test_normalize_table_name_low() } } } - -/********************************************************************* -Test ut_format_name(). */ -static -void -test_ut_format_name() -/*=================*/ -{ - char buf[NAME_LEN * 3]; - - struct { - const char* name; - ulint buf_size; - const char* expected; - } test_data[] = { - {"test/t1", sizeof(buf), "`test`.`t1`"}, - {"test/t1", 12, "`test`.`t1`"}, - {"test/t1", 11, "`test`.`t1"}, - {"test/t1", 10, "`test`.`t"}, - {"test/t1", 9, "`test`.`"}, - {"test/t1", 8, "`test`."}, - {"test/t1", 7, "`test`"}, - {"test/t1", 6, "`test"}, - {"test/t1", 5, "`tes"}, - {"test/t1", 4, "`te"}, - {"test/t1", 3, "`t"}, - {"test/t1", 2, "`"}, - {"test/t1", 1, ""}, - {"test/t1", 0, "BUF_NOT_CHANGED"}, - {"table", sizeof(buf), "`table`"}, - {"ta'le", sizeof(buf), "`ta'le`"}, - {"ta\"le", sizeof(buf), "`ta\"le`"}, - {"ta`le", sizeof(buf), "`ta``le`"}, - }; - - for (size_t i = 0; i < UT_ARR_SIZE(test_data); i++) { - - memcpy(buf, "BUF_NOT_CHANGED", strlen("BUF_NOT_CHANGED") + 1); - - char* ret; - - ret = ut_format_name(test_data[i].name, - buf, - test_data[i].buf_size); - - ut_a(ret == buf); - - if (strcmp(buf, test_data[i].expected) == 0) { - ib::info() << "ut_format_name(" << test_data[i].name - << ", buf, " << test_data[i].buf_size << ")," - " expected " << test_data[i].expected - << ", OK"; - } else { - ib::error() << "ut_format_name(" << test_data[i].name - << ", buf, " << test_data[i].buf_size << ")," - " expected " << test_data[i].expected - << ", ERROR: got " << buf; - ut_error; - } - } -} #endif /* !DBUG_OFF */ /** Match index columns between MySQL and InnoDB. @@ -5763,9 +5715,9 @@ func_exit: return ret; } -/********************************************************************//** -Get the upper limit of the MySQL integral and floating-point type. -@return maximum allowed value for the field */ +/** Get the maximum integer value of a numeric column. +@param field column definition +@return maximum allowed integer value */ ulonglong innobase_get_int_col_max_value(const Field *field) { ulonglong max_value = 0; @@ -5830,46 +5782,45 @@ ha_innobase::open(). @param[in,out] table persistent table @param[in] field the AUTO_INCREMENT column */ -static -void -initialize_auto_increment(dict_table_t* table, const Field* field) -{ - ut_ad(!table->is_temporary()); - - const unsigned col_no = innodb_col_no(field); - - table->autoinc_mutex.wr_lock(); - - table->persistent_autoinc = static_cast<uint16_t>( - dict_table_get_nth_col_pos(table, col_no, NULL) + 1) - & dict_index_t::MAX_N_FIELDS; +static void initialize_auto_increment(dict_table_t *table, const Field& field, + const TABLE_SHARE &s) +{ + ut_ad(!table->is_temporary()); + const unsigned col_no= innodb_col_no(&field); + table->autoinc_mutex.wr_lock(); + table->persistent_autoinc= + uint16_t(dict_table_get_nth_col_pos(table, col_no, nullptr) + 1) & + dict_index_t::MAX_N_FIELDS; + if (table->autoinc) + /* Already initialized. Our caller checked + table->persistent_autoinc without + autoinc_mutex protection, and there might be multiple + ha_innobase::open() executing concurrently. */; + else if (srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) + /* If innodb_force_recovery is set so high that writes + are disabled we force the AUTOINC counter to 0 + value effectively disabling writes to the table. + Secondly, we avoid reading the table in case the read + results in failure due to a corrupted table/index. + + We will not return an error to the client, so that the + tables can be dumped with minimal hassle. If an error + were returned in this case, the first attempt to read + the table would fail and subsequent SELECTs would succeed. */; + else if (table->persistent_autoinc) + { + uint64_t max_value= innobase_get_int_col_max_value(&field); + table->autoinc= + innobase_next_autoinc(btr_read_autoinc_with_fallback(table, col_no, + s.mysql_version, + max_value), + 1 /* need */, + 1 /* auto_increment_increment */, + 0 /* auto_increment_offset */, + max_value); + } - if (table->autoinc) { - /* Already initialized. Our caller checked - table->persistent_autoinc without - autoinc_mutex protection, and there might be multiple - ha_innobase::open() executing concurrently. */ - } else if (srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) { - /* If the recovery level is set so high that writes - are disabled we force the AUTOINC counter to 0 - value effectively disabling writes to the table. - Secondly, we avoid reading the table in case the read - results in failure due to a corrupted table/index. - - We will not return an error to the client, so that the - tables can be dumped with minimal hassle. If an error - were returned in this case, the first attempt to read - the table would fail and subsequent SELECTs would succeed. */ - } else if (table->persistent_autoinc) { - table->autoinc = innobase_next_autoinc( - btr_read_autoinc_with_fallback(table, col_no), - 1 /* need */, - 1 /* auto_increment_increment */, - 0 /* auto_increment_offset */, - innobase_get_int_col_max_value(field)); - } - - table->autoinc_mutex.wr_unlock(); + table->autoinc_mutex.wr_unlock(); } /** Open an InnoDB table @@ -6105,7 +6056,7 @@ ha_innobase::open(const char* name, int, uint) || m_prebuilt->table->persistent_autoinc || !m_prebuilt->table->is_readable()) { } else if (const Field* ai = table->found_next_number_field) { - initialize_auto_increment(m_prebuilt->table, ai); + initialize_auto_increment(m_prebuilt->table, *ai, *table->s); } /* Set plugin parser for fulltext index */ @@ -7394,26 +7345,55 @@ ha_innobase::build_template( ulint num_v = 0; - if (active_index != MAX_KEY - && active_index == pushed_idx_cond_keyno) { - m_prebuilt->idx_cond = this; - goto icp; - } else if (pushed_rowid_filter && rowid_filter_is_active) { -icp: - /* Push down an index condition or an end_range check. */ + /* MDEV-31154: For pushed down index condition we don't support virtual + column and idx_cond_push() does check for it. For row ID filtering we + don't need such restrictions but we get into trouble trying to use the + ICP path. + + 1. It should be fine to follow no_icp path if primary key is generated. + However, with user specified primary key(PK), the row is identified by + the PK and those columns need to be converted to mysql format in + row_search_idx_cond_check before doing the comparison. Since secondary + indexes always have PK appended in innodb, it works with current ICP + handling code when fetch_primary_key_cols is set to TRUE. + + 2. Although ICP comparison and Row ID comparison works on different + columns the current ICP code can be shared by both. + + 3. In most cases, it works today by jumping to goto no_icp when we + encounter a virtual column. This is hackish and already have some + issues as it cannot handle PK and all states are not reset properly, + for example, idx_cond_n_cols is not reset. + + 4. We already encountered MDEV-28747 m_prebuilt->idx_cond was being set. + + Neither ICP nor row ID comparison needs virtual columns and the code is + simplified to handle both. It should handle the issues. */ + + const bool pushed_down = active_index != MAX_KEY + && active_index == pushed_idx_cond_keyno; + + m_prebuilt->idx_cond = pushed_down ? this : nullptr; + + if (m_prebuilt->idx_cond || m_prebuilt->pk_filter) { + /* Push down an index condition, end_range check or row ID + filter */ for (ulint i = 0; i < n_fields; i++) { const Field* field = table->field[i]; const bool is_v = !field->stored_in_db(); - if (is_v && skip_virtual) { - num_v++; - continue; - } + bool index_contains = index->contains_col_or_prefix( is_v ? num_v : i - num_v, is_v); - if (is_v && index_contains) { - m_prebuilt->n_template = 0; - num_v = 0; - goto no_icp; + + if (is_v) { + if (index_contains) { + /* We want to ensure that ICP is not + used with virtual columns. */ + ut_ad(!pushed_down); + m_prebuilt->idx_cond = nullptr; + } + num_v++; + continue; } /* Test if an end_range or an index condition @@ -7433,7 +7413,7 @@ icp: which would be acceptable if end_range==NULL. */ if (build_template_needs_field_in_icp( index, m_prebuilt, index_contains, - is_v ? num_v : i - num_v, is_v)) { + i - num_v, false)) { if (!whole_row) { field = build_template_needs_field( index_contains, @@ -7442,15 +7422,10 @@ icp: fetch_primary_key_cols, index, table, i, num_v); if (!field) { - if (is_v) { - num_v++; - } continue; } } - ut_ad(!is_v); - mysql_row_templ_t* templ= build_template_field( m_prebuilt, clust_index, index, table, field, i - num_v, 0); @@ -7527,15 +7502,16 @@ icp: */ } - if (is_v) { - num_v++; - } } - ut_ad(m_prebuilt->idx_cond_n_cols > 0); - ut_ad(m_prebuilt->idx_cond_n_cols == m_prebuilt->n_template); - num_v = 0; + ut_ad(m_prebuilt->idx_cond_n_cols == m_prebuilt->n_template); + if (m_prebuilt->idx_cond_n_cols == 0) { + /* No columns to push down. It is safe to jump to np ICP + path. */ + m_prebuilt->idx_cond = nullptr; + goto no_icp; + } /* Include the fields that are not needed in index condition pushdown. */ @@ -7550,7 +7526,7 @@ icp: bool index_contains = index->contains_col_or_prefix( is_v ? num_v : i - num_v, is_v); - if (!build_template_needs_field_in_icp( + if (is_v || !build_template_needs_field_in_icp( index, m_prebuilt, index_contains, is_v ? num_v : i - num_v, is_v)) { /* Not needed in ICP */ @@ -7583,7 +7559,7 @@ icp: } else { no_icp: /* No index condition pushdown */ - m_prebuilt->idx_cond = NULL; + ut_ad(!m_prebuilt->idx_cond); ut_ad(num_v == 0); for (ulint i = 0; i < n_fields; i++) { @@ -8737,6 +8713,7 @@ ha_innobase::delete_row( : PLAIN_DELETE; trx->fts_next_doc_id = 0; + ut_ad(!trx->is_bulk_insert()); error = row_update_for_mysql(m_prebuilt); #ifdef WITH_WSREP @@ -8844,47 +8821,63 @@ ha_innobase::index_end(void) DBUG_RETURN(0); } -/*********************************************************************//** -Converts a search mode flag understood by MySQL to a flag understood -by InnoDB. */ -page_cur_mode_t -convert_search_mode_to_innobase( -/*============================*/ - ha_rkey_function find_flag) -{ - switch (find_flag) { - case HA_READ_KEY_EXACT: - /* this does not require the index to be UNIQUE */ - case HA_READ_KEY_OR_NEXT: - return(PAGE_CUR_GE); - case HA_READ_AFTER_KEY: - return(PAGE_CUR_G); - case HA_READ_BEFORE_KEY: - return(PAGE_CUR_L); - case HA_READ_KEY_OR_PREV: - case HA_READ_PREFIX_LAST: - case HA_READ_PREFIX_LAST_OR_PREV: - return(PAGE_CUR_LE); - case HA_READ_MBR_CONTAIN: - return(PAGE_CUR_CONTAIN); - case HA_READ_MBR_INTERSECT: - return(PAGE_CUR_INTERSECT); - case HA_READ_MBR_WITHIN: - return(PAGE_CUR_WITHIN); - case HA_READ_MBR_DISJOINT: - return(PAGE_CUR_DISJOINT); - case HA_READ_MBR_EQUAL: - return(PAGE_CUR_MBR_EQUAL); - case HA_READ_PREFIX: - return(PAGE_CUR_UNSUPP); - /* do not use "default:" in order to produce a gcc warning: - enumeration value '...' not handled in switch - (if -Wswitch or -Wall is used) */ - } - - my_error(ER_CHECK_NOT_IMPLEMENTED, MYF(0), "this functionality"); - - return(PAGE_CUR_UNSUPP); +/** Convert a MariaDB search mode to an InnoDB search mode. +@tparam last_match whether last_match_mode is to be set +@param find_flag MariaDB search mode +@param mode InnoDB search mode +@param last_match_mode pointer to ha_innobase::m_last_match_mode +@return whether the search mode is unsupported */ +template<bool last_match= false> +static bool convert_search_mode_to_innobase(ha_rkey_function find_flag, + page_cur_mode_t &mode, + uint *last_match_mode= nullptr) +{ + mode= PAGE_CUR_LE; + if (last_match) + *last_match_mode= 0; + + switch (find_flag) { + case HA_READ_KEY_EXACT: + /* this does not require the index to be UNIQUE */ + if (last_match) + *last_match_mode= ROW_SEL_EXACT; + /* fall through */ + case HA_READ_KEY_OR_NEXT: + mode= PAGE_CUR_GE; + return false; + case HA_READ_AFTER_KEY: + mode= PAGE_CUR_G; + return false; + case HA_READ_BEFORE_KEY: + mode= PAGE_CUR_L; + return false; + case HA_READ_PREFIX_LAST: + if (last_match) + *last_match_mode= ROW_SEL_EXACT_PREFIX; + /* fall through */ + case HA_READ_KEY_OR_PREV: + case HA_READ_PREFIX_LAST_OR_PREV: + return false; + case HA_READ_MBR_CONTAIN: + mode= PAGE_CUR_CONTAIN; + return false; + case HA_READ_MBR_INTERSECT: + mode= PAGE_CUR_INTERSECT; + return false; + case HA_READ_MBR_WITHIN: + mode= PAGE_CUR_WITHIN; + return false; + case HA_READ_MBR_DISJOINT: + mode= PAGE_CUR_DISJOINT; + return false; + case HA_READ_MBR_EQUAL: + mode= PAGE_CUR_MBR_EQUAL; + return false; + case HA_READ_PREFIX: + break; + } + + return true; } /* @@ -8962,8 +8955,7 @@ ha_innobase::index_read( mariadb_set_stats set_stats_temporary(handler_stats); DEBUG_SYNC_C("ha_innobase_index_read_begin"); - ut_a(m_prebuilt->trx == thd_to_trx(m_user_thd)); - ut_ad(key_len != 0 || find_flag != HA_READ_KEY_EXACT); + ut_ad(m_prebuilt->trx == thd_to_trx(m_user_thd)); dict_index_t* index = m_prebuilt->index; @@ -8999,7 +8991,8 @@ ha_innobase::index_read( build_template(false); } - if (key_ptr != NULL) { + if (key_len) { + ut_ad(key_ptr); /* Convert the search key value to InnoDB format into m_prebuilt->search_tuple */ @@ -9009,84 +9002,58 @@ ha_innobase::index_read( m_prebuilt->srch_key_val_len, index, (byte*) key_ptr, - (ulint) key_len); + key_len); DBUG_ASSERT(m_prebuilt->search_tuple->n_fields > 0); } else { + ut_ad(find_flag != HA_READ_KEY_EXACT); /* We position the cursor to the last or the first entry in the index */ dtuple_set_n_fields(m_prebuilt->search_tuple, 0); } - page_cur_mode_t mode = convert_search_mode_to_innobase(find_flag); - - ulint match_mode = 0; + page_cur_mode_t mode; - if (find_flag == HA_READ_KEY_EXACT) { - - match_mode = ROW_SEL_EXACT; - - } else if (find_flag == HA_READ_PREFIX_LAST) { - - match_mode = ROW_SEL_EXACT_PREFIX; + if (convert_search_mode_to_innobase<true>(find_flag, mode, + &m_last_match_mode)) { + table->status = STATUS_NOT_FOUND; + DBUG_RETURN(HA_ERR_UNSUPPORTED); } - m_last_match_mode = (uint) match_mode; - - dberr_t ret = mode == PAGE_CUR_UNSUPP ? DB_UNSUPPORTED - : row_search_mvcc(buf, mode, m_prebuilt, match_mode, 0); + dberr_t ret = + row_search_mvcc(buf, mode, m_prebuilt, m_last_match_mode, 0); DBUG_EXECUTE_IF("ib_select_query_failure", ret = DB_ERROR;); - int error; - - switch (ret) { - case DB_SUCCESS: - error = 0; + if (UNIV_LIKELY(ret == DB_SUCCESS)) { table->status = 0; - break; - - case DB_RECORD_NOT_FOUND: - error = HA_ERR_KEY_NOT_FOUND; - table->status = STATUS_NOT_FOUND; - break; + DBUG_RETURN(0); + } - case DB_END_OF_INDEX: - error = HA_ERR_KEY_NOT_FOUND; - table->status = STATUS_NOT_FOUND; - break; + table->status = STATUS_NOT_FOUND; + switch (ret) { case DB_TABLESPACE_DELETED: ib_senderrf( m_prebuilt->trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_TABLESPACE_DISCARDED, table->s->table_name.str); - - table->status = STATUS_NOT_FOUND; - error = HA_ERR_TABLESPACE_MISSING; - break; - + DBUG_RETURN(HA_ERR_TABLESPACE_MISSING); + case DB_RECORD_NOT_FOUND: + case DB_END_OF_INDEX: + DBUG_RETURN(HA_ERR_KEY_NOT_FOUND); case DB_TABLESPACE_NOT_FOUND: - ib_senderrf( m_prebuilt->trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_TABLESPACE_MISSING, table->s->table_name.str); - - table->status = STATUS_NOT_FOUND; - error = HA_ERR_TABLESPACE_MISSING; - break; - + DBUG_RETURN(HA_ERR_TABLESPACE_MISSING); default: - error = convert_error_code_to_mysql( - ret, m_prebuilt->table->flags, m_user_thd); - - table->status = STATUS_NOT_FOUND; - break; + DBUG_RETURN(convert_error_code_to_mysql( + ret, m_prebuilt->table->flags, + m_user_thd)); } - - DBUG_RETURN(error); } /*******************************************************************//** @@ -9513,8 +9480,6 @@ ha_innobase::rnd_pos( DBUG_ENTER("rnd_pos"); DBUG_DUMP("key", pos, ref_length); - ut_a(m_prebuilt->trx == thd_to_trx(ha_thd())); - /* Note that we assume the length of the row reference is fixed for the table, and it is == ref_length */ @@ -9938,7 +9903,8 @@ wsrep_append_foreign_key( } ulint rcode = DB_SUCCESS; - char cache_key[513] = {'\0'}; + char cache_key[MAX_FULL_NAME_LEN] = {'\0'}; + char db_name[MAX_DATABASE_NAME_LEN+1] = {'\0'}; size_t cache_key_len = 0; if ( !((referenced) ? @@ -10028,14 +9994,38 @@ wsrep_append_foreign_key( return DB_ERROR; } - strncpy(cache_key, + char * fk_table = (wsrep_protocol_version > 1) ? ((referenced) ? foreign->referenced_table->name.m_name : foreign->foreign_table->name.m_name) : - foreign->foreign_table->name.m_name, sizeof(cache_key) - 1); - cache_key_len = strlen(cache_key); + foreign->foreign_table->name.m_name; + + /* convert db and table name parts separately to system charset */ + ulint db_name_len = dict_get_db_name_len(fk_table); + strmake(db_name, fk_table, db_name_len); + uint errors; + cache_key_len= innobase_convert_to_system_charset(cache_key, + db_name, sizeof(cache_key), &errors); + if (errors) { + WSREP_WARN("unexpected foreign key table %s %s", + foreign->referenced_table->name.m_name, + foreign->foreign_table->name.m_name); + return DB_ERROR; + } + /* after db name adding 0 and then converted table name */ + cache_key[db_name_len]= '\0'; + cache_key_len++; + + cache_key_len+= innobase_convert_to_system_charset(cache_key+cache_key_len, + fk_table+db_name_len+1, sizeof(cache_key), &errors); + if (errors) { + WSREP_WARN("unexpected foreign key table %s %s", + foreign->referenced_table->name.m_name, + foreign->foreign_table->name.m_name); + return DB_ERROR; + } #ifdef WSREP_DEBUG_PRINT ulint j; fprintf(stderr, "FK parent key, table: %s %s len: %lu ", @@ -10045,16 +10035,6 @@ wsrep_append_foreign_key( } fprintf(stderr, "\n"); #endif - char *p = strchr(cache_key, '/'); - - if (p) { - *p = '\0'; - } else { - WSREP_WARN("unexpected foreign key table %s %s", - foreign->referenced_table->name.m_name, - foreign->foreign_table->name.m_name); - } - wsrep_buf_t wkey_part[3]; wsrep_key_t wkey = {wkey_part, 3}; @@ -12269,7 +12249,7 @@ create_table_info_t::create_foreign_keys() dict_index_t* index = NULL; fkerr_t index_error = FK_SUCCESS; dict_index_t* err_index = NULL; - ulint err_col; + ulint err_col = 0; const bool tmp_table = m_flags2 & DICT_TF2_TEMPORARY; const CHARSET_INFO* cs = thd_charset(m_thd); const char* operation = "Create "; @@ -13383,6 +13363,49 @@ ha_innobase::discard_or_import_tablespace( DBUG_RETURN(0); } +/** Report a DROP TABLE failure due to a FOREIGN KEY constraint. +@param name table name +@param foreign constraint */ +ATTRIBUTE_COLD +static void delete_table_cannot_drop_foreign(const table_name_t &name, + const dict_foreign_t &foreign) +{ + mysql_mutex_lock(&dict_foreign_err_mutex); + rewind(dict_foreign_err_file); + ut_print_timestamp(dict_foreign_err_file); + fputs(" Cannot drop table ", dict_foreign_err_file); + ut_print_name(dict_foreign_err_file, nullptr, name.m_name); + fputs("\nbecause it is referenced by ", dict_foreign_err_file); + ut_print_name(dict_foreign_err_file, nullptr, foreign.foreign_table_name); + putc('\n', dict_foreign_err_file); + mysql_mutex_unlock(&dict_foreign_err_mutex); +} + +/** Check if DROP TABLE would fail due to a FOREIGN KEY constraint. +@param table table to be dropped +@param sqlcom thd_sql_command(current_thd) +@return whether child tables that refer to this table exist */ +static bool delete_table_check_foreigns(const dict_table_t &table, + enum_sql_command sqlcom) +{ + const bool drop_db{sqlcom == SQLCOM_DROP_DB}; + for (const auto foreign : table.referenced_set) + { + /* We should allow dropping a referenced table if creating + that referenced table has failed for some reason. For example + if referenced table is created but it column types that are + referenced do not match. */ + if (foreign->foreign_table == &table || + (drop_db && + dict_tables_have_same_db(table.name.m_name, + foreign->foreign_table_name_lookup))) + continue; + delete_table_cannot_drop_foreign(table.name, *foreign); + return true; + } + + return false; +} /** DROP TABLE (possibly as part of DROP DATABASE, CREATE/ALTER TABLE) @param name table name @@ -13397,8 +13420,8 @@ int ha_innobase::delete_table(const char *name) DBUG_EXECUTE_IF("test_normalize_table_name_low", test_normalize_table_name_low();); - DBUG_EXECUTE_IF("test_ut_format_name", test_ut_format_name();); + const enum_sql_command sqlcom= enum_sql_command(thd_sql_command(thd)); trx_t *parent_trx= check_trx_exists(thd); dict_table_t *table; @@ -13435,6 +13458,13 @@ int ha_innobase::delete_table(const char *name) DBUG_RETURN(0); } + if (parent_trx->check_foreigns && + delete_table_check_foreigns(*table, sqlcom)) + { + dict_sys.unlock(); + DBUG_RETURN(HA_ERR_ROW_IS_REFERENCED); + } + table->acquire(); dict_sys.unlock(); @@ -13467,14 +13497,7 @@ int ha_innobase::delete_table(const char *name) /* FOREIGN KEY constraints cannot exist on partitioned tables. */; #endif else - { - dict_sys.freeze(SRW_LOCK_CALL); - for (const dict_foreign_t* f : table->referenced_set) - if (dict_table_t* child= f->foreign_table) - if ((err= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS) - break; - dict_sys.unfreeze(); - } + err= lock_table_children(table, trx); } dict_table_t *table_stats= nullptr, *index_stats= nullptr; @@ -13484,7 +13507,6 @@ int ha_innobase::delete_table(const char *name) const bool fts= err == DB_SUCCESS && (table->flags2 & (DICT_TF2_FTS_HAS_DOC_ID | DICT_TF2_FTS)); - const enum_sql_command sqlcom= enum_sql_command(thd_sql_command(thd)); if (fts) { @@ -13642,36 +13664,16 @@ err_exit: DBUG_RETURN(convert_error_code_to_mysql(err, 0, NULL)); } - if (!table->no_rollback() && trx->check_foreigns) + if (!table->no_rollback()) { - const bool drop_db= sqlcom == SQLCOM_DROP_DB; - for (auto foreign : table->referenced_set) + if (trx->check_foreigns && delete_table_check_foreigns(*table, sqlcom)) { - /* We should allow dropping a referenced table if creating - that referenced table has failed for some reason. For example - if referenced table is created but it column types that are - referenced do not match. */ - if (foreign->foreign_table == table || - (drop_db && - dict_tables_have_same_db(table->name.m_name, - foreign->foreign_table_name_lookup))) - continue; - mysql_mutex_lock(&dict_foreign_err_mutex); - rewind(dict_foreign_err_file); - ut_print_timestamp(dict_foreign_err_file); - fputs(" Cannot drop table ", dict_foreign_err_file); - ut_print_name(dict_foreign_err_file, trx, table->name.m_name); - fputs("\nbecause it is referenced by ", dict_foreign_err_file); - ut_print_name(dict_foreign_err_file, trx, foreign->foreign_table_name); - putc('\n', dict_foreign_err_file); - mysql_mutex_unlock(&dict_foreign_err_mutex); err= DB_CANNOT_DROP_CONSTRAINT; goto err_exit; } - } - if (!table->no_rollback()) err= trx->drop_table_foreign(table->name); + } if (err == DB_SUCCESS && table_stats && index_stats) err= trx->drop_table_statistics(table->name); @@ -13790,6 +13792,19 @@ int ha_innobase::truncate() update_thd(); +#ifdef UNIV_DEBUG + if (!thd_test_options(m_user_thd, OPTION_NO_FOREIGN_KEY_CHECKS)) + { + /* fk_truncate_illegal_if_parent() should have failed in + Sql_cmd_truncate_table::handler_truncate() if foreign_key_checks=ON + and child tables exist. */ + dict_sys.freeze(SRW_LOCK_CALL); + for (const auto foreign : m_prebuilt->table->referenced_set) + ut_ad(foreign->foreign_table == m_prebuilt->table); + dict_sys.unfreeze(); + } +#endif + if (is_read_only()) DBUG_RETURN(HA_ERR_TABLE_READONLY); @@ -13872,14 +13887,7 @@ int ha_innobase::truncate() dict_table_t *table_stats = nullptr, *index_stats = nullptr; MDL_ticket *mdl_table = nullptr, *mdl_index = nullptr; - dberr_t error= DB_SUCCESS; - - dict_sys.freeze(SRW_LOCK_CALL); - for (const dict_foreign_t *f : ib_table->referenced_set) - if (dict_table_t *child= f->foreign_table) - if ((error= lock_table_for_trx(child, trx, LOCK_X)) != DB_SUCCESS) - break; - dict_sys.unfreeze(); + dberr_t error= lock_table_children(ib_table, trx); if (error == DB_SUCCESS) error= lock_table_for_trx(ib_table, trx, LOCK_X); @@ -14070,16 +14078,7 @@ ha_innobase::rename_table( /* There is no need to lock any FOREIGN KEY child tables. */ } else if (dict_table_t *table = dict_table_open_on_name( norm_from, false, DICT_ERR_IGNORE_FK_NOKEY)) { - dict_sys.freeze(SRW_LOCK_CALL); - for (const dict_foreign_t* f : table->referenced_set) { - if (dict_table_t* child = f->foreign_table) { - error = lock_table_for_trx(child, trx, LOCK_X); - if (error != DB_SUCCESS) { - break; - } - } - } - dict_sys.unfreeze(); + error = lock_table_children(table, trx); if (error == DB_SUCCESS) { error = lock_table_for_trx(table, trx, LOCK_X); } @@ -14218,14 +14217,14 @@ ha_innobase::records_in_range( dict_index_t* index; dtuple_t* range_start; dtuple_t* range_end; - ha_rows n_rows; + ha_rows n_rows = HA_POS_ERROR; page_cur_mode_t mode1; page_cur_mode_t mode2; mem_heap_t* heap; DBUG_ENTER("records_in_range"); - ut_a(m_prebuilt->trx == thd_to_trx(ha_thd())); + ut_ad(m_prebuilt->trx == thd_to_trx(ha_thd())); m_prebuilt->trx->op_info = "estimating records in index range"; @@ -14238,12 +14237,7 @@ ha_innobase::records_in_range( /* There exists possibility of not being able to find requested index due to inconsistency between MySQL and InoDB dictionary info. Necessary message should have been printed in innobase_get_index() */ - if (!m_prebuilt->table->space) { - n_rows = HA_POS_ERROR; - goto func_exit; - } - if (!index) { - n_rows = HA_POS_ERROR; + if (!index || !m_prebuilt->table->space) { goto func_exit; } if (index->is_corrupted()) { @@ -14259,61 +14253,50 @@ ha_innobase::records_in_range( + sizeof(dtuple_t))); range_start = dtuple_create(heap, key->ext_key_parts); - dict_index_copy_types(range_start, index, key->ext_key_parts); range_end = dtuple_create(heap, key->ext_key_parts); - dict_index_copy_types(range_end, index, key->ext_key_parts); - - row_sel_convert_mysql_key_to_innobase( - range_start, - m_prebuilt->srch_key_val1, - m_prebuilt->srch_key_val_len, - index, - (byte*) (min_key ? min_key->key : (const uchar*) 0), - (ulint) (min_key ? min_key->length : 0)); - - DBUG_ASSERT(min_key - ? range_start->n_fields > 0 - : range_start->n_fields == 0); - - row_sel_convert_mysql_key_to_innobase( - range_end, - m_prebuilt->srch_key_val2, - m_prebuilt->srch_key_val_len, - index, - (byte*) (max_key ? max_key->key : (const uchar*) 0), - (ulint) (max_key ? max_key->length : 0)); - - DBUG_ASSERT(max_key - ? range_end->n_fields > 0 - : range_end->n_fields == 0); - - mode1 = convert_search_mode_to_innobase( - min_key ? min_key->flag : HA_READ_KEY_EXACT); - - mode2 = convert_search_mode_to_innobase( - max_key ? max_key->flag : HA_READ_KEY_EXACT); - - if (mode1 != PAGE_CUR_UNSUPP && mode2 != PAGE_CUR_UNSUPP) { - - if (dict_index_is_spatial(index)) { - /*Only min_key used in spatial index. */ - n_rows = rtr_estimate_n_rows_in_range( - index, range_start, mode1); - } else { - btr_pos_t tuple1(range_start, mode1, pages->first_page); - btr_pos_t tuple2(range_end, mode2, pages->last_page); - n_rows = btr_estimate_n_rows_in_range( - index, &tuple1, &tuple2); - pages->first_page= tuple1.page_id.raw(); - pages->last_page= tuple2.page_id.raw(); - } + + if (!min_key) { + mode1 = PAGE_CUR_GE; + dtuple_set_n_fields(range_start, 0); + } else if (convert_search_mode_to_innobase(min_key->flag, mode1)) { + goto unsupported; } else { + dict_index_copy_types(range_start, index, key->ext_key_parts); + row_sel_convert_mysql_key_to_innobase( + range_start, + m_prebuilt->srch_key_val1, + m_prebuilt->srch_key_val_len, + index, min_key->key, min_key->length); + DBUG_ASSERT(range_start->n_fields > 0); + } - n_rows = HA_POS_ERROR; + if (!max_key) { + mode2 = PAGE_CUR_GE; + dtuple_set_n_fields(range_end, 0); + } else if (convert_search_mode_to_innobase(max_key->flag, mode2)) { + goto unsupported; + } else { + dict_index_copy_types(range_end, index, key->ext_key_parts); + row_sel_convert_mysql_key_to_innobase( + range_end, + m_prebuilt->srch_key_val2, + m_prebuilt->srch_key_val_len, + index, max_key->key, max_key->length); + DBUG_ASSERT(range_end->n_fields > 0); } - mem_heap_free(heap); + if (dict_index_is_spatial(index)) { + /*Only min_key used in spatial index. */ + n_rows = rtr_estimate_n_rows_in_range( + index, range_start, mode1); + } else { + btr_pos_t tuple1(range_start, mode1, pages->first_page); + btr_pos_t tuple2(range_end, mode2, pages->last_page); + n_rows = btr_estimate_n_rows_in_range(index, &tuple1, &tuple2); + pages->first_page= tuple1.page_id.raw(); + pages->last_page= tuple2.page_id.raw(); + } DBUG_EXECUTE_IF( "print_btr_estimate_n_rows_in_range_return_value", @@ -14324,11 +14307,7 @@ ha_innobase::records_in_range( (longlong) n_rows); ); -func_exit: - - m_prebuilt->trx->op_info = (char*)""; - - /* The MySQL optimizer seems to believe an estimate of 0 rows is + /* The MariaDB optimizer seems to believe an estimate of 0 rows is always accurate and may return the result 'Empty set' based on that. The accuracy is not guaranteed, and even if it were, for a locking read we should anyway perform the search to set the next-key lock. @@ -14338,6 +14317,10 @@ func_exit: n_rows = 1; } +unsupported: + mem_heap_free(heap); +func_exit: + m_prebuilt->trx->op_info = ""; DBUG_RETURN((ha_rows) n_rows); } @@ -15140,6 +15123,7 @@ ha_innobase::check( ulint n_rows_in_table = ULINT_UNDEFINED; bool is_ok = true; dberr_t ret; + uint handler_flags= check_opt->handler_flags; DBUG_ENTER("ha_innobase::check"); DBUG_ASSERT(thd == ha_thd()); @@ -15148,6 +15132,27 @@ ha_innobase::check( ut_a(m_prebuilt->trx == thd_to_trx(thd)); ut_ad(m_prebuilt->trx->mysql_thd == thd); + if (handler_flags || check_for_upgrade(check_opt)) { + /* The file was already checked and fixed as part of open */ + print_check_msg(thd, table->s->db.str, table->s->table_name.str, + "check", "note", + (opt_readonly || high_level_read_only + || !(check_opt->sql_flags & TT_FOR_UPGRADE)) + ? "Auto_increment will be" + " checked on each open until" + " CHECK TABLE FOR UPGRADE is executed" + : "Auto_increment checked and" + " .frm file version updated", 1); + if (handler_flags && (check_opt->sql_flags & TT_FOR_UPGRADE)) { + /* + No other issues found (as handler_flags was only + set if there as not other problems with the table + than auto_increment). + */ + DBUG_RETURN(HA_ADMIN_OK); + } + } + if (m_prebuilt->mysql_template == NULL) { /* Build the template; we will use a dummy template in index scans done in checking */ @@ -15330,7 +15335,7 @@ ha_innobase::check( } /* Restore the original isolation level */ - m_prebuilt->trx->isolation_level = old_isolation_level; + m_prebuilt->trx->isolation_level = old_isolation_level & 3; #ifdef BTR_CUR_HASH_ADAPT # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG /* We validate the whole adaptive hash index for all tables @@ -15351,6 +15356,35 @@ func_exit: DBUG_RETURN(is_ok ? HA_ADMIN_OK : HA_ADMIN_CORRUPT); } +/** +Check if we there is a problem with the InnoDB table. +@param check_opt check options +@retval HA_ADMIN_OK if Table is ok +@retval HA_ADMIN_NEEDS_ALTER User should run ALTER TABLE FOR UPGRADE +@retval HA_ADMIN_NEEDS_CHECK User should run CHECK TABLE FOR UPGRADE +@retval HA_ADMIN_FAILED if InnoDB is in read-only mode */ +int ha_innobase::check_for_upgrade(HA_CHECK_OPT *check_opt) +{ + /* + Check if there is a possibility that the auto increment value + stored in PAGE_ROOT_AUTO_INC could be corrupt. + */ + if (table->s->mysql_version >= 100210); + else if (const Field *auto_increment= table->found_next_number_field) + { + uint col_no= innodb_col_no(auto_increment); + const dict_col_t *autoinc_col= + dict_table_get_nth_col(m_prebuilt->table, col_no); + if (m_prebuilt->table->get_index(*autoinc_col)) + { + check_opt->handler_flags= 1; + return (high_level_read_only && !opt_readonly) + ? HA_ADMIN_FAILED : HA_ADMIN_NEEDS_CHECK; + } + } + return HA_ADMIN_OK; +} + /*******************************************************************//** Gets the foreign key create info for a table stored in InnoDB. @return own: character string in the form which can be inserted to the @@ -15408,7 +15442,6 @@ get_foreign_key_info( char tmp_buff[NAME_LEN+1]; char name_buff[NAME_LEN+1]; const char* ptr; - LEX_CSTRING* referenced_key_name; LEX_CSTRING* name = NULL; if (dict_table_t::is_temporary_name(foreign->foreign_table_name)) { @@ -15509,18 +15542,16 @@ get_foreign_key_info( if (foreign->referenced_index && foreign->referenced_index->name != NULL) { - referenced_key_name = thd_make_lex_string( + f_key_info.referenced_key_name = thd_make_lex_string( thd, - f_key_info.referenced_key_name, + nullptr, foreign->referenced_index->name, strlen(foreign->referenced_index->name), 1); } else { - referenced_key_name = NULL; + f_key_info.referenced_key_name = NULL; } - f_key_info.referenced_key_name = referenced_key_name; - pf_key_info = (FOREIGN_KEY_INFO*) thd_memdup(thd, &f_key_info, sizeof(FOREIGN_KEY_INFO)); @@ -15806,7 +15837,7 @@ ha_innobase::start_stmt( } /* fall through */ default: - trx->end_bulk_insert(*m_prebuilt->table); + trx->bulk_insert_apply_for_table(m_prebuilt->table); if (!trx->bulk_insert) { break; } @@ -16000,7 +16031,7 @@ ha_innobase::external_lock( } /* fall through */ default: - trx->end_bulk_insert(*m_prebuilt->table); + trx->bulk_insert_apply_for_table(m_prebuilt->table); if (!trx->bulk_insert) { break; } @@ -16370,7 +16401,7 @@ ha_innobase::store_lock( if (lock_type != TL_IGNORE && trx->n_mysql_tables_in_use == 0) { trx->isolation_level = innobase_map_isolation_level( - (enum_tx_isolation) thd_tx_isolation(thd)); + (enum_tx_isolation) thd_tx_isolation(thd)) & 3; if (trx->isolation_level <= TRX_ISO_READ_COMMITTED) { @@ -16600,6 +16631,13 @@ ha_innobase::get_auto_increment( if (error != DB_SUCCESS) { *first_value = (~(ulonglong) 0); + /* This is an error case. We do the error handling by calling + the error code conversion function. Specifically, we need to + call thd_mark_transaction_to_rollback() to inform sql that we + have rolled back innodb transaction after a deadlock error. We + ignore the returned mysql error code here. */ + std::ignore = convert_error_code_to_mysql( + error, m_prebuilt->table->flags, m_user_thd); return; } @@ -17535,6 +17573,7 @@ innodb_make_page_dirty(THD*, st_mysql_sys_var*, void*, const void* save) { mtr_t mtr; uint space_id = *static_cast<const uint*>(save); + srv_fil_make_page_dirty_debug= space_id; mysql_mutex_unlock(&LOCK_global_system_variables); fil_space_t* space = fil_space_t::get(space_id); @@ -18261,13 +18300,15 @@ buf_flush_list_now_set(THD*, st_mysql_sys_var*, void*, const void* save) return; const uint s= srv_fil_make_page_dirty_debug; mysql_mutex_unlock(&LOCK_global_system_variables); - if (s) - buf_flush_sync(); - else + if (s == 0 || srv_is_undo_tablespace(s)) { - while (buf_flush_list_space(fil_system.sys_space, nullptr)); + fil_space_t *space= fil_system.sys_space; + if (s) { space= fil_space_get(s); } + while (buf_flush_list_space(space, nullptr)); os_aio_wait_until_no_pending_writes(true); } + else + buf_flush_sync(); mysql_mutex_lock(&LOCK_global_system_variables); } @@ -18452,7 +18493,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, const bool in_progress(buf_pool.get_oldest_modification(LSN_MAX) < log_sys.resize_in_progress()); if (in_progress) - my_cond_timedwait(&buf_pool.do_flush_list, + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex.m_mutex, &abstime); mysql_mutex_unlock(&buf_pool.flush_list_mutex); if (!log_sys.resize_in_progress()) @@ -18463,6 +18504,15 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, mysql_mutex_lock(&LOCK_global_system_variables); } +static void innodb_log_spin_wait_delay_update(THD *, st_mysql_sys_var*, + void *, const void *save) +{ + log_sys.latch.wr_lock(SRW_LOCK_CALL); + mtr_t::spin_wait_delay= *static_cast<const unsigned*>(save); + mtr_t::finisher_update(); + log_sys.latch.wr_unlock(); +} + /** Update innodb_status_output or innodb_status_output_locks, which control InnoDB "status monitor" output to the error log. @param[out] var current value @@ -19279,10 +19329,10 @@ static MYSQL_SYSVAR_ULONG(page_size, srv_page_size, NULL, NULL, UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE_MIN, UNIV_PAGE_SIZE_MAX, 0); -static MYSQL_SYSVAR_SIZE_T(log_buffer_size, log_sys.buf_size, +static MYSQL_SYSVAR_UINT(log_buffer_size, log_sys.buf_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Redo log buffer size in bytes.", - NULL, NULL, 16U << 20, 2U << 20, SIZE_T_MAX, 4096); + NULL, NULL, 16U << 20, 2U << 20, log_sys.buf_size_max, 4096); #if defined __linux__ || defined _WIN32 static MYSQL_SYSVAR_BOOL(log_file_buffering, log_sys.log_buffered, @@ -19297,6 +19347,12 @@ static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, nullptr, innodb_log_file_size_update, 96 << 20, 4 << 20, std::numeric_limits<ulonglong>::max(), 4096); +static MYSQL_SYSVAR_UINT(log_spin_wait_delay, mtr_t::spin_wait_delay, + PLUGIN_VAR_OPCMDARG, + "Delay between log buffer spin lock polls (0 to use a blocking latch)", + nullptr, innodb_log_spin_wait_delay_update, + 0, 0, 6000, 0); + static MYSQL_SYSVAR_UINT(old_blocks_pct, innobase_old_blocks_pct, PLUGIN_VAR_RQCMDARG, "Percentage of the buffer pool to reserve for 'old' blocks.", @@ -19756,6 +19812,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_file_buffering), #endif MYSQL_SYSVAR(log_file_size), + MYSQL_SYSVAR(log_spin_wait_delay), MYSQL_SYSVAR(log_group_home_dir), MYSQL_SYSVAR(max_dirty_pages_pct), MYSQL_SYSVAR(max_dirty_pages_pct_lwm), @@ -19776,6 +19833,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(ft_server_stopword_table), MYSQL_SYSVAR(ft_user_stopword_table), MYSQL_SYSVAR(disable_sort_file_cache), + MYSQL_SYSVAR(snapshot_isolation), MYSQL_SYSVAR(stats_on_metadata), MYSQL_SYSVAR(stats_transient_sample_pages), MYSQL_SYSVAR(stats_persistent), diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 1f42bf18..50ac423f 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -209,6 +209,7 @@ public: int rename_table(const char* from, const char* to) override; inline int defragment_table(); int check(THD* thd, HA_CHECK_OPT* check_opt) override; + int check_for_upgrade(HA_CHECK_OPT* check_opt) override; inline void reload_statistics(); @@ -909,6 +910,12 @@ unsigned innodb_col_no(const Field* field) MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Get the maximum integer value of a numeric column. +@param field column definition +@return maximum allowed integer value */ +ulonglong innobase_get_int_col_max_value(const Field *field) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + /********************************************************************//** Helper function to push frm mismatch error to error log and if needed to sql-layer. */ diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 1401136f..6689b9ef 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -864,6 +864,9 @@ my_error_innodb( case DB_DEADLOCK: my_error(ER_LOCK_DEADLOCK, MYF(0)); break; + case DB_RECORD_CHANGED: + my_error(ER_CHECKREAD, MYF(0), table); + break; case DB_LOCK_WAIT_TIMEOUT: my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0)); break; @@ -1458,11 +1461,6 @@ struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx } }; -/********************************************************************//** -Get the upper limit of the MySQL integral and floating-point type. -@return maximum allowed value for the field */ -ulonglong innobase_get_int_col_max_value(const Field *field); - /** Determine if fulltext indexes exist in a given table. @param table MySQL table @return number of fulltext indexes */ @@ -1730,11 +1728,9 @@ instant_alter_column_possible( ut_ad(!is_null || nullable); n_nullable += nullable; n_add++; - uint l; + uint l = (*af)->pack_length(); switch ((*af)->type()) { case MYSQL_TYPE_VARCHAR: - l = reinterpret_cast<const Field_varstring*> - (*af)->get_length(); variable_length: if (l >= min_local_len) { max_size += blob_prefix @@ -1748,7 +1744,6 @@ instant_alter_column_possible( if (!is_null) { min_size += l; } - l = (*af)->pack_length(); max_size += l; lenlen += l > 255 ? 2 : 1; } @@ -1762,7 +1757,6 @@ instant_alter_column_possible( ((*af))->get_length(); goto variable_length; default: - l = (*af)->pack_length(); if (l > 255 && ib_table.not_redundant()) { goto variable_length; } @@ -2748,6 +2742,9 @@ cannot_create_many_fulltext_index: online = false; } + static constexpr const char *not_implemented + = "Not implemented for system-versioned operations"; + if (ha_alter_info->handler_flags & ALTER_ADD_NON_UNIQUE_NON_PRIM_INDEX) { /* ADD FULLTEXT|SPATIAL INDEX requires a lock. @@ -2775,6 +2772,12 @@ cannot_create_many_fulltext_index: goto cannot_create_many_fulltext_index; } + if (altered_table->versioned()) { + ha_alter_info->unsupported_reason + = not_implemented; + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + } + add_fulltext = true; if (ha_alter_info->online && !ha_alter_info->unsupported_reason) { @@ -2811,12 +2814,18 @@ cannot_create_many_fulltext_index: } } + if (m_prebuilt->table->is_stats_table()) { + if (ha_alter_info->online) { + ha_alter_info->unsupported_reason = + table_share->table_name.str; + } + online= false; + } + // FIXME: implement Online DDL for system-versioned operations if (ha_alter_info->handler_flags & INNOBASE_ALTER_VERSIONED_REBUILD) { - if (ha_alter_info->online) { - ha_alter_info->unsupported_reason = - "Not implemented for system-versioned operations"; + ha_alter_info->unsupported_reason = not_implemented; } online = false; @@ -7444,6 +7453,7 @@ error_handled: row_mysql_lock_data_dictionary(ctx->trx); } else { row_merge_drop_indexes(ctx->trx, user_table, true); + user_table->indexes.start->online_log = nullptr; ctx->trx->commit(); } @@ -9865,13 +9875,7 @@ commit_set_autoinc( const dict_col_t* autoinc_col = dict_table_get_nth_col(ctx->old_table, innodb_col_no(ai)); - dict_index_t* index - = dict_table_get_first_index(ctx->old_table); - while (index != NULL - && index->fields[0].col != autoinc_col) { - index = dict_table_get_next_index(index); - } - + auto index = ctx->old_table->get_index(*autoinc_col); ut_ad(index); ib_uint64_t max_in_table = index @@ -10246,6 +10250,7 @@ when rebuilding the table. @param ctx In-place ALTER TABLE context @param altered_table MySQL table that is being altered @param old_table MySQL table as it is before the ALTER operation +@param statistics_exist whether to update InnoDB persistent statistics @param trx Data dictionary transaction @param table_name Table name in MySQL @retval true Failure @@ -10519,6 +10524,7 @@ when not rebuilding the table. @param ha_alter_info Data used during in-place alter @param ctx In-place ALTER TABLE context @param old_table MySQL table as it is before the ALTER operation +@param statistics_exist whether to update InnoDB persistent statistics @param trx Data dictionary transaction @param table_name Table name in MySQL @retval true Failure @@ -10532,6 +10538,7 @@ commit_try_norebuild( ha_innobase_inplace_ctx*ctx, TABLE* altered_table, const TABLE* old_table, + bool statistics_exist, trx_t* trx, const char* table_name) { @@ -10646,6 +10653,10 @@ commit_try_norebuild( goto handle_error; } + if (!statistics_exist) { + continue; + } + error = dict_stats_delete_from_index_stats(db, table, index->name, trx); switch (error) { @@ -10657,7 +10668,8 @@ commit_try_norebuild( } } - if (const size_t size = ha_alter_info->rename_keys.size()) { + if (!statistics_exist) { + } else if (const size_t size = ha_alter_info->rename_keys.size()) { char tmp_name[5]; char db[MAX_DB_UTF8_LEN], table[MAX_TABLE_UTF8_LEN]; @@ -11224,16 +11236,7 @@ ha_innobase::commit_inplace_alter_table( fts_optimize_remove_table(ctx->old_table); } - dict_sys.freeze(SRW_LOCK_CALL); - for (auto f : ctx->old_table->referenced_set) { - if (dict_table_t* child = f->foreign_table) { - error = lock_table_for_trx(child, trx, LOCK_X); - if (error != DB_SUCCESS) { - break; - } - } - } - dict_sys.unfreeze(); + error = lock_table_children(ctx->old_table, trx); if (ctx->new_table->fts) { ut_ad(!ctx->new_table->fts->add_wq); @@ -11413,6 +11416,8 @@ err_index: } } + DEBUG_SYNC(m_user_thd, "innodb_commit_inplace_before_lock"); + DBUG_EXECUTE_IF("stats_lock_fail", error = DB_LOCK_WAIT_TIMEOUT; trx_rollback_for_mysql(trx);); @@ -11496,7 +11501,9 @@ fail: goto fail; } } else if (commit_try_norebuild(ha_alter_info, ctx, - altered_table, table, trx, + altered_table, table, + table_stats && index_stats, + trx, table_share->table_name.str)) { goto fail; } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 4ec07b81..e77401ed 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -24,6 +24,7 @@ Insert buffer Created 7/19/1997 Heikki Tuuri *******************************************************/ +#include <tuple> #include "ibuf0ibuf.h" #include "btr0sea.h" @@ -1833,7 +1834,7 @@ corrupted: err = flst_add_last(ibuf_root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, block, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, - &mtr); + fil_system.sys_space->free_limit, &mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { goto corrupted; } @@ -1864,7 +1865,6 @@ Removes a page from the free list and frees it to the fsp system. */ static void ibuf_remove_free_page() { mtr_t mtr; - mtr_t mtr2; page_t* header_page; log_free_check(); @@ -1891,26 +1891,28 @@ early_exit: return; } - ibuf_mtr_start(&mtr2); - - buf_block_t* root = ibuf_tree_root_get(&mtr2); + buf_block_t* root = ibuf_tree_root_get(&mtr); if (UNIV_UNLIKELY(!root)) { - ibuf_mtr_commit(&mtr2); goto early_exit; } - mysql_mutex_unlock(&ibuf_mutex); - + const auto root_savepoint = mtr.get_savepoint() - 1; const uint32_t page_no = flst_get_last(PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST + root->page.frame).page; + if (page_no >= fil_system.sys_space->free_limit) { + goto early_exit; + } + + mysql_mutex_unlock(&ibuf_mutex); + /* NOTE that we must release the latch on the ibuf tree root because in fseg_free_page we access level 1 pages, and the root is a level 2 page. */ - - ibuf_mtr_commit(&mtr2); + root->page.lock.u_unlock(); + mtr.lock_register(root_savepoint, MTR_MEMO_BUF_FIX); ibuf_exit(&mtr); /* Since pessimistic inserts were prevented, we know that the @@ -1933,15 +1935,7 @@ early_exit: ibuf_enter(&mtr); mysql_mutex_lock(&ibuf_mutex); - - root = ibuf_tree_root_get(&mtr, &err); - if (UNIV_UNLIKELY(!root)) { - mysql_mutex_unlock(&ibuf_pessimistic_insert_mutex); - goto func_exit; - } - - ut_ad(page_no == flst_get_last(PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST - + root->page.frame).page); + mtr.upgrade_buffer_fix(root_savepoint, RW_X_LATCH); /* Remove the page from the free list and update the ibuf size data */ if (buf_block_t* block = @@ -1950,7 +1944,7 @@ early_exit: err = flst_remove(root, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST, block, PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST_NODE, - &mtr); + fil_system.sys_space->free_limit, &mtr); } mysql_mutex_unlock(&ibuf_pessimistic_insert_mutex); @@ -2332,7 +2326,8 @@ func_exit: /** Merge the change buffer to some pages. */ static void ibuf_read_merge_pages(const uint32_t* space_ids, - const uint32_t* page_nos, ulint n_stored) + const uint32_t* page_nos, ulint n_stored, + bool slow_shutdown_cleanup) { for (ulint i = 0; i < n_stored; i++) { const uint32_t space_id = space_ids[i]; @@ -2357,30 +2352,28 @@ tablespace_deleted: if (UNIV_LIKELY(page_nos[i] < size)) { mtr.start(); dberr_t err; - buf_block_t *block = + /* Load the page and apply change buffers. */ + std::ignore = buf_page_get_gen(page_id_t(space_id, page_nos[i]), zip_size, RW_X_LATCH, nullptr, BUF_GET_POSSIBLY_FREED, &mtr, &err, true); - bool remove = !block - || fil_page_get_type(block->page.frame) - != FIL_PAGE_INDEX - || !page_is_leaf(block->page.frame); mtr.commit(); if (err == DB_TABLESPACE_DELETED) { s->x_unlock(); goto tablespace_deleted; } - if (!remove) { - s->x_unlock(); - continue; - } } s->x_unlock(); - if (srv_shutdown_state == SRV_SHUTDOWN_NONE - || srv_fast_shutdown) { + /* During slow shutdown cleanup, we apply all pending IBUF + changes and need to cleanup any left-over IBUF records. There + are a few cases when the changes are already discarded and IBUF + bitmap is cleaned but we miss to delete the record. Even after + the issues are fixed, we need to keep this safety measure for + upgraded DBs with such left over records. */ + if (!slow_shutdown_cleanup) { continue; } @@ -2451,7 +2444,7 @@ ATTRIBUTE_COLD ulint ibuf_contract() space_ids, page_nos, &n_pages); ibuf_mtr_commit(&mtr); - ibuf_read_merge_pages(space_ids, page_nos, n_pages); + ibuf_read_merge_pages(space_ids, page_nos, n_pages, true); return(sum_sizes + 1); } @@ -2532,7 +2525,7 @@ ibuf_merge_space( } #endif /* UNIV_DEBUG */ - ibuf_read_merge_pages(spaces, pages, n_pages); + ibuf_read_merge_pages(spaces, pages, n_pages, false); } return(n_pages); diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index b42c543c..83bdaa97 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -189,13 +189,16 @@ btr_read_autoinc(dict_index_t* index) /** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, or fall back to MAX(auto_increment_column). -@param[in] table table containing an AUTO_INCREMENT column -@param[in] col_no index of the AUTO_INCREMENT column -@return the AUTO_INCREMENT value -@retval 0 on error or if no AUTO_INCREMENT value was used yet */ -ib_uint64_t -btr_read_autoinc_with_fallback(const dict_table_t* table, unsigned col_no) - MY_ATTRIBUTE((nonnull, warn_unused_result)); +@param table table containing an AUTO_INCREMENT column +@param col_no index of the AUTO_INCREMENT column +@param mysql_version TABLE_SHARE::mysql_version +@param max the maximum value of the AUTO_INCREMENT column +@return the AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +uint64_t btr_read_autoinc_with_fallback(const dict_table_t *table, + unsigned col_no, ulong mysql_version, + uint64_t max) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Write the next available AUTO_INCREMENT value to PAGE_ROOT_AUTO_INC. @param[in,out] index clustered index diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index c66a3bfa..5f84328d 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -28,7 +28,6 @@ Created 2/23/1996 Heikki Tuuri #include "dict0dict.h" #include "btr0cur.h" -#include "buf0block_hint.h" #include "btr0btr.h" #include "gis0rtree.h" @@ -332,8 +331,8 @@ struct btr_pcur_t /** BTR_PCUR_ON, BTR_PCUR_BEFORE, or BTR_PCUR_AFTER, depending on whether cursor was on, before, or after the old_rec record */ btr_pcur_pos_t rel_pos= btr_pcur_pos_t(0); - /** buffer block when the position was stored */ - buf::Block_hint block_when_stored; + /** the page identifier of old_rec */ + page_id_t old_page_id{0,0}; /** the modify clock value of the buffer block when the cursor position was stored */ ib_uint64_t modify_clock= 0; @@ -432,7 +431,8 @@ btr_pcur_open( } /** Open a cursor on the first user record satisfying the search condition; -in case of no match, after the last index record. */ +in case of no match, after the last index record. +@return DB_SUCCESS or error code */ MY_ATTRIBUTE((nonnull, warn_unused_result)) inline dberr_t diff --git a/storage/innobase/include/buf0block_hint.h b/storage/innobase/include/buf0block_hint.h deleted file mode 100644 index d4fee7c1..00000000 --- a/storage/innobase/include/buf0block_hint.h +++ /dev/null @@ -1,76 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2020, MariaDB Corporation. -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License, version 2.0, as published by the -Free Software Foundation. - -This program is also distributed with certain software (including but not -limited to OpenSSL) that is licensed under separate terms, as designated in a -particular file or component or in included license documentation. The authors -of MySQL hereby grant you an additional permission to link the program and -your derivative works with the separately licensed software that they have -included with MySQL. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, -for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ -#pragma once -#include "buf0buf.h" - -namespace buf { -class Block_hint { -public: - /** Stores the pointer to the block, which is currently buffer-fixed. - @param block a pointer to a buffer-fixed block to be stored */ - inline void store(buf_block_t *block) - { - ut_ad(block->page.buf_fix_count()); - m_block= block; - m_page_id= block->page.id(); - } - - /** Clears currently stored pointer. */ - inline void clear() { m_block= nullptr; } - - /** Invoke f on m_block(which may be null) - @param f The function to be executed. It will be passed the pointer. - If you wish to use the block pointer subsequently, - you need to ensure you buffer-fix it before returning from f. - @return the return value of f - */ - template <typename F> - bool run_with_hint(const F &f) - { - buffer_fix_block_if_still_valid(); - /* m_block could be changed during f() call, so we use local - variable to remember which block we need to unfix */ - buf_block_t *block= m_block; - bool res= f(block); - if (block) - block->page.unfix(); - return res; - } - - buf_block_t *block() const { return m_block; } - - private: - /** The block pointer stored by store(). */ - buf_block_t *m_block= nullptr; - /** If m_block is non-null, the m_block->page.id at time it was stored. */ - page_id_t m_page_id{0, 0}; - - /** A helper function which checks if m_block is not a dangling pointer and - still points to block with page with m_page_id and if so, buffer-fixes it, - otherwise clear()s it */ - void buffer_fix_block_if_still_valid(); -}; -} // namespace buf diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index cd7cc294..c291615c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -158,14 +158,25 @@ buf_block_free( #define buf_page_get(ID, SIZE, LA, MTR) \ buf_page_get_gen(ID, SIZE, LA, NULL, BUF_GET, MTR) -/** Try to acquire a page latch. -@param rw_latch RW_S_LATCH or RW_X_LATCH +/** Try to buffer-fix a page. @param block guessed block +@param id expected block->page.id() +@return block if it was buffer-fixed +@retval nullptr if the block no longer is valid */ +buf_block_t *buf_page_optimistic_fix(buf_block_t *block, page_id_t id) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + +/** Try to acquire a page latch after buf_page_optimistic_fix(). +@param block buffer-fixed block +@param rw_latch RW_S_LATCH or RW_X_LATCH @param modify_clock expected value of block->modify_clock @param mtr mini-transaction -@return whether the latch was acquired (the page is an allocated file page) */ -bool buf_page_optimistic_get(ulint rw_latch, buf_block_t *block, - uint64_t modify_clock, mtr_t *mtr); +@return block if the latch was acquired +@retval nullptr if block->unfix() was called because it no longer is valid */ +buf_block_t *buf_page_optimistic_get(buf_block_t *block, + rw_lock_type_t rw_latch, + uint64_t modify_clock, mtr_t *mtr) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Try to S-latch a page. Suitable for using when holding the lock_sys latches (as it avoids deadlock). @@ -292,15 +303,6 @@ void buf_block_modify_clock_inc( /*=======================*/ buf_block_t* block); /*!< in: block */ -/********************************************************************//** -Returns the value of the modify clock. The caller must have an s-lock -or x-lock on the block. -@return value */ -UNIV_INLINE -ib_uint64_t -buf_block_get_modify_clock( -/*=======================*/ - buf_block_t* block); /*!< in: block */ #endif /* !UNIV_INNOCHECKSUM */ /** Check if a buffer is all zeroes. @@ -771,17 +773,16 @@ public: @retval DB_FAIL if the page contains the wrong ID */ dberr_t read_complete(const fil_node_t &node); - /** Note that a block is no longer dirty, while not removing - it from buf_pool.flush_list - @param temporary whether the page belongs to the temporary tablespace - @param error whether an error may have occurred while writing */ - inline void write_complete(bool temporary, bool error); + /** Release a write fix after a page write was completed. + @param persistent whether the page belongs to a persistent tablespace + @param error whether an error may have occurred while writing + @param state recently read state() value with the correct io-fix */ + void write_complete(bool persistent, bool error, uint32_t state); /** Write a flushable page to a file or free a freeable block. - @param evict whether to evict the page on write completion @param space tablespace @return whether a page write was initiated and buf_pool.mutex released */ - bool flush(bool evict, fil_space_t *space); + bool flush(fil_space_t *space); /** Notify that a page in a temporary tablespace has been modified. */ void set_temp_modified() @@ -1756,10 +1757,6 @@ public: /** Decrement the number of pending LRU flush */ inline void n_flush_dec(); - /** Decrement the number of pending LRU flush - while holding flush_list_mutex */ - inline void n_flush_dec_holding_mutex(); - /** @return whether flush_list flushing is active */ bool flush_list_active() const { @@ -1912,6 +1909,9 @@ public: /** Free a page whose underlying file page has been freed. */ ATTRIBUTE_COLD void release_freed_page(buf_page_t *bpage) noexcept; + /** Issue a warning that we could not free up buffer pool pages. */ + ATTRIBUTE_COLD void LRU_warn(); + private: /** Temporary memory for page_compressed and encrypted I/O */ struct io_buf_t diff --git a/storage/innobase/include/buf0buf.inl b/storage/innobase/include/buf0buf.inl index b3158cf1..050c8493 100644 --- a/storage/innobase/include/buf0buf.inl +++ b/storage/innobase/include/buf0buf.inl @@ -116,17 +116,3 @@ buf_block_modify_clock_inc( block->modify_clock++; } - -/********************************************************************//** -Returns the value of the modify clock. The caller must have an s-lock -or x-lock on the block. -@return value */ -UNIV_INLINE -ib_uint64_t -buf_block_get_modify_clock( -/*=======================*/ - buf_block_t* block) /*!< in: block */ -{ - ut_ad(block->page.lock.have_any()); - return(block->modify_clock); -} diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h index 0cce514b..cc32a38a 100644 --- a/storage/innobase/include/buf0flu.h +++ b/storage/innobase/include/buf0flu.h @@ -85,16 +85,6 @@ buf_flush_init_for_writing( bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed= nullptr) MY_ATTRIBUTE((warn_unused_result)); -/** Write out dirty blocks from buf_pool.LRU, -and move clean blocks to buf_pool.free. -The caller must invoke buf_dblwr.flush_buffered_writes() -after releasing buf_pool.mutex. -@param max_n wished maximum mumber of blocks flushed -@param evict whether to evict pages after flushing -@return evict ? number of processed pages : number of pages written -@retval 0 if a buf_pool.LRU batch is already running */ -ulint buf_flush_LRU(ulint max_n, bool evict); - /** Wait until a LRU flush batch ends. */ void buf_flush_wait_LRU_batch_end(); /** Wait until all persistent pages are flushed up to a limit. diff --git a/storage/innobase/include/cache.h b/storage/innobase/include/cache.h new file mode 100644 index 00000000..0647cbe6 --- /dev/null +++ b/storage/innobase/include/cache.h @@ -0,0 +1,33 @@ +/***************************************************************************** + +Copyright (c) 2024, MariaDB plc + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once +#include <cstddef> + +#if defined __x86_64__ || defined __aarch64__ || defined __powerpc64__ +struct pmem_control +{ + void (*persist)(const void *, size_t); +public: + pmem_control(); +}; +extern const pmem_control pmem; +# define pmem_persist(buf, size) pmem.persist(buf, size) +#else +void pmem_persist(const void *buf, size_t size); +#endif diff --git a/storage/innobase/include/data0data.h b/storage/innobase/include/data0data.h index a5356e0d..fcb543eb 100644 --- a/storage/innobase/include/data0data.h +++ b/storage/innobase/include/data0data.h @@ -339,15 +339,12 @@ dtuple_set_types_binary( dtuple_t* tuple, /*!< in: data tuple */ ulint n) /*!< in: number of fields to set */ MY_ATTRIBUTE((nonnull)); -/**********************************************************************//** -Checks if a dtuple contains an SQL null value. -@return TRUE if some field is SQL null */ +/** Checks if a dtuple contains an SQL null value. +@param tuple tuple +@param fields_number number of fields in the tuple to check +@return true if some field is SQL null */ UNIV_INLINE -ibool -dtuple_contains_null( -/*=================*/ - const dtuple_t* tuple) /*!< in: dtuple */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); +bool dtuple_contains_null(const dtuple_t *tuple, ulint fields_number = 0); /**********************************************************//** Checks that a data field is typed. Asserts an error if not. @return TRUE if ok */ diff --git a/storage/innobase/include/data0data.inl b/storage/innobase/include/data0data.inl index 2d1bf5a2..b6c6ace8 100644 --- a/storage/innobase/include/data0data.inl +++ b/storage/innobase/include/data0data.inl @@ -596,28 +596,18 @@ data_write_sql_null( memset(data, 0, len); } -/**********************************************************************//** -Checks if a dtuple contains an SQL null value. -@return TRUE if some field is SQL null */ +/** Checks if a dtuple contains an SQL null value. +@param tuple tuple +@param fields_number number of fields in the tuple to check +@return true if some field is SQL null */ UNIV_INLINE -ibool -dtuple_contains_null( -/*=================*/ - const dtuple_t* tuple) /*!< in: dtuple */ +bool dtuple_contains_null(const dtuple_t *tuple, ulint fields_number) { - ulint n; - ulint i; - - n = dtuple_get_n_fields(tuple); - - for (i = 0; i < n; i++) { - if (dfield_is_null(dtuple_get_nth_field(tuple, i))) { - - return(TRUE); - } - } - - return(FALSE); + ulint n= fields_number ? fields_number : dtuple_get_n_fields(tuple); + for (ulint i= 0; i < n; i++) + if (dfield_is_null(dtuple_get_nth_field(tuple, i))) + return true; + return false; } /**************************************************************//** diff --git a/storage/innobase/include/db0err.h b/storage/innobase/include/db0err.h index 64182aab..960ec390 100644 --- a/storage/innobase/include/db0err.h +++ b/storage/innobase/include/db0err.h @@ -32,23 +32,25 @@ Created 5/24/1996 Heikki Tuuri enum dberr_t { DB_SUCCESS, - DB_SUCCESS_LOCKED_REC = 9, /*!< like DB_SUCCESS, but a new + DB_SUCCESS_LOCKED_REC= 9, /*!< like DB_SUCCESS, but a new explicit record lock was created */ /* The following are error codes */ - DB_ERROR = 11, + DB_RECORD_CHANGED, + DB_ERROR, DB_INTERRUPTED, DB_OUT_OF_MEMORY, DB_OUT_OF_FILE_SPACE, DB_LOCK_WAIT, DB_DEADLOCK, - DB_ROLLBACK, DB_DUPLICATE_KEY, DB_MISSING_HISTORY, /*!< required history data has been deleted due to lack of space in rollback segment */ - DB_CLUSTER_NOT_FOUND = 30, - DB_TABLE_NOT_FOUND, +#ifdef WITH_WSREP + DB_ROLLBACK, +#endif + DB_TABLE_NOT_FOUND= 31, DB_TOO_BIG_RECORD, /*!< a record in an index would not fit on a compressed page, or it would become bigger than 1/2 free space in diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 5fafb2c5..3baac658 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -35,6 +35,7 @@ Created 1/8/1996 Heikki Tuuri #include <my_sys.h> #include <deque> +class MDL_context; class MDL_ticket; /** the first table or index ID for other than hard-coded system tables */ @@ -139,6 +140,21 @@ dict_acquire_mdl_shared(dict_table_t *table, MDL_ticket **mdl, dict_table_op_t table_op= DICT_TABLE_OP_NORMAL); +/** Acquire MDL shared for the table name. +@tparam trylock whether to use non-blocking operation +@param[in,out] table table object +@param[in,out] mdl_context MDL context +@param[out] mdl MDL ticket +@param[in] table_op operation to perform when opening +@return table object after locking MDL shared +@retval nullptr if the table is not readable, or if trylock && MDL blocked */ +template<bool trylock> +__attribute__((nonnull, warn_unused_result)) +dict_table_t* +dict_acquire_mdl_shared(dict_table_t *table, + MDL_context *mdl_context, MDL_ticket **mdl, + dict_table_op_t table_op); + /** Look up a table by numeric identifier. @param[in] table_id table identifier @param[in] dict_locked data dictionary locked @@ -1314,13 +1330,7 @@ class dict_sys_t std::atomic<ulonglong> latch_ex_wait_start; /** the rw-latch protecting the data dictionary cache */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_lock latch; -#ifdef UNIV_DEBUG - /** whether latch is being held in exclusive mode (by any thread) */ - Atomic_relaxed<pthread_t> latch_ex; - /** number of S-latch holders */ - Atomic_counter<uint32_t> latch_readers; -#endif + alignas(CPU_LEVEL1_DCACHE_LINESIZE) IF_DBUG(srw_lock_debug,srw_lock) latch; public: /** Indexes of SYS_TABLE[] */ enum @@ -1488,15 +1498,12 @@ public: } #ifdef UNIV_DEBUG - /** @return whether any thread (not necessarily the current thread) - is holding the latch; that is, this check may return false - positives */ - bool frozen() const { return latch_readers || latch_ex; } - /** @return whether any thread (not necessarily the current thread) - is holding a shared latch */ - bool frozen_not_locked() const { return latch_readers; } + /** @return whether the current thread is holding the latch */ + bool frozen() const { return latch.have_any(); } + /** @return whether the current thread is holding a shared latch */ + bool frozen_not_locked() const { return latch.have_rd(); } /** @return whether the current thread holds the exclusive latch */ - bool locked() const { return latch_ex == pthread_self(); } + bool locked() const { return latch.have_wr(); } #endif private: /** Acquire the exclusive latch */ @@ -1511,13 +1518,7 @@ public: /** Exclusively lock the dictionary cache. */ void lock(SRW_LOCK_ARGS(const char *file, unsigned line)) { - if (latch.wr_lock_try()) - { - ut_ad(!latch_readers); - ut_ad(!latch_ex); - ut_d(latch_ex= pthread_self()); - } - else + if (!latch.wr_lock_try()) lock_wait(SRW_LOCK_ARGS(file, line)); } @@ -1530,27 +1531,11 @@ public: ATTRIBUTE_NOINLINE void unfreeze(); #else /** Unlock the data dictionary cache. */ - void unlock() - { - ut_ad(latch_ex == pthread_self()); - ut_ad(!latch_readers); - ut_d(latch_ex= 0); - latch.wr_unlock(); - } + void unlock() { latch.wr_unlock(); } /** Acquire a shared lock on the dictionary cache. */ - void freeze() - { - latch.rd_lock(); - ut_ad(!latch_ex); - ut_d(latch_readers++); - } + void freeze() { latch.rd_lock(); } /** Release a shared lock on the dictionary cache. */ - void unfreeze() - { - ut_ad(!latch_ex); - ut_ad(latch_readers--); - latch.rd_unlock(); - } + void unfreeze() { latch.rd_unlock(); } #endif /** Estimate the used memory occupied by the data dictionary diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index fde2a714..0268a280 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1010,8 +1010,6 @@ struct dict_index_t { /*!< number of columns the user defined to be in the index: in the internal representation we add more columns */ - unsigned nulls_equal:1; - /*!< if true, SQL NULL == SQL NULL */ unsigned n_uniq:10;/*!< number of fields from the beginning which are enough to determine an index entry uniquely */ @@ -2448,6 +2446,9 @@ public: /** @return number of unique columns in FTS_DOC_ID index */ unsigned fts_n_uniq() const { return versioned() ? 2 : 1; } + /** @return the index for that starts with a specific column */ + dict_index_t *get_index(const dict_col_t &col) const; + /** Create metadata. @param name table name @param space tablespace diff --git a/storage/innobase/include/dict0mem.inl b/storage/innobase/include/dict0mem.inl index d60ee5d9..edb7cf92 100644 --- a/storage/innobase/include/dict0mem.inl +++ b/storage/innobase/include/dict0mem.inl @@ -63,6 +63,5 @@ dict_mem_fill_index_struct( & index->MAX_N_FIELDS; /* The '1 +' above prevents allocation of an empty mem block */ - index->nulls_equal = false; ut_d(index->magic_n = DICT_INDEX_MAGIC_N); } diff --git a/storage/innobase/include/dyn0buf.h b/storage/innobase/include/dyn0buf.h index 06af4dcc..7a4e6760 100644 --- a/storage/innobase/include/dyn0buf.h +++ b/storage/innobase/include/dyn0buf.h @@ -57,11 +57,7 @@ public: /** Gets the number of used bytes in a block. @return number of bytes used */ - ulint used() const - MY_ATTRIBUTE((warn_unused_result)) - { - return(static_cast<ulint>(m_used & ~DYN_BLOCK_FULL_FLAG)); - } + uint32_t used() const { return m_used; } /** Gets pointer to the start of data. @@ -153,8 +149,7 @@ public: /** Storage */ byte m_data[MAX_DATA_SIZE]; - /** number of data bytes used in this block; - DYN_BLOCK_FULL_FLAG is set when the block becomes full */ + /** number of data bytes used in this block */ uint32_t m_used; friend class mtr_buf_t; diff --git a/storage/innobase/include/dyn0types.h b/storage/innobase/include/dyn0types.h index 83d0b0d6..af7f663d 100644 --- a/storage/innobase/include/dyn0types.h +++ b/storage/innobase/include/dyn0types.h @@ -33,7 +33,4 @@ Created 2013-03-16 Sunny Bains /** This is the initial 'payload' size of a dynamic array */ #define DYN_ARRAY_DATA_SIZE 512 -/** Flag for dyn_block_t::used that indicates a full block */ -#define DYN_BLOCK_FULL_FLAG 0x1000000UL - #endif /* dyn0types_h */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index cdc32515..dfda1178 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -63,7 +63,7 @@ enum srv_flush_t SRV_LITTLESYNC, /** do not flush after writing */ SRV_NOSYNC, - /** invoke os_file_set_nocache() on data files. This implies using + /** Open or create files with O_DIRECT. This implies using unbuffered I/O but still fdatasync(), because some filesystems might not flush meta-data on write completion */ SRV_O_DIRECT, @@ -347,7 +347,6 @@ struct fil_space_t final ~fil_space_t() { ut_ad(!latch_owner); - ut_ad(!latch_count); latch.destroy(); } @@ -411,9 +410,9 @@ private: /** The reference count */ static constexpr uint32_t PENDING= ~(STOPPING | CLOSING | NEEDS_FSYNC); /** latch protecting all page allocation bitmap pages */ - srw_lock latch; + IF_DBUG(srw_lock_debug, srw_lock) latch; + /** the thread that holds the exclusive latch, or 0 */ pthread_t latch_owner; - ut_d(Atomic_relaxed<uint32_t> latch_count;) public: /** MariaDB encryption data */ fil_space_crypt_t *crypt_data; @@ -1004,40 +1003,32 @@ public: bool recheck, bool encrypt); #ifdef UNIV_DEBUG - bool is_latched() const { return latch_count != 0; } + bool is_latched() const { return latch.have_any(); } #endif - bool is_owner() const { return latch_owner == pthread_self(); } + bool is_owner() const + { + const bool owner{latch_owner == pthread_self()}; + ut_ad(owner == latch.have_wr()); + return owner; + } /** Acquire the allocation latch in exclusive mode */ void x_lock() { latch.wr_lock(SRW_LOCK_CALL); ut_ad(!latch_owner); latch_owner= pthread_self(); - ut_ad(!latch_count.fetch_add(1)); } /** Release the allocation latch from exclusive mode */ void x_unlock() { - ut_ad(latch_count.fetch_sub(1) == 1); ut_ad(latch_owner == pthread_self()); latch_owner= 0; latch.wr_unlock(); } /** Acquire the allocation latch in shared mode */ - void s_lock() - { - ut_ad(!is_owner()); - latch.rd_lock(SRW_LOCK_CALL); - ut_ad(!latch_owner); - ut_d(latch_count.fetch_add(1)); - } + void s_lock() { latch.rd_lock(SRW_LOCK_CALL); } /** Release the allocation latch from shared mode */ - void s_unlock() - { - ut_ad(latch_count.fetch_sub(1)); - ut_ad(!latch_owner); - latch.rd_unlock(); - } + void s_unlock() { latch.rd_unlock(); } typedef span<const char> name_type; @@ -1637,17 +1628,34 @@ void fil_close_tablespace(uint32_t id); /*******************************************************************//** Allocates and builds a file name from a path, a table or tablespace name and a suffix. The string must be freed by caller with ut_free(). -@param[in] path NULL or the directory path or the full path and filename. +@param[in] path nullptr or the directory path or the full path and filename @param[in] name {} if path is full, or Table/Tablespace name -@param[in] ext the file extension to use -@param[in] trim_name true if the last name on the path should be trimmed. +@param[in] extension the file extension to use +@param[in] trim_name true if the last name on the path should be trimmed @return own: file name */ -char* fil_make_filepath(const char *path, const fil_space_t::name_type &name, - ib_extention ext, bool trim_name); +char* fil_make_filepath_low(const char *path, + const fil_space_t::name_type &name, + ib_extention extension, bool trim_name); char *fil_make_filepath(const char* path, const table_name_t name, ib_extention suffix, bool strip_name); +/** Wrapper function over fil_make_filepath_low to build file name. +@param path nullptr or the directory path or the full path and filename +@param name {} if path is full, or Table/Tablespace name +@param extension the file extension to use +@param trim_name true if the last name on the path should be trimmed +@return own: file name */ +static inline char* +fil_make_filepath(const char* path, const fil_space_t::name_type &name, + ib_extention extension, bool trim_name) +{ + /* If we are going to strip a name off the path, there better be a + path and a new name to put back on. */ + ut_ad(!trim_name || (path && name.data())); + return fil_make_filepath_low(path, name, extension, trim_name); +} + /** Create a tablespace file. @param[in] space_id Tablespace ID @param[in] name Tablespace name in dbname/tablename format. diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 26261554..99459bcb 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -209,24 +209,6 @@ typedef byte fseg_inode_t; static constexpr byte FSEG_MAGIC_N_BYTES[4]={0x05,0xd6,0x69,0xd2}; -#define FSEG_FILLFACTOR 8 /* If the number of unused but reserved - pages in a segment is less than - reserved pages / FSEG_FILLFACTOR, - and there are - at least FSEG_FRAG_LIMIT used pages, - then we allow a new empty extent to - be added to the segment in - fseg_alloc_free_page_general(). - Otherwise, we - use unused pages of the segment. */ - -#define FSEG_FRAG_LIMIT FSEG_FRAG_ARR_N_SLOTS - /* If the segment has >= this many - used pages, it may be expanded by - allocating extents to the segment; - until that only individual fragment - pages are allocated from the space */ - #define FSEG_FREE_LIST_LIMIT 40 /* If the reserved size of a segment is at least this many extents, we allow extents to be put to the free @@ -294,7 +276,7 @@ Determine if a page is marked free. @param[in] descr extent descriptor @param[in] offset page offset within extent @return whether the page is free */ -inline bool xdes_is_free(const xdes_t *descr, ulint offset) +inline bool xdes_is_free(const xdes_t *descr, uint32_t offset) { ut_ad(offset < FSP_EXTENT_SIZE); ulint index= XDES_FREE_BIT + XDES_BITS_PER_PAGE * offset; diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index c0151b44..1d2b409b 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -163,6 +163,9 @@ struct fts_token_t; struct fts_doc_ids_t; struct fts_index_cache_t; +/** Compare two DOC_ID. */ +int fts_doc_id_cmp(const void *p1, const void *p2) + __attribute__((nonnull, warn_unused_result)); /** Initialize the "fts_table" for internal query into FTS auxiliary tables */ @@ -412,6 +415,9 @@ inline void fts_doc_ids_free(fts_doc_ids_t* doc_ids) mem_heap_free(static_cast<mem_heap_t*>(doc_ids->self_heap->arg)); } +/** Sort an array of doc_id */ +void fts_doc_ids_sort(ib_vector_t *doc_ids); + /******************************************************************//** Notify the FTS system about an operation on an FTS-indexed table. */ void diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index ae0bb036..04faceb9 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -271,27 +271,6 @@ fts_index_fetch_nodes( word, /*!< in: the word to fetch */ fts_fetch_t* fetch) /*!< in: fetch callback.*/ MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Compare two fts_trx_table_t instances, we actually compare the -table id's here. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int -fts_trx_table_cmp( -/*==============*/ - const void* v1, /*!< in: id1 */ - const void* v2) /*!< in: id2 */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); -/******************************************************************//** -Compare a table id with a trx_table_t table id. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int -fts_trx_table_id_cmp( -/*=================*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() /******************************************************************//** diff --git a/storage/innobase/include/fts0priv.inl b/storage/innobase/include/fts0priv.inl index 3cb09c92..3d937bb3 100644 --- a/storage/innobase/include/fts0priv.inl +++ b/storage/innobase/include/fts0priv.inl @@ -52,47 +52,3 @@ fts_read_object_id( if the id is HEX or DEC and do the right thing with it. */ return(sscanf(str, UINT64PFx, id) == 1); } - -/******************************************************************//** -Compare two fts_trx_table_t instances. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int -fts_trx_table_cmp( -/*==============*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ -{ - const dict_table_t* table1 - = (*static_cast<const fts_trx_table_t* const*>(p1))->table; - - const dict_table_t* table2 - = (*static_cast<const fts_trx_table_t* const*>(p2))->table; - - return((table1->id > table2->id) - ? 1 - : (table1->id == table2->id) - ? 0 - : -1); -} - -/******************************************************************//** -Compare a table id with a fts_trx_table_t table id. -@return < 0 if n1 < n2, 0 if n1 == n2,> 0 if n1 > n2 */ -UNIV_INLINE -int -fts_trx_table_id_cmp( -/*=================*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ -{ - const uintmax_t* table_id = static_cast<const uintmax_t*>(p1); - const dict_table_t* table2 - = (*static_cast<const fts_trx_table_t* const*>(p2))->table; - - return((*table_id > table2->id) - ? 1 - : (*table_id == table2->id) - ? 0 - : -1); -} diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index fb278d54..7b95348b 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -278,44 +278,6 @@ struct fts_token_t { extern const fts_index_selector_t fts_index_selector[]; /******************************************************************//** -Compare two fts_trx_row_t instances doc_ids. */ -UNIV_INLINE -int -fts_trx_row_doc_id_cmp( -/*===================*/ - /*!< out: - < 0 if n1 < n2, - 0 if n1 == n2, - > 0 if n1 > n2 */ - const void* p1, /*!< in: id1 */ - const void* p2); /*!< in: id2 */ - -/******************************************************************//** -Compare two fts_ranking_t instances doc_ids. */ -UNIV_INLINE -int -fts_ranking_doc_id_cmp( -/*===================*/ - /*!< out: - < 0 if n1 < n2, - 0 if n1 == n2, - > 0 if n1 > n2 */ - const void* p1, /*!< in: id1 */ - const void* p2); /*!< in: id2 */ - -/******************************************************************//** -Compare two doc_ids. */ -UNIV_INLINE -int fts_doc_id_cmp( -/*==================*/ - /*!< out: - < 0 if n1 < n2, - 0 if n1 == n2, - > 0 if n1 > n2 */ - const void* p1, /*!< in: id1 */ - const void* p2); /*!< in: id2 */ - -/******************************************************************//** Duplicate a string. */ UNIV_INLINE void diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index facc1e5c..5b57cad7 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -47,53 +47,6 @@ fts_string_dup( } /******************************************************************//** -Compare two fts_trx_row_t doc_ids. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int -fts_trx_row_doc_id_cmp( -/*===================*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ -{ - const fts_trx_row_t* tr1 = (const fts_trx_row_t*) p1; - const fts_trx_row_t* tr2 = (const fts_trx_row_t*) p2; - - return((int)(tr1->doc_id - tr2->doc_id)); -} - -/******************************************************************//** -Compare two fts_ranking_t doc_ids. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int -fts_ranking_doc_id_cmp( -/*===================*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ -{ - const fts_ranking_t* rk1 = (const fts_ranking_t*) p1; - const fts_ranking_t* rk2 = (const fts_ranking_t*) p2; - - return((int)(rk1->doc_id - rk2->doc_id)); -} - -/******************************************************************//** -Compare two doc_ids. -@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */ -UNIV_INLINE -int fts_doc_id_cmp( -/*==================*/ - const void* p1, /*!< in: id1 */ - const void* p2) /*!< in: id2 */ -{ - const doc_id_t* up1 = static_cast<const doc_id_t*>(p1); - const doc_id_t* up2 = static_cast<const doc_id_t*>(p2); - - return static_cast<int>(*up1 - *up2); -} - -/******************************************************************//** Get the first character's code position for FTS index partition */ extern ulint diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index 746dab80..1adec365 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -78,34 +78,40 @@ void flst_init(const buf_block_t &block, byte *base, mtr_t *mtr) MY_ATTRIBUTE((nonnull)); /** Append a file list node to a list. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] add block to be added -@param[in] aoffset byte offset of the node to be added -@param[in,out] mtr mini-transaction +@param base base node block +@param boffset byte offset of the base node +@param add block to be added +@param aoffset byte offset of the node to be added +@param limit fil_space_t::free_limit +@param mtr mini-transaction @return error code */ dberr_t flst_add_last(buf_block_t *base, uint16_t boffset, - buf_block_t *add, uint16_t aoffset, mtr_t *mtr) + buf_block_t *add, uint16_t aoffset, + uint32_t limit, mtr_t *mtr) MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Prepend a file list node to a list. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] add block to be added -@param[in] aoffset byte offset of the node to be added -@param[in,out] mtr mini-transaction +@param base base node block +@param boffset byte offset of the base node +@param add block to be added +@param aoffset byte offset of the node to be added +@param limit fil_space_t::free_limit +@param mtr mini-transaction @return error code */ dberr_t flst_add_first(buf_block_t *base, uint16_t boffset, - buf_block_t *add, uint16_t aoffset, mtr_t *mtr) + buf_block_t *add, uint16_t aoffset, + uint32_t limit, mtr_t *mtr) MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Remove a file list node. -@param[in,out] base base node block -@param[in] boffset byte offset of the base node -@param[in,out] cur block to be removed -@param[in] coffset byte offset of the current record to be removed -@param[in,out] mtr mini-transaction +@param base base node block +@param boffset byte offset of the base node +@param cur block to be removed +@param coffset byte offset of the current record to be removed +@param limit fil_space_t::free_limit +@param mtr mini-transaction @return error code */ dberr_t flst_remove(buf_block_t *base, uint16_t boffset, - buf_block_t *cur, uint16_t coffset, mtr_t *mtr) + buf_block_t *cur, uint16_t coffset, + uint32_t limit, mtr_t *mtr) MY_ATTRIBUTE((nonnull, warn_unused_result)); /** @return the length of a list */ @@ -117,11 +123,9 @@ inline uint32_t flst_get_len(const flst_base_node_t *base) /** @return a file address */ inline fil_addr_t flst_read_addr(const byte *faddr) { - fil_addr_t addr= { mach_read_from_4(faddr + FIL_ADDR_PAGE), - mach_read_from_2(faddr + FIL_ADDR_BYTE) }; - ut_a(addr.page == FIL_NULL || addr.boffset >= FIL_PAGE_DATA); - ut_a(ut_align_offset(faddr, srv_page_size) >= FIL_PAGE_DATA); - return addr; + ut_ad(ut_align_offset(faddr, srv_page_size) >= FIL_PAGE_DATA); + return fil_addr_t{mach_read_from_4(faddr + FIL_ADDR_PAGE), + mach_read_from_2(faddr + FIL_ADDR_BYTE)}; } /** @return list first node address */ diff --git a/storage/innobase/include/gis0type.h b/storage/innobase/include/gis0type.h index d6a4ef67..2dc25a89 100644 --- a/storage/innobase/include/gis0type.h +++ b/storage/innobase/include/gis0type.h @@ -66,10 +66,7 @@ typedef std::vector<rtr_rec_t, ut_allocator<rtr_rec_t> > rtr_rec_vector; /* Structure for matched records on the leaf page */ typedef struct matched_rec { - byte* bufp; /*!< aligned buffer point */ - byte rec_buf[UNIV_PAGE_SIZE_MAX * 2]; - /*!< buffer used to copy matching rec */ - buf_block_t block; /*!< the shadow buffer block */ + buf_block_t* block; /*!< the shadow buffer block */ ulint used; /*!< memory used */ rtr_rec_vector* matched_recs; /*!< vector holding the matching rec */ mysql_mutex_t rtr_match_mutex;/*!< mutex protect the match_recs @@ -107,7 +104,6 @@ typedef struct rtr_info{ /*!< mutex protect the "path" vector */ rtr_mbr_t mbr; /*!< the search MBR */ que_thr_t* thr; /*!< the search thread */ - mem_heap_t* heap; /*!< memory heap */ btr_cur_t* cursor; /*!< cursor used for search */ dict_index_t* index; /*!< index it is searching */ bool need_prdt_lock; diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 59ee7f55..08b9f4bc 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -438,6 +438,13 @@ dberr_t lock_table_for_trx(dict_table_t *table, trx_t *trx, lock_mode mode, bool no_wait= false) MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Lock the child tables of a table. +@param table parent table +@param trx transaction +@return error code */ +dberr_t lock_table_children(dict_table_t *table, trx_t *trx) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + /** Exclusively lock the data dictionary tables. @param trx dictionary transaction @return error code @@ -724,13 +731,8 @@ private: bool m_initialised; /** mutex proteting the locks */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch; -#ifdef UNIV_DEBUG - /** The owner of exclusive latch (0 if none); protected by latch */ - std::atomic<pthread_t> writer{0}; - /** Number of shared latches */ - std::atomic<ulint> readers{0}; -#endif + alignas(CPU_LEVEL1_DCACHE_LINESIZE) + IF_DBUG(srw_lock_debug,srw_spin_lock) latch; #ifdef SUX_LOCK_GENERIC protected: /** mutex for hash_latch::wait() */ @@ -789,71 +791,35 @@ public: void wr_lock() { mysql_mutex_assert_not_owner(&wait_mutex); - ut_ad(!is_writer()); latch.wr_lock(); - ut_ad(!writer.exchange(pthread_self(), - std::memory_order_relaxed)); } /** Release exclusive lock_sys.latch */ - void wr_unlock() - { - ut_ad(writer.exchange(0, std::memory_order_relaxed) == - pthread_self()); - latch.wr_unlock(); - } + void wr_unlock() { latch.wr_unlock(); } /** Acquire shared lock_sys.latch */ void rd_lock() { mysql_mutex_assert_not_owner(&wait_mutex); - ut_ad(!is_writer()); latch.rd_lock(); - ut_ad(!writer.load(std::memory_order_relaxed)); - ut_d(readers.fetch_add(1, std::memory_order_relaxed)); } /** Release shared lock_sys.latch */ - void rd_unlock() - { - ut_ad(!is_writer()); - ut_ad(readers.fetch_sub(1, std::memory_order_relaxed)); - latch.rd_unlock(); - } + void rd_unlock() { latch.rd_unlock(); } #endif /** Try to acquire exclusive lock_sys.latch @return whether the latch was acquired */ - bool wr_lock_try() - { - ut_ad(!is_writer()); - if (!latch.wr_lock_try()) return false; - ut_ad(!writer.exchange(pthread_self(), - std::memory_order_relaxed)); - return true; - } + bool wr_lock_try() { return latch.wr_lock_try(); } /** Try to acquire shared lock_sys.latch @return whether the latch was acquired */ - bool rd_lock_try() - { - ut_ad(!is_writer()); - if (!latch.rd_lock_try()) return false; - ut_ad(!writer.load(std::memory_order_relaxed)); - ut_d(readers.fetch_add(1, std::memory_order_relaxed)); - return true; - } + bool rd_lock_try() { return latch.rd_lock_try(); } /** Assert that wr_lock() has been invoked by this thread */ - void assert_locked() const { ut_ad(is_writer()); } + void assert_locked() const { ut_ad(latch.have_wr()); } /** Assert that wr_lock() has not been invoked by this thread */ - void assert_unlocked() const { ut_ad(!is_writer()); } + void assert_unlocked() const { ut_ad(!latch.have_wr()); } #ifdef UNIV_DEBUG /** @return whether the current thread is the lock_sys.latch writer */ - bool is_writer() const - { -# ifdef SUX_LOCK_GENERIC - return writer.load(std::memory_order_relaxed) == pthread_self(); -# else - return writer.load(std::memory_order_relaxed) == pthread_self() || - (xtest() && !latch.is_locked_or_waiting()); -# endif - } + bool is_writer() const { return latch.have_wr(); } + /** @return whether the current thread is holding lock_sys.latch */ + bool is_holder() const { return latch.have_any(); } /** Assert that a lock shard is exclusively latched (by some thread) */ void assert_locked(const lock_t &lock) const; /** Assert that a table lock shard is exclusively latched by this thread */ @@ -965,14 +931,14 @@ extern lock_sys_t lock_sys; /** @return the index of an array element */ inline ulint lock_sys_t::hash_table::calc_hash(ulint fold) const { - ut_ad(lock_sys.is_writer() || lock_sys.readers); + ut_ad(lock_sys.is_holder()); return calc_hash(fold, n_cells); } /** Get a hash table cell. */ inline hash_cell_t *lock_sys_t::hash_table::cell_get(ulint fold) const { - ut_ad(lock_sys.is_writer() || lock_sys.readers); + ut_ad(lock_sys.is_holder()); return &array[calc_hash(fold)]; } diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 22c0c963..2500ac05 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -28,6 +28,9 @@ MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation. #include "log0log.h" +/** innodb_encrypt_log: whether to encrypt the redo log */ +extern my_bool srv_encrypt_log; + /** Initialize the redo log encryption key and random parameters when creating a new redo log. The random parameters will be persisted in the log header. diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 54851ca0..cef0dcae 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -132,6 +132,9 @@ public: /** Redo log buffer */ struct log_t { + /** The maximum buf_size */ + static constexpr unsigned buf_size_max= os_file_request_size_max; + /** The original (not version-tagged) InnoDB redo log format */ static constexpr uint32_t FORMAT_3_23= 0; /** The MySQL 5.7.9/MariaDB 10.2.2 log format */ @@ -165,60 +168,92 @@ struct log_t static constexpr lsn_t FIRST_LSN= START_OFFSET; private: - /** The log sequence number of the last change of durable InnoDB files */ + /** the lock bit in buf_free */ + static constexpr size_t buf_free_LOCK= ~(~size_t{0} >> 1); alignas(CPU_LEVEL1_DCACHE_LINESIZE) + /** first free offset within buf used; + the most significant bit is set by lock_lsn() to protect this field + as well as write_to_buf, waits */ + std::atomic<size_t> buf_free; +public: + /** number of write requests (to buf); protected by lock_lsn() or lsn_lock */ + size_t write_to_buf; + /** log record buffer, written to by mtr_t::commit() */ + byte *buf; +private: + /** The log sequence number of the last change of durable InnoDB files; + protected by lock_lsn() or lsn_lock or latch.wr_lock() */ std::atomic<lsn_t> lsn; /** the first guaranteed-durable log sequence number */ std::atomic<lsn_t> flushed_to_disk_lsn; - /** log sequence number when log resizing was initiated, or 0 */ - std::atomic<lsn_t> resize_lsn; - /** set when there may be need to initiate a log checkpoint. - This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */ - std::atomic<bool> need_checkpoint; +public: + /** number of append_prepare_wait(); protected by lock_lsn() or lsn_lock */ + size_t waits; + /** innodb_log_buffer_size (size of buf,flush_buf if !is_pmem(), in bytes) */ + unsigned buf_size; + /** log file size in bytes, including the header */ + lsn_t file_size; + +#ifdef LOG_LATCH_DEBUG + typedef srw_lock_debug log_rwlock; + typedef srw_mutex log_lsn_lock; -#if defined(__aarch64__) - /* On ARM, we do more spinning */ + bool latch_have_wr() const { return latch.have_wr(); } + bool latch_have_rd() const { return latch.have_rd(); } + bool latch_have_any() const { return latch.have_any(); } +#else +# ifndef UNIV_DEBUG +# elif defined SUX_LOCK_GENERIC + bool latch_have_wr() const { return true; } + bool latch_have_rd() const { return true; } + bool latch_have_any() const { return true; } +# else + bool latch_have_wr() const { return latch.is_write_locked(); } + bool latch_have_rd() const { return latch.is_locked(); } + bool latch_have_any() const { return latch.is_locked(); } +# endif +# ifdef __aarch64__ + /* On ARM, we spin more */ typedef srw_spin_lock log_rwlock; typedef pthread_mutex_wrapper<true> log_lsn_lock; -#else +# else typedef srw_lock log_rwlock; typedef srw_mutex log_lsn_lock; +# endif #endif - -public: - /** rw-lock protecting writes to buf; normal mtr_t::commit() - outside any log checkpoint is covered by a shared latch */ + /** exclusive latch for checkpoint, shared for mtr_t::commit() to buf */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock latch; -private: - /** mutex protecting buf_free et al, together with latch */ - log_lsn_lock lsn_lock; -public: - /** first free offset within buf use; protected by lsn_lock */ - Atomic_relaxed<size_t> buf_free; - /** number of write requests (to buf); protected by lsn_lock */ - size_t write_to_buf; - /** number of append_prepare_wait(); protected by lsn_lock */ - size_t waits; -private: + + /** number of std::swap(buf, flush_buf) and writes from buf to log; + protected by latch.wr_lock() */ + ulint write_to_log; + /** Last written LSN */ lsn_t write_lsn; -public: - /** log record buffer, written to by mtr_t::commit() */ - byte *buf; + /** buffer for writing data to ib_logfile0, or nullptr if is_pmem() In write_buf(), buf and flush_buf are swapped */ byte *flush_buf; - /** number of std::swap(buf, flush_buf) and writes from buf to log; - protected by latch.wr_lock() */ - ulint write_to_log; - + /** set when there may be need to initiate a log checkpoint. + This must hold if lsn - last_checkpoint_lsn > max_checkpoint_age. */ + std::atomic<bool> need_checkpoint; + /** whether a checkpoint is pending; protected by latch.wr_lock() */ + Atomic_relaxed<bool> checkpoint_pending; + /** next checkpoint number (protected by latch.wr_lock()) */ + byte next_checkpoint_no; + /** recommended maximum buf_free size, after which the buffer is flushed */ + unsigned max_buf_free; /** Log sequence number when a log file overwrite (broken crash recovery) was noticed. Protected by latch.wr_lock(). */ lsn_t overwrite_warned; - /** innodb_log_buffer_size (size of buf,flush_buf if !is_pmem(), in bytes) */ - size_t buf_size; + /** latest completed checkpoint (protected by latch.wr_lock()) */ + Atomic_relaxed<lsn_t> last_checkpoint_lsn; + /** next checkpoint LSN (protected by latch.wr_lock()) */ + lsn_t next_checkpoint_lsn; + /** Log file */ + log_file_t log; private: /** Log file being constructed during resizing; protected by latch */ log_file_t resize_log; @@ -229,18 +264,14 @@ private: /** Buffer for writing to resize_log; @see flush_buf */ byte *resize_flush_buf; - void init_lsn_lock() {lsn_lock.init(); } - void lock_lsn() { lsn_lock.wr_lock(); } - void unlock_lsn() {lsn_lock.wr_unlock(); } - void destroy_lsn_lock() { lsn_lock.destroy(); } - -public: - /** recommended maximum size of buf, after which the buffer is flushed */ - size_t max_buf_free; + /** Special implementation of lock_lsn() for IA-32 and AMD64 */ + void lsn_lock_bts() noexcept; + /** Acquire a lock for updating buf_free and related fields. + @return the value of buf_free */ + size_t lock_lsn() noexcept; - /** log file size in bytes, including the header */ - lsn_t file_size; -private: + /** log sequence number when log resizing was initiated, or 0 */ + std::atomic<lsn_t> resize_lsn; /** the log sequence number at the start of the log file */ lsn_t first_lsn; #if defined __linux__ || defined _WIN32 @@ -250,8 +281,6 @@ private: public: /** format of the redo log: e.g., FORMAT_10_8 */ uint32_t format; - /** Log file */ - log_file_t log; #if defined __linux__ || defined _WIN32 /** whether file system caching is enabled for the log */ my_bool log_buffered; @@ -279,21 +308,29 @@ public: /*!< this is the maximum allowed value for lsn - last_checkpoint_lsn when a new query step is started */ - /** latest completed checkpoint (protected by latch.wr_lock()) */ - Atomic_relaxed<lsn_t> last_checkpoint_lsn; - /** next checkpoint LSN (protected by log_sys.latch) */ - lsn_t next_checkpoint_lsn; - /** next checkpoint number (protected by latch.wr_lock()) */ - ulint next_checkpoint_no; - /** whether a checkpoint is pending */ - Atomic_relaxed<bool> checkpoint_pending; /** buffer for checkpoint header */ byte *checkpoint_buf; /* @} */ +private: + /** A lock when the spin-only lock_lsn() is not being used */ + log_lsn_lock lsn_lock; +public: + bool is_initialised() const noexcept { return max_buf_free != 0; } + /** whether there is capacity in the log buffer */ + bool buf_free_ok() const noexcept + { + ut_ad(!is_pmem()); + return (buf_free.load(std::memory_order_relaxed) & ~buf_free_LOCK) < + max_buf_free; + } + + void set_buf_free(size_t f) noexcept + { ut_ad(f < buf_free_LOCK); buf_free.store(f, std::memory_order_relaxed); } + #ifdef HAVE_PMEM bool is_pmem() const noexcept { return !flush_buf; } #else @@ -302,7 +339,7 @@ public: bool is_opened() const noexcept { return log.is_opened(); } - /** @return target write LSN to react on buf_free >= max_buf_free */ + /** @return target write LSN to react on !buf_free_ok() */ inline lsn_t get_write_target() const; /** @return LSN at which log resizing was started and is still in progress @@ -402,9 +439,7 @@ public: void set_recovered_lsn(lsn_t lsn) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); -#endif /* SUX_LOCK_GENERIC */ + ut_ad(latch_have_wr()); write_lsn= lsn; this->lsn.store(lsn, std::memory_order_relaxed); flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); @@ -444,17 +479,23 @@ public: private: /** Wait in append_prepare() for buffer to become available - @param lsn log sequence number to write up to - @param ex whether log_sys.latch is exclusively locked */ - ATTRIBUTE_COLD void append_prepare_wait(lsn_t lsn, bool ex) noexcept; + @tparam spin whether to use the spin-only lock_lsn() + @param b the value of buf_free + @param ex whether log_sys.latch is exclusively locked + @param lsn log sequence number to write up to + @return the new value of buf_free */ + template<bool spin> + ATTRIBUTE_COLD size_t append_prepare_wait(size_t b, bool ex, lsn_t lsn) + noexcept; public: /** Reserve space in the log buffer for appending data. + @tparam spin whether to use the spin-only lock_lsn() @tparam pmem log_sys.is_pmem() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ - template<bool pmem> - inline std::pair<lsn_t,byte*> append_prepare(size_t size, bool ex) noexcept; + template<bool spin,bool pmem> + std::pair<lsn_t,byte*> append_prepare(size_t size, bool ex) noexcept; /** Append a string of bytes to the redo log. @param d destination @@ -462,9 +503,7 @@ public: @param size length of str, in bytes */ void append(byte *&d, const void *s, size_t size) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_locked()); -#endif + ut_ad(latch_have_any()); ut_ad(d + size <= buf + (is_pmem() ? file_size : buf_size)); memcpy(d, s, size); d+= size; diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index c916edc9..bfa66216 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -695,14 +695,40 @@ private: /** Encrypt the log */ ATTRIBUTE_NOINLINE void encrypt(); + /** Commit the mini-transaction log. + @tparam pmem log_sys.is_pmem() + @param mtr mini-transaction + @param lsns {start_lsn,flush_ahead} */ + template<bool pmem> + static void commit_log(mtr_t *mtr, std::pair<lsn_t,page_flush_ahead> lsns); + /** Append the redo log records to the redo log buffer. @return {start_lsn,flush_ahead} */ std::pair<lsn_t,page_flush_ahead> do_write(); /** Append the redo log records to the redo log buffer. + @tparam spin whether to use the spin-only log_sys.lock_lsn() + @tparam pmem log_sys.is_pmem() + @param mtr mini-transaction @param len number of bytes to write @return {start_lsn,flush_ahead} */ - std::pair<lsn_t,page_flush_ahead> finish_write(size_t len); + template<bool spin,bool pmem> static + std::pair<lsn_t,page_flush_ahead> finish_writer(mtr_t *mtr, size_t len); + + /** The applicable variant of commit_log() */ + static void (*commit_logger)(mtr_t *, std::pair<lsn_t,page_flush_ahead>); + /** The applicable variant of finish_writer() */ + static std::pair<lsn_t,page_flush_ahead> (*finisher)(mtr_t *, size_t); + + std::pair<lsn_t,page_flush_ahead> finish_write(size_t len) + { return finisher(this, len); } +public: + /** Poll interval in log_sys.lock_lsn(); 0 to use log_sys.lsn_lock. + Protected by LOCK_global_system_variables and log_sys.latch. */ + static unsigned spin_wait_delay; + /** Update finisher when spin_wait_delay is changing to or from 0. */ + static void finisher_update(); +private: /** Release all latches. */ void release(); diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index c8374515..7eba359f 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -46,6 +46,18 @@ Created 10/21/1995 Heikki Tuuri #include <time.h> #endif /* !_WIN32 */ +/** The maximum size of a read or write request. + +According to Linux "man 2 read" and "man 2 write" this applies to +both 32-bit and 64-bit systems. + +On FreeBSD, the limit is close to the Linux one, INT_MAX. + +On Microsoft Windows, the limit is UINT_MAX (4 GiB - 1). + +On other systems, the limit typically is up to SSIZE_T_MAX. */ +static constexpr unsigned os_file_request_size_max= 0x7ffff000; + extern bool os_has_said_disk_full; /** File offset in bytes */ @@ -109,25 +121,21 @@ struct pfs_os_file_t /** Options for os_file_create_func @{ */ enum os_file_create_t { - OS_FILE_OPEN = 51, /*!< to open an existing file (if - doesn't exist, error) */ - OS_FILE_CREATE, /*!< to create new file (if - exists, error) */ - OS_FILE_OVERWRITE, /*!< to create a new file, if exists - the overwrite old file */ - OS_FILE_OPEN_RAW, /*!< to open a raw device or disk - partition */ - OS_FILE_CREATE_PATH, /*!< to create the directories */ - OS_FILE_OPEN_RETRY, /*!< open with retry */ - - /** Flags that can be combined with the above values. Please ensure - that the above values stay below 128. */ - - OS_FILE_ON_ERROR_NO_EXIT = 128, /*!< do not exit on unknown errors */ - OS_FILE_ON_ERROR_SILENT = 256 /*!< don't print diagnostic messages to - the log unless it is a fatal error, - this flag is only used if - ON_ERROR_NO_EXIT is set */ + /** create a new file */ + OS_FILE_CREATE= 0, + /** open an existing file */ + OS_FILE_OPEN, + /** retry opening an existing file */ + OS_FILE_OPEN_RETRY, + /** open a raw block device */ + OS_FILE_OPEN_RAW, + + /** do not display diagnostic messages */ + OS_FILE_ON_ERROR_SILENT= 4, + + OS_FILE_CREATE_SILENT= OS_FILE_CREATE | OS_FILE_ON_ERROR_SILENT, + OS_FILE_OPEN_SILENT= OS_FILE_OPEN | OS_FILE_ON_ERROR_SILENT, + OS_FILE_OPEN_RETRY_SILENT= OS_FILE_OPEN_RETRY | OS_FILE_ON_ERROR_SILENT }; static const ulint OS_FILE_READ_ONLY = 333; @@ -144,7 +152,7 @@ static const ulint OS_FILE_NORMAL = 62; /** Types for file create @{ */ static constexpr ulint OS_DATA_FILE = 100; static constexpr ulint OS_LOG_FILE = 101; -#if defined _WIN32 || defined HAVE_FCNTL_DIRECT +#if defined _WIN32 || defined O_DIRECT static constexpr ulint OS_DATA_FILE_NO_O_DIRECT = 103; #endif /* @} */ @@ -191,14 +199,10 @@ public: WRITE_ASYNC= WRITE_SYNC | 1, /** A doublewrite batch */ DBLWR_BATCH= WRITE_ASYNC | 8, - /** Write data; evict the block on write completion */ - WRITE_LRU= WRITE_ASYNC | 32, /** Write data and punch hole for the rest */ - PUNCH= WRITE_ASYNC | 64, - /** Write data and punch hole; evict the block on write completion */ - PUNCH_LRU= PUNCH | WRITE_LRU, + PUNCH= WRITE_ASYNC | 16, /** Zero out a range of bytes in fil_space_t::io() */ - PUNCH_RANGE= WRITE_SYNC | 128, + PUNCH_RANGE= WRITE_SYNC | 32, }; constexpr IORequest(buf_page_t *bpage, buf_tmp_buffer_t *slot, @@ -211,7 +215,6 @@ public: bool is_read() const { return (type & READ_SYNC) != 0; } bool is_write() const { return (type & WRITE_SYNC) != 0; } - bool is_LRU() const { return (type & (WRITE_LRU ^ WRITE_ASYNC)) != 0; } bool is_async() const { return (type & (READ_SYNC ^ READ_ASYNC)) != 0; } void write_complete(int io_error) const; @@ -349,7 +352,7 @@ A simple function to open or create a file. pfs_os_file_t os_file_create_simple_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success); @@ -358,7 +361,7 @@ os_file_create_simple_func( os_file_create_simple_no_error_handling(), not directly this function! A simple function to open or create a file. @param[in] name name of the file or path as a null-terminated string -@param[in] create_mode create mode +@param[in] create_mode OS_FILE_CREATE or OS_FILE_OPEN @param[in] access_type OS_FILE_READ_ONLY, OS_FILE_READ_WRITE, or OS_FILE_READ_ALLOW_DELETE; the last option is used by a backup program reading the file @@ -369,28 +372,12 @@ A simple function to open or create a file. pfs_os_file_t os_file_create_simple_no_error_handling_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success) MY_ATTRIBUTE((warn_unused_result)); -#ifndef HAVE_FCNTL_DIRECT -#define os_file_set_nocache(fd, file_name, operation_name) do{}while(0) -#else -/** Tries to disable OS caching on an opened file descriptor. -@param[in] fd file descriptor to alter -@param[in] file_name file name, used in the diagnostic message -@param[in] name "open" or "create"; used in the diagnostic - message */ -void -os_file_set_nocache( -/*================*/ - int fd, /*!< in: file descriptor to alter */ - const char* file_name, - const char* operation_name); -#endif - #ifndef _WIN32 /* On Microsoft Windows, mandatory locking is used */ /** Obtain an exclusive lock on a file. @param fd file descriptor @@ -419,7 +406,7 @@ Opens an existing file or creates a new. pfs_os_file_t os_file_create_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint purpose, ulint type, bool read_only, @@ -617,7 +604,7 @@ pfs_os_file_t pfs_os_file_create_simple_func( mysql_pfs_key_t key, const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success, @@ -633,7 +620,7 @@ monitor file creation/open. @param[in] key Performance Schema Key @param[in] name name of the file or path as a null-terminated string -@param[in] create_mode create mode +@param[in] create_mode OS_FILE_CREATE or OS_FILE_OPEN @param[in] access_type OS_FILE_READ_ONLY, OS_FILE_READ_WRITE, or OS_FILE_READ_ALLOW_DELETE; the last option is used by a backup program reading the file @@ -648,7 +635,7 @@ pfs_os_file_t pfs_os_file_create_simple_no_error_handling_func( mysql_pfs_key_t key, const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success, @@ -681,7 +668,7 @@ pfs_os_file_t pfs_os_file_create_func( mysql_pfs_key_t key, const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint purpose, ulint type, bool read_only, diff --git a/storage/innobase/include/os0file.inl b/storage/innobase/include/os0file.inl index 7de31505..a7603028 100644 --- a/storage/innobase/include/os0file.inl +++ b/storage/innobase/include/os0file.inl @@ -45,7 +45,7 @@ pfs_os_file_t pfs_os_file_create_simple_func( mysql_pfs_key_t key, const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success, @@ -80,7 +80,7 @@ monitor file creation/open. @param[in] key Performance Schema Key @param[in] name name of the file or path as a null-terminated string -@param[in] create_mode create mode +@param[in] create_mode OS_FILE_CREATE or OS_FILE_OPEN @param[in] access_type OS_FILE_READ_ONLY, OS_FILE_READ_WRITE, or OS_FILE_READ_ALLOW_DELETE; the last option is used by a backup program reading the file @@ -95,7 +95,7 @@ pfs_os_file_t pfs_os_file_create_simple_no_error_handling_func( mysql_pfs_key_t key, const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success, @@ -146,7 +146,7 @@ pfs_os_file_t pfs_os_file_create_func( mysql_pfs_key_t key, const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint purpose, ulint type, bool read_only, diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index 93ea650d..1c2af128 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -165,14 +165,11 @@ row_merge_drop_indexes( prepare_inplace_alter_table_dict(). */ void row_merge_drop_temp_indexes(); -/** Create temporary merge files in the given paramater path, and if -UNIV_PFS_IO defined, register the file descriptor with Performance Schema. -@param[in] path location for creating temporary merge files, or NULL +/** Create a temporary file at the specified path. +@param path location for creating temporary merge files, or nullptr @return File descriptor */ -pfs_os_file_t -row_merge_file_create_low( - const char* path) - MY_ATTRIBUTE((warn_unused_result)); +pfs_os_file_t row_merge_file_create_low(const char *path) + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Destroy a merge file. And de-register the file from Performance Schema if UNIV_PFS_IO is defined. */ diff --git a/storage/innobase/include/row0row.h b/storage/innobase/include/row0row.h index a1350740..7056c77f 100644 --- a/storage/innobase/include/row0row.h +++ b/storage/innobase/include/row0row.h @@ -370,6 +370,12 @@ row_search_index_entry( mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Get the byte offset of the DB_TRX_ID column +@param[in] rec clustered index record +@param[in] index clustered index +@return the byte offset of DB_TRX_ID, from the start of rec */ +ulint row_trx_id_offset(const rec_t* rec, const dict_index_t* index); + #define ROW_COPY_DATA 1 #define ROW_COPY_POINTERS 2 diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 8134c60f..54e4a1d2 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -115,8 +115,8 @@ row_sel_convert_mysql_key_to_innobase( ulint buf_len, /*!< in: buffer length */ dict_index_t* index, /*!< in: index of the key value */ const byte* key_ptr, /*!< in: MySQL key value */ - ulint key_len); /*!< in: MySQL key value length */ - + ulint key_len) /*!< in: MySQL key value length */ + MY_ATTRIBUTE((nonnull(1,4,5))); /** Search for rows in the database using cursor. Function is mainly used for tables that are shared across connections and diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 51f3049b..2ed26748 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -194,7 +194,6 @@ enum monitor_id_t { MONITOR_FLUSH_ADAPTIVE_AVG_PASS, MONITOR_LRU_GET_FREE_LOOPS, - MONITOR_LRU_GET_FREE_WAITS, MONITOR_FLUSH_AVG_PAGE_RATE, MONITOR_FLUSH_LSN_AVG_RATE, @@ -215,7 +214,6 @@ enum monitor_id_t { MONITOR_LRU_BATCH_SCANNED_PER_CALL, MONITOR_LRU_BATCH_FLUSH_TOTAL_PAGE, MONITOR_LRU_BATCH_EVICT_TOTAL_PAGE, - MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT, MONITOR_LRU_GET_FREE_SEARCH, MONITOR_LRU_SEARCH_SCANNED, MONITOR_LRU_SEARCH_SCANNED_NUM_CALL, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 457d9ab5..5e6bfc33 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -121,10 +121,6 @@ struct srv_stats_t ulint_ctr_n_t n_temp_blocks_decrypted; }; -/** We are prepared for a situation that we have this many threads waiting for -a transactional lock inside InnoDB. srv_start() sets the value. */ -extern ulint srv_max_n_threads; - extern const char* srv_main_thread_op_info; /** Prefix used by MySQL to indicate pre-5.1 table name encoding */ diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h index 01067322..98c256d3 100644 --- a/storage/innobase/include/srw_lock.h +++ b/storage/innobase/include/srw_lock.h @@ -153,7 +153,7 @@ template<bool spinloop> class srw_lock_impl; /** Slim shared-update-exclusive lock with no recursion */ template<bool spinloop> -class ssux_lock_impl final +class ssux_lock_impl { #ifdef UNIV_PFS_RWLOCK friend class ssux_lock; @@ -550,3 +550,51 @@ typedef srw_lock_impl<false> srw_lock; typedef srw_lock_impl<true> srw_spin_lock; #endif + +#ifdef UNIV_DEBUG +# include <unordered_set> + +class srw_lock_debug : private srw_lock +{ + /** The owner of the exclusive lock (0 if none) */ + std::atomic<pthread_t> writer; + /** Protects readers */ + mutable srw_mutex readers_lock; + /** Threads that hold the lock in shared mode */ + std::atomic<std::unordered_multiset<pthread_t>*> readers; + + /** Register a read lock. */ + void readers_register(); + +public: + void SRW_LOCK_INIT(mysql_pfs_key_t key); + void destroy(); + +#ifndef SUX_LOCK_GENERIC + /** @return whether any lock may be held by any thread */ + bool is_locked_or_waiting() const noexcept + { return srw_lock::is_locked_or_waiting(); } + /** @return whether an exclusive lock may be held by any thread */ + bool is_write_locked() const noexcept { return srw_lock::is_write_locked(); } +#endif + + /** Acquire an exclusive lock */ + void wr_lock(SRW_LOCK_ARGS(const char *file, unsigned line)); + /** @return whether an exclusive lock was acquired */ + bool wr_lock_try(); + /** Release after wr_lock() */ + void wr_unlock(); + /** Acquire a shared lock */ + void rd_lock(SRW_LOCK_ARGS(const char *file, unsigned line)); + /** @return whether a shared lock was acquired */ + bool rd_lock_try(); + /** Release after rd_lock() */ + void rd_unlock(); + /** @return whether this thread is between rd_lock() and rd_unlock() */ + bool have_rd() const noexcept; + /** @return whether this thread is between wr_lock() and wr_unlock() */ + bool have_wr() const noexcept; + /** @return whether this thread is holding rd_lock() or wr_lock() */ + bool have_any() const noexcept; +}; +#endif diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index 0f4f8afa..1fb6cd68 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -55,80 +55,74 @@ Run a purge batch. @return number of undo log pages handled in the batch */ ulint trx_purge(ulint n_tasks, ulint history_size); -/** Rollback segements from a given transaction with trx-no -scheduled for purge. */ -class TrxUndoRsegs { -private: - typedef std::vector<trx_rseg_t*, ut_allocator<trx_rseg_t*> > - trx_rsegs_t; -public: - typedef trx_rsegs_t::iterator iterator; - typedef trx_rsegs_t::const_iterator const_iterator; - - TrxUndoRsegs() = default; - - /** Constructor */ - TrxUndoRsegs(trx_rseg_t& rseg) - : trx_no(rseg.last_trx_no()), m_rsegs(1, &rseg) {} - /** Constructor */ - TrxUndoRsegs(trx_id_t trx_no, trx_rseg_t& rseg) - : trx_no(trx_no), m_rsegs(1, &rseg) {} - - bool operator!=(const TrxUndoRsegs& other) const - { return trx_no != other.trx_no; } - bool empty() const { return m_rsegs.empty(); } - void erase(iterator& it) { m_rsegs.erase(it); } - iterator begin() { return(m_rsegs.begin()); } - iterator end() { return(m_rsegs.end()); } - const_iterator begin() const { return m_rsegs.begin(); } - const_iterator end() const { return m_rsegs.end(); } - - /** Compare two TrxUndoRsegs based on trx_no. - @param elem1 first element to compare - @param elem2 second element to compare - @return true if elem1 > elem2 else false.*/ - bool operator()(const TrxUndoRsegs& lhs, const TrxUndoRsegs& rhs) - { - return(lhs.trx_no > rhs.trx_no); - } - - /** Copy of trx_rseg_t::last_trx_no() */ - trx_id_t trx_no= 0; -private: - /** Rollback segments of a transaction, scheduled for purge. */ - trx_rsegs_t m_rsegs{}; -}; - -typedef std::priority_queue< - TrxUndoRsegs, - std::vector<TrxUndoRsegs, ut_allocator<TrxUndoRsegs> >, - TrxUndoRsegs> purge_pq_t; - -/** Chooses the rollback segment with the oldest committed transaction */ -struct TrxUndoRsegsIterator { - /** Constructor */ - TrxUndoRsegsIterator(); - /** Sets the next rseg to purge in purge_sys. - Executed in the purge coordinator thread. - @retval false when nothing is to be purged - @retval true when purge_sys.rseg->latch was locked */ - inline bool set_next(); - -private: - // Disable copying - TrxUndoRsegsIterator(const TrxUndoRsegsIterator&); - TrxUndoRsegsIterator& operator=(const TrxUndoRsegsIterator&); - - /** The current element to process */ - TrxUndoRsegs m_rsegs; - /** Track the current element in m_rsegs */ - TrxUndoRsegs::const_iterator m_iter; -}; - /** The control structure used in the purge operation */ class purge_sys_t { - friend TrxUndoRsegsIterator; + /** Min-heap based priority queue of (trx_no, trx_sys.rseg_array index) + pairs, ordered on trx_no. The highest 64-TRX_NO_SHIFT bits of each element is + trx_no, the lowest 8 bits is rseg's index in trx_sys.rseg_array. */ + class purge_queue + { + public: + typedef std::vector<uint64_t, ut_allocator<uint64_t>> container_type; + /** Number of bits reseved to shift trx_no in purge queue element */ + static constexpr unsigned TRX_NO_SHIFT= 8; + + bool empty() const { return m_array.empty(); } + void clear() { m_array.clear(); } + + /** Push (trx_no, trx_sys.rseg_array index) into min-heap. + @param trx_no_rseg (trx_no << TRX_NO_SHIFT | (trx_sys.rseg_array index)) */ + void push_trx_no_rseg(container_type::value_type trx_no_rseg) + { + m_array.push_back(trx_no_rseg); + std::push_heap(m_array.begin(), m_array.end(), + std::greater<container_type::value_type>()); + } + + /** Push rseg to priority queue. + @param trx_no trx_no of committed transaction + @param rseg rseg of committed transaction*/ + void push(trx_id_t trx_no, const trx_rseg_t &rseg) + { + ut_ad(trx_no < 1ULL << (DATA_TRX_ID_LEN * CHAR_BIT)); + ut_ad(&rseg >= trx_sys.rseg_array); + ut_ad(&rseg < trx_sys.rseg_array + TRX_SYS_N_RSEGS); + push_trx_no_rseg(trx_no << TRX_NO_SHIFT | + byte(&rseg - trx_sys.rseg_array)); + } + + /** Extracts rseg from (trx_no, trx_sys.rseg_array index) pair. + @param trx_no_rseg (trx_no << TRX_NO_SHIFT | (trx_sys.rseg_array index) + @return pointer to rseg in trx_sys.rseg_array */ + static trx_rseg_t *rseg(container_type::value_type trx_no_rseg) { + byte i= static_cast<byte>(trx_no_rseg); + ut_ad(i < TRX_SYS_N_RSEGS); + return &trx_sys.rseg_array[i]; + } + + /** Pop rseg from priority queue. + @return pointer to popped trx_rseg_t object */ + trx_rseg_t *pop() + { + ut_ad(!empty()); + std::pop_heap(m_array.begin(), m_array.end(), + std::greater<container_type::value_type>()); + trx_rseg_t *r = rseg(m_array.back()); + m_array.pop_back(); + return r; + } + + /** Clone m_array. + @return m_array clone */ + container_type clone_container() const{ return m_array; } + + private: + /** Array of (trx_no, trx_sys.rseg_array index) pairs. */ + container_type m_array; + }; + + public: /** latch protecting view, m_enabled */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) mutable srw_spin_lock latch; @@ -244,15 +238,36 @@ private: record */ uint16_t hdr_offset; /*!< Header byte offset on the page */ + /** Binary min-heap of (trx_no, trx_sys.rseg_array index) pairs, ordered on + trx_no. It is protected by the pq_mutex */ + purge_queue purge_queue; + + /** Mutex protecting purge_queue */ + mysql_mutex_t pq_mutex; - TrxUndoRsegsIterator - rseg_iter; /*!< Iterator to get the next rseg - to process */ public: - purge_pq_t purge_queue; /*!< Binary min-heap, ordered on - TrxUndoRsegs::trx_no. It is protected - by the pq_mutex */ - mysql_mutex_t pq_mutex; /*!< Mutex protecting purge_queue */ + + void enqueue(trx_id_t trx_no, const trx_rseg_t &rseg) { + mysql_mutex_assert_owner(&pq_mutex); + purge_queue.push(trx_no, rseg); + } + + /** Push to purge queue without acquiring pq_mutex. + @param rseg rseg to push */ + void enqueue(const trx_rseg_t &rseg) { enqueue(rseg.last_trx_no(), rseg); } + + /** Clone purge queue container. + @return purge queue container clone */ + purge_queue::container_type clone_queue_container() const { + mysql_mutex_assert_owner(&pq_mutex); + return purge_queue.clone_container(); + } + + /** Acquare purge_queue_mutex */ + void queue_lock() { mysql_mutex_lock(&pq_mutex); } + + /** Release purge queue mutex */ + void queue_unlock() { mysql_mutex_unlock(&pq_mutex); } /** innodb_undo_log_truncate=ON state; only modified by purge_coordinator_callback() */ @@ -332,8 +347,9 @@ private: /** Update the last not yet purged history log info in rseg when we have purged a whole undo log. Advances also purge_trx_no - past the purged log. */ - void rseg_get_next_history_log(); + past the purged log. + @return whether anything is to be purged */ + bool rseg_get_next_history_log(); public: /** @@ -438,6 +454,11 @@ public: @param already_stopped True indicates purge threads were already stopped */ void stop_FTS(const dict_table_t &table, bool already_stopped=false); + + /** Cleanse purge queue to remove the rseg that reside in undo-tablespace + marked for truncate. + @param space undo tablespace being truncated */ + void cleanse_purge_queue(const fil_space_t &space); }; /** The global data structure coordinating a purge */ diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 7fa43047..e0051b2a 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -59,7 +59,7 @@ struct alignas(CPU_LEVEL1_DCACHE_LINESIZE) trx_rseg_t /** tablespace containing the rollback segment; constant after init() */ fil_space_t *space; /** latch protecting everything except page_no, space */ - srw_spin_lock latch; + IF_DBUG(srw_lock_debug,srw_spin_lock) latch; /** rollback segment header page number; constant after init() */ uint32_t page_no; /** length of the TRX_RSEG_HISTORY list (number of transactions) */ @@ -170,19 +170,21 @@ public: /** Last not yet purged undo log header; FIL_NULL if all purged */ uint32_t last_page_no; - /** trx_t::no | last_offset << 48 */ + /** trx_t::no << 16 | last_offset */ uint64_t last_commit_and_offset; /** @return the commit ID of the last committed transaction */ trx_id_t last_trx_no() const - { return last_commit_and_offset & ((1ULL << 48) - 1); } + { return last_commit_and_offset >> 16; } /** @return header offset of the last committed transaction */ uint16_t last_offset() const - { return static_cast<uint16_t>(last_commit_and_offset >> 48); } + { + return static_cast<uint16_t>(last_commit_and_offset); + } void set_last_commit(uint16_t last_offset, trx_id_t trx_no) { - last_commit_and_offset= static_cast<uint64_t>(last_offset) << 48 | trx_no; + last_commit_and_offset= trx_no << 16 | static_cast<uint64_t>(last_offset); } /** @return the page identifier */ diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 0a3e0d62..15255354 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -785,13 +785,19 @@ public: const char* op_info; /*!< English text describing the current operation, or an empty string */ - uint isolation_level;/*!< TRX_ISO_REPEATABLE_READ, ... */ - bool check_foreigns; /*!< normally TRUE, but if the user - wants to suppress foreign key checks, - (in table imports, for example) we - set this FALSE */ + /** TRX_ISO_REPEATABLE_READ, ... */ + unsigned isolation_level:2; + /** when set, REPEATABLE READ will actually be Snapshot Isolation, due to + detecting write/write conflicts and disabling "semi-consistent read" */ + unsigned snapshot_isolation:1; + /** normally set; "SET foreign_key_checks=0" can be issued to suppress + foreign key checks, in table imports, for example */ + unsigned check_foreigns:1; + /** normally set; "SET unique_checks=0, foreign_key_checks=0" + enables bulk insert into an empty table */ + unsigned check_unique_secondary:1; /** whether an insert into an empty table is active */ - bool bulk_insert; + unsigned bulk_insert:1; /*------------------------------*/ /* MySQL has a transaction coordinator to coordinate two phase commit between multiple storage engines and the binary log. When @@ -805,13 +811,6 @@ public: /** whether this is holding the prepare mutex */ bool active_commit_ordered; /*------------------------------*/ - bool check_unique_secondary; - /*!< normally TRUE, but if the user - wants to speed up inserts by - suppressing unique key checks - for secondary indexes when we decide - if we can use the insert buffer for - them, we set this FALSE */ bool flush_log_later;/* In 2PC, we hold the prepare_commit mutex across both phases. In that case, we @@ -1189,10 +1188,16 @@ public: return UNIV_UNLIKELY(bulk_insert) ? bulk_insert_apply_low(): DB_SUCCESS; } + /** Do the bulk insert for the buffered insert operation of a table. + @param table bulk insert operation + @return DB_SUCCESS or error code. */ + dberr_t bulk_insert_apply_for_table(dict_table_t *table); private: /** Apply the buffered bulk inserts. */ dberr_t bulk_insert_apply_low(); + /** Rollback the bulk insert operation for the transaction */ + void bulk_rollback_low(); /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ trx_rseg_t *assign_temp_rseg(); diff --git a/storage/innobase/include/trx0undo.inl b/storage/innobase/include/trx0undo.inl index 9f05989f..023e2b98 100644 --- a/storage/innobase/include/trx0undo.inl +++ b/storage/innobase/include/trx0undo.inl @@ -125,5 +125,6 @@ trx_undo_page_get_next_rec(const buf_block_t *undo_page, uint16_t rec, { uint16_t end= trx_undo_page_get_end(undo_page, page_no, offset); uint16_t next= mach_read_from_2(undo_page->page.frame + rec); - return next == end ? nullptr : undo_page->page.frame + next; + ut_ad(next <= end); + return next >= end ? nullptr : undo_page->page.frame + next; } diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index f4183e4c..3ff5f885 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -1071,9 +1071,8 @@ static inline void *ut_malloc_dontdump(size_t n_bytes, ...) { void *ptr = my_large_malloc(&n_bytes, MYF(0)); - ut_dontdump(ptr, n_bytes, true); - if (ptr) { + ut_dontdump(ptr, n_bytes, true); os_total_large_mem_allocated += n_bytes; } return ptr; diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index fe16ce14..500b6455 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -242,20 +242,6 @@ ut_print_name( FILE* ef, /*!< in: stream */ const trx_t* trx, /*!< in: transaction */ const char* name); /*!< in: table name to print */ -/** Format a table name, quoted as an SQL identifier. -If the name contains a slash '/', the result will contain two -identifiers separated by a period (.), as in SQL -database_name.table_name. -@see table_name_t -@param[in] name table or index name -@param[out] formatted formatted result, will be NUL-terminated -@param[in] formatted_size size of the buffer in bytes -@return pointer to 'formatted' */ -char* -ut_format_name( - const char* name, - char* formatted, - ulint formatted_size); /**********************************************************************//** Catenate files. */ diff --git a/storage/innobase/include/ut0vec.h b/storage/innobase/include/ut0vec.h index f4660f96..ad43e1c8 100644 --- a/storage/innobase/include/ut0vec.h +++ b/storage/innobase/include/ut0vec.h @@ -201,15 +201,6 @@ ib_vector_last_const( const ib_vector_t* vec); /* in: vector */ /******************************************************************** -Sort the vector elements. */ -UNIV_INLINE -void -ib_vector_sort( -/*===========*/ - ib_vector_t* vec, /* in/out: vector */ - ib_compare_t compare); /* in: the comparator to use for sort */ - -/******************************************************************** The default ib_vector_t heap free. Does nothing. */ UNIV_INLINE void diff --git a/storage/innobase/include/ut0vec.inl b/storage/innobase/include/ut0vec.inl index 531f0f22..1a844dd8 100644 --- a/storage/innobase/include/ut0vec.inl +++ b/storage/innobase/include/ut0vec.inl @@ -305,19 +305,6 @@ ib_vector_remove( } /******************************************************************** -Sort the vector elements. */ -UNIV_INLINE -void -ib_vector_sort( -/*===========*/ - /* out: void */ - ib_vector_t* vec, /* in: vector */ - ib_compare_t compare)/* in: the comparator to use for sort */ -{ - qsort(vec->data, vec->used, vec->sizeof_value, compare); -} - -/******************************************************************** Destroy the vector. Make sure the vector owns the allocator, e.g., the heap in the the heap allocator. */ UNIV_INLINE diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index df51ceb1..a2107007 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -47,6 +47,7 @@ Created 5/7/1996 Heikki Tuuri #include "que0que.h" #include "scope.h" #include <debug_sync.h> +#include <mysql/service_thd_mdl.h> #include <set> @@ -173,7 +174,7 @@ void lock_sys_t::assert_locked(const dict_table_t &table) const ut_ad(!table.is_temporary()); if (is_writer()) return; - ut_ad(readers); + ut_ad(latch.have_rd()); ut_ad(table.lock_mutex_is_owner()); } @@ -182,7 +183,7 @@ void lock_sys_t::hash_table::assert_locked(const page_id_t id) const { if (lock_sys.is_writer()) return; - ut_ad(lock_sys.readers); + ut_ad(lock_sys.is_holder()); ut_ad(latch(cell_get(id.fold()))->is_locked()); } @@ -191,7 +192,7 @@ void lock_sys_t::assert_locked(const hash_cell_t &cell) const { if (is_writer()) return; - ut_ad(lock_sys.readers); + ut_ad(lock_sys.is_holder()); ut_ad(hash_table::latch(const_cast<hash_cell_t*>(&cell))->is_locked()); } #endif @@ -426,13 +427,10 @@ void lock_sys_t::wr_lock(const char *file, unsigned line) { mysql_mutex_assert_not_owner(&wait_mutex); latch.wr_lock(file, line); - ut_ad(!writer.exchange(pthread_self(), std::memory_order_relaxed)); } /** Release exclusive lock_sys.latch */ void lock_sys_t::wr_unlock() { - ut_ad(writer.exchange(0, std::memory_order_relaxed) == - pthread_self()); latch.wr_unlock(); } @@ -441,15 +439,11 @@ void lock_sys_t::rd_lock(const char *file, unsigned line) { mysql_mutex_assert_not_owner(&wait_mutex); latch.rd_lock(file, line); - ut_ad(!writer.load(std::memory_order_relaxed)); - ut_d(readers.fetch_add(1, std::memory_order_relaxed)); } /** Release shared lock_sys.latch */ void lock_sys_t::rd_unlock() { - ut_ad(!writer.load(std::memory_order_relaxed)); - ut_ad(readers.fetch_sub(1, std::memory_order_relaxed)); latch.rd_unlock(); } #endif @@ -976,8 +970,31 @@ func_exit: for (lock_t *lock= UT_LIST_GET_FIRST(table->locks); lock; lock= UT_LIST_GET_NEXT(un_member.tab_lock.locks, lock)) { - /* if victim has also BF status, but has earlier seqno, we have to wait */ - if (lock->trx != trx && + /* Victim trx needs to be different from BF trx and it has to have a + THD so that we can kill it. Victim might not have THD in two cases: + + (1) An incomplete transaction that was recovered from undo logs + on server startup (and not yet rolled back). + + (2) Transaction that is in XA PREPARE state and whose client + connection was disconnected. + + Neither of these can complete before lock_wait_wsrep() releases + lock_sys.latch. + + (1) trx_t::commit_in_memory() is clearing both + trx_t::state and trx_t::is_recovered before it invokes + lock_release(trx_t*) (which would be blocked by the exclusive + lock_sys.latch that we are holding here). Hence, it is not + possible to write a debug assertion to document this scenario. + + (2) If is in XA PREPARE state, it would eventually be rolled + back and the lock conflict would be resolved when an XA COMMIT + or XA ROLLBACK statement is executed in some other connection. + + If victim has also BF status, but has earlier seqno, we have to wait. + */ + if (lock->trx != trx && lock->trx->mysql_thd && !(wsrep_thd_is_BF(lock->trx->mysql_thd, false) && wsrep_thd_order_before(lock->trx->mysql_thd, trx->mysql_thd))) { @@ -1009,8 +1026,11 @@ func_exit: lock= lock_rec_get_next(heap_no, lock); do { - /* if victim has also BF status, but has earlier seqno, we have to wait */ - if (lock->trx != trx && + /* This is similar case as above except here we have + record-locks instead of table locks. See details + from comment above. + */ + if (lock->trx != trx && lock->trx->mysql_thd && !(wsrep_thd_is_BF(lock->trx->mysql_thd, false) && wsrep_thd_order_before(lock->trx->mysql_thd, trx->mysql_thd))) { @@ -1036,8 +1056,12 @@ func_exit: std::vector<std::pair<ulong,trx_id_t>> victim_id; for (trx_t *v : victims) + { + /* Victim must have THD */ + ut_ad(v->mysql_thd); victim_id.emplace_back(std::pair<ulong,trx_id_t> {thd_get_thread_id(v->mysql_thd), v->id}); + } DBUG_EXECUTE_IF("sync.before_wsrep_thd_abort", { @@ -3940,6 +3964,8 @@ static void lock_table_dequeue(lock_t *in_lock, bool owns_wait_mutex) dberr_t lock_table_for_trx(dict_table_t *table, trx_t *trx, lock_mode mode, bool no_wait) { + ut_ad(!dict_sys.frozen()); + mem_heap_t *heap= mem_heap_create(512); sel_node_t *node= sel_node_create(heap); que_thr_t *thr= pars_complete_graph_for_exec(node, trx, heap, nullptr); @@ -3976,6 +4002,67 @@ run_again: return err; } +/** Lock the child tables of a table. +@param table parent table +@param trx transaction +@return error code */ +dberr_t lock_table_children(dict_table_t *table, trx_t *trx) +{ + MDL_context *mdl_context= + static_cast<MDL_context*>(thd_mdl_context(trx->mysql_thd)); + ut_ad(mdl_context); + struct table_mdl{dict_table_t* table; MDL_ticket *mdl;}; + std::vector<table_mdl> children; + children.emplace_back(table_mdl{table, nullptr}); + + dberr_t err= DB_SUCCESS; + dict_sys.freeze(SRW_LOCK_CALL); + + rescan: + for (auto f : table->referenced_set) + if (dict_table_t *child= f->foreign_table) + { + if (std::find_if(children.begin(), children.end(), + [&](const table_mdl &c){ return c.table == child; }) != + children.end()) + continue; /* We already acquired MDL on this child table. */ + MDL_ticket *mdl= nullptr; + child->acquire(); + child= dict_acquire_mdl_shared<false>(child, mdl_context, &mdl, + DICT_TABLE_OP_NORMAL); + if (child) + { + if (!mdl) + child->release(); + children.emplace_back(table_mdl{child, mdl}); + goto rescan; + } + err= DB_LOCK_WAIT_TIMEOUT; + break; + } + dict_sys.unfreeze(); + + if (err == DB_SUCCESS) + for (const table_mdl &child : children) + if (child.mdl) + if ((err= lock_table_for_trx(child.table, trx, LOCK_X)) != DB_SUCCESS) + break; + + dict_sys.freeze(SRW_LOCK_CALL); + for (table_mdl &child : children) + { + if (child.mdl) + { + child.table->release(); + mdl_context->release_lock(child.mdl); + } + } + dict_sys.unfreeze(); + + return err; +} + + /** Exclusively lock the data dictionary tables. @param trx dictionary transaction @return error code @@ -4125,7 +4212,7 @@ restart: ulint count= 1000; /* We will not attempt hardware lock elision (memory transaction) here. Both lock_rec_dequeue_from_page() and lock_table_dequeue() - would likely lead to a memory transaction due to a system call, to + would likely lead to a memory transaction abort due to a system call, to wake up a waiting transaction. */ lock_sys.rd_lock(SRW_LOCK_CALL); trx->mutex_lock(); @@ -4295,28 +4382,82 @@ void lock_release_on_drop(trx_t *trx) } } -/** Reset lock bit for supremum and rebuild waiting queue. +/** Reset a lock bit and rebuild waiting queue. @param cell rec hash cell of in_lock @param lock the lock with supemum bit set */ -static void lock_rec_unlock_supremum(hash_cell_t &cell, lock_t *lock) +static void lock_rec_unlock(hash_cell_t &cell, lock_t *lock, ulint heap_no) { - ut_ad(lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM)); + ut_ad(lock_rec_get_nth_bit(lock, heap_no)); #ifdef SAFE_MUTEX ut_ad(!mysql_mutex_is_owner(&lock_sys.wait_mutex)); #endif /* SAFE_MUTEX */ ut_ad(!lock->is_table()); ut_ad(lock_sys.is_writer() || lock->trx->mutex_is_owner()); - lock_rec_reset_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM); + lock_rec_reset_nth_bit(lock, heap_no); - lock_t *first_lock= lock_sys_t::get_first( - cell, lock->un_member.rec_lock.page_id, PAGE_HEAP_NO_SUPREMUM); + lock_t *first_lock= + lock_sys_t::get_first(cell, lock->un_member.rec_lock.page_id, heap_no); lock_rec_rebuild_waiting_queue( #if defined(UNIV_DEBUG) || !defined(DBUG_OFF) lock->trx, #endif /* defined(UNIV_DEBUG) || !defined(DBUG_OFF) */ - cell, first_lock, PAGE_HEAP_NO_SUPREMUM); + cell, first_lock, heap_no); +} + +/** Release locks to unmodified records on a clustered index page. +@param cell lock_sys.rec_hash cell of lock +@param lock record lock +@param offsets storage for rec_get_offsets() +@param heap storage for rec_get_offsets() +@param mtr mini-transaction (will be started and committed) */ +static void lock_rec_unlock_unmodified(hash_cell_t &cell, lock_t *lock, + rec_offs *&offsets, mem_heap_t *&heap, + mtr_t &mtr) +{ + ut_ad(!lock->is_waiting()); + + dict_index_t *const index= lock->index; + + mtr.start(); + if (buf_block_t *block= + btr_block_get(*index, lock->un_member.rec_lock.page_id.page_no(), + RW_S_LATCH, true, &mtr)) + { + if (UNIV_UNLIKELY(!page_is_leaf(block->page.frame))) + { + ut_ad("corrupted lock system" == 0); + goto func_exit; + } + + for (ulint i= PAGE_HEAP_NO_USER_LOW; i < lock_rec_get_n_bits(lock); ++i) + { + if (!lock_rec_get_nth_bit(lock, i)); + else if (const rec_t *rec= + page_find_rec_with_heap_no(block->page.frame, i)) + { + if (index->is_clust()) + { + if (trx_read_trx_id(rec + row_trx_id_offset(rec, index)) == + lock->trx->id) + continue; + unlock_rec: + lock_rec_unlock(cell, lock, i); + } + else + { + offsets= rec_get_offsets(rec, index, offsets, index->n_core_fields, + ULINT_UNDEFINED, &heap); + if (lock->trx != + lock_sec_rec_some_has_impl(lock->trx, rec, index, offsets)) + goto unlock_rec; + } + } + } + } +func_exit: + mtr.commit(); } /** Release non-exclusive locks on XA PREPARE, @@ -4334,6 +4475,12 @@ static bool lock_release_on_prepare_try(trx_t *trx) DBUG_ASSERT(trx->state == TRX_STATE_PREPARED); bool all_released= true; + mtr_t mtr; + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs *offsets= offsets_; + mem_heap_t *heap= nullptr; + rec_offs_init(offsets_); + lock_sys.rd_lock(SRW_LOCK_CALL); trx->mutex_lock(); @@ -4350,20 +4497,24 @@ static bool lock_release_on_prepare_try(trx_t *trx) if (!lock->is_table()) { ut_ad(!lock->index->table->is_temporary()); - bool supremum_bit = lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM); - bool rec_granted_exclusive_not_gap = + bool supremum_bit= lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM); + bool rec_granted_exclusive_not_gap= lock->is_rec_granted_exclusive_not_gap(); if (!supremum_bit && rec_granted_exclusive_not_gap) continue; - auto &lock_hash= lock_sys.hash_get(lock->type_mode); - auto cell= lock_hash.cell_get(lock->un_member.rec_lock.page_id.fold()); + if (UNIV_UNLIKELY(lock->type_mode & (LOCK_PREDICATE | LOCK_PRDT_PAGE))) + continue; /* SPATIAL INDEX locking is broken. */ + auto cell= + lock_sys.rec_hash.cell_get(lock->un_member.rec_lock.page_id.fold()); auto latch= lock_sys_t::hash_table::latch(cell); if (latch->try_acquire()) { if (!rec_granted_exclusive_not_gap) lock_rec_dequeue_from_page(lock, false); else if (supremum_bit) - lock_rec_unlock_supremum(*cell, lock); + lock_rec_unlock(*cell, lock, PAGE_HEAP_NO_SUPREMUM); + else + lock_rec_unlock_unmodified(*cell, lock, offsets, heap, mtr); latch->release(); } else @@ -4396,6 +4547,8 @@ static bool lock_release_on_prepare_try(trx_t *trx) lock_sys.rd_unlock(); trx->mutex_unlock(); + if (UNIV_LIKELY_NULL(heap)) + mem_heap_free(heap); return all_released; } @@ -4409,52 +4562,71 @@ void lock_release_on_prepare(trx_t *trx) if (lock_release_on_prepare_try(trx)) return; - LockMutexGuard g{SRW_LOCK_CALL}; - trx->mutex_lock(); + mtr_t mtr; + rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; + rec_offs *offsets= offsets_; + mem_heap_t *heap= nullptr; + + rec_offs_init(offsets_); - for (lock_t *prev, *lock= UT_LIST_GET_LAST(trx->lock.trx_locks); lock; - lock= prev) { - ut_ad(lock->trx == trx); - prev= UT_LIST_GET_PREV(trx_locks, lock); - if (!lock->is_table()) + LockMutexGuard g{SRW_LOCK_CALL}; + trx->mutex_lock(); + + for (lock_t *prev, *lock= UT_LIST_GET_LAST(trx->lock.trx_locks); lock; + lock= prev) { - ut_ad(!lock->index->table->is_temporary()); - if (!lock->is_rec_granted_exclusive_not_gap()) - lock_rec_dequeue_from_page(lock, false); - else if (lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM)) + ut_ad(lock->trx == trx); + prev= UT_LIST_GET_PREV(trx_locks, lock); + if (!lock->is_table()) { - auto &lock_hash= lock_sys.hash_get(lock->type_mode); - auto cell= lock_hash.cell_get(lock->un_member.rec_lock.page_id.fold()); - lock_rec_unlock_supremum(*cell, lock); + ut_ad(!lock->index->table->is_temporary()); + if (!lock->is_rec_granted_exclusive_not_gap()) + lock_rec_dequeue_from_page(lock, false); + else if (UNIV_UNLIKELY(lock->type_mode & + (LOCK_PREDICATE | LOCK_PRDT_PAGE))) + /* SPATIAL INDEX locking is broken. */; + else + { + auto cell= lock_sys.rec_hash.cell_get(lock->un_member.rec_lock. + page_id.fold()); + if (lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM)) + lock_rec_unlock(*cell, lock, PAGE_HEAP_NO_SUPREMUM); + else + { + ut_ad(lock->trx->isolation_level > TRX_ISO_READ_COMMITTED || + /* Insert-intention lock is valid for supremum for isolation + level > TRX_ISO_READ_COMMITTED */ + lock->mode() == LOCK_X || + !lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM)); + lock_rec_unlock_unmodified(*cell, lock, offsets, heap, mtr); + } + } } else - ut_ad(lock->trx->isolation_level > TRX_ISO_READ_COMMITTED || - /* Insert-intention lock is valid for supremum for isolation - level > TRX_ISO_READ_COMMITTED */ - lock->mode() == LOCK_X || - !lock_rec_get_nth_bit(lock, PAGE_HEAP_NO_SUPREMUM)); - } - else - { - ut_d(dict_table_t *table= lock->un_member.tab_lock.table); - ut_ad(!table->is_temporary()); - switch (lock->mode()) { - case LOCK_IS: - case LOCK_S: - lock_table_dequeue(lock, false); - break; - case LOCK_IX: - case LOCK_X: - ut_ad(table->id >= DICT_HDR_FIRST_ID || trx->dict_operation); - /* fall through */ - default: - break; + { + ut_d(dict_table_t *table= lock->un_member.tab_lock.table); + ut_ad(!table->is_temporary()); + switch (lock->mode()) { + case LOCK_IS: + case LOCK_S: + lock_table_dequeue(lock, false); + break; + case LOCK_IX: + case LOCK_X: + ut_ad(table->id >= DICT_HDR_FIRST_ID || trx->dict_operation); + /* fall through */ + default: + break; + } } } } trx->mutex_unlock(); + + if (UNIV_LIKELY_NULL(heap)) + mem_heap_free(heap); } /** Release locks on a table whose creation is being rolled back */ @@ -5414,47 +5586,43 @@ lock_rec_insert_check_and_lock( return err; } -/*********************************************************************//** -Creates an explicit record lock for a running transaction that currently only -has an implicit lock on the record. The transaction instance must have a -reference count > 0 so that it can't be committed and freed before this -function has completed. */ -static -bool -lock_rec_convert_impl_to_expl_for_trx( -/*==================================*/ - trx_t* trx, /*!< in/out: active transaction */ - const page_id_t id, /*!< in: page identifier */ - const rec_t* rec, /*!< in: user record on page */ - dict_index_t* index) /*!< in: index of record */ +/** Create an explicit record lock for a transaction that currently only +has an implicit lock on the record. +@param trx referenced, active transaction, or nullptr +@param id page identifier +@param rec record in the page +@param index the index B-tree that the record belongs to +@return trx, with the reference released */ +static trx_t *lock_rec_convert_impl_to_expl_for_trx(trx_t *trx, + const page_id_t id, + const rec_t *rec, + dict_index_t *index) { - if (!trx) - return false; - - ut_ad(trx->is_referenced()); - ut_ad(page_rec_is_leaf(rec)); - ut_ad(!rec_is_metadata(rec, *index)); + if (trx) + { + ut_ad(trx->is_referenced()); + ut_ad(page_rec_is_leaf(rec)); + ut_ad(!rec_is_metadata(rec, *index)); - DEBUG_SYNC_C("before_lock_rec_convert_impl_to_expl_for_trx"); - ulint heap_no= page_rec_get_heap_no(rec); + ulint heap_no= page_rec_get_heap_no(rec); - { - LockGuard g{lock_sys.rec_hash, id}; - trx->mutex_lock(); - ut_ad(!trx_state_eq(trx, TRX_STATE_NOT_STARTED)); + { + LockGuard g{lock_sys.rec_hash, id}; + trx->mutex_lock(); + ut_ad(!trx_state_eq(trx, TRX_STATE_NOT_STARTED)); + + if (!trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY) && + !lock_rec_has_expl(LOCK_X | LOCK_REC_NOT_GAP, g.cell(), id, heap_no, + trx)) + lock_rec_add_to_queue(LOCK_X | LOCK_REC_NOT_GAP, g.cell(), id, + page_align(rec), heap_no, index, trx, true); + } - if (!trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY) && - !lock_rec_has_expl(LOCK_X | LOCK_REC_NOT_GAP, g.cell(), id, heap_no, - trx)) - lock_rec_add_to_queue(LOCK_X | LOCK_REC_NOT_GAP, g.cell(), id, - page_align(rec), heap_no, index, trx, true); + trx->release_reference(); + trx->mutex_unlock(); } - trx->mutex_unlock(); - trx->release_reference(); - - DEBUG_SYNC_C("after_lock_rec_convert_impl_to_expl_for_trx"); - return false; + return trx; } @@ -5545,10 +5713,11 @@ should be created. @param[in] rec record on the leaf page @param[in] index the index of the record @param[in] offsets rec_get_offsets(rec,index) -@return whether caller_trx already holds an exclusive lock on rec */ +@return unsafe pointer to a transaction that held an exclusive lock on rec +@retval nullptr if no transaction held an exclusive lock */ template<bool is_primary> static -bool +const trx_t * lock_rec_convert_impl_to_expl( trx_t* caller_trx, page_id_t id, @@ -5572,10 +5741,10 @@ lock_rec_convert_impl_to_expl( trx_id = lock_clust_rec_some_has_impl(rec, index, offsets); if (trx_id == 0) { - return false; + return nullptr; } if (UNIV_UNLIKELY(trx_id == caller_trx->id)) { - return true; + return caller_trx; } trx = trx_sys.find(caller_trx, trx_id); @@ -5586,7 +5755,7 @@ lock_rec_convert_impl_to_expl( offsets); if (trx == caller_trx) { trx->release_reference(); - return true; + return trx; } ut_d(lock_rec_other_trx_holds_expl(caller_trx, trx, rec, id)); @@ -5631,11 +5800,18 @@ lock_clust_rec_modify_check_and_lock( /* If a transaction has no explicit x-lock set on the record, set one for it */ - if (lock_rec_convert_impl_to_expl<true>(thr_get_trx(thr), - block->page.id(), + trx_t *trx = thr_get_trx(thr); + if (const trx_t *owner = + lock_rec_convert_impl_to_expl<true>(trx, block->page.id(), rec, index, offsets)) { - /* We already hold an implicit exclusive lock. */ - return DB_SUCCESS; + if (owner == trx) { + /* We already hold an exclusive lock. */ + return DB_SUCCESS; + } + + if (trx->snapshot_isolation && trx->read_view.is_open()) { + return DB_RECORD_CHANGED; + } } err = lock_rec_lock(true, LOCK_X | LOCK_REC_NOT_GAP, @@ -5798,12 +5974,19 @@ lock_sec_rec_read_check_and_lock( return DB_SUCCESS; } - if (!page_rec_is_supremum(rec) - && lock_rec_convert_impl_to_expl<false>( - trx, block->page.id(), rec, index, offsets) - && gap_mode == LOCK_REC_NOT_GAP) { - /* We already hold an implicit exclusive lock. */ - return DB_SUCCESS; + if (page_rec_is_supremum(rec)) { + } else if (const trx_t *owner = + lock_rec_convert_impl_to_expl<false>(trx, block->page.id(), + rec, index, offsets)) { + if (owner == trx) { + if (gap_mode == LOCK_REC_NOT_GAP) { + /* We already hold an exclusive lock. */ + return DB_SUCCESS; + } + } else if (trx->snapshot_isolation + && trx->read_view.is_open()) { + return DB_RECORD_CHANGED; + } } #ifdef WITH_WSREP @@ -5883,13 +6066,28 @@ lock_clust_rec_read_check_and_lock( ulint heap_no = page_rec_get_heap_no(rec); trx_t *trx = thr_get_trx(thr); - if (!lock_table_has(trx, index->table, LOCK_X) - && heap_no != PAGE_HEAP_NO_SUPREMUM - && lock_rec_convert_impl_to_expl<true>(trx, id, - rec, index, offsets) - && gap_mode == LOCK_REC_NOT_GAP) { - /* We already hold an implicit exclusive lock. */ - return DB_SUCCESS; + if (lock_table_has(trx, index->table, LOCK_X) + || heap_no == PAGE_HEAP_NO_SUPREMUM) { + } else if (const trx_t *owner = + lock_rec_convert_impl_to_expl<true>(trx, id, + rec, index, offsets)) { + if (owner == trx) { + if (gap_mode == LOCK_REC_NOT_GAP) { + /* We already hold an exclusive lock. */ + return DB_SUCCESS; + } + } else if (trx->snapshot_isolation + && trx->read_view.is_open()) { + return DB_RECORD_CHANGED; + } + } + + if (heap_no > PAGE_HEAP_NO_SUPREMUM && gap_mode != LOCK_GAP + && trx->snapshot_isolation + && trx->read_view.is_open() + && !trx->read_view.changes_visible( + trx_read_trx_id(rec + row_trx_id_offset(rec, index)))) { + return DB_RECORD_CHANGED; } dberr_t err = lock_rec_lock(false, gap_mode | mode, @@ -6656,6 +6854,7 @@ and less modified rows. Bit 0 is used to prefer orig_trx in case of a tie. print(buf); } + DBUG_EXECUTE_IF("innodb_deadlock_victim_self", victim= trx;); ut_ad(victim->state == TRX_STATE_ACTIVE); /* victim->lock.was_chosen_as_deadlock_victim must always be set before diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 9f39b303..ea717de2 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -69,9 +69,7 @@ log_t log_sys; void log_t::set_capacity() { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); /* Margin for the free space in the smallest log, before a new query step which modifies the database, is started */ @@ -134,7 +132,6 @@ bool log_t::create() #endif latch.SRW_LOCK_INIT(log_latch_key); - init_lsn_lock(); last_checkpoint_lsn= FIRST_LSN; log_capacity= 0; @@ -143,7 +140,7 @@ bool log_t::create() next_checkpoint_lsn= 0; checkpoint_pending= false; - buf_free= 0; + set_buf_free(0); ut_ad(is_initialised()); #ifndef HAVE_PMEM @@ -175,11 +172,13 @@ void log_file_t::write(os_offset_t offset, span<const byte> buf) noexcept ut_ad(is_opened()); if (dberr_t err= os_file_write_func(IORequestWrite, "ib_logfile0", m_file, buf.data(), offset, buf.size())) - ib::fatal() << "write(\"ib_logfile0\") returned " << err; + ib::fatal() << "write(\"ib_logfile0\") returned " << err + << ". Operating system error number " + << IF_WIN(GetLastError(), errno) << "."; } #ifdef HAVE_PMEM -# include <libpmem.h> +# include "cache.h" /** Attempt to memory map a file. @param file log file handle @@ -236,12 +235,13 @@ void log_t::attach_low(log_file_t file, os_offset_t size) log.close(); mprotect(ptr, size_t(size), PROT_READ); buf= static_cast<byte*>(ptr); - max_buf_free= size; + max_buf_free= 1; # if defined __linux__ || defined _WIN32 set_block_size(CPU_LEVEL1_DCACHE_LINESIZE); # endif log_maybe_unbuffered= true; log_buffered= false; + mtr_t::finisher_update(); return true; } } @@ -276,6 +276,7 @@ void log_t::attach_low(log_file_t file, os_offset_t size) block_size); #endif + mtr_t::finisher_update(); #ifdef HAVE_PMEM checkpoint_buf= static_cast<byte*>(aligned_malloc(block_size, block_size)); memset_aligned<64>(checkpoint_buf, 0, block_size); @@ -311,9 +312,7 @@ void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) void log_t::create(lsn_t lsn) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); -#endif + ut_ad(latch_have_wr()); ut_ad(!recv_no_log_write); ut_ad(is_latest()); ut_ad(this == &log_sys); @@ -330,12 +329,12 @@ void log_t::create(lsn_t lsn) noexcept { mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); memset_aligned<4096>(buf, 0, 4096); - buf_free= START_OFFSET; + set_buf_free(START_OFFSET); } else #endif { - buf_free= 0; + set_buf_free(0); memset_aligned<4096>(flush_buf, 0, buf_size); memset_aligned<4096>(buf, 0, buf_size); } @@ -464,8 +463,7 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size) noexcept resize_lsn.store(1, std::memory_order_relaxed); resize_target= 0; resize_log.m_file= - os_file_create_func(path.c_str(), - OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT, + os_file_create_func(path.c_str(), OS_FILE_CREATE, OS_FILE_NORMAL, OS_LOG_FILE, false, &success); if (success) { @@ -812,9 +810,7 @@ ATTRIBUTE_COLD void log_t::resize_write_buf(size_t length) noexcept @return the current log sequence number */ template<bool release_latch> inline lsn_t log_t::write_buf() noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_write_locked()); -#endif + ut_ad(latch_have_wr()); ut_ad(!is_pmem()); ut_ad(!srv_read_only_mode); @@ -930,7 +926,7 @@ wait and check if an already running write is covering the request. void log_write_up_to(lsn_t lsn, bool durable, const completion_callback *callback) { - ut_ad(!srv_read_only_mode || (log_sys.buf_free < log_sys.max_buf_free)); + ut_ad(!srv_read_only_mode || log_sys.buf_free_ok()); ut_ad(lsn != LSN_MAX); ut_ad(lsn != 0); @@ -1067,7 +1063,7 @@ NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ void log_free_check() { - ut_ad(!lock_sys.is_writer()); + ut_ad(!lock_sys.is_holder()); if (log_sys.check_for_checkpoint()) { ut_ad(!recv_no_log_write); @@ -1291,6 +1287,7 @@ log_print( void log_t::close() { ut_ad(this == &log_sys); + ut_ad(!(buf_free & buf_free_LOCK)); if (!is_initialised()) return; close_file(); @@ -1308,7 +1305,6 @@ void log_t::close() #endif latch.destroy(); - destroy_lsn_lock(); recv_sys.close(); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index e72f842f..6b6a6868 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -844,8 +844,7 @@ processed: inside recv_sys_t::recover_deferred(). */ bool success; handle= os_file_create(innodb_data_file_key, filename, - OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT | - OS_FILE_ON_ERROR_SILENT, + OS_FILE_CREATE_SILENT, OS_FILE_AIO, OS_DATA_FILE, false, &success); } space->add(filename, handle, size, false, false); @@ -1714,7 +1713,7 @@ dberr_t recv_sys_t::find_checkpoint() std::string path{get_log_file_path()}; bool success; os_file_t file{os_file_create_func(path.c_str(), - OS_FILE_OPEN | OS_FILE_ON_ERROR_NO_EXIT, + OS_FILE_OPEN, OS_FILE_NORMAL, OS_LOG_FILE, srv_read_only_mode, &success)}; if (file == OS_FILE_CLOSED) @@ -1744,8 +1743,7 @@ dberr_t recv_sys_t::find_checkpoint() { path= get_log_file_path(LOG_FILE_NAME_PREFIX).append(std::to_string(i)); file= os_file_create_func(path.c_str(), - OS_FILE_OPEN | OS_FILE_ON_ERROR_NO_EXIT | - OS_FILE_ON_ERROR_SILENT, + OS_FILE_OPEN_SILENT, OS_FILE_NORMAL, OS_LOG_FILE, true, &success); if (file == OS_FILE_CLOSED) break; @@ -2520,11 +2518,9 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse(source &l, bool if_exists) noexcept { restart: -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked() || + ut_ad(log_sys.latch_have_wr() || srv_operation == SRV_OPERATION_BACKUP || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER); -#endif mysql_mutex_assert_owner(&mutex); ut_ad(log_sys.next_checkpoint_lsn); ut_ad(log_sys.is_latest()); @@ -4052,9 +4048,7 @@ static bool recv_scan_log(bool last_phase) lsn_t rewound_lsn= 0; for (ut_d(lsn_t source_offset= 0);;) { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); #ifdef UNIV_DEBUG const bool wrap{source_offset + recv_sys.len == log_sys.file_size}; #endif @@ -4120,9 +4114,10 @@ static bool recv_scan_log(bool last_phase) const lsn_t end{recv_sys.file_checkpoint}; ut_ad(!end || end == recv_sys.lsn); + bool corrupt_fs= recv_sys.is_corrupt_fs(); mysql_mutex_unlock(&recv_sys.mutex); - if (!end) + if (!end && !corrupt_fs) { recv_sys.set_corrupt_log(); sql_print_error("InnoDB: Missing FILE_CHECKPOINT(" LSN_PF @@ -4448,9 +4443,7 @@ recv_init_crash_recovery_spaces(bool rescan, bool& missing_tablespace) static dberr_t recv_rename_files() { mysql_mutex_assert_owner(&recv_sys.mutex); -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); dberr_t err= DB_SUCCESS; @@ -4602,6 +4595,9 @@ read_only_recovery: LSN_PF, recv_sys.lsn); goto err_exit; } + if (recv_sys.is_corrupt_fs()) { + goto err_exit; + } ut_ad(recv_sys.file_checkpoint); if (rewind) { recv_sys.lsn = log_sys.next_checkpoint_lsn; @@ -4640,9 +4636,9 @@ read_only_recovery: do { rescan = recv_scan_log(false); - ut_ad(!recv_sys.is_corrupt_fs()); - if (recv_sys.is_corrupt_log()) { + if (recv_sys.is_corrupt_log() || + recv_sys.is_corrupt_fs()) { goto err_exit; } @@ -4730,7 +4726,7 @@ err_exit: PROT_READ | PROT_WRITE); #endif } - log_sys.buf_free = recv_sys.offset; + log_sys.set_buf_free(recv_sys.offset); if (recv_needed_recovery && srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { /* Write a FILE_CHECKPOINT marker as the first thing, diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 01641f74..74d3adb2 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -37,6 +37,31 @@ Created 11/26/1995 Heikki Tuuri #include "srv0start.h" #include "log.h" #include "mariadb_stats.h" +#include "my_cpu.h" + +#ifdef HAVE_PMEM +void (*mtr_t::commit_logger)(mtr_t *, std::pair<lsn_t,page_flush_ahead>); +#endif +std::pair<lsn_t,mtr_t::page_flush_ahead> (*mtr_t::finisher)(mtr_t *, size_t); +unsigned mtr_t::spin_wait_delay; + +void mtr_t::finisher_update() +{ + ut_ad(log_sys.latch_have_wr()); +#ifdef HAVE_PMEM + if (log_sys.is_pmem()) + { + commit_logger= mtr_t::commit_log<true>; + finisher= spin_wait_delay + ? mtr_t::finish_writer<true,true> : mtr_t::finish_writer<false,true>; + return; + } + commit_logger= mtr_t::commit_log<false>; +#endif + finisher= + (spin_wait_delay + ? mtr_t::finish_writer<true,false> : mtr_t::finish_writer<false,false>); +} void mtr_memo_slot_t::release() const { @@ -82,9 +107,7 @@ void mtr_memo_slot_t::release() const inline buf_page_t *buf_pool_t::prepare_insert_into_flush_list(lsn_t lsn) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(recv_recovery_is_on() || log_sys.latch.is_locked()); -#endif + ut_ad(recv_recovery_is_on() || log_sys.latch_have_any()); ut_ad(lsn >= log_sys.last_checkpoint_lsn); mysql_mutex_assert_owner(&flush_list_mutex); static_assert(log_t::FIRST_LSN >= 2, "compatibility"); @@ -234,7 +257,14 @@ static void insert_imported(buf_block_t *block) if (block->page.oldest_modification() <= 1) { log_sys.latch.rd_lock(SRW_LOCK_CALL); - const lsn_t lsn= log_sys.last_checkpoint_lsn; + /* For unlogged mtrs (MTR_LOG_NO_REDO), we use the current system LSN. The + mtr that generated the LSN is either already committed or in mtr_t::commit. + Shared latch and relaxed atomics should be fine here as it is guaranteed + that both the current mtr and the mtr that generated the LSN would have + added the dirty pages to flush list before we access the minimum LSN during + checkpoint. log_checkpoint_low() acquires exclusive log_sys.latch before + commencing. */ + const lsn_t lsn= log_sys.get_lsn(); mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_pool.insert_into_flush_list (buf_pool.prepare_insert_into_flush_list(lsn), block, lsn); @@ -310,12 +340,9 @@ void mtr_t::release() inline lsn_t log_t::get_write_target() const { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_locked()); -#endif - if (UNIV_LIKELY(buf_free < max_buf_free)) + ut_ad(latch_have_any()); + if (UNIV_LIKELY(buf_free_ok())) return 0; - ut_ad(!is_pmem()); /* The LSN corresponding to the end of buf is write_lsn - (first_lsn & 4095) + buf_free, but we use simpler arithmetics to return a smaller write target in @@ -324,151 +351,161 @@ inline lsn_t log_t::get_write_target() const return write_lsn + max_buf_free / 2; } -/** Commit a mini-transaction. */ -void mtr_t::commit() +template<bool pmem> +void mtr_t::commit_log(mtr_t *mtr, std::pair<lsn_t,page_flush_ahead> lsns) { - ut_ad(is_active()); - ut_ad(!is_inside_ibuf()); - - /* This is a dirty read, for debugging. */ - ut_ad(!m_modifications || !recv_no_log_write); - ut_ad(!m_modifications || m_log_mode != MTR_LOG_NONE); - ut_ad(!m_latch_ex); + size_t modified= 0; + const lsn_t write_lsn= pmem ? 0 : log_sys.get_write_target(); - if (m_modifications && (m_log_mode == MTR_LOG_NO_REDO || !m_log.empty())) + if (mtr->m_made_dirty) { - if (UNIV_UNLIKELY(!is_logged())) + auto it= mtr->m_memo.rbegin(); + + mysql_mutex_lock(&buf_pool.flush_list_mutex); + + buf_page_t *const prev= + buf_pool.prepare_insert_into_flush_list(lsns.first); + + while (it != mtr->m_memo.rend()) { - release_unlogged(); - goto func_exit; + const mtr_memo_slot_t &slot= *it++; + if (slot.type & MTR_MEMO_MODIFY) + { + ut_ad(slot.type == MTR_MEMO_PAGE_X_MODIFY || + slot.type == MTR_MEMO_PAGE_SX_MODIFY); + modified++; + buf_block_t *b= static_cast<buf_block_t*>(slot.object); + ut_ad(b->page.id() < end_page_id); + ut_d(const auto s= b->page.state()); + ut_ad(s > buf_page_t::FREED); + ut_ad(s < buf_page_t::READ_FIX); + ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= + mtr->m_commit_lsn); + mach_write_to_8(b->page.frame + FIL_PAGE_LSN, mtr->m_commit_lsn); + if (UNIV_LIKELY_NULL(b->page.zip.data)) + memcpy_aligned<8>(FIL_PAGE_LSN + b->page.zip.data, + FIL_PAGE_LSN + b->page.frame, 8); + buf_pool.insert_into_flush_list(prev, b, lsns.first); + } } - ut_ad(!srv_read_only_mode); - std::pair<lsn_t,page_flush_ahead> lsns{do_write()}; - process_freed_pages(); - size_t modified= 0; - const lsn_t write_lsn= log_sys.get_write_target(); + ut_ad(modified); + buf_pool.flush_list_requests+= modified; + buf_pool.page_cleaner_wakeup(); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (m_made_dirty) + if (mtr->m_latch_ex) { - auto it= m_memo.rbegin(); - - mysql_mutex_lock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_unlock(); + mtr->m_latch_ex= false; + } + else + log_sys.latch.rd_unlock(); - buf_page_t *const prev= - buf_pool.prepare_insert_into_flush_list(lsns.first); + mtr->release(); + } + else + { + if (mtr->m_latch_ex) + { + log_sys.latch.wr_unlock(); + mtr->m_latch_ex= false; + } + else + log_sys.latch.rd_unlock(); - while (it != m_memo.rend()) - { - const mtr_memo_slot_t &slot= *it++; + for (auto it= mtr->m_memo.rbegin(); it != mtr->m_memo.rend(); ) + { + const mtr_memo_slot_t &slot= *it++; + ut_ad(slot.object); + switch (slot.type) { + case MTR_MEMO_S_LOCK: + static_cast<index_lock*>(slot.object)->s_unlock(); + break; + case MTR_MEMO_SPACE_X_LOCK: + static_cast<fil_space_t*>(slot.object)->set_committed_size(); + static_cast<fil_space_t*>(slot.object)->x_unlock(); + break; + case MTR_MEMO_X_LOCK: + case MTR_MEMO_SX_LOCK: + static_cast<index_lock*>(slot.object)-> + u_or_x_unlock(slot.type == MTR_MEMO_SX_LOCK); + break; + default: + buf_page_t *bpage= static_cast<buf_page_t*>(slot.object); + ut_d(const auto s=) + bpage->unfix(); if (slot.type & MTR_MEMO_MODIFY) { ut_ad(slot.type == MTR_MEMO_PAGE_X_MODIFY || slot.type == MTR_MEMO_PAGE_SX_MODIFY); - modified++; - buf_block_t *b= static_cast<buf_block_t*>(slot.object); - ut_ad(b->page.id() < end_page_id); - ut_d(const auto s= b->page.state()); - ut_ad(s > buf_page_t::FREED); + ut_ad(bpage->oldest_modification() > 1); + ut_ad(bpage->oldest_modification() < mtr->m_commit_lsn); + ut_ad(bpage->id() < end_page_id); + ut_ad(s >= buf_page_t::FREED); ut_ad(s < buf_page_t::READ_FIX); - ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= - m_commit_lsn); - mach_write_to_8(b->page.frame + FIL_PAGE_LSN, m_commit_lsn); - if (UNIV_LIKELY_NULL(b->page.zip.data)) - memcpy_aligned<8>(FIL_PAGE_LSN + b->page.zip.data, - FIL_PAGE_LSN + b->page.frame, 8); - buf_pool.insert_into_flush_list(prev, b, lsns.first); + ut_ad(mach_read_from_8(bpage->frame + FIL_PAGE_LSN) <= + mtr->m_commit_lsn); + mach_write_to_8(bpage->frame + FIL_PAGE_LSN, mtr->m_commit_lsn); + if (UNIV_LIKELY_NULL(bpage->zip.data)) + memcpy_aligned<8>(FIL_PAGE_LSN + bpage->zip.data, + FIL_PAGE_LSN + bpage->frame, 8); + modified++; + } + switch (auto latch= slot.type & ~MTR_MEMO_MODIFY) { + case MTR_MEMO_PAGE_S_FIX: + bpage->lock.s_unlock(); + continue; + case MTR_MEMO_PAGE_SX_FIX: + case MTR_MEMO_PAGE_X_FIX: + bpage->lock.u_or_x_unlock(latch == MTR_MEMO_PAGE_SX_FIX); + continue; + default: + ut_ad(latch == MTR_MEMO_BUF_FIX); } } + } - ut_ad(modified); - buf_pool.flush_list_requests+= modified; - buf_pool.page_cleaner_wakeup(); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); + buf_pool.add_flush_list_requests(modified); + mtr->m_memo.clear(); + } - if (m_latch_ex) - { - log_sys.latch.wr_unlock(); - m_latch_ex= false; - } - else - log_sys.latch.rd_unlock(); + mariadb_increment_pages_updated(modified); - release(); - } - else - { - if (m_latch_ex) - { - log_sys.latch.wr_unlock(); - m_latch_ex= false; - } - else - log_sys.latch.rd_unlock(); + if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO)) + buf_flush_ahead(mtr->m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC); - for (auto it= m_memo.rbegin(); it != m_memo.rend(); ) - { - const mtr_memo_slot_t &slot= *it++; - ut_ad(slot.object); - switch (slot.type) { - case MTR_MEMO_S_LOCK: - static_cast<index_lock*>(slot.object)->s_unlock(); - break; - case MTR_MEMO_SPACE_X_LOCK: - static_cast<fil_space_t*>(slot.object)->set_committed_size(); - static_cast<fil_space_t*>(slot.object)->x_unlock(); - break; - case MTR_MEMO_X_LOCK: - case MTR_MEMO_SX_LOCK: - static_cast<index_lock*>(slot.object)-> - u_or_x_unlock(slot.type == MTR_MEMO_SX_LOCK); - break; - default: - buf_page_t *bpage= static_cast<buf_page_t*>(slot.object); - ut_d(const auto s=) - bpage->unfix(); - if (slot.type & MTR_MEMO_MODIFY) - { - ut_ad(slot.type == MTR_MEMO_PAGE_X_MODIFY || - slot.type == MTR_MEMO_PAGE_SX_MODIFY); - ut_ad(bpage->oldest_modification() > 1); - ut_ad(bpage->oldest_modification() < m_commit_lsn); - ut_ad(bpage->id() < end_page_id); - ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); - ut_ad(mach_read_from_8(bpage->frame + FIL_PAGE_LSN) <= - m_commit_lsn); - mach_write_to_8(bpage->frame + FIL_PAGE_LSN, m_commit_lsn); - if (UNIV_LIKELY_NULL(bpage->zip.data)) - memcpy_aligned<8>(FIL_PAGE_LSN + bpage->zip.data, - FIL_PAGE_LSN + bpage->frame, 8); - modified++; - } - switch (auto latch= slot.type & ~MTR_MEMO_MODIFY) { - case MTR_MEMO_PAGE_S_FIX: - bpage->lock.s_unlock(); - continue; - case MTR_MEMO_PAGE_SX_FIX: - case MTR_MEMO_PAGE_X_FIX: - bpage->lock.u_or_x_unlock(latch == MTR_MEMO_PAGE_SX_FIX); - continue; - default: - ut_ad(latch == MTR_MEMO_BUF_FIX); - } - } - } + if (!pmem && UNIV_UNLIKELY(write_lsn != 0)) + log_write_up_to(write_lsn, false); +} - buf_pool.add_flush_list_requests(modified); - m_memo.clear(); - } +/** Commit a mini-transaction. */ +void mtr_t::commit() +{ + ut_ad(is_active()); + ut_ad(!is_inside_ibuf()); - mariadb_increment_pages_updated(modified); + /* This is a dirty read, for debugging. */ + ut_ad(!m_modifications || !recv_no_log_write); + ut_ad(!m_modifications || m_log_mode != MTR_LOG_NONE); + ut_ad(!m_latch_ex); - if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO)) - buf_flush_ahead(m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC); + if (m_modifications && (m_log_mode == MTR_LOG_NO_REDO || !m_log.empty())) + { + if (UNIV_UNLIKELY(!is_logged())) + { + release_unlogged(); + goto func_exit; + } - if (UNIV_UNLIKELY(write_lsn != 0)) - log_write_up_to(write_lsn, false); + ut_ad(!srv_read_only_mode); + std::pair<lsn_t,page_flush_ahead> lsns{do_write()}; + process_freed_pages(); +#ifdef HAVE_PMEM + commit_logger(this, lsns); +#else + commit_log<false>(this, lsns); +#endif } else { @@ -513,10 +550,8 @@ void mtr_t::rollback_to_savepoint(ulint begin, ulint end) /** Set create_lsn. */ inline void fil_space_t::set_create_lsn(lsn_t lsn) { -#ifndef SUX_LOCK_GENERIC /* Concurrent log_checkpoint_low() must be impossible. */ - ut_ad(latch.is_write_locked()); -#endif + ut_ad(latch.have_wr()); create_lsn= lsn; } @@ -529,7 +564,6 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) ut_ad(!is_inside_ibuf()); ut_ad(!high_level_read_only); ut_ad(m_modifications); - ut_ad(m_made_dirty); ut_ad(!m_memo.empty()); ut_ad(!recv_recovery_is_on()); ut_ad(m_log_mode == MTR_LOG_ALL); @@ -554,9 +588,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) /* Durably write the reduced FSP_SIZE before truncating the data file. */ log_write_and_flush(); -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); os_file_truncate(space.chain.start->name, space.chain.start->handle, os_offset_t{size} << srv_page_size_shift, true); @@ -713,9 +745,7 @@ This is to be used at log_checkpoint(). @return current LSN */ ATTRIBUTE_COLD lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_write_locked()); -#endif + ut_ad(log_sys.latch_have_wr()); ut_ad(is_active()); ut_ad(!is_inside_ibuf()); ut_ad(m_log_mode == MTR_LOG_ALL); @@ -870,13 +900,111 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) ? ". Shutdown is in progress" : ""); } -/** Wait in append_prepare() for buffer to become available -@param lsn log sequence number to write up to -@param ex whether log_sys.latch is exclusively locked */ -ATTRIBUTE_COLD void log_t::append_prepare_wait(lsn_t lsn, bool ex) noexcept +static ATTRIBUTE_NOINLINE void lsn_delay(size_t delay, size_t mult) noexcept +{ + delay*= mult * 2; // GCC 13.2.0 -O2 targeting AMD64 wants to unroll twice + HMT_low(); + do + MY_RELAX_CPU(); + while (--delay); + HMT_medium(); +} + +#if defined __clang_major__ && __clang_major__ < 10 +/* Only clang-10 introduced support for asm goto */ +#elif defined __APPLE__ +/* At least some versions of Apple Xcode do not support asm goto */ +#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) +# if SIZEOF_SIZE_T == 8 +# define LOCK_TSET \ + __asm__ goto("lock btsq $63, %0\n\t" "jnc %l1" \ + : : "m"(buf_free) : "cc", "memory" : got) +# else +# define LOCK_TSET \ + __asm__ goto("lock btsl $31, %0\n\t" "jnc %l1" \ + : : "m"(buf_free) : "cc", "memory" : got) +# endif +#elif defined _MSC_VER && (defined _M_IX86 || defined _M_X64) +# if SIZEOF_SIZE_T == 8 +# define LOCK_TSET \ + if (!_interlockedbittestandset64 \ + (reinterpret_cast<volatile LONG64*>(&buf_free), 63)) return +# else +# define LOCK_TSET \ + if (!_interlockedbittestandset \ + (reinterpret_cast<volatile long*>(&buf_free), 31)) return +# endif +#endif + +#ifdef LOCK_TSET +ATTRIBUTE_NOINLINE +void log_t::lsn_lock_bts() noexcept +{ + LOCK_TSET; + { + const size_t m= mtr_t::spin_wait_delay; + constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; + for (size_t delay_count= DELAY, delay_iterations= 1;; + lsn_delay(delay_iterations, m)) + { + if (!(buf_free.load(std::memory_order_relaxed) & buf_free_LOCK)) + LOCK_TSET; + if (!delay_count); + else if (delay_iterations < MAX_ITERATIONS) + delay_count= DELAY, delay_iterations++; + else + delay_count--; + } + } + +# ifdef __GNUC__ + got: + return; +# endif +} + +inline +#else +ATTRIBUTE_NOINLINE +#endif +size_t log_t::lock_lsn() noexcept +{ +#ifdef LOCK_TSET + lsn_lock_bts(); + return ~buf_free_LOCK & buf_free.load(std::memory_order_relaxed); +# undef LOCK_TSET +#else + size_t b= buf_free.fetch_or(buf_free_LOCK, std::memory_order_acquire); + if (b & buf_free_LOCK) + { + const size_t m= mtr_t::spin_wait_delay; + constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; + for (size_t delay_count= DELAY, delay_iterations= 1; + ((b= buf_free.load(std::memory_order_relaxed)) & buf_free_LOCK) || + (buf_free_LOCK & (b= buf_free.fetch_or(buf_free_LOCK, + std::memory_order_acquire))); + lsn_delay(delay_iterations, m)) + if (!delay_count); + else if (delay_iterations < MAX_ITERATIONS) + delay_count= DELAY, delay_iterations++; + else + delay_count--; + } + return b; +#endif +} + +template<bool spin> +ATTRIBUTE_COLD size_t log_t::append_prepare_wait(size_t b, bool ex, lsn_t lsn) + noexcept { waits++; - unlock_lsn(); + ut_ad(buf_free.load(std::memory_order_relaxed) == + (spin ? (b | buf_free_LOCK) : b)); + if (spin) + buf_free.store(b, std::memory_order_release); + else + lsn_lock.wr_unlock(); if (ex) latch.wr_unlock(); @@ -890,51 +1018,57 @@ ATTRIBUTE_COLD void log_t::append_prepare_wait(lsn_t lsn, bool ex) noexcept else latch.rd_lock(SRW_LOCK_CALL); - lock_lsn(); + if (spin) + return lock_lsn(); + + lsn_lock.wr_lock(); + return buf_free.load(std::memory_order_relaxed); } /** Reserve space in the log buffer for appending data. +@tparam spin whether to use the spin-only lock_lsn() @tparam pmem log_sys.is_pmem() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ -template<bool pmem> +template<bool spin,bool pmem> inline std::pair<lsn_t,byte*> log_t::append_prepare(size_t size, bool ex) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_locked()); -# ifndef _WIN32 // there is no accurate is_write_locked() on SRWLOCK - ut_ad(ex == latch.is_write_locked()); -# endif -#endif + ut_ad(ex ? latch_have_wr() : latch_have_rd()); ut_ad(pmem == is_pmem()); - lock_lsn(); + if (!spin) + lsn_lock.wr_lock(); + size_t b{spin ? lock_lsn() : buf_free.load(std::memory_order_relaxed)}; write_to_buf++; const lsn_t l{lsn.load(std::memory_order_relaxed)}, end_lsn{l + size}; - size_t b{buf_free}; if (UNIV_UNLIKELY(pmem ? (end_lsn - get_flushed_lsn(std::memory_order_relaxed)) > capacity() : b + size >= buf_size)) - { - append_prepare_wait(l, ex); - b= buf_free; - } + b= append_prepare_wait<spin>(b, ex, l); - lsn.store(end_lsn, std::memory_order_relaxed); size_t new_buf_free= b + size; if (pmem && new_buf_free >= file_size) new_buf_free-= size_t(capacity()); - buf_free= new_buf_free; - unlock_lsn(); + + lsn.store(end_lsn, std::memory_order_relaxed); if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) - set_check_for_checkpoint(); + set_check_for_checkpoint(true); + + byte *our_buf= buf; + if (spin) + buf_free.store(new_buf_free, std::memory_order_release); + else + { + buf_free.store(new_buf_free, std::memory_order_relaxed); + lsn_lock.wr_unlock(); + } - return {l, &buf[b]}; + return {l, our_buf + b}; } /** Finish appending data to the log. @@ -942,9 +1076,7 @@ std::pair<lsn_t,byte*> log_t::append_prepare(size_t size, bool ex) noexcept @return whether buf_flush_ahead() will have to be invoked */ static mtr_t::page_flush_ahead log_close(lsn_t lsn) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(log_sys.latch.is_locked()); -#endif + ut_ad(log_sys.latch_have_any()); const lsn_t checkpoint_age= lsn - log_sys.last_checkpoint_lsn; @@ -1009,9 +1141,7 @@ std::pair<lsn_t,mtr_t::page_flush_ahead> mtr_t::do_write() ut_ad(!recv_no_log_write); ut_ad(is_logged()); ut_ad(m_log.size()); -#ifndef SUX_LOCK_GENERIC - ut_ad(!m_latch_ex || log_sys.latch.is_write_locked()); -#endif + ut_ad(!m_latch_ex || log_sys.latch_have_wr()); #ifndef DBUG_OFF do @@ -1069,9 +1199,7 @@ func_exit: inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, size_t seq) noexcept { -#ifndef SUX_LOCK_GENERIC - ut_ad(latch.is_locked()); -#endif + ut_ad(latch_have_any()); if (UNIV_LIKELY_NULL(resize_buf)) { @@ -1176,50 +1304,47 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, } } +template<bool spin,bool pmem> std::pair<lsn_t,mtr_t::page_flush_ahead> -mtr_t::finish_write(size_t len) +mtr_t::finish_writer(mtr_t *mtr, size_t len) { + ut_ad(log_sys.is_latest()); ut_ad(!recv_no_log_write); - ut_ad(is_logged()); -#ifndef SUX_LOCK_GENERIC -# ifndef _WIN32 // there is no accurate is_write_locked() on SRWLOCK - ut_ad(m_latch_ex == log_sys.latch.is_write_locked()); -# endif -#endif + ut_ad(mtr->is_logged()); + ut_ad(mtr->m_latch_ex ? log_sys.latch_have_wr() : log_sys.latch_have_rd()); - const size_t size{m_commit_lsn ? 5U + 8U : 5U}; - std::pair<lsn_t, byte*> start; + const size_t size{mtr->m_commit_lsn ? 5U + 8U : 5U}; + std::pair<lsn_t, byte*> start= + log_sys.append_prepare<spin,pmem>(len, mtr->m_latch_ex); - if (!log_sys.is_pmem()) + if (!pmem) { - start= log_sys.append_prepare<false>(len, m_latch_ex); - m_log.for_each_block([&start](const mtr_buf_t::block_t *b) + mtr->m_log.for_each_block([&start](const mtr_buf_t::block_t *b) { log_sys.append(start.second, b->begin(), b->used()); return true; }); #ifdef HAVE_PMEM write_trailer: #endif *start.second++= log_sys.get_sequence_bit(start.first + len - size); - if (m_commit_lsn) + if (mtr->m_commit_lsn) { - mach_write_to_8(start.second, m_commit_lsn); - m_crc= my_crc32c(m_crc, start.second, 8); + mach_write_to_8(start.second, mtr->m_commit_lsn); + mtr->m_crc= my_crc32c(mtr->m_crc, start.second, 8); start.second+= 8; } - mach_write_to_4(start.second, m_crc); + mach_write_to_4(start.second, mtr->m_crc); start.second+= 4; } #ifdef HAVE_PMEM else { - start= log_sys.append_prepare<true>(len, m_latch_ex); if (UNIV_LIKELY(start.second + len <= &log_sys.buf[log_sys.file_size])) { - m_log.for_each_block([&start](const mtr_buf_t::block_t *b) + mtr->m_log.for_each_block([&start](const mtr_buf_t::block_t *b) { log_sys.append(start.second, b->begin(), b->used()); return true; }); goto write_trailer; } - m_log.for_each_block([&start](const mtr_buf_t::block_t *b) + mtr->m_log.for_each_block([&start](const mtr_buf_t::block_t *b) { size_t size{b->used()}; const size_t size_left(&log_sys.buf[log_sys.file_size] - start.second); @@ -1242,14 +1367,14 @@ mtr_t::finish_write(size_t len) byte tail[5 + 8]; tail[0]= log_sys.get_sequence_bit(start.first + len - size); - if (m_commit_lsn) + if (mtr->m_commit_lsn) { - mach_write_to_8(tail + 1, m_commit_lsn); - m_crc= my_crc32c(m_crc, tail + 1, 8); - mach_write_to_4(tail + 9, m_crc); + mach_write_to_8(tail + 1, mtr->m_commit_lsn); + mtr->m_crc= my_crc32c(mtr->m_crc, tail + 1, 8); + mach_write_to_4(tail + 9, mtr->m_crc); } else - mach_write_to_4(tail + 1, m_crc); + mach_write_to_4(tail + 1, mtr->m_crc); ::memcpy(start.second, tail, size_left); ::memcpy(log_sys.buf + log_sys.START_OFFSET, tail + size_left, @@ -1258,12 +1383,14 @@ mtr_t::finish_write(size_t len) ((size >= size_left) ? log_sys.START_OFFSET : log_sys.file_size) + (size - size_left); } +#else + static_assert(!pmem, ""); #endif log_sys.resize_write(start.first, start.second, len, size); - m_commit_lsn= start.first + len; - return {start.first, log_close(m_commit_lsn)}; + mtr->m_commit_lsn= start.first + len; + return {start.first, log_close(mtr->m_commit_lsn)}; } bool mtr_t::have_x_latch(const buf_block_t &block) const @@ -1385,7 +1512,7 @@ void mtr_t::upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) ut_ad(slot.type == MTR_MEMO_BUF_FIX); buf_block_t *block= static_cast<buf_block_t*>(slot.object); ut_d(const auto state= block->page.state()); - ut_ad(state > buf_page_t::UNFIXED); + ut_ad(state > buf_page_t::FREED); ut_ad(state > buf_page_t::WRITE_FIX || state < buf_page_t::READ_FIX); static_assert(int{MTR_MEMO_PAGE_S_FIX} == int{RW_S_LATCH}, ""); static_assert(int{MTR_MEMO_PAGE_X_FIX} == int{RW_X_LATCH}, ""); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 31bec346..3293db12 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -200,17 +200,10 @@ os_file_handle_error_cond_exit( bool on_error_silent); /** Does error handling when a file operation fails. -@param[in] name name of a file or NULL -@param[in] operation operation name that failed -@return true if we should retry the operation */ -static -bool -os_file_handle_error( - const char* name, - const char* operation) +@param operation name of operation that failed */ +static void os_file_handle_error(const char *operation) { - /* Exit in case of unknown error */ - return(os_file_handle_error_cond_exit(name, operation, true, false)); + os_file_handle_error_cond_exit(nullptr, operation, true, false); } /** Does error handling when a file operation fails. @@ -327,6 +320,12 @@ private: ssize_t m_n; /** Offset from where to read/write */ os_offset_t m_offset; + + /** Do the read/write + @param request The IO context and type + @param n Number of bytes to read/write + @return the number of bytes read/written or negative value on error */ + ssize_t execute_low(const IORequest& request, ssize_t n); }; #ifndef _WIN32 /* On Microsoft Windows, mandatory locking is used */ @@ -680,28 +679,46 @@ os_file_create_subdirs_if_needed( /** Do the read/write @param[in] request The IO context and type +@param[in] n Number of bytes to read/write +@return the number of bytes read/written or negative value on error */ +ssize_t +SyncFileIO::execute_low(const IORequest& request, ssize_t n) +{ + ut_ad(n > 0); + ut_ad(size_t(n) <= os_file_request_size_max); + + if (request.is_read()) + return IF_WIN(tpool::pread(m_fh, m_buf, n, m_offset), pread(m_fh, m_buf, n, m_offset)); + return IF_WIN(tpool::pwrite(m_fh, m_buf, n, m_offset), pwrite(m_fh, m_buf, n, m_offset)); +} + +/** Do the read/write +@param[in] request The IO context and type @return the number of bytes read/written or negative value on error */ ssize_t SyncFileIO::execute(const IORequest& request) { - ssize_t n_bytes; + ssize_t n_bytes= 0; + ut_ad(m_n > 0); - if (request.is_read()) { -#ifdef _WIN32 - n_bytes = tpool::pread(m_fh, m_buf, m_n, m_offset); -#else - n_bytes = pread(m_fh, m_buf, m_n, m_offset); -#endif - } else { - ut_ad(request.is_write()); -#ifdef _WIN32 - n_bytes = tpool::pwrite(m_fh, m_buf, m_n, m_offset); -#else - n_bytes = pwrite(m_fh, m_buf, m_n, m_offset); -#endif - } + while (size_t(m_n) > os_file_request_size_max) + { + ssize_t n_partial_bytes= execute_low(request, os_file_request_size_max); + if (n_partial_bytes < 0) + return n_partial_bytes; + n_bytes+= n_partial_bytes; + if (n_partial_bytes != os_file_request_size_max) + return n_bytes; + advance(os_file_request_size_max); + } - return(n_bytes); + if (ssize_t n= execute_low(request, m_n)) + { + if (n < 0) + return n; + n_bytes += n; + } + return n_bytes; } #ifndef _WIN32 @@ -942,7 +959,7 @@ os_file_flush_func( ib::error() << "The OS said file flush did not succeed"; - os_file_handle_error(NULL, "flush"); + os_file_handle_error("flush"); /* It is a fatal error if a file flush does not succeed, because then the database can get corrupt on disk */ @@ -965,7 +982,7 @@ A simple function to open or create a file. pfs_os_file_t os_file_create_simple_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success) @@ -974,76 +991,52 @@ os_file_create_simple_func( *success = false; - int create_flag; - const char* mode_str __attribute__((unused)); - - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); - - if (create_mode == OS_FILE_OPEN) { - mode_str = "OPEN"; - - if (access_type == OS_FILE_READ_ONLY) { - - create_flag = O_RDONLY; - - } else if (read_only) { - - create_flag = O_RDONLY; - - } else { - create_flag = O_RDWR; - } - - } else if (read_only) { - - mode_str = "OPEN"; - create_flag = O_RDONLY; + int create_flag = O_RDONLY | O_CLOEXEC; + if (read_only) { } else if (create_mode == OS_FILE_CREATE) { - - mode_str = "CREATE"; - create_flag = O_RDWR | O_CREAT | O_EXCL; - - } else if (create_mode == OS_FILE_CREATE_PATH) { - - mode_str = "CREATE PATH"; - /* Create subdirs along the path if needed. */ - - *success = os_file_create_subdirs_if_needed(name); - - if (!*success) { - - ib::error() - << "Unable to create subdirectories '" - << name << "'"; - - return(OS_FILE_CLOSED); - } - - create_flag = O_RDWR | O_CREAT | O_EXCL; - create_mode = OS_FILE_CREATE; + create_flag = O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC; } else { - - ib::error() - << "Unknown file create mode (" - << create_mode - << " for file '" << name << "'"; - - return(OS_FILE_CLOSED); + ut_ad(create_mode == OS_FILE_OPEN); + if (access_type != OS_FILE_READ_ONLY) { + create_flag = O_RDWR | O_CLOEXEC; + } } bool retry; +#ifdef O_DIRECT + int direct_flag = 0; + /* This function is always called for data files, we should disable + OS caching (O_DIRECT) here as we do in os_file_create_func(), so + we open the same file in the same mode, see man page of open(2). */ + switch (srv_file_flush_method) { + case SRV_O_DSYNC: + case SRV_O_DIRECT: + case SRV_O_DIRECT_NO_FSYNC: + direct_flag = O_DIRECT; + break; + } +#else + constexpr int direct_flag = 0; +#endif + do { - file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); + file = open(name, create_flag | direct_flag, os_innodb_umask); if (file == -1) { +#ifdef O_DIRECT + if (direct_flag && errno == EINVAL) { + direct_flag = 0; + retry = true; + continue; + } +#endif *success = false; - retry = os_file_handle_error( + retry = os_file_handle_error_no_exit( name, - create_mode == OS_FILE_OPEN - ? "open" : "create"); + create_mode == OS_FILE_CREATE + ? "create" : "open", false); } else { *success = true; retry = false; @@ -1051,24 +1044,6 @@ os_file_create_simple_func( } while (retry); -#ifdef HAVE_FCNTL_DIRECT - /* This function is always called for data files, we should disable - OS caching (O_DIRECT) here as we do in os_file_create_func(), so - we open the same file in the same mode, see man page of open(2). */ - if (!srv_read_only_mode && *success) { - switch (srv_file_flush_method) { - case SRV_O_DSYNC: - case SRV_O_DIRECT: - case SRV_O_DIRECT_NO_FSYNC: - os_file_set_nocache(file, name, mode_str); - break; - default: - break; - } - } -#endif - -#ifndef _WIN32 if (!read_only && *success && access_type == OS_FILE_READ_WRITE @@ -1079,7 +1054,6 @@ os_file_create_simple_func( close(file); file = -1; } -#endif /* !_WIN32 */ return(file); } @@ -1113,6 +1087,61 @@ os_file_create_directory( return(true); } +#ifdef O_DIRECT +# if defined __linux +/** Note that the log file uses buffered I/O. */ +static ATTRIBUTE_COLD void os_file_log_buffered() +{ + log_sys.log_maybe_unbuffered= false; + log_sys.log_buffered= true; + log_sys.set_block_size(512); +} +# endif + +/** @return whether the log file may work with unbuffered I/O. */ +static ATTRIBUTE_COLD bool os_file_log_maybe_unbuffered(const struct stat &st) +{ + MSAN_STAT_WORKAROUND(&st); +# ifdef __linux__ + char b[20 + sizeof "/sys/dev/block/" ":" "/../queue/physical_block_size"]; + if (snprintf(b, sizeof b, "/sys/dev/block/%u:%u/queue/physical_block_size", + major(st.st_dev), minor(st.st_dev)) >= + static_cast<int>(sizeof b)) + return false; + int f= open(b, O_RDONLY); + if (f == -1) + { + if (snprintf(b, sizeof b, "/sys/dev/block/%u:%u/../queue/" + "physical_block_size", + major(st.st_dev), minor(st.st_dev)) >= + static_cast<int>(sizeof b)) + return false; + f= open(b, O_RDONLY); + } + unsigned long s= 0; + if (f != -1) + { + ssize_t l= read(f, b, sizeof b); + if (l > 0 && size_t(l) < sizeof b && b[l - 1] == '\n') + { + char *end= b; + s= strtoul(b, &end, 10); + if (b == end || *end != '\n') + s = 0; + } + close(f); + } + if (s > 4096 || s < 64 || !ut_is_2pow(s)) + return false; + log_sys.set_block_size(uint32_t(s)); +# else + constexpr unsigned long s= 4096; +# endif + + return !(st.st_size & (s - 1)); +} +#endif + /** NOTE! Use the corresponding macro os_file_create(), not directly this function! Opens an existing file or creates a new. @@ -1133,73 +1162,83 @@ Opens an existing file or creates a new. pfs_os_file_t os_file_create_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint purpose, ulint type, bool read_only, bool* success) { - bool on_error_no_exit; - bool on_error_silent; - *success = false; DBUG_EXECUTE_IF( "ib_create_table_fail_disk_full", - *success = false; errno = ENOSPC; return(OS_FILE_CLOSED); ); - int create_flag; - const char* mode_str __attribute__((unused)); - - on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT - ? true : false; - on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT - ? true : false; + int create_flag; - create_mode &= ulint(~(OS_FILE_ON_ERROR_NO_EXIT - | OS_FILE_ON_ERROR_SILENT)); - - if (create_mode == OS_FILE_OPEN - || create_mode == OS_FILE_OPEN_RAW - || create_mode == OS_FILE_OPEN_RETRY) { - - mode_str = "OPEN"; - - create_flag = read_only ? O_RDONLY : O_RDWR; - - } else if (read_only) { - - mode_str = "OPEN"; - - create_flag = O_RDONLY; - - } else if (create_mode == OS_FILE_CREATE) { - - mode_str = "CREATE"; - create_flag = O_RDWR | O_CREAT | O_EXCL; + if (read_only) { + create_flag = O_RDONLY | O_CLOEXEC; + } else if (create_mode == OS_FILE_CREATE + || create_mode == OS_FILE_CREATE_SILENT) { + create_flag = O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC; + } else { + ut_ad(create_mode == OS_FILE_OPEN + || create_mode == OS_FILE_OPEN_SILENT + || create_mode == OS_FILE_OPEN_RETRY + || create_mode == OS_FILE_OPEN_RETRY_SILENT + || create_mode == OS_FILE_OPEN_RAW); + create_flag = O_RDWR | O_CLOEXEC; + } - } else if (create_mode == OS_FILE_OVERWRITE) { +#ifdef O_DIRECT + struct stat st; + ut_a(type == OS_LOG_FILE + || type == OS_DATA_FILE || type == OS_DATA_FILE_NO_O_DIRECT); + int direct_flag = 0; - mode_str = "OVERWRITE"; - create_flag = O_RDWR | O_CREAT | O_TRUNC; + if (type == OS_DATA_FILE) { + switch (srv_file_flush_method) { + case SRV_O_DSYNC: + case SRV_O_DIRECT: + case SRV_O_DIRECT_NO_FSYNC: + direct_flag = O_DIRECT; + break; + default: + break; + } +# ifdef __linux__ + } else if (type != OS_LOG_FILE) { + } else if (log_sys.log_buffered) { + skip_o_direct: + os_file_log_buffered(); + } else if (create_mode != OS_FILE_CREATE + && create_mode != OS_FILE_CREATE_SILENT + && !log_sys.is_opened()) { + if (stat(name, &st)) { + if (errno == ENOENT) { + if (create_mode & OS_FILE_ON_ERROR_SILENT) { + goto not_found; + } + sql_print_error( + "InnoDB: File %s was not found", name); + goto not_found; + } + goto skip_o_direct; + } - } else { - ib::error() - << "Unknown file create mode (" << create_mode << ")" - << " for file '" << name << "'"; + if (!os_file_log_maybe_unbuffered(st)) { + goto skip_o_direct; + } - return(OS_FILE_CLOSED); + direct_flag = O_DIRECT; + log_sys.log_maybe_unbuffered= true; +# endif } - -#ifdef HAVE_FCNTL_DIRECT - ut_a(type == OS_LOG_FILE - || type == OS_DATA_FILE - || type == OS_DATA_FILE_NO_O_DIRECT); #else ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE); + constexpr int direct_flag = 0; #endif ut_a(purpose == OS_FILE_AIO || purpose == OS_FILE_NORMAL); @@ -1216,115 +1255,66 @@ os_file_create_func( } os_file_t file; - bool retry; - do { - file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); + for (;;) { + file = open(name, create_flag | direct_flag, os_innodb_umask); if (file == -1) { - const char* operation; - - operation = (create_mode == OS_FILE_CREATE - && !read_only) ? "create" : "open"; - - *success = false; - - if (on_error_no_exit) { - retry = os_file_handle_error_no_exit( - name, operation, on_error_silent); - } else { - retry = os_file_handle_error(name, operation); +#ifdef O_DIRECT + if (direct_flag && errno == EINVAL) { + direct_flag = 0; +# ifdef __linux__ + if (type == OS_LOG_FILE) { + os_file_log_buffered(); + } +# endif + if (create_mode == OS_FILE_CREATE + || create_mode == OS_FILE_CREATE_SILENT) { + /* Linux may create the file + before rejecting the O_DIRECT. */ + unlink(name); + } + continue; + } +#endif + if (!os_file_handle_error_no_exit( + name, (create_flag & O_CREAT) + ? "create" : "open", + create_mode & OS_FILE_ON_ERROR_SILENT)) { + break; } } else { *success = true; - retry = false; + break; } - - } while (retry); + } if (!*success) { - return file; +#ifdef __linux__ +not_found: +#endif + return OS_FILE_CLOSED; } -#ifdef HAVE_FCNTL_DIRECT - if (type == OS_DATA_FILE) { - switch (srv_file_flush_method) { - case SRV_O_DSYNC: - case SRV_O_DIRECT: - case SRV_O_DIRECT_NO_FSYNC: -# ifdef __linux__ -use_o_direct: -# endif - os_file_set_nocache(file, name, mode_str); - break; - default: - break; - } - } -# ifdef __linux__ - else if (type == OS_LOG_FILE && !log_sys.is_opened()) { - struct stat st; - char b[20 + sizeof "/sys/dev/block/" ":" - "/../queue/physical_block_size"]; - int f; - if (fstat(file, &st)) { - goto skip_o_direct; - } - MSAN_STAT_WORKAROUND(&st); - if (snprintf(b, sizeof b, - "/sys/dev/block/%u:%u/queue/physical_block_size", - major(st.st_dev), minor(st.st_dev)) - >= static_cast<int>(sizeof b)) { - goto skip_o_direct; - } - if ((f = open(b, O_RDONLY)) == -1) { - if (snprintf(b, sizeof b, - "/sys/dev/block/%u:%u/../queue/" - "physical_block_size", - major(st.st_dev), minor(st.st_dev)) - >= static_cast<int>(sizeof b)) { - goto skip_o_direct; - } - f = open(b, O_RDONLY); - } - if (f != -1) { - ssize_t l = read(f, b, sizeof b); - unsigned long s = 0; - - if (l > 0 && static_cast<size_t>(l) < sizeof b - && b[l - 1] == '\n') { - char* end = b; - s = strtoul(b, &end, 10); - if (b == end || *end != '\n') { - s = 0; - } - } - close(f); - if (s > 4096 || s < 64 || !ut_is_2pow(s)) { - goto skip_o_direct; - } - log_sys.log_maybe_unbuffered= true; - log_sys.set_block_size(uint32_t(s)); - if (!log_sys.log_buffered && !(st.st_size & (s - 1))) { - goto use_o_direct; - } +#ifdef __linux__ + if ((create_flag & O_CREAT) && type == OS_LOG_FILE) { + if (fstat(file, &st) || !os_file_log_maybe_unbuffered(st)) { + os_file_log_buffered(); } else { -skip_o_direct: - log_sys.log_maybe_unbuffered= false; - log_sys.log_buffered= true; - log_sys.set_block_size(512); + close(file); + return os_file_create_func(name, OS_FILE_OPEN, purpose, + type, false, success); } } -# endif #endif -#ifndef _WIN32 if (!read_only && create_mode != OS_FILE_OPEN_RAW && !my_disable_locking && os_file_lock(file, name)) { - if (create_mode == OS_FILE_OPEN_RETRY) { + if (create_mode == OS_FILE_OPEN_RETRY + || create_mode == OS_FILE_OPEN_RETRY_SILENT) { ib::info() << "Retrying to lock the first data file"; @@ -1346,7 +1336,6 @@ skip_o_direct: close(file); file = -1; } -#endif /* !_WIN32 */ return(file); } @@ -1356,7 +1345,7 @@ os_file_create_simple_no_error_handling(), not directly this function! A simple function to open or create a file. @param[in] name name of the file or path as a null-terminated string -@param[in] create_mode create mode +@param[in] create_mode OS_FILE_CREATE or OS_FILE_OPEN @param[in] access_type OS_FILE_READ_ONLY, OS_FILE_READ_WRITE, or OS_FILE_READ_ALLOW_DELETE; the last option is used by a backup program reading the file @@ -1367,59 +1356,33 @@ A simple function to open or create a file. pfs_os_file_t os_file_create_simple_no_error_handling_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success) { os_file_t file; - int create_flag; - - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); + int create_flag = O_RDONLY | O_CLOEXEC; *success = false; - if (create_mode == OS_FILE_OPEN) { - - if (access_type == OS_FILE_READ_ONLY) { - - create_flag = O_RDONLY; - - } else if (read_only) { - - create_flag = O_RDONLY; - - } else { - + if (read_only) { + } else if (create_mode == OS_FILE_CREATE) { + create_flag = O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC; + } else { + ut_ad(create_mode == OS_FILE_OPEN); + if (access_type != OS_FILE_READ_ONLY) { ut_a(access_type == OS_FILE_READ_WRITE || access_type == OS_FILE_READ_ALLOW_DELETE); create_flag = O_RDWR; } - - } else if (read_only) { - - create_flag = O_RDONLY; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = O_RDWR | O_CREAT | O_EXCL; - - } else { - - ib::error() - << "Unknown file create mode " - << create_mode << " for file '" << name << "'"; - - return(OS_FILE_CLOSED); } - file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); + file = open(name, create_flag, os_innodb_umask); *success = (file != -1); -#ifndef _WIN32 if (!read_only && *success && access_type == OS_FILE_READ_WRITE @@ -1431,7 +1394,6 @@ os_file_create_simple_no_error_handling_func( file = -1; } -#endif /* !_WIN32 */ return(file); } @@ -1537,7 +1499,7 @@ bool os_file_close_func(os_file_t file) if (!ret) return true; - os_file_handle_error(NULL, "close"); + os_file_handle_error("close"); return false; } @@ -1810,7 +1772,7 @@ bool os_file_flush_func(os_file_t file) if (srv_start_raw_disk_in_use && GetLastError() == ERROR_INVALID_FUNCTION) return true; - os_file_handle_error(nullptr, "flush"); + os_file_handle_error("flush"); /* It is a fatal error if a file flush does not succeed, because then the database can get corrupt on disk */ @@ -1924,7 +1886,7 @@ A simple function to open or create a file. pfs_os_file_t os_file_create_simple_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success) @@ -1933,82 +1895,31 @@ os_file_create_simple_func( *success = false; - DWORD access; + DWORD access = GENERIC_READ; DWORD create_flag; DWORD attributes = 0; - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); ut_ad(srv_operation == SRV_OPERATION_NORMAL); - if (create_mode == OS_FILE_OPEN) { - - create_flag = OPEN_EXISTING; - - } else if (read_only) { - + if (read_only || create_mode == OS_FILE_OPEN) { create_flag = OPEN_EXISTING; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = CREATE_NEW; - - } else if (create_mode == OS_FILE_CREATE_PATH) { - - /* Create subdirs along the path if needed. */ - *success = os_file_create_subdirs_if_needed(name); - - if (!*success) { - - ib::error() - << "Unable to create subdirectories '" - << name << "'"; - - return(OS_FILE_CLOSED); - } - - create_flag = CREATE_NEW; - create_mode = OS_FILE_CREATE; - } else { - - ib::error() - << "Unknown file create mode (" - << create_mode << ") for file '" - << name << "'"; - - return(OS_FILE_CLOSED); + ut_ad(create_mode == OS_FILE_CREATE); + create_flag = CREATE_NEW; } if (access_type == OS_FILE_READ_ONLY) { - - access = GENERIC_READ; - } else if (read_only) { - ib::info() << "Read only mode set. Unable to" " open file '" << name << "' in RW mode, " << "trying RO mode"; - - access = GENERIC_READ; - - } else if (access_type == OS_FILE_READ_WRITE) { - - access = GENERIC_READ | GENERIC_WRITE; - } else { - - ib::error() - << "Unknown file access type (" << access_type << ") " - "for file '" << name << "'"; - - return(OS_FILE_CLOSED); + ut_ad(access_type == OS_FILE_READ_WRITE); + access = GENERIC_READ | GENERIC_WRITE; } - bool retry; - - do { + for (;;) { /* Use default security attributes and no template file. */ file = CreateFile( @@ -2016,22 +1927,18 @@ os_file_create_simple_func( FILE_SHARE_READ | FILE_SHARE_DELETE, my_win_file_secattr(), create_flag, attributes, NULL); - if (file == INVALID_HANDLE_VALUE) { - - *success = false; - - retry = os_file_handle_error( - name, create_mode == OS_FILE_OPEN ? - "open" : "create"); - - } else { - - retry = false; - + if (file != INVALID_HANDLE_VALUE) { *success = true; + break; } - } while (retry); + if (!os_file_handle_error_no_exit(name, + create_flag == CREATE_NEW + ? "create" : "open", + false)) { + break; + } + } return(file); } @@ -2100,16 +2007,13 @@ Opens an existing file or creates a new. pfs_os_file_t os_file_create_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint purpose, ulint type, bool read_only, bool* success) { os_file_t file; - bool retry; - bool on_error_no_exit; - bool on_error_silent; *success = false; @@ -2120,54 +2024,30 @@ os_file_create_func( return(OS_FILE_CLOSED); ); - DWORD create_flag; + DWORD create_flag = OPEN_EXISTING; DWORD share_mode = read_only ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; - on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT - ? true : false; - - on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT - ? true : false; - - create_mode &= ~(OS_FILE_ON_ERROR_NO_EXIT | OS_FILE_ON_ERROR_SILENT); - - if (create_mode == OS_FILE_OPEN_RAW) { - + switch (create_mode) { + case OS_FILE_OPEN_RAW: ut_a(!read_only); - /* On Windows Physical devices require admin privileges and have to have the write-share mode set. See the remarks section for the CreateFile() function documentation in MSDN. */ share_mode |= FILE_SHARE_WRITE; - - create_flag = OPEN_EXISTING; - - } else if (create_mode == OS_FILE_OPEN - || create_mode == OS_FILE_OPEN_RETRY) { - - create_flag = OPEN_EXISTING; - - } else if (read_only) { - - create_flag = OPEN_EXISTING; - - } else if (create_mode == OS_FILE_CREATE) { - + break; + case OS_FILE_CREATE_SILENT: + case OS_FILE_CREATE: create_flag = CREATE_NEW; - - } else if (create_mode == OS_FILE_OVERWRITE) { - - create_flag = CREATE_ALWAYS; - - } else { - ib::error() - << "Unknown file create mode (" << create_mode << ") " - << " for file '" << name << "'"; - - return(OS_FILE_CLOSED); + break; + default: + ut_ad(create_mode == OS_FILE_OPEN + || create_mode == OS_FILE_OPEN_SILENT + || create_mode == OS_FILE_OPEN_RETRY_SILENT + || create_mode == OS_FILE_OPEN_RETRY); + break; } DWORD attributes = (purpose == OS_FILE_AIO && srv_use_native_aio) @@ -2225,18 +2105,11 @@ os_file_create_func( break; } - operation = (create_mode == OS_FILE_CREATE && !read_only) ? - "create" : "open"; + operation = create_flag == CREATE_NEW ? "create" : "open"; - if (on_error_no_exit) { - retry = os_file_handle_error_no_exit( - name, operation, on_error_silent); - } - else { - retry = os_file_handle_error(name, operation); - } - - if (!retry) { + if (!os_file_handle_error_no_exit(name, operation, + create_mode + & OS_FILE_ON_ERROR_SILENT)) { break; } } @@ -2263,79 +2136,42 @@ A simple function to open or create a file. pfs_os_file_t os_file_create_simple_no_error_handling_func( const char* name, - ulint create_mode, + os_file_create_t create_mode, ulint access_type, bool read_only, bool* success) { os_file_t file; - *success = false; - - DWORD access; - DWORD create_flag; + DWORD access = GENERIC_READ; + DWORD create_flag = OPEN_EXISTING; DWORD attributes = 0; - DWORD share_mode = read_only - ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE - : FILE_SHARE_READ | FILE_SHARE_DELETE; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; ut_a(name); - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); - - if (create_mode == OS_FILE_OPEN) { - - create_flag = OPEN_EXISTING; - - } else if (read_only) { - - create_flag = OPEN_EXISTING; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = CREATE_NEW; - + if (read_only) { + share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE + | FILE_SHARE_DELETE; } else { + if (create_mode == OS_FILE_CREATE) { + create_flag = CREATE_NEW; + } else { + ut_ad(create_mode == OS_FILE_OPEN); + } - ib::error() - << "Unknown file create mode (" << create_mode << ") " - << " for file '" << name << "'"; - - return(OS_FILE_CLOSED); - } - - if (access_type == OS_FILE_READ_ONLY) { - - access = GENERIC_READ; - - } else if (read_only) { - - access = GENERIC_READ; - - } else if (access_type == OS_FILE_READ_WRITE) { - - access = GENERIC_READ | GENERIC_WRITE; - - } else if (access_type == OS_FILE_READ_ALLOW_DELETE) { - - ut_a(!read_only); - - access = GENERIC_READ; - - /*!< A backup program has to give mysqld the maximum - freedom to do what it likes with the file */ - - share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE - | FILE_SHARE_READ; - - } else { - - ib::error() - << "Unknown file access type (" << access_type << ") " - << "for file '" << name << "'"; - - return(OS_FILE_CLOSED); + switch (access_type) { + case OS_FILE_READ_ONLY: break; + case OS_FILE_READ_WRITE: + access = GENERIC_READ | GENERIC_WRITE; + break; + default: + ut_ad(access_type == OS_FILE_READ_ALLOW_DELETE); + /* A backup program has to give mariadbd the maximum + freedom to do what it likes with the file */ + share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE + | FILE_SHARE_READ; + } } file = CreateFile((LPCTSTR) name, @@ -2503,7 +2339,7 @@ bool os_file_close_func(os_file_t file) ut_ad(file); if (!CloseHandle(file)) { - os_file_handle_error(NULL, "close"); + os_file_handle_error("close"); return false; } @@ -2941,8 +2777,8 @@ os_file_read_func( if (ulint(n_bytes) == n || err != DB_SUCCESS) return err; - os_file_handle_error_cond_exit(type.node ? type.node->name : nullptr, "read", - false, false); + os_file_handle_error_no_exit(type.node ? type.node->name : nullptr, "read", + false); sql_print_error("InnoDB: Tried to read %zu bytes at offset %llu" " of file %s, but was only able to read %zd", n, offset, type.node ? type.node->name : "(unknown)", @@ -3045,36 +2881,6 @@ os_file_handle_error_cond_exit( return(false); } -#ifdef HAVE_FCNTL_DIRECT -/** Tries to disable OS caching on an opened file descriptor. -@param[in] fd file descriptor to alter -@param[in] file_name file name, used in the diagnostic message -@param[in] name "open" or "create"; used in the diagnostic - message */ -void -os_file_set_nocache(int fd, const char *file_name, const char *operation_name) -{ - if (fcntl(fd, F_SETFL, O_DIRECT) == -1) { - int errno_save = errno; - static bool warning_message_printed = false; - if (errno_save == EINVAL) { - if (!warning_message_printed) { - warning_message_printed = true; - ib::info() - << "Setting O_DIRECT on file " - << file_name << " failed"; - } - } else { - ib::warn() - << "Failed to set O_DIRECT on file " - << file_name << "; " << operation_name - << " : " << strerror(errno_save) - << ", continuing anyway."; - } - } -} -#endif /* HAVE_FCNTL_DIRECT */ - /** Check if the file system supports sparse files. @param fh file handle @return true if the file system supports sparse files */ @@ -3859,8 +3665,9 @@ func_exit: if (srv_thread_pool->submit_io(cb)) { slots->release(cb); - os_file_handle_error(type.node->name, type.is_read() - ? "aio read" : "aio write"); + os_file_handle_error_no_exit(type.node->name, type.is_read() + ? "aio read" : "aio write", + false); err = DB_IO_ERROR; type.node->space->release(); } diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 89e6d149..e375fbfb 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -3269,7 +3269,6 @@ page_zip_validate_low( ibool sloppy) /*!< in: FALSE=strict, TRUE=ignore the MIN_REC_FLAG */ { - page_zip_des_t temp_page_zip; ibool valid; if (memcmp(page_zip->data + FIL_PAGE_PREV, page + FIL_PAGE_PREV, @@ -3310,7 +3309,7 @@ page_zip_validate_low( MEM_CHECK_DEFINED(page, srv_page_size); MEM_CHECK_DEFINED(page_zip->data, page_zip_get_size(page_zip)); - temp_page_zip = *page_zip; + page_zip_des_t temp_page_zip(*page_zip); valid = page_zip_decompress_low(&temp_page_zip, temp_page, TRUE); if (!valid) { fputs("page_zip_validate(): failed to decompress\n", stderr); diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index a862edd7..47be5e44 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -427,7 +427,7 @@ start: } if (!field->fixed_len - || (format == REC_LEAF_TEMP + || (format <= REC_LEAF_TEMP_INSTANT && !dict_col_get_fixed_size(col, true))) { /* Variable-length field: read the length */ len = *lens--; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 2516e24e..6194e9c3 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -45,9 +45,11 @@ Created 2012-02-08 by Sunny Bains. #include "lzo/lzo1x.h" #include "snappy-c.h" #include "log.h" +#include "table.h" +#include "ha_innodb.h" #include "scope.h" - +#include "dict0crea.h" #include <vector> #ifdef HAVE_MY_AES_H @@ -192,6 +194,60 @@ struct row_import { dberr_t match_flags(THD *thd) const ; + ulint find_fts_idx_offset() const + { + for (ulint i= 0; i < m_n_indexes; i++) + { + const char* index_name= + reinterpret_cast<const char*>(m_indexes[i].m_name); + if (!strcmp(index_name, FTS_DOC_ID_INDEX_NAME)) + return i; + } + return ULINT_UNDEFINED; + } + + const row_index_t *find_index_by_name(const char *name) const + { + for (ulint i= 0; i < m_n_indexes; i++) + { + const char* index_name= + reinterpret_cast<const char*>(m_indexes[i].m_name); + if (!strcmp(index_name, name)) + return &m_indexes[i]; + } + return nullptr; + } + + /** @return whether cfg file has FTS_DOC_ID + & FTS_DOC_ID_INDEX*/ + bool has_hidden_fts() const + { + if (m_missing) return false; + ulint col_offset= find_col(FTS_DOC_ID_COL_NAME); + if (col_offset == ULINT_UNDEFINED) return false; + + const dict_col_t *col= &m_cols[col_offset]; + if (col->mtype != DATA_INT + || (col->prtype & ~(DATA_NOT_NULL + | DATA_UNSIGNED | DATA_BINARY_TYPE + | DATA_FTS_DOC_ID)) + || col->len != sizeof(doc_id_t)) + return false; + + return find_index_by_name(FTS_DOC_ID_INDEX_NAME) != nullptr; + } + + /** Need to check whether the table need to add system + generated fts column and system generated fts document index + @param table table to be imported + @return whether the table has to add system generated + fts column and fts index */ + bool need_hidden_fts(dict_table_t *table) const + { + return has_hidden_fts() && !table->fts_doc_id_index && + m_n_cols == static_cast<ulint>(table->n_cols + 1) && + m_n_indexes == UT_LIST_GET_LEN(table->indexes) + 1; + } dict_table_t* m_table; /*!< Table instance */ @@ -547,7 +603,7 @@ protected: if (m_xdes != 0) { const xdes_t* xdesc = xdes(page_no, m_xdes); - ulint pos = page_no % FSP_EXTENT_SIZE; + uint32_t pos = page_no % FSP_EXTENT_SIZE; return xdes_is_free(xdesc, pos); } @@ -1079,7 +1135,6 @@ row_import::find_col( return(i); } } - return(ULINT_UNDEFINED); } @@ -1800,14 +1855,39 @@ PageConverter::update_records( bool clust_index = m_index->m_srv_index == m_cluster_index; /* This will also position the cursor on the first user record. */ + rec_t* rec = m_rec_iter.open(block, m_index->m_srv_index); - if (!m_rec_iter.open(block, m_index->m_srv_index)) { + if (!rec) { return DB_CORRUPTION; } + ulint deleted; + + if (!page_has_prev(block->page.frame) + && m_index->m_srv_index->is_instant()) { + /* Expect to find the hidden metadata record */ + if (page_rec_is_supremum(rec)) { + return DB_CORRUPTION; + } + + const ulint info_bits = rec_get_info_bits(rec, comp); + + if (!(info_bits & REC_INFO_MIN_REC_FLAG)) { + return DB_CORRUPTION; + } + + if (!(info_bits & REC_INFO_DELETED_FLAG) + != !m_index->m_srv_index->table->instant) { + return DB_CORRUPTION; + } + + deleted = 0; + goto first; + } + while (!m_rec_iter.end()) { - rec_t* rec = m_rec_iter.current(); - ibool deleted = rec_get_deleted_flag(rec, comp); + rec = m_rec_iter.current(); + deleted = rec_get_deleted_flag(rec, comp); /* For the clustered index we have to adjust the BLOB reference and the system fields irrespective of the @@ -1815,6 +1895,7 @@ PageConverter::update_records( cluster records is required for purge to work later. */ if (deleted || clust_index) { +first: m_offsets = rec_get_offsets( rec, m_index->m_srv_index, m_offsets, m_index->m_srv_index->n_core_fields, @@ -2101,14 +2182,30 @@ dberr_t PageConverter::operator()(buf_block_t* block) UNIV_NOTHROW return DB_SUCCESS; } -/*****************************************************************//** -Clean up after import tablespace. */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +static void reload_fts_table(row_prebuilt_t *prebuilt, + dict_table_t* table) +{ + ut_ad(prebuilt->table != table); + /* Reload the table in case of hidden fts column */ + const table_id_t id= prebuilt->table->id; + prebuilt->table->release(); + dict_sys.remove(prebuilt->table); + prebuilt->table= + dict_table_open_on_id(id, true, DICT_TABLE_OP_NORMAL); + prebuilt->table->space= table->space; +} + +/** Clean up after import tablespace. +@param prebuilt prebuilt from handler +@param err error code +@param fts_table constructed table which has system generated + fulltext document id +@return error code or DB_SUCCESS */ +static dberr_t -row_import_cleanup( -/*===============*/ - row_prebuilt_t* prebuilt, /*!< in/out: prebuilt from handler */ - dberr_t err) /*!< in: error code */ +row_import_cleanup(row_prebuilt_t* prebuilt, + dberr_t err, + dict_table_t* fts_table = nullptr) { if (err != DB_SUCCESS) { dict_table_t* table = prebuilt->table; @@ -2128,11 +2225,44 @@ row_import_cleanup( index = UT_LIST_GET_NEXT(indexes, index)) { index->page = FIL_NULL; } + + prebuilt->trx->rollback(); + } + else { + DBUG_EXECUTE_IF("ib_import_before_commit_crash", DBUG_SUICIDE();); + prebuilt->trx->commit(); } - DBUG_EXECUTE_IF("ib_import_before_commit_crash", DBUG_SUICIDE();); + if (fts_table && fts_table != prebuilt->table) { - prebuilt->trx->commit(); + if (err == DB_SUCCESS) { + reload_fts_table(prebuilt, fts_table); + ib::warn() << "Added system generated FTS_DOC_ID " + "and FTS_DOC_ID_INDEX while importing " + "the tablespace " << prebuilt->table->name; + } else if (fts_table->space) { + fil_close_tablespace(fts_table->space_id); + fts_table->space = NULL; + } + + if (!prebuilt->trx->dict_operation_lock_mode) { + dict_sys.lock(SRW_LOCK_CALL); + } + + dict_index_t* index = UT_LIST_GET_FIRST( + fts_table->indexes); + while (index) { + dict_index_t* next_index = + UT_LIST_GET_NEXT(indexes, index); + dict_index_remove_from_cache(fts_table, index); + index = next_index; + } + dict_mem_table_free(fts_table); + + if (!prebuilt->trx->dict_operation_lock_mode) { + dict_sys.unlock(); + } + } if (prebuilt->trx->dict_operation_lock_mode) { row_mysql_unlock_data_dictionary(prebuilt->trx); @@ -2145,14 +2275,17 @@ row_import_cleanup( return(err); } -/*****************************************************************//** -Report error during tablespace import. */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +/** Report error during tablespace import. +@param prebuilt prebuilt from the handler +@param err error code +@param fts_table table definition containing hidden FTS_DOC_ID column +@return error code or DB_SUCCESS */ +static dberr_t row_import_error( -/*=============*/ - row_prebuilt_t* prebuilt, /*!< in/out: prebuilt from handler */ - dberr_t err) /*!< in: error code */ + row_prebuilt_t* prebuilt, + dberr_t err, + dict_table_t* fts_table=nullptr) { if (!trx_is_interrupted(prebuilt->trx)) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -2167,7 +2300,7 @@ row_import_error( table_name, (ulong) err, ut_strerr(err)); } - return row_import_cleanup(prebuilt, err); + return row_import_cleanup(prebuilt, err, fts_table); } /*****************************************************************//** @@ -3059,7 +3192,139 @@ static size_t get_buf_size() ); } -/* find, parse instant metadata, performing variaous checks, +/** Add fts index to the table +@param table fts index to be added on the table */ +static void add_fts_index(dict_table_t *table) +{ + dict_index_t *fts_index= dict_mem_index_create( + table, FTS_DOC_ID_INDEX_NAME, DICT_UNIQUE, 2); + fts_index->page= FIL_NULL; + fts_index->cached= 1; + fts_index->n_uniq= 1; + /* Add fields for FTS_DOC_ID_INDEX */ + dict_index_add_col( + fts_index, table, + &table->cols[table->n_cols - (DATA_N_SYS_COLS + 1)], 0); + dict_index_t *clust_index= UT_LIST_GET_FIRST(table->indexes); + for (ulint i= 0; i < clust_index->n_uniq; i++) + dict_index_add_col(fts_index, table, clust_index->fields[i].col, + clust_index->fields[i].prefix_len); +#ifdef BTR_CUR_HASH_ADAPT + fts_index->search_info= btr_search_info_create(fts_index->heap); + fts_index->search_info->ref_count= 0; +#endif /* BTR_CUR_HASH_ADAPT */ + UT_LIST_ADD_LAST(fts_index->table->indexes, fts_index); +} + +/** Append the hidden fts column and fts doc index to the +existing table +@param table table to be imported +@param thd thread +@param cfg metadata required by import +@return table which has fts doc id and fts doc id index */ +static dict_table_t *build_fts_hidden_table( + dict_table_t *table, const row_import &cfg) +{ + dict_table_t *new_table= dict_table_t::create( + {table->name.m_name, strlen(table->name.m_name)}, + table->space, table->n_t_cols - (DATA_N_SYS_COLS - 1), + table->n_v_cols, table->flags, + table->flags2); + + new_table->id= table->id; + new_table->space_id= table->space_id; + const char* col_name= &table->col_names[0]; + /* Copy columns from old table to new fts table */ + for (ulint new_i= 0; + new_i < ulint(new_table->n_cols - (DATA_N_SYS_COLS + 1)); + new_i++) + { + dict_mem_table_add_col(new_table, new_table->heap, col_name, + table->cols[new_i].mtype, + table->cols[new_i].prtype, + table->cols[new_i].len); + col_name+= strlen(col_name) + 1; + } + + unsigned fts_col_ind= unsigned(table->n_cols - DATA_N_SYS_COLS); + fts_add_doc_id_column(new_table, new_table->heap); + new_table->cols[fts_col_ind].ind= + fts_col_ind & dict_index_t::MAX_N_FIELDS; + new_table->cols[fts_col_ind].ord_part= 1; + dict_table_add_system_columns(new_table, new_table->heap); + + col_name= &table->v_col_names[0]; + for (ulint new_i= 0; new_i < new_table->n_v_cols; new_i++) + { + dict_col_t old_vcol= table->v_cols[new_i].m_col; + dict_mem_table_add_v_col(new_table, new_table->heap, col_name, + old_vcol.mtype, old_vcol.prtype, + old_vcol.len, old_vcol.ind + 1, + table->v_cols[new_i].num_base); + for (ulint i= 0; i < table->v_cols[new_i].num_base; i++) + { + dict_col_t *base_col= dict_table_get_nth_col( + new_table, table->v_cols[new_i].base_col[i]->ind); + new_table->v_cols[new_i].base_col[i]= base_col; + } + col_name+= strlen(col_name) + 1; + } + + bool is_clustered= true; + /* Copy indexes from old table to new table */ + for (dict_index_t *old_index= UT_LIST_GET_FIRST(table->indexes); + old_index; is_clustered= false) + { + dict_index_t *new_index= dict_mem_index_create( + new_table, old_index->name, old_index->type, + old_index->n_fields + is_clustered); + + new_index->id= old_index->id; + new_index->n_uniq= old_index->n_uniq; + new_index->type= old_index->type; + new_index->cached= 1; + new_index->n_user_defined_cols= old_index->n_user_defined_cols; + new_index->n_core_null_bytes= old_index->n_core_null_bytes; + /* Copy all fields from old index to new index */ + for (ulint i= 0; i < old_index->n_fields; i++) + { + dict_field_t *field= dict_index_get_nth_field(old_index, i); + dict_col_t *col= field->col; + if (col->is_virtual()) + { + dict_v_col_t *v_col= reinterpret_cast<dict_v_col_t*>(col); + col= &new_table->v_cols[v_col->v_pos].m_col; + } + else + { + unsigned ind= field->col->ind; + if (ind >= fts_col_ind) ind++; + col= &new_table->cols[ind]; + } + dict_index_add_col(new_index, new_table, col, + field->prefix_len); + if (i < old_index->n_uniq) col->ord_part= 1; + } + + if (is_clustered) + { + /* Add fts doc id in clustered index */ + dict_index_add_col( + new_index, new_table, &table->cols[fts_col_ind], 0); + new_index->fields[old_index->n_fields].fixed_len= sizeof(doc_id_t); + } + + new_index->search_info= old_index->search_info; + UT_LIST_ADD_LAST(new_index->table->indexes, new_index); + old_index= UT_LIST_GET_NEXT(indexes, old_index); + if (UT_LIST_GET_LEN(new_table->indexes) + == cfg.find_fts_idx_offset()) + add_fts_index(new_table); + } + return new_table; +} + +/* find, parse instant metadata, performing various checks, and apply it to dict_table_t @return DB_SUCCESS or some error */ static dberr_t handle_instant_metadata(dict_table_t *table, @@ -4228,6 +4493,107 @@ fil_tablespace_iterate( return(err); } +static void row_import_autoinc(dict_table_t *table, row_prebuilt_t *prebuilt, + uint64_t autoinc) +{ + if (!table->persistent_autoinc) + { + ut_ad(!autoinc); + return; + } + + if (autoinc) + { + btr_write_autoinc(dict_table_get_first_index(table), autoinc - 1); + autoinc_set: + table->autoinc= autoinc; + sql_print_information("InnoDB: %`.*s.%`s autoinc value set to " UINT64PF, + int(table->name.dblen()), table->name.m_name, + table->name.basename(), autoinc); + } + else if (TABLE *t= prebuilt->m_mysql_table) + { + if (const Field *ai= t->found_next_number_field) + { + autoinc= 1 + + btr_read_autoinc_with_fallback(table, innodb_col_no(ai), + t->s->mysql_version, + innobase_get_int_col_max_value(ai)); + goto autoinc_set; + } + } +} + +/** Update the virtual column position in SYS_COLUMNS and SYS_VIRTUAL +@param table_id table identifier +@param new_pos position value +@param trx transaction +@return DB_SUCCESS or error code */ +dberr_t update_vcol_pos(table_id_t table_id, ulint new_pos, trx_t *trx) +{ + pars_info_t *info= pars_info_create(); + pars_info_add_ull_literal(info, "id", table_id); + pars_info_add_int4_literal(info, "old_pos", new_pos - 1); + DBUG_EXECUTE_IF("ib_import_vcol_update_fail", + return DB_DUPLICATE_KEY;); + return que_eval_sql(info, + "PROCEDURE UPDATE_VCOL () IS\n" + "BEGIN\n" + "UPDATE SYS_COLUMNS SET POS = POS + 1 " + "WHERE TABLE_ID= :id AND POS = :old_pos;\n" + "UPDATE SYS_VIRTUAL SET POS = POS + 1 " + "WHERE TABLE_ID= :id AND POS = :old_pos;\n" + "END\n;", trx); +} + +/** +1) Update the position of the columns and +2) Insert the hidden fts doc id in the sys columns table +3) Insert the hidden fts doc id in the sys indexes and +sys_fields table +@param table table to be imported +@param fts_pos position of fts doc id column +@param trx transaction +@return DB_SUCCESS or error code */ +static +dberr_t innodb_insert_hidden_fts_col(dict_table_t* table, + ulint fts_pos, + trx_t* trx) +{ + dict_index_t* fts_idx= + dict_table_get_index_on_name(table, FTS_DOC_ID_INDEX_NAME); + if (!fts_idx) return DB_ERROR; + for (ulint new_i= 0; new_i < table->n_v_cols; new_i++) + { + ulint pos= dict_create_v_col_pos( + table->v_cols[new_i].v_pos, + table->v_cols[new_i].m_col.ind); + if (dberr_t err= update_vcol_pos(table->id, pos, trx)) + return err; + } + pars_info_t *info= pars_info_create(); + pars_info_add_ull_literal(info, "id", table->id); + dict_hdr_get_new_id(NULL, &fts_idx->id, NULL); + pars_info_add_ull_literal(info, "idx_id", fts_idx->id); + pars_info_add_int4_literal(info, "pos", fts_pos); + pars_info_add_int4_literal(info, "space", fts_idx->table->space_id); + pars_info_add_int4_literal(info, "page_no", fts_idx->page); + + return que_eval_sql(info, + "PROCEDURE ADD_FTS_COL () IS\n" + "BEGIN\n" + "INSERT INTO SYS_COLUMNS VALUES" + "(:id,:pos,'FTS_DOC_ID',6, 1795, 8, 0);\n" + "UPDATE SYS_TABLES SET N_COLS = N_COLS + 1" + " WHERE ID = :id;\n" + "INSERT INTO SYS_INDEXES VALUES" + "(:id, :idx_id, 'FTS_DOC_ID_INDEX', 1," + " 2, :space, :page_no, 50);\n" + "INSERT INTO SYS_FIELDS VALUES" + "(:idx_id, 1, 'FTS_DOC_ID');\n" + "END;\n", trx); +} + /*****************************************************************//** Imports a tablespace. The space id in the .ibd file must match the space id of the table in the data dictionary. @@ -4253,9 +4619,27 @@ row_import_for_mysql( ut_ad(trx); ut_ad(trx->state == TRX_STATE_ACTIVE); ut_ad(!table->is_readable()); + ut_ad(prebuilt->table == table); ibuf_delete_for_discarded_space(table->space_id); +#ifdef BTR_CUR_HASH_ADAPT + /* On DISCARD TABLESPACE, we did not drop any adaptive hash + index entries. If we replaced the discarded tablespace with a + smaller one here, there could still be some adaptive hash + index entries that point to cached garbage pages in the buffer + pool, because PageConverter::operator() only evicted those + pages that were replaced by the imported pages. We must + detach any remaining adaptive hash index entries, because the + adaptive hash index must be a subset of the table contents; + false positives are not tolerated. */ + for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); index; + index = UT_LIST_GET_NEXT(indexes, index)) { + index = index->clone_if_needed(); + } +#endif /* BTR_CUR_HASH_ADAPT */ + UT_LIST_GET_FIRST(table->indexes)->clear_instant_alter(); + /* Assign an undo segment for the transaction, so that the transaction will be recovered after a crash. */ @@ -4281,7 +4665,6 @@ row_import_for_mysql( row_import cfg; THD* thd = trx->mysql_thd; - err = row_import_read_cfg(table, thd, cfg); /* Check if the table column definitions match the contents @@ -4289,8 +4672,16 @@ row_import_for_mysql( if (err == DB_SUCCESS) { - if (dberr_t err = handle_instant_metadata(table, cfg)) { - return row_import_error(prebuilt, err); + if (cfg.need_hidden_fts(table)) { + cfg.m_table = table = build_fts_hidden_table( + table, cfg); + } + + err = handle_instant_metadata(table, cfg); + if (err != DB_SUCCESS) { +import_error: + return row_import_error( + prebuilt, err, table); } /* We have a schema file, try and match it with our @@ -4326,7 +4717,7 @@ row_import_for_mysql( "table %s when .cfg file is missing.", table->name.m_name); err = DB_ERROR; - return row_import_error(prebuilt, err); + goto import_error; } FetchIndexRootPages fetchIndexRootPages(table, trx); @@ -4355,7 +4746,7 @@ row_import_for_mysql( } if (err != DB_SUCCESS) { - return row_import_error(prebuilt, err); + goto import_error; } trx->op_info = "importing tablespace"; @@ -4375,21 +4766,6 @@ row_import_for_mysql( DBUG_EXECUTE_IF("ib_import_reset_space_and_lsn_failure", err = DB_TOO_MANY_CONCURRENT_TRXS;); -#ifdef BTR_CUR_HASH_ADAPT - /* On DISCARD TABLESPACE, we did not drop any adaptive hash - index entries. If we replaced the discarded tablespace with a - smaller one here, there could still be some adaptive hash - index entries that point to cached garbage pages in the buffer - pool, because PageConverter::operator() only evicted those - pages that were replaced by the imported pages. We must - detach any remaining adaptive hash index entries, because the - adaptive hash index must be a subset of the table contents; - false positives are not tolerated. */ - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); index; - index = UT_LIST_GET_NEXT(indexes, index)) { - index = index->clone_if_needed(); - } -#endif /* BTR_CUR_HASH_ADAPT */ if (err != DB_SUCCESS) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -4406,7 +4782,7 @@ row_import_for_mysql( table_name, ut_strerr(err)); } - return row_import_cleanup(prebuilt, err); + goto import_error; } /* If the table is stored in a remote tablespace, we need to @@ -4469,7 +4845,8 @@ row_import_for_mysql( dict_index_t* index = dict_table_get_first_index(table); if (!dict_index_is_clust(index)) { - return row_import_error(prebuilt, DB_CORRUPTION); + err = DB_CORRUPTION; + goto import_error; } /* Update the Btree segment headers for index node and @@ -4481,7 +4858,7 @@ row_import_for_mysql( err = DB_CORRUPTION;); if (err != DB_SUCCESS) { - return row_import_error(prebuilt, err); + goto import_error; } else if (cfg.requires_purge(index->name)) { /* Purge any delete-marked records that couldn't be @@ -4500,7 +4877,7 @@ row_import_for_mysql( DBUG_EXECUTE_IF("ib_import_cluster_failure", err = DB_CORRUPTION;); if (err != DB_SUCCESS) { - return row_import_error(prebuilt, err); + goto import_error; } /* For secondary indexes, purge any records that couldn't be purged @@ -4513,7 +4890,7 @@ row_import_for_mysql( err = DB_CORRUPTION;); if (err != DB_SUCCESS) { - return row_import_error(prebuilt, err); + goto import_error; } /* Ensure that the next available DB_ROW_ID is not smaller than @@ -4528,9 +4905,9 @@ row_import_for_mysql( /* Ensure that all pages dirtied during the IMPORT make it to disk. The only dirty pages generated should be from the pessimistic purge of delete marked records that couldn't be purged in Phase I. */ - while (buf_flush_list_space(prebuilt->table->space)); + while (buf_flush_list_space(table->space)); - for (ulint count = 0; prebuilt->table->space->referenced(); count++) { + for (ulint count = 0; table->space->referenced(); count++) { /* Issue a warning every 10.24 seconds, starting after 2.56 seconds */ if ((count & 511) == 128) { @@ -4541,38 +4918,48 @@ row_import_for_mysql( } ib::info() << "Phase IV - Flush complete"; - prebuilt->table->space->set_imported(); + /* Set tablespace purpose as FIL_TYPE_TABLESPACE, + so that rollback can go ahead smoothly */ + table->space->set_imported(); + err = lock_sys_tables(trx); + if (err != DB_SUCCESS) { + goto import_error; + } /* The dictionary latches will be released in in row_import_cleanup() after the transaction commit, for both success and error. */ row_mysql_lock_data_dictionary(trx); + if (prebuilt->table != table) { + /* Add fts_doc_id and fts_doc_idx in data dictionary */ + err = innodb_insert_hidden_fts_col( + table, cfg.find_col(FTS_DOC_ID_COL_NAME), trx); + DBUG_EXECUTE_IF("ib_import_fts_error", + err= DB_DUPLICATE_KEY;); + if (err != DB_SUCCESS) { + goto import_error; + } + } /* Update the root pages of the table's indexes. */ err = row_import_update_index_root(trx, table, false); if (err != DB_SUCCESS) { - return row_import_error(prebuilt, err); + goto import_error; } err = row_import_update_discarded_flag(trx, table->id, false); if (err != DB_SUCCESS) { - return row_import_error(prebuilt, err); + goto import_error; } table->file_unreadable = false; table->flags2 &= ~DICT_TF2_DISCARDED & ((1U << DICT_TF2_BITS) - 1); /* Set autoinc value read from .cfg file, if one was specified. - Otherwise, keep the PAGE_ROOT_AUTO_INC as is. */ - if (autoinc) { - ib::info() << table->name << " autoinc value set to " - << autoinc; - - table->autoinc = autoinc--; - btr_write_autoinc(dict_table_get_first_index(table), autoinc); - } + Otherwise, read the PAGE_ROOT_AUTO_INC and set it to table autoinc. */ + row_import_autoinc(table, prebuilt, autoinc); - return row_import_cleanup(prebuilt, err); + return row_import_cleanup(prebuilt, err, table); } diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 9c3c5d22..1f319aae 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2000,7 +2000,7 @@ row_ins_dupl_error_with_rec( /* In a unique secondary index we allow equal key values if they contain SQL NULLs */ - if (!dict_index_is_clust(index) && !index->nulls_equal) { + if (!dict_index_is_clust(index)) { for (i = 0; i < n_unique; i++) { if (dfield_is_null(dtuple_get_nth_field(entry, i))) { @@ -2102,16 +2102,8 @@ row_ins_scan_sec_index_for_duplicate( /* If the secondary index is unique, but one of the fields in the n_unique first fields is NULL, a unique key violation cannot occur, since we define NULL != NULL in this case */ - - if (!index->nulls_equal) { - for (ulint i = 0; i < n_unique; i++) { - if (UNIV_SQL_NULL == dfield_get_len( - dtuple_get_nth_field(entry, i))) { - - DBUG_RETURN(DB_SUCCESS); - } - } - } + if (index->n_nullable && dtuple_contains_null(entry, n_unique)) + DBUG_RETURN(DB_SUCCESS); /* Store old value on n_fields_cmp */ @@ -2569,12 +2561,6 @@ row_ins_index_entry_big_rec( return(error); } -#ifdef HAVE_REPLICATION /* Working around MDEV-24622 */ -extern "C" int thd_is_slave(const MYSQL_THD thd); -#else -# define thd_is_slave(thd) 0 -#endif - #if defined __aarch64__&&defined __GNUC__&&__GNUC__==4&&!defined __clang__ /* Avoid GCC 4.8.5 internal compiler error due to srw_mutex::wr_unlock(). We would only need this for row_ins_clust_index_entry_low(), @@ -2728,7 +2714,8 @@ err_exit: && !index->table->is_active_ddl() && !index->table->has_spatial_index() && !index->table->versioned() - && !thd_is_slave(trx->mysql_thd) /* FIXME: MDEV-24622 */) { + && (!dict_table_is_partition(index->table) + || thd_sql_command(trx->mysql_thd) == SQLCOM_INSERT)) { DEBUG_SYNC_C("empty_root_page_insert"); trx->bulk_insert = true; diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 188d8ba5..6fb530f0 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -526,8 +526,6 @@ static ulint row_merge_bulk_buf_add(row_merge_buf_t* buf, @param[in,out] row table row @param[in] ext cache of externally stored column prefixes, or NULL -@param[in] history_fts row is historical in a system-versioned table - on which a FTS_DOC_ID_INDEX(FTS_DOC_ID) exists @param[in,out] doc_id Doc ID if we are creating FTS index @param[in,out] conv_heap memory heap where to allocate data when @@ -550,7 +548,6 @@ row_merge_buf_add( fts_psort_t* psort_info, dtuple_t* row, const row_ext_t* ext, - const bool history_fts, doc_id_t* doc_id, mem_heap_t* conv_heap, dberr_t* err, @@ -615,7 +612,7 @@ error: : NULL; /* Process the Doc ID column */ - if (!v_col && (history_fts || *doc_id) + if (!v_col && *doc_id && col->ind == index->table->fts->doc_col) { fts_write_doc_id((byte*) &write_doc_id, *doc_id); @@ -676,7 +673,7 @@ error: } /* Tokenize and process data for FTS */ - if (!history_fts && (index->type & DICT_FTS)) { + if (index->type & DICT_FTS) { fts_doc_item_t* doc_item; byte* value; void* ptr; @@ -1895,6 +1892,7 @@ row_merge_read_clustered_index( DBUG_ENTER("row_merge_read_clustered_index"); ut_ad((old_table == new_table) == !col_map); + ut_ad(old_table->fts || !new_table->fts || !new_table->versioned()); ut_ad(!defaults || col_map); ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)); ut_ad(trx->id); @@ -2120,7 +2118,6 @@ corrupted_metadata: dtuple_t* row; row_ext_t* ext; page_cur_t* cur = btr_pcur_get_page_cur(&pcur); - bool history_row, history_fts = false; stage->n_pk_recs_inc(); @@ -2382,11 +2379,6 @@ end_of_index: row_heap); ut_ad(row); - history_row = new_table->versioned() - && dtuple_get_nth_field(row, new_table->vers_end) - ->vers_history_row(); - history_fts = history_row && new_table->fts; - for (ulint i = 0; i < n_nonnull; i++) { dfield_t* field = &row->fields[nonnull[i]]; @@ -2415,7 +2407,7 @@ end_of_index: } /* Get the next Doc ID */ - if (add_doc_id && !history_fts) { + if (add_doc_id) { doc_id++; } else { doc_id = 0; @@ -2455,7 +2447,9 @@ end_of_index: add_autoinc); if (new_table->versioned()) { - if (history_row) { + if (dtuple_get_nth_field(row, + new_table->vers_end) + ->vers_history_row()) { if (dfield_get_type(dfield)->prtype & DATA_NOT_NULL) { err = DB_UNSUPPORTED; my_error(ER_UNSUPPORTED_EXTENSION, MYF(0), @@ -2571,7 +2565,7 @@ write_buffers: if (UNIV_LIKELY (row && (rows_added = row_merge_buf_add( buf, fts_index, old_table, new_table, - psort_info, row, ext, history_fts, + psort_info, row, ext, &doc_id, conv_heap, &err, &v_heap, eval_table, trx, col_collate)))) { @@ -2904,7 +2898,7 @@ write_buffers: (!(rows_added = row_merge_buf_add( buf, fts_index, old_table, new_table, psort_info, - row, ext, history_fts, &doc_id, + row, ext, &doc_id, conv_heap, &err, &v_heap, eval_table, trx, col_collate)))) { /* An empty buffer should have enough @@ -4355,9 +4349,7 @@ void row_merge_drop_temp_indexes() UNIV_PFS_IO defined, register the file descriptor with Performance Schema. @param[in] path location for creating temporary merge files, or NULL @return File descriptor */ -pfs_os_file_t -row_merge_file_create_low( - const char* path) +static pfs_os_file_t row_merge_file_create_mode(const char *path, int mode) { if (!path) { path = mysql_tmpdir; @@ -4398,6 +4390,13 @@ row_merge_file_create_low( return(fd); } +/** Create a temporary file at the specified path. +@param path location for creating temporary merge files, or nullptr +@return File descriptor */ +pfs_os_file_t row_merge_file_create_low(const char *path) +{ + return row_merge_file_create_mode(path, O_BINARY | O_SEQUENTIAL); +} /** Create a merge file in the given location. @param[out] merge_file merge file structure @@ -4408,17 +4407,16 @@ row_merge_file_create( merge_file_t* merge_file, const char* path) { - merge_file->fd = row_merge_file_create_low(path); merge_file->offset = 0; merge_file->n_rec = 0; -#ifdef HAVE_FCNTL_DIRECT - if (merge_file->fd != OS_FILE_CLOSED) { - if (srv_disable_sort_file_cache) { - os_file_set_nocache(merge_file->fd, - "row0merge.cc", "sort"); - } - } + merge_file->fd = + row_merge_file_create_mode(path, +#if !defined _WIN32 && defined O_DIRECT + srv_disable_sort_file_cache + ? O_DIRECT | O_BINARY | O_SEQUENTIAL + : #endif + O_BINARY | O_SEQUENTIAL); return(merge_file->fd); } @@ -5353,18 +5351,8 @@ dberr_t trx_mod_table_time_t::write_bulk(dict_table_t *table, trx_t *trx) return err; } -dberr_t trx_t::bulk_insert_apply_low() +void trx_t::bulk_rollback_low() { - ut_ad(bulk_insert); - ut_ad(!check_unique_secondary); - ut_ad(!check_foreigns); - dberr_t err; - for (auto& t : mod_tables) - if (t.second.is_bulk_insert()) - if ((err= t.second.write_bulk(t.first, this)) != DB_SUCCESS) - goto bulk_rollback; - return DB_SUCCESS; -bulk_rollback: undo_no_t low_limit= UINT64_MAX; for (auto& t : mod_tables) { @@ -5374,9 +5362,37 @@ bulk_rollback: low_limit= t.second.get_first(); delete t.second.bulk_store; t.second.bulk_store= nullptr; + t.second.end_bulk_insert(); } } trx_savept_t bulk_save{low_limit}; rollback(&bulk_save); - return err; +} + +dberr_t trx_t::bulk_insert_apply_for_table(dict_table_t *table) +{ + auto it= mod_tables.find(table); + if (it != mod_tables.end()) + { + if (dberr_t err= it->second.write_bulk(table, this)) + { + bulk_rollback_low(); + return err; + } + it->second.end_bulk_insert(); + } + return DB_SUCCESS; +} + +dberr_t trx_t::bulk_insert_apply_low() +{ + ut_ad(bulk_insert); + for (auto& t : mod_tables) + if (t.second.is_bulk_insert()) + if (dberr_t err= t.second.write_bulk(t.first, this)) + { + bulk_rollback_low(); + return err; + } + return DB_SUCCESS; } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index c5ee3be7..6a71cf3a 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -695,6 +695,7 @@ handle_new_error: DBUG_RETURN(true); case DB_DEADLOCK: + case DB_RECORD_CHANGED: case DB_LOCK_TABLE_FULL: rollback: /* Roll back the whole transaction; this resolution was added @@ -1585,7 +1586,8 @@ init_fts_doc_id_for_ref( for (dict_foreign_t* foreign : table->referenced_set) { ut_ad(foreign->foreign_table); - if (foreign->foreign_table->fts) { + if (foreign->foreign_table->space + && foreign->foreign_table->fts) { fts_init_doc_id(foreign->foreign_table); } @@ -2383,7 +2385,6 @@ row_discard_tablespace( dict_table_change_id_in_cache(table, new_id); dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - if (index) index->clear_instant_alter(); /* Reset the root page numbers. */ for (; index; index = UT_LIST_GET_NEXT(indexes, index)) { diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index e927096f..057b20c7 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -431,6 +431,10 @@ row_quiesce_write_header( /*********************************************************************//** Write the table meta data after quiesce. @return DB_SUCCESS or error code */ + +/* Stack size 20904 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t row_quiesce_write_cfg( @@ -488,6 +492,7 @@ row_quiesce_write_cfg( return(err); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /*********************************************************************//** Check whether a table has an FTS index defined on it. diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 33f4d81f..944f7358 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -864,6 +864,11 @@ row_sel_build_committed_vers_for_mysql( column version if any */ mtr_t* mtr) /*!< in: mtr */ { + if (prebuilt->trx->snapshot_isolation) { + *old_vers = rec; + return; + } + if (prebuilt->old_vers_heap) { mem_heap_empty(prebuilt->old_vers_heap); } else { @@ -1184,11 +1189,11 @@ sel_set_rtr_rec_lock( ut_ad(page_align(first_rec) == cur_block->page.frame); ut_ad(match->valid); - match->block.page.lock.x_lock(); + match->block->page.lock.x_lock(); retry: cur_block = btr_pcur_get_block(pcur); - ut_ad(match->block.page.lock.have_x() - || match->block.page.lock.have_s()); + ut_ad(match->block->page.lock.have_x() + || match->block->page.lock.have_s()); ut_ad(page_is_leaf(cur_block->page.frame)); err = lock_sec_rec_read_check_and_lock( @@ -1288,7 +1293,7 @@ re_scan: ULINT_UNDEFINED, &heap); err = lock_sec_rec_read_check_and_lock( - 0, &match->block, rtr_rec->r_rec, index, + 0, match->block, rtr_rec->r_rec, index, my_offsets, static_cast<lock_mode>(mode), type, thr); @@ -1304,7 +1309,7 @@ re_scan: match->locked = true; func_end: - match->block.page.lock.x_unlock(); + match->block->page.lock.x_unlock(); if (heap != NULL) { mem_heap_free(heap); } @@ -3401,7 +3406,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( if (dict_index_is_spatial(sec_index) && btr_cur->rtr_info->matches && (page_align(rec) - == btr_cur->rtr_info->matches->block.page.frame + == btr_cur->rtr_info->matches->block->page.frame || rec != btr_pcur_get_rec(prebuilt->pcur))) { #ifdef UNIV_DEBUG rtr_info_t* rtr_info = btr_cur->rtr_info; @@ -4456,13 +4461,11 @@ early_not_found: DBUG_RETURN(DB_RECORD_NOT_FOUND); } +#if SIZEOF_SIZE_T < 8 + if (UNIV_LIKELY(~prebuilt->n_rows_fetched)) +#endif prebuilt->n_rows_fetched++; - if (prebuilt->n_rows_fetched > 1000000000) { - /* Prevent wrap-over */ - prebuilt->n_rows_fetched = 500000000; - } - mode = pcur->search_mode; } diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index a01eaea5..52f54443 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -190,7 +190,7 @@ row_undo_mod_clust_low( @param[in] rec clustered index record @param[in] index clustered index @return the byte offset of DB_TRX_ID, from the start of rec */ -static ulint row_trx_id_offset(const rec_t* rec, const dict_index_t* index) +ulint row_trx_id_offset(const rec_t* rec, const dict_index_t* index) { ut_ad(index->n_uniq <= MAX_REF_PARTS); ulint trx_id_offset = index->trx_id_offset; diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 75798241..62229842 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -364,11 +364,6 @@ static monitor_info_t innodb_counter_info[] = MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_LRU_GET_FREE_LOOPS}, - {"buffer_LRU_get_free_waits", "buffer", - "Total sleep waits in LRU get free.", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_LRU_GET_FREE_WAITS}, - {"buffer_flush_avg_page_rate", "buffer", "Average number of pages at which flushing is happening", MONITOR_NONE, @@ -472,11 +467,6 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_LRU_BATCH_EVICT_TOTAL_PAGE}, - {"buffer_LRU_single_flush_failure_count", "Buffer", - "Number of times attempt to flush a single page from LRU failed", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT}, - {"buffer_LRU_get_free_search", "Buffer", "Number of searches performed for a clean page", MONITOR_NONE, diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 738e0a7e..fc557673 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -91,10 +91,6 @@ Created 2/16/1996 Heikki Tuuri #include "zlib.h" #include "log.h" -/** We are prepared for a situation that we have this many threads waiting for -a transactional lock inside InnoDB. srv_start() sets the value. */ -ulint srv_max_n_threads; - /** Log sequence number at shutdown */ lsn_t srv_shutdown_lsn; @@ -201,7 +197,7 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) bool ret; os_file_t file{ os_file_create_func(logfile0.c_str(), - OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT, + OS_FILE_CREATE, OS_FILE_NORMAL, OS_LOG_FILE, false, &ret) }; @@ -632,15 +628,15 @@ static uint32_t srv_undo_tablespace_open(bool create, const char* name, } } - pfs_os_file_t fh= os_file_create(innodb_data_file_key, name, OS_FILE_OPEN | - OS_FILE_ON_ERROR_NO_EXIT | - OS_FILE_ON_ERROR_SILENT, + pfs_os_file_t fh= os_file_create(innodb_data_file_key, name, + OS_FILE_OPEN_SILENT, OS_FILE_AIO, OS_DATA_FILE, srv_read_only_mode, &success); if (!success) return 0; + ulint n_retries = 5; os_offset_t size= os_file_get_size(fh); ut_a(size != os_offset_t(-1)); @@ -648,15 +644,25 @@ static uint32_t srv_undo_tablespace_open(bool create, const char* name, { page_t *page= static_cast<byte*>(aligned_malloc(srv_page_size, srv_page_size)); +undo_retry: if (os_file_read(IORequestRead, fh, page, 0, srv_page_size, nullptr) != DB_SUCCESS) { err_exit: + if (n_retries && srv_operation == SRV_OPERATION_BACKUP) + { + sql_print_information("InnoDB: Retrying to read undo " + "tablespace %s", name); + n_retries--; + goto undo_retry; + } ib::error() << "Unable to read first page of file " << name; aligned_free(page); return ~0U; } + DBUG_EXECUTE_IF("undo_space_read_fail", goto err_exit;); + uint32_t id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); if (id == 0 || id >= SRV_SPACE_ID_UPPER_BOUND || memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, @@ -731,9 +737,7 @@ srv_check_undo_redo_logs_exists() fh = os_file_create_func( name, - OS_FILE_OPEN_RETRY - | OS_FILE_ON_ERROR_NO_EXIT - | OS_FILE_ON_ERROR_SILENT, + OS_FILE_OPEN_RETRY_SILENT, OS_FILE_NORMAL, OS_DATA_FILE, srv_read_only_mode, @@ -755,8 +759,7 @@ srv_check_undo_redo_logs_exists() auto logfilename = get_log_file_path(); fh = os_file_create_func(logfilename.c_str(), - OS_FILE_OPEN_RETRY | OS_FILE_ON_ERROR_NO_EXIT - | OS_FILE_ON_ERROR_SILENT, + OS_FILE_OPEN_RETRY_SILENT, OS_FILE_NORMAL, OS_LOG_FILE, srv_read_only_mode, &ret); @@ -1179,12 +1182,6 @@ dberr_t srv_start(bool create_new_db) mysql_stage_register("innodb", srv_stages, static_cast<int>(UT_ARR_SIZE(srv_stages))); - srv_max_n_threads = - 1 /* dict_stats_thread */ - + 1 /* fts_optimize_thread */ - + 128 /* safety margin */ - + max_connections; - srv_boot(); ib::info() << my_crc32c_implementation(); @@ -1523,6 +1520,71 @@ dberr_t srv_start(bool create_new_db) fil_system.space_id_reuse_warned = false; + if (srv_operation > SRV_OPERATION_EXPORT_RESTORED) { + ut_ad(srv_operation == SRV_OPERATION_RESTORE_EXPORT + || srv_operation == SRV_OPERATION_RESTORE); + return(err); + } + + /* Upgrade or resize or rebuild the redo logs before + generating any dirty pages, so that the old redo log + file will not be written to. */ + + if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) { + /* Completely ignore the redo log. */ + } else if (srv_read_only_mode) { + /* Leave the redo log alone. */ + } else if (log_sys.file_size == srv_log_file_size + && log_sys.format + == (srv_encrypt_log + ? log_t::FORMAT_ENC_10_8 + : log_t::FORMAT_10_8)) { + /* No need to add or remove encryption, + upgrade, or resize. */ + delete_log_files(); + } else { + /* Prepare to delete the old redo log file */ + const lsn_t lsn{srv_prepare_to_delete_redo_log_file()}; + + DBUG_EXECUTE_IF("innodb_log_abort_1", + return(srv_init_abort(DB_ERROR));); + /* Prohibit redo log writes from any other + threads until creating a log checkpoint at the + end of create_log_file(). */ + ut_d(recv_no_log_write = true); + ut_ad(!os_aio_pending_reads()); + ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); + ut_ad(!buf_pool.get_oldest_modification(0)); + ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); + /* os_aio_pending_writes() may hold here if + some write_io_callback() did not release the + slot yet. However, the page write itself must + have completed, because the buf_pool.flush_list + is empty. In debug builds, we wait for this to + happen, hoping to get a hung process if this + assumption does not hold. */ + ut_d(os_aio_wait_until_no_pending_writes(false)); + + /* Close the redo log file, so that we can replace it */ + log_sys.close_file(); + + DBUG_EXECUTE_IF("innodb_log_abort_5", + return(srv_init_abort(DB_ERROR));); + DBUG_PRINT("ib_log", ("After innodb_log_abort_5")); + + err = create_log_file(false, lsn); + + if (err == DB_SUCCESS && log_sys.resize_rename()) { + err = DB_ERROR; + } + + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); + } + } + + recv_sys.debug_free(); + if (!srv_read_only_mode) { const uint32_t flags = FSP_FLAGS_PAGE_SSIZE(); for (uint32_t id = srv_undo_space_id_start; @@ -1607,71 +1669,6 @@ dberr_t srv_start(bool create_new_db) return(srv_init_abort(DB_ERROR)); } } - - if (srv_operation > SRV_OPERATION_EXPORT_RESTORED) { - ut_ad(srv_operation == SRV_OPERATION_RESTORE_EXPORT - || srv_operation == SRV_OPERATION_RESTORE); - return(err); - } - - /* Upgrade or resize or rebuild the redo logs before - generating any dirty pages, so that the old redo log - file will not be written to. */ - - if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) { - /* Completely ignore the redo log. */ - } else if (srv_read_only_mode) { - /* Leave the redo log alone. */ - } else if (log_sys.file_size == srv_log_file_size - && log_sys.format - == (srv_encrypt_log - ? log_t::FORMAT_ENC_10_8 - : log_t::FORMAT_10_8)) { - /* No need to add or remove encryption, - upgrade, or resize. */ - delete_log_files(); - } else { - /* Prepare to delete the old redo log file */ - const lsn_t lsn{srv_prepare_to_delete_redo_log_file()}; - - DBUG_EXECUTE_IF("innodb_log_abort_1", - return(srv_init_abort(DB_ERROR));); - /* Prohibit redo log writes from any other - threads until creating a log checkpoint at the - end of create_log_file(). */ - ut_d(recv_no_log_write = true); - ut_ad(!os_aio_pending_reads()); - ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); - ut_ad(!buf_pool.get_oldest_modification(0)); - ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); - /* os_aio_pending_writes() may hold here if - some write_io_callback() did not release the - slot yet. However, the page write itself must - have completed, because the buf_pool.flush_list - is empty. In debug builds, we wait for this to - happen, hoping to get a hung process if this - assumption does not hold. */ - ut_d(os_aio_wait_until_no_pending_writes(false)); - - /* Close the redo log file, so that we can replace it */ - log_sys.close_file(); - - DBUG_EXECUTE_IF("innodb_log_abort_5", - return(srv_init_abort(DB_ERROR));); - DBUG_PRINT("ib_log", ("After innodb_log_abort_5")); - - err = create_log_file(false, lsn); - - if (err == DB_SUCCESS && log_sys.resize_rename()) { - err = DB_ERROR; - } - - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); - } - } - - recv_sys.debug_free(); } ut_ad(err == DB_SUCCESS); diff --git a/storage/innobase/sync/cache.cc b/storage/innobase/sync/cache.cc new file mode 100644 index 00000000..43d642d0 --- /dev/null +++ b/storage/innobase/sync/cache.cc @@ -0,0 +1,160 @@ +/***************************************************************************** + +Copyright (c) 2024, MariaDB plc + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/* This is based on the implementation of pmem_persist() in +https://github.com/pmem/pmdk/, Copyright 2014-2020, Intel Corporation, +last revised in libpmem-1.12.0. */ + +#include "my_global.h" +#include "cache.h" +#include <cstdint> + +#if defined __x86_64__ || defined __aarch64__ || defined __powerpc64__ +# ifdef __x86_64__ +static void pmem_clflush(const void *buf, size_t size) +{ + for (uintptr_t u= uintptr_t(buf) & ~(CPU_LEVEL1_DCACHE_LINESIZE), + end= uintptr_t(buf) + size; + u < end; u+= CPU_LEVEL1_DCACHE_LINESIZE) + __asm__ __volatile__("clflush %0" :: + "m"(*reinterpret_cast<const char*>(u)) : "memory"); +} + +static void pmem_clflushopt(const void *buf, size_t size) +{ + for (uintptr_t u= uintptr_t(buf) & ~(CPU_LEVEL1_DCACHE_LINESIZE), + end= uintptr_t(buf) + size; + u < end; u+= CPU_LEVEL1_DCACHE_LINESIZE) + __asm__ __volatile__(".byte 0x66; clflush %0" /* clflushopt */ :: + "m"(*reinterpret_cast<const char*>(u)) : "memory"); + __asm__ __volatile__("sfence" ::: "memory"); +} + +static void pmem_clwb(const void *buf, size_t size) +{ + for (uintptr_t u= uintptr_t(buf) & ~(CPU_LEVEL1_DCACHE_LINESIZE), + end= uintptr_t(buf) + size; + u < end; u+= CPU_LEVEL1_DCACHE_LINESIZE) + __asm__ __volatile__(".byte 0x66; xsaveopt %0" /* clwb */ :: + "m"(*reinterpret_cast<const char*>(u)) : "memory"); + __asm__ __volatile__("sfence" ::: "memory"); +} + +# include <cpuid.h> +static decltype(pmem_control::persist) pmem_persist_init() +{ + uint32_t eax= 0, ebx= 0, ecx= 0, edx= 0; + __cpuid_count(7, 0, eax, ebx, ecx, edx); + if (ebx & 1U<<24 /* CLWB */) + return pmem_clwb; + else if (ebx & 1U<<23 /* CLFLUSHOPT */) + return pmem_clflushopt; + else + return pmem_clflush; +} +# elif defined __aarch64__ +static void pmem_cvac(const void* buf, size_t size) +{ + for (uintptr_t u= uintptr_t(buf) & ~(CPU_LEVEL1_DCACHE_LINESIZE), + end= uintptr_t(buf) + size; + u < end; u+= CPU_LEVEL1_DCACHE_LINESIZE) + __asm__ __volatile__("dc cvac, %0" :: "r"(u) : "memory"); + __asm__ __volatile__("dmb ishst" ::: "memory"); +} + +static void pmem_cvap(const void* buf, size_t size) +{ + for (uintptr_t u= uintptr_t(buf) & ~(CPU_LEVEL1_DCACHE_LINESIZE), + end= uintptr_t(buf) + size; + u < end; u+= CPU_LEVEL1_DCACHE_LINESIZE) + __asm__ __volatile__(".arch armv8.2-a\n dc cvap, %0" :: "r"(u) : "memory"); + __asm__ __volatile__("dmb ishst" ::: "memory"); +} + +# include <sys/auxv.h> +# include <asm/hwcap.h> +# ifndef HWCAP_DCPOP +# define HWCAP_DCPOP (1 << 16) +# endif + +static decltype(pmem_control::persist) pmem_persist_init() +{ + return (getauxval(AT_HWCAP) & HWCAP_DCPOP) ? pmem_cvap : pmem_cvac; +} +# elif defined __powerpc64__ +static void pmem_phwsync(const void* buf, size_t size) +{ + for (uintptr_t u= uintptr_t(buf) & ~(CPU_LEVEL1_DCACHE_LINESIZE), + end= uintptr_t(buf) + size; + u < end; u+= CPU_LEVEL1_DCACHE_LINESIZE) + { + /* GCC is just passing the inline asm snippets to the assembler, + and it does not even define these mnemonics by itself. Clang does, + and it includes a built-in assembler. + + Let us hope that having a recent enough GCC is an adequate proxy + for having a recent enough assembler. */ +# if __GNUC__ >= 11 || (defined __clang_major__ && __clang_major__ >= 12) + __asm__ __volatile__("dcbstps 0,%0" :: "r"(u) : "memory"); +# else + __asm__ __volatile__(".long (0x7cc000AC | %0 << 11)" :: "r"(u) : "memory"); +# endif + } + +# if __GNUC__ >= 11 || (defined __clang_major__ && __clang_major__ >= 18) + __asm__ __volatile__("phwsync" ::: "memory"); +# else + __asm__ __volatile__(".long 0x7c80040a" ::: "memory"); +# endif +} + +# include <atomic> +static void pmem_fence(const void*, size_t) +{ + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +# include <sys/auxv.h> +# ifndef AT_HWCAP2 +# define AT_HWCAP2 26 +# endif +# ifndef PPC_FEATURE2_ARCH_3_1 +# define PPC_FEATURE2_ARCH_3_1 4 +# endif + +static decltype(pmem_control::persist) pmem_persist_init() +{ + return (getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_3_1) + ? pmem_phwsync : pmem_fence; +} +# endif + +pmem_control::pmem_control() : persist(pmem_persist_init()) {} +const pmem_control pmem; +#else +void pmem_persist(const void *buf, size_t size) +{ +# if defined __riscv && __riscv_xlen == 64 + __asm__ __volatile__("fence w,w" ::: "memory"); +# elif defined __loongarch64 + __asm__ __volatile__("dbar 0" ::: "memory"); +# else +# error "Missing implementation; recompile with cmake -DWITH_INNODB_PMEM=OFF" +# endif +} +#endif diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index 5afb79f2..19db1245 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -548,3 +548,124 @@ template void ssux_lock_impl<false>::rd_unlock(); template void ssux_lock_impl<false>::u_unlock(); template void ssux_lock_impl<false>::wr_unlock(); #endif /* UNIV_PFS_RWLOCK */ + +#ifdef UNIV_DEBUG +void srw_lock_debug::SRW_LOCK_INIT(mysql_pfs_key_t key) +{ + srw_lock::SRW_LOCK_INIT(key); + readers_lock.init(); + ut_ad(!readers.load(std::memory_order_relaxed)); + ut_ad(!have_any()); +} + +void srw_lock_debug::destroy() +{ + ut_ad(!writer); + if (auto r= readers.load(std::memory_order_relaxed)) + { + readers.store(0, std::memory_order_relaxed); + ut_ad(r->empty()); + delete r; + } + srw_lock::destroy(); +} + +bool srw_lock_debug::wr_lock_try() +{ + ut_ad(!have_any()); + if (!srw_lock::wr_lock_try()) + return false; + ut_ad(!writer); + writer.store(pthread_self(), std::memory_order_relaxed); + return true; +} + +void srw_lock_debug::wr_lock(SRW_LOCK_ARGS(const char *file, unsigned line)) +{ + ut_ad(!have_any()); + srw_lock::wr_lock(SRW_LOCK_ARGS(file, line)); + ut_ad(!writer); + writer.store(pthread_self(), std::memory_order_relaxed); +} + +void srw_lock_debug::wr_unlock() +{ + ut_ad(have_wr()); + writer.store(0, std::memory_order_relaxed); + srw_lock::wr_unlock(); +} + +void srw_lock_debug::readers_register() +{ + readers_lock.wr_lock(); + auto r= readers.load(std::memory_order_relaxed); + if (!r) + { + r= new std::unordered_multiset<pthread_t>(); + readers.store(r, std::memory_order_relaxed); + } + r->emplace(pthread_self()); + readers_lock.wr_unlock(); +} + +bool srw_lock_debug::rd_lock_try() +{ + ut_ad(!have_any()); + if (!srw_lock::rd_lock_try()) + return false; + readers_register(); + return true; +} + +void srw_lock_debug::rd_lock(SRW_LOCK_ARGS(const char *file, unsigned line)) +{ + ut_ad(!have_any()); + srw_lock::rd_lock(SRW_LOCK_ARGS(file, line)); + readers_register(); +} + +void srw_lock_debug::rd_unlock() +{ + const pthread_t self= pthread_self(); + ut_ad(writer != self); + readers_lock.wr_lock(); + auto r= readers.load(std::memory_order_relaxed); + ut_ad(r); + auto i= r->find(self); + ut_ad(i != r->end()); + r->erase(i); + readers_lock.wr_unlock(); + + srw_lock::rd_unlock(); +} + +bool srw_lock_debug::have_rd() const noexcept +{ + if (auto r= readers.load(std::memory_order_relaxed)) + { + readers_lock.wr_lock(); + bool found= r->find(pthread_self()) != r->end(); + readers_lock.wr_unlock(); +# ifndef SUX_LOCK_GENERIC + ut_ad(!found || is_locked()); +# endif + return found; + } + return false; +} + +bool srw_lock_debug::have_wr() const noexcept +{ + if (writer != pthread_self()) + return false; +# ifndef SUX_LOCK_GENERIC + ut_ad(is_write_locked()); +# endif + return true; +} + +bool srw_lock_debug::have_any() const noexcept +{ + return have_wr() || have_rd(); +} +#endif diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index cff16d9c..f32f4de5 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -56,84 +56,6 @@ purge_sys_t purge_sys; my_bool srv_purge_view_update_only_debug; #endif /* UNIV_DEBUG */ -/** Sentinel value */ -static const TrxUndoRsegs NullElement; - -/** Default constructor */ -TrxUndoRsegsIterator::TrxUndoRsegsIterator() - : m_rsegs(NullElement), m_iter(m_rsegs.begin()) -{ -} - -/** Sets the next rseg to purge in purge_sys. -Executed in the purge coordinator thread. -@retval false when nothing is to be purged -@retval true when purge_sys.rseg->latch was locked */ -inline bool TrxUndoRsegsIterator::set_next() -{ - ut_ad(!purge_sys.next_stored); - mysql_mutex_lock(&purge_sys.pq_mutex); - - /* Only purge consumes events from the priority queue, user - threads only produce the events. */ - - /* Check if there are more rsegs to process in the - current element. */ - if (m_iter != m_rsegs.end()) { - /* We are still processing rollback segment from - the same transaction and so expected transaction - number shouldn't increase. Undo the increment of - expected commit done by caller assuming rollback - segments from given transaction are done. */ - purge_sys.tail.trx_no = (*m_iter)->last_trx_no(); - } else if (!purge_sys.purge_queue.empty()) { - m_rsegs = purge_sys.purge_queue.top(); - purge_sys.purge_queue.pop(); - ut_ad(purge_sys.purge_queue.empty() - || purge_sys.purge_queue.top() != m_rsegs); - m_iter = m_rsegs.begin(); - } else { - /* Queue is empty, reset iterator. */ - purge_sys.rseg = NULL; - mysql_mutex_unlock(&purge_sys.pq_mutex); - m_rsegs = NullElement; - m_iter = m_rsegs.begin(); - return false; - } - - purge_sys.rseg = *m_iter++; - mysql_mutex_unlock(&purge_sys.pq_mutex); - - /* We assume in purge of externally stored fields that space - id is in the range of UNDO tablespace space ids */ - ut_ad(purge_sys.rseg->space->id == TRX_SYS_SPACE - || srv_is_undo_tablespace(purge_sys.rseg->space->id)); - - purge_sys.rseg->latch.wr_lock(SRW_LOCK_CALL); - trx_id_t last_trx_no = purge_sys.rseg->last_trx_no(); - purge_sys.hdr_offset = purge_sys.rseg->last_offset(); - purge_sys.hdr_page_no = purge_sys.rseg->last_page_no; - - /* Only the purge_coordinator_task will access this object - purge_sys.rseg_iter, or any of purge_sys.hdr_page_no, - purge_sys.tail. - The field purge_sys.head and purge_sys.view are modified by - purge_sys_t::clone_end_view() - in the purge_coordinator_task - while holding exclusive purge_sys.latch. - The purge_sys.view may also be modified by - purge_sys_t::wake_if_not_active() while holding exclusive - purge_sys.latch. - The purge_sys.head may be read by - purge_truncation_callback(). */ - ut_ad(last_trx_no == m_rsegs.trx_no); - ut_a(purge_sys.hdr_page_no != FIL_NULL); - ut_a(purge_sys.tail.trx_no <= last_trx_no); - purge_sys.tail.trx_no = last_trx_no; - - return(true); -} - /** Build a purge 'query' graph. The actual purge is performed by executing this query graph. @return own: the query graph */ @@ -345,7 +267,8 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) that is known to be corrupted. */ ut_a(flst_add_first(rseg_header, TRX_RSEG + TRX_RSEG_HISTORY, undo_page, uint16_t(page_offset(undo_header) + - TRX_UNDO_HISTORY_NODE), mtr) == DB_SUCCESS); + TRX_UNDO_HISTORY_NODE), rseg->space->free_limit, + mtr) == DB_SUCCESS); mtr->write<2>(*undo_page, TRX_UNDO_SEG_HDR + TRX_UNDO_STATE + undo_page->page.frame, undo_state); @@ -396,9 +319,7 @@ static void trx_purge_free_segment(buf_block_t *rseg_hdr, buf_block_t *block, void purge_sys_t::rseg_enable(trx_rseg_t &rseg) { ut_ad(this == &purge_sys); -#ifndef SUX_LOCK_GENERIC - ut_ad(rseg.latch.is_write_locked()); -#endif + ut_ad(rseg.latch.have_wr()); uint8_t skipped= skipped_rseg; ut_ad(skipped < TRX_SYS_N_RSEGS); if (&rseg == &trx_sys.rseg_array[skipped]) @@ -437,6 +358,19 @@ inline dberr_t purge_sys_t::iterator::free_history_rseg(trx_rseg_t &rseg) const mtr_t mtr; bool freed= false; uint32_t rseg_ref= 0; + const auto last_boffset= srv_page_size - TRX_UNDO_LOG_OLD_HDR_SIZE; + /* Technically, rseg.space->free_limit is not protected by + rseg.latch, which we are holding, but rseg.space->latch. The value + that we are reading may become stale (too small) if other pages are + being allocated in this tablespace, for other rollback + segments. Nothing can be added to this rseg without holding + rseg.latch, and hence we can validate the entire file-based list + against the limit that we are reading here. + + Note: The read here may look like a data race. On none of our target + architectures this should be an actual problem, because the uint32_t + value should always fit in a register and be correctly aligned. */ + const auto last_page= rseg.space->free_limit; mtr.start(); @@ -452,13 +386,23 @@ func_exit: } hdr_addr= flst_get_last(TRX_RSEG + TRX_RSEG_HISTORY + rseg_hdr->page.frame); - hdr_addr.boffset= static_cast<uint16_t>(hdr_addr.boffset - - TRX_UNDO_HISTORY_NODE); -loop: if (hdr_addr.page == FIL_NULL) goto func_exit; + if (hdr_addr.page >= last_page || + hdr_addr.boffset < TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE || + hdr_addr.boffset >= last_boffset) + { + corrupted: + err= DB_CORRUPTION; + goto func_exit; + } + + hdr_addr.boffset= static_cast<uint16_t>(hdr_addr.boffset - + TRX_UNDO_HISTORY_NODE); + +loop: buf_block_t *b= buf_page_get_gen(page_id_t(rseg.space->id, hdr_addr.page), 0, RW_X_LATCH, nullptr, BUF_GET_POSSIBLY_FREED, @@ -507,11 +451,18 @@ loop: fil_addr_t prev_hdr_addr= flst_get_prev_addr(b->page.frame + hdr_addr.boffset + TRX_UNDO_HISTORY_NODE); + if (prev_hdr_addr.page == FIL_NULL); + else if (prev_hdr_addr.page >= last_page || + prev_hdr_addr.boffset < TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE || + prev_hdr_addr.boffset >= last_boffset) + goto corrupted; + prev_hdr_addr.boffset= static_cast<uint16_t>(prev_hdr_addr.boffset - TRX_UNDO_HISTORY_NODE); err= flst_remove(rseg_hdr, TRX_RSEG + TRX_RSEG_HISTORY, b, - uint16_t(hdr_addr.boffset + TRX_UNDO_HISTORY_NODE), &mtr); + uint16_t(hdr_addr.boffset + TRX_UNDO_HISTORY_NODE), + last_page, &mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) goto func_exit; @@ -571,45 +522,21 @@ loop: ut_ad(rseg_hdr->page.id() == rseg.page_id()); mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_FIX); + if (hdr_addr.page == FIL_NULL) + goto func_exit; + goto loop; } -/** Cleanse purge queue to remove the rseg that reside in undo-tablespace -marked for truncate. -@param[in] space undo tablespace being truncated */ -static void trx_purge_cleanse_purge_queue(const fil_space_t& space) +void purge_sys_t::cleanse_purge_queue(const fil_space_t &space) { - typedef std::vector<TrxUndoRsegs> purge_elem_list_t; - purge_elem_list_t purge_elem_list; - - mysql_mutex_lock(&purge_sys.pq_mutex); - - /* Remove rseg instances that are in the purge queue before we start - truncate of corresponding UNDO truncate. */ - while (!purge_sys.purge_queue.empty()) { - purge_elem_list.push_back(purge_sys.purge_queue.top()); - purge_sys.purge_queue.pop(); - } - - for (purge_elem_list_t::iterator it = purge_elem_list.begin(); - it != purge_elem_list.end(); - ++it) { - - for (TrxUndoRsegs::iterator it2 = it->begin(); - it2 != it->end(); - ++it2) { - if ((*it2)->space == &space) { - it->erase(it2); - break; - } - } - - if (!it->empty()) { - purge_sys.purge_queue.push(*it); - } - } - - mysql_mutex_unlock(&purge_sys.pq_mutex); + mysql_mutex_lock(&pq_mutex); + auto purge_elem_list= clone_queue_container(); + purge_queue.clear(); + for (auto elem : purge_elem_list) + if (purge_queue::rseg(elem)->space != &space) + purge_queue.push_trx_no_rseg(elem); + mysql_mutex_unlock(&pq_mutex); } dberr_t purge_sys_t::iterator::free_history() const @@ -672,7 +599,9 @@ fil_space_t *purge_sys_t::truncating_tablespace() if (space || srv_undo_tablespaces_active < 2 || !srv_undo_log_truncate) return space; - const uint32_t size= uint32_t(srv_max_undo_log_size >> srv_page_size_shift); + const uint32_t size= + uint32_t(std::min(ulonglong{std::numeric_limits<uint32_t>::max()}, + srv_max_undo_log_size >> srv_page_size_shift)); for (uint32_t i= truncate_undo_space.last, j= i;; ) { if (fil_space_t *s= undo_truncate_try(srv_undo_space_id_start + i, size)) @@ -751,7 +680,7 @@ not_free: const char *file_name= UT_LIST_GET_FIRST(space->chain)->name; sql_print_information("InnoDB: Truncating %s", file_name); - trx_purge_cleanse_purge_queue(*space); + purge_sys.cleanse_purge_queue(*space); /* Lock all modified pages of the tablespace. @@ -870,13 +799,11 @@ buf_block_t *purge_sys_t::get_page(page_id_t id) return nullptr; } -void purge_sys_t::rseg_get_next_history_log() +bool purge_sys_t::rseg_get_next_history_log() { fil_addr_t prev_log_addr; -#ifndef SUX_LOCK_GENERIC - ut_ad(rseg->latch.is_write_locked()); -#endif + ut_ad(rseg->latch.have_wr()); ut_a(rseg->last_page_no != FIL_NULL); tail.trx_no= rseg->last_trx_no() + 1; @@ -888,21 +815,24 @@ void purge_sys_t::rseg_get_next_history_log() { const byte *log_hdr= undo_page->page.frame + rseg->last_offset(); prev_log_addr= flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE); + if (prev_log_addr.boffset < TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE || + prev_log_addr.boffset >= srv_page_size - TRX_UNDO_LOG_OLD_HDR_SIZE) + goto corrupted; prev_log_addr.boffset = static_cast<uint16_t>(prev_log_addr.boffset - TRX_UNDO_HISTORY_NODE); } else - prev_log_addr.page= FIL_NULL; + goto corrupted; - if (prev_log_addr.page == FIL_NULL) + if (prev_log_addr.page >= rseg->space->free_limit) + corrupted: rseg->last_page_no= FIL_NULL; else { /* Read the previous log header. */ trx_id_t trx_no= 0; if (const buf_block_t* undo_page= - get_page(page_id_t(rseg->space->id, - prev_log_addr.page))) + get_page(page_id_t(rseg->space->id, prev_log_addr.page))) { const byte *log_hdr= undo_page->page.frame + prev_log_addr.boffset; trx_no= mach_read_from_8(log_hdr + TRX_UNDO_TRX_NO); @@ -920,12 +850,13 @@ void purge_sys_t::rseg_get_next_history_log() can never produce events from an empty rollback segment. */ mysql_mutex_lock(&pq_mutex); - purge_queue.push(*rseg); + enqueue(*rseg); mysql_mutex_unlock(&pq_mutex); } } rseg->latch.wr_unlock(); + return choose_next_log(); } /** Position the purge sys "iterator" on the undo record to use for purging. @@ -933,11 +864,37 @@ void purge_sys_t::rseg_get_next_history_log() @retval true when purge_sys.rseg->latch was locked */ bool purge_sys_t::choose_next_log() { - if (!rseg_iter.set_next()) - return false; + ut_ad(!next_stored); - hdr_offset= rseg->last_offset(); - hdr_page_no= rseg->last_page_no; + mysql_mutex_lock(&pq_mutex); + if (purge_queue.empty()) { + rseg = nullptr; + mysql_mutex_unlock(&purge_sys.pq_mutex); + return false; + } + rseg= purge_queue.pop(); + mysql_mutex_unlock(&purge_sys.pq_mutex); + + /* We assume in purge of externally stored fields that space + id is in the range of UNDO tablespace space ids */ + ut_ad(rseg->space == fil_system.sys_space || + srv_is_undo_tablespace(rseg->space->id)); + + rseg->latch.wr_lock(SRW_LOCK_CALL); + trx_id_t last_trx_no = rseg->last_trx_no(); + hdr_offset = rseg->last_offset(); + hdr_page_no = rseg->last_page_no; + + /* Only the purge_coordinator_task will access this any of + purge_sys.hdr_page_no, purge_sys.tail. The field purge_sys.head and + purge_sys.view are modified by clone_end_view() in the + purge_coordinator_task while holding exclusive purge_sys.latch. The + purge_sys.view may also be modified by wake_if_not_active() while holding + exclusive purge_sys.latch. The purge_sys.head may be read by + purge_truncation_callback(). */ + ut_a(hdr_page_no != FIL_NULL); + ut_a(tail.trx_no <= last_trx_no); + tail.trx_no = last_trx_no; if (!rseg->needs_purge) { @@ -968,7 +925,7 @@ bool purge_sys_t::choose_next_log() if (!b) goto purge_nothing; undo_rec= - trx_undo_page_get_first_rec(b, page_no, hdr_offset); + trx_undo_page_get_first_rec(b, hdr_page_no, hdr_offset); if (!undo_rec) goto purge_nothing; } @@ -992,18 +949,13 @@ inline trx_purge_rec_t purge_sys_t::get_next_rec(roll_ptr_t roll_ptr) { ut_ad(next_stored); ut_ad(tail.trx_no < low_limit_no()); -#ifndef SUX_LOCK_GENERIC - ut_ad(rseg->latch.is_write_locked()); -#endif + ut_ad(rseg->latch.have_wr()); if (!offset) { - /* It is the dummy undo log record, which means that there is no - need to purge this undo log */ - rseg_get_next_history_log(); - - /* Look for the next undo log and record to purge */ - if (choose_next_log()) + /* It is the dummy undo log record, which means that there is no need to + purge this undo log. Look for the next undo log and record to purge */ + if (rseg_get_next_history_log()) rseg->latch.wr_unlock(); return {nullptr, 1}; } @@ -1051,9 +1003,8 @@ inline trx_purge_rec_t purge_sys_t::get_next_rec(roll_ptr_t roll_ptr) else { got_no_rec: - rseg_get_next_history_log(); /* Look for the next undo log and record to purge */ - locked= choose_next_log(); + locked= rseg_get_next_history_log(); } if (locked) diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index 87a2ac7b..964dca94 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -201,7 +201,7 @@ bool trx_rseg_read_wsrep_checkpoint(const buf_block_t *rseg_header, XID &xid) memcpy(xid.data, TRX_RSEG + TRX_RSEG_WSREP_XID_DATA + rseg_header->page.frame, XIDDATASIZE); - return true; + return wsrep_is_wsrep_xid(&xid); } /** Read the WSREP XID from the TRX_SYS page (in case of upgrade). @@ -210,6 +210,11 @@ bool trx_rseg_read_wsrep_checkpoint(const buf_block_t *rseg_header, XID &xid) @return whether the WSREP XID is present */ static bool trx_rseg_init_wsrep_xid(const page_t* page, XID& xid) { + if (memcmp(TRX_SYS + TRX_SYS_WSREP_XID_INFO + page, + field_ref_zero, TRX_SYS_WSREP_XID_LEN) == 0) { + return false; + } + if (mach_read_from_4(TRX_SYS + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_MAGIC_N_FLD + page) @@ -232,7 +237,8 @@ static bool trx_rseg_init_wsrep_xid(const page_t* page, XID& xid) memcpy(xid.data, TRX_SYS + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_DATA + page, XIDDATASIZE); - return true; + + return wsrep_is_wsrep_xid(&xid); } /** Recover the latest WSREP checkpoint XID. @@ -448,7 +454,14 @@ static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, mtr_t *mtr) { if (!rseg->space) return DB_TABLESPACE_NOT_FOUND; + + /* Access the tablespace header page to recover rseg->space->free_limit */ + page_id_t page_id{rseg->space->id, 0}; dberr_t err; + if (!buf_page_get_gen(page_id, 0, RW_S_LATCH, nullptr, BUF_GET, mtr, &err)) + return err; + mtr->release_last_page(); + page_id.set_page_no(rseg->page_no); const buf_block_t *rseg_hdr= buf_page_get_gen(rseg->page_id(), 0, RW_S_LATCH, nullptr, BUF_GET, mtr, &err); @@ -493,10 +506,17 @@ static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, mtr_t *mtr) trx_sys.recovered_binlog_offset= binlog_offset; trx_sys.recovered_binlog_is_legacy_pos= false; } + } #ifdef WITH_WSREP - trx_rseg_read_wsrep_checkpoint(rseg_hdr, trx_sys.recovered_wsrep_xid); + XID tmp_xid; + tmp_xid.null(); + /* Update recovered wsrep xid only if we found wsrep xid from + rseg header page and read xid seqno is larger than currently + recovered xid seqno. */ + if (trx_rseg_read_wsrep_checkpoint(rseg_hdr, tmp_xid) && + wsrep_xid_seqno(&tmp_xid) > wsrep_xid_seqno(&trx_sys.recovered_wsrep_xid)) + trx_sys.recovered_wsrep_xid.set(&tmp_xid); #endif - } } if (srv_operation == SRV_OPERATION_RESTORE) @@ -518,6 +538,11 @@ static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, mtr_t *mtr) fil_addr_t node_addr= flst_get_last(TRX_RSEG + TRX_RSEG_HISTORY + rseg_hdr->page.frame); + if (node_addr.page >= rseg->space->free_limit || + node_addr.boffset < TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE || + node_addr.boffset >= srv_page_size - TRX_UNDO_LOG_OLD_HDR_SIZE) + return DB_CORRUPTION; + node_addr.boffset= static_cast<uint16_t>(node_addr.boffset - TRX_UNDO_HISTORY_NODE); rseg->last_page_no= node_addr.page; @@ -544,7 +569,7 @@ static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, mtr_t *mtr) if (rseg->last_page_no != FIL_NULL) /* There is no need to cover this operation by the purge mutex because we are still bootstrapping. */ - purge_sys.purge_queue.push(*rseg); + purge_sys.enqueue(*rseg); } trx_sys.set_undo_non_empty(rseg->history_size > 0); @@ -567,10 +592,6 @@ static void trx_rseg_init_binlog_info(const page_t* page) + TRX_SYS + page); trx_sys.recovered_binlog_is_legacy_pos= true; } - -#ifdef WITH_WSREP - trx_rseg_init_wsrep_xid(page, trx_sys.recovered_wsrep_xid); -#endif } /** Initialize or recover the rollback segments at startup. */ @@ -589,7 +610,17 @@ dberr_t trx_rseg_array_init() #endif mtr_t mtr; dberr_t err = DB_SUCCESS; - + /* mariabackup --prepare only deals with the redo log and the data + files, not with transactions or the data dictionary, that's why + trx_lists_init_at_db_start() does not invoke purge_sys.create() and + purge queue mutex stays uninitialized, and trx_rseg_mem_restore() quits + before initializing undo log lists. */ + if (srv_operation != SRV_OPERATION_RESTORE) + /* Acquiring purge queue mutex here should be fine from the + deadlock prevention point of view, because executing that + function is a prerequisite for starting the purge subsystem or + any transactions. */ + purge_sys.queue_lock(); for (ulint rseg_id = 0; rseg_id < TRX_SYS_N_RSEGS; rseg_id++) { mtr.start(); if (const buf_block_t* sys = trx_sysf_get(&mtr, false)) { @@ -602,7 +633,11 @@ dberr_t trx_rseg_array_init() + sys->page.frame); trx_rseg_init_binlog_info(sys->page.frame); #ifdef WITH_WSREP - wsrep_sys_xid.set(&trx_sys.recovered_wsrep_xid); + if (trx_rseg_init_wsrep_xid( + sys->page.frame, trx_sys.recovered_wsrep_xid)) { + wsrep_sys_xid.set( + &trx_sys.recovered_wsrep_xid); + } #endif } @@ -655,7 +690,8 @@ dberr_t trx_rseg_array_init() mtr.commit(); } - + if (srv_operation != SRV_OPERATION_RESTORE) + purge_sys.queue_unlock(); if (err != DB_SUCCESS) { for (auto& rseg : trx_sys.rseg_array) { while (auto u = UT_LIST_GET_FIRST(rseg.undo_list)) { @@ -667,7 +703,7 @@ dberr_t trx_rseg_array_init() } #ifdef WITH_WSREP - if (!wsrep_sys_xid.is_null()) { + if (srv_operation == SRV_OPERATION_NORMAL && !wsrep_sys_xid.is_null()) { /* Upgrade from a version prior to 10.3.5, where WSREP XID was stored in TRX_SYS page. If no rollback segment has a WSREP XID set, diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 942b8bd4..1d22b853 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -412,12 +412,12 @@ void trx_t::free() #endif read_view.mem_noaccess(); MEM_NOACCESS(&lock, sizeof lock); - MEM_NOACCESS(&op_info, sizeof op_info); - MEM_NOACCESS(&isolation_level, sizeof isolation_level); - MEM_NOACCESS(&check_foreigns, sizeof check_foreigns); + MEM_NOACCESS(&op_info, sizeof op_info + + sizeof(unsigned) /* isolation_level, snapshot_isolation, + check_foreigns, check_unique_secondary, + bulk_insert */); MEM_NOACCESS(&is_registered, sizeof is_registered); MEM_NOACCESS(&active_commit_ordered, sizeof active_commit_ordered); - MEM_NOACCESS(&check_unique_secondary, sizeof check_unique_secondary); MEM_NOACCESS(&flush_log_later, sizeof flush_log_later); MEM_NOACCESS(&duplicates, sizeof duplicates); MEM_NOACCESS(&dict_operation, sizeof dict_operation); @@ -1142,15 +1142,23 @@ inline void trx_t::write_serialisation_history(mtr_t *mtr) } else if (rseg->last_page_no == FIL_NULL) { - mysql_mutex_lock(&purge_sys.pq_mutex); + /* trx_sys.assign_new_trx_no() and + purge_sys.enqueue() must be invoked in the same + critical section protected with purge queue mutex to avoid rseg with + greater last commit number to be pushed to purge queue prior to rseg with + lesser last commit number. In other words pushing to purge queue must be + serialized along with assigning trx_no. Otherwise purge coordinator + thread can also fetch redo log records from rseg with greater last commit + number before rseg with lesser one. */ + purge_sys.queue_lock(); trx_sys.assign_new_trx_no(this); const trx_id_t end{rw_trx_hash_element->no}; + rseg->last_page_no= undo->hdr_page_no; /* end cannot be less than anything in rseg. User threads only produce events when a rollback segment is empty. */ - purge_sys.purge_queue.push(TrxUndoRsegs{end, *rseg}); - mysql_mutex_unlock(&purge_sys.pq_mutex); - rseg->last_page_no= undo->hdr_page_no; rseg->set_last_commit(undo->hdr_offset, end); + purge_sys.enqueue(end, *rseg); + purge_sys.queue_unlock(); } else trx_sys.assign_new_trx_no(this); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index ccc68dfe..c0f5b1fb 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -134,8 +134,9 @@ trx_undo_page_get_first_rec(const buf_block_t *block, uint32_t page_no, uint16_t offset) { uint16_t start= trx_undo_page_get_start(block, page_no, offset); - return start == trx_undo_page_get_end(block, page_no, offset) - ? nullptr : block->page.frame + start; + uint16_t end= trx_undo_page_get_end(block, page_no, offset); + ut_ad(start <= end); + return start >= end ? nullptr : block->page.frame + start; } /** Get the last undo log record on a page. @@ -149,8 +150,10 @@ trx_undo_rec_t* trx_undo_page_get_last_rec(const buf_block_t *block, uint32_t page_no, uint16_t offset) { + uint16_t start= trx_undo_page_get_start(block, page_no, offset); uint16_t end= trx_undo_page_get_end(block, page_no, offset); - return trx_undo_page_get_start(block, page_no, offset) == end + ut_ad(start <= end); + return start >= end ? nullptr : block->page.frame + mach_read_from_2(block->page.frame + end - 2); } @@ -510,7 +513,7 @@ trx_undo_seg_create(fil_space_t *space, buf_block_t *rseg_hdr, ulint *id, *err = flst_add_last(block, TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST, block, TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE, - mtr); + space->free_limit, mtr); *id = slot_no; mtr->write<4>(*rseg_hdr, TRX_RSEG + TRX_RSEG_UNDO_SLOTS @@ -693,7 +696,8 @@ buf_block_t *trx_undo_add_page(trx_undo_t *undo, mtr_t *mtr, dberr_t *err) mtr->undo_create(*new_block); trx_undo_page_init(*new_block); *err= flst_add_last(header_block, TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST, - new_block, TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE, mtr); + new_block, TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE, + rseg->space->free_limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) new_block= nullptr; else @@ -744,9 +748,11 @@ trx_undo_free_page( buf_page_make_young_if_needed(&header_block->page); + const uint32_t limit = rseg->space->free_limit; + *err = flst_remove(header_block, TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST, undo_block, TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE, - mtr); + limit, mtr); if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { return FIL_NULL; @@ -755,7 +761,13 @@ trx_undo_free_page( const fil_addr_t last_addr = flst_get_last( TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + header_block->page.frame); - if (UNIV_UNLIKELY(last_addr.page == page_no)) { + if (UNIV_UNLIKELY(last_addr.page == page_no) + || UNIV_UNLIKELY(last_addr.page != FIL_NULL + && last_addr.page >= limit) + || UNIV_UNLIKELY(last_addr.boffset < TRX_UNDO_PAGE_HDR + + TRX_UNDO_PAGE_NODE) + || UNIV_UNLIKELY(last_addr.boffset >= srv_page_size + - TRX_UNDO_LOG_OLD_HDR_SIZE)) { *err = DB_CORRUPTION; return FIL_NULL; } @@ -972,8 +984,8 @@ trx_undo_mem_create_at_db_start(trx_rseg_t *rseg, ulint id, uint32_t page_no) ut_ad(id < TRX_RSEG_N_SLOTS); mtr.start(); - const buf_block_t* block = buf_page_get( - page_id_t(rseg->space->id, page_no), 0, RW_X_LATCH, &mtr); + const page_id_t page_id{rseg->space->id, page_no}; + const buf_block_t* block = buf_page_get(page_id, 0, RW_X_LATCH, &mtr); if (UNIV_UNLIKELY(!block)) { corrupted: mtr.commit(); @@ -1075,6 +1087,15 @@ corrupted_type: fil_addr_t last_addr = flst_get_last( TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + block->page.frame); + if (last_addr.page >= rseg->space->free_limit + || last_addr.boffset < TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE + || last_addr.boffset >= srv_page_size + - TRX_UNDO_LOG_OLD_HDR_SIZE) { + corrupted_undo: + ut_free(undo); + goto corrupted; + } + undo->last_page_no = last_addr.page; undo->top_page_no = last_addr.page; @@ -1083,8 +1104,7 @@ corrupted_type: RW_X_LATCH, &mtr); if (UNIV_UNLIKELY(!last)) { - ut_free(undo); - goto corrupted; + goto corrupted_undo; } if (const trx_undo_rec_t* rec = trx_undo_page_get_last_rec( diff --git a/storage/innobase/unittest/CMakeLists.txt b/storage/innobase/unittest/CMakeLists.txt index 7dd7c111..9330d231 100644 --- a/storage/innobase/unittest/CMakeLists.txt +++ b/storage/innobase/unittest/CMakeLists.txt @@ -17,6 +17,10 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/unittest/mytap ${CMAKE_SOURCE_DIR}/storage/innobase/include ${CMAKE_SOURCE_DIR}/tpool) +ADD_EXECUTABLE(innodb_rbt-t innodb_rbt-t.cc ../ut/ut0rbt.cc) +TARGET_LINK_LIBRARIES(innodb_rbt-t mysys mytap) +ADD_DEPENDENCIES(innodb_rbt-t GenError) +MY_ADD_TEST(innodb_rbt) ADD_EXECUTABLE(innodb_fts-t innodb_fts-t.cc) TARGET_LINK_LIBRARIES(innodb_fts-t mysys mytap) ADD_DEPENDENCIES(innodb_fts-t GenError) diff --git a/storage/innobase/unittest/innodb_rbt-t.cc b/storage/innobase/unittest/innodb_rbt-t.cc new file mode 100644 index 00000000..38b980da --- /dev/null +++ b/storage/innobase/unittest/innodb_rbt-t.cc @@ -0,0 +1,83 @@ +#include "tap.h" +#include "ut0rbt.h" +#include "ut0new.h" + +const size_t alloc_max_retries= 0; +void os_thread_sleep(ulint) { abort(); } +void ut_dbg_assertion_failed(const char *, const char *, unsigned) +{ abort(); } +namespace ib { fatal_or_error::~fatal_or_error() { abort(); } } +#ifdef UNIV_PFS_MEMORY +PSI_memory_key mem_key_other, mem_key_std; +PSI_memory_key ut_new_get_key_by_file(uint32_t) { return mem_key_std; } +#endif + +static const uint64_t doc_ids[]= +{ + 103571, 104018, 106821, 108647, 109352, 109379, + 110325, 122868, 210682130, 231275441, 234172769, 366236849, + 526467159, 1675241735, 1675243405, 1947751899, 1949940363, 2033691953, + 2148227299, 2256289791, 2294223591, 2367501260, 2792700091, 2792701220, + 2817121627, 2820680352, 2821165664, 3253312130, 3404918378, 3532599429, + 3538712078, 3539373037, 3546479309, 3566641838, 3580209634, 3580871267, + 3693930556, 3693932734, 3693932983, 3781949558, 3839877411, 3930968983 +}; + +static int fts_doc_id_cmp(const void *p1, const void *p2) +{ + uint64_t a= *static_cast<const uint64_t*>(p1), + b= *static_cast<const uint64_t*>(p2); + return b > a ? -1 : a > b; +} + + +static int fts_doc_id_buggy_cmp(const void *p1, const void *p2) +{ + return int(*static_cast<const uint64_t*>(p1) - + *static_cast<const uint64_t*>(p2)); +} + +typedef int (*comparator) (const void*, const void*); + +static void rbt_populate(ib_rbt_t *rbt) +{ + ib_rbt_bound_t parent; + for (const uint64_t &doc_id : doc_ids) + { + if (rbt_search(rbt, &parent, &doc_id)) + rbt_add_node(rbt, &parent, &doc_id); + } +} + +static void rbt_populate2(ib_rbt_t *rbt) +{ + for (const uint64_t &doc_id : doc_ids) + rbt_insert(rbt, &doc_id, &doc_id); +} + +static bool rbt_search_all(ib_rbt_t *rbt) +{ + ib_rbt_bound_t parent; + for (const uint64_t &doc_id : doc_ids) + if (rbt_search(rbt, &parent, &doc_id)) + return false; + return true; +} + +static void rbt_test(comparator cmp, bool buggy) +{ + ib_rbt_t *rbt= rbt_create(sizeof(uint64_t), cmp); + rbt_populate(rbt); + ok(rbt_search_all(rbt) != buggy, "search after populate"); + rbt_free(rbt); + rbt= rbt_create(sizeof(uint64_t), cmp); + rbt_populate2(rbt); + ok(rbt_search_all(rbt) != buggy, "search after populate2"); + rbt_free(rbt); +} + +int main () +{ + rbt_test(fts_doc_id_buggy_cmp, true); + rbt_test(fts_doc_id_cmp, false); +} diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index 7b69042c..5b3bc185 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -258,47 +258,6 @@ ut_print_name( } } -/** Format a table name, quoted as an SQL identifier. -If the name contains a slash '/', the result will contain two -identifiers separated by a period (.), as in SQL -database_name.table_name. -@see table_name_t -@param[in] name table or index name -@param[out] formatted formatted result, will be NUL-terminated -@param[in] formatted_size size of the buffer in bytes -@return pointer to 'formatted' */ -char* -ut_format_name( - const char* name, - char* formatted, - ulint formatted_size) -{ - switch (formatted_size) { - case 1: - formatted[0] = '\0'; - /* FALL-THROUGH */ - case 0: - return(formatted); - } - - char* end; - - end = innobase_convert_name(formatted, formatted_size, - name, strlen(name), NULL); - - /* If the space in 'formatted' was completely used, then sacrifice - the last character in order to write '\0' at the end. */ - if ((ulint) (end - formatted) == formatted_size) { - end--; - } - - ut_a((ulint) (end - formatted) < formatted_size); - - *end = '\0'; - - return(formatted); -} - /**********************************************************************//** Catenate files. */ void @@ -353,14 +312,16 @@ ut_strerr( return("Lock wait"); case DB_DEADLOCK: return("Deadlock"); + case DB_RECORD_CHANGED: + return("Record changed"); +#ifdef WITH_WSREP case DB_ROLLBACK: return("Rollback"); +#endif case DB_DUPLICATE_KEY: return("Duplicate key"); case DB_MISSING_HISTORY: return("Required history data has been deleted"); - case DB_CLUSTER_NOT_FOUND: - return("Cluster not found"); case DB_TABLE_NOT_FOUND: return("Table not found"); case DB_TOO_BIG_RECORD: diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index f55d78f0..426a19f0 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -124,7 +124,7 @@ ENDIF() IF (CURL_FOUND) INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIRS}) MYSQL_ADD_PLUGIN(s3 ha_s3.cc ${S3_SOURCES} COMPONENT s3-engine - LINK_LIBRARIES ${CURL_LIBRARIES} z STORAGE_ENGINE NOT_EMBEDDED CONFIG s3.cnf) + LINK_LIBRARIES ${CURL_LIBRARIES} ${ZLIB_LIBRARIES} STORAGE_ENGINE NOT_EMBEDDED CONFIG s3.cnf) ENDIF() SET(CPACK_RPM_s3-engine_PACKAGE_SUMMARY "Amazon S3 archival storage engine for MariaDB" PARENT_SCOPE) @@ -132,7 +132,7 @@ SET(CPACK_RPM_s3-engine_PACKAGE_DESCRIPTION "The S3 storage engine allows one to IF(TARGET s3) MYSQL_ADD_EXECUTABLE(aria_s3_copy aria_s3_copy.cc ${S3_SOURCES} COMPONENT s3-engine) - TARGET_LINK_LIBRARIES(aria_s3_copy aria myisam mysys mysys_ssl ${CURL_LIBRARIES} ${ZLIB_LIBRARY}) + TARGET_LINK_LIBRARIES(aria_s3_copy aria myisam mysys mysys_ssl ${CURL_LIBRARIES} ${ZLIB_LIBRARIES}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/libmarias3) ADD_DEFINITIONS(-DWITH_S3_STORAGE_ENGINE) ENDIF() diff --git a/storage/maria/aria_chk.c b/storage/maria/aria_chk.c index 143110dc..4bbe513b 100644 --- a/storage/maria/aria_chk.c +++ b/storage/maria/aria_chk.c @@ -145,7 +145,8 @@ int main(int argc, char **argv) { if ((ma_control_file_open(FALSE, opt_require_control_file || !(check_param.testflag & T_SILENT), - TRUE))) + TRUE, + control_file_open_flags))) { if (opt_require_control_file || (opt_transaction_logging && (check_param.testflag & T_REP_ANY))) diff --git a/storage/maria/aria_pack.c b/storage/maria/aria_pack.c index 40e7e399..464a08fb 100644 --- a/storage/maria/aria_pack.c +++ b/storage/maria/aria_pack.c @@ -239,7 +239,8 @@ int main(int argc, char **argv) if (!opt_ignore_control_file && (no_control_file= ma_control_file_open(FALSE, (opt_require_control_file || - !silent), FALSE)) && + !silent), FALSE, + control_file_open_flags)) && opt_require_control_file) { error= 1; diff --git a/storage/maria/aria_read_log.c b/storage/maria/aria_read_log.c index c0c76ed5..2997314d 100644 --- a/storage/maria/aria_read_log.c +++ b/storage/maria/aria_read_log.c @@ -104,7 +104,7 @@ int main(int argc, char **argv) goto end; } /* we don't want to create a control file, it MUST exist */ - if (ma_control_file_open(FALSE, TRUE, TRUE)) + if (ma_control_file_open(FALSE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't open control file (%d)\n", errno); goto err; diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 66dd9867..b3b0ba0f 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -38,6 +38,7 @@ C_MODE_START #include "ma_recovery.h" C_MODE_END #include "ma_trnman.h" +#include "ma_loghandler.h" //#include "sql_priv.h" #include "protocol.h" @@ -45,6 +46,7 @@ C_MODE_END #include "key.h" #include "log.h" #include "sql_parse.h" +#include "mysql/service_print_check_msg.h" #include "debug.h" /* @@ -428,10 +430,8 @@ static void _ma_check_print_msg(HA_CHECK *param, const LEX_CSTRING *msg_type, const char *fmt, va_list args) { THD *thd= (THD *) param->thd; - Protocol *protocol= thd->protocol; - size_t length, msg_length; + size_t msg_length __attribute__((unused)); char msgbuf[MYSQL_ERRMSG_SIZE]; - char name[NAME_LEN * 2 + 2]; if (param->testflag & T_SUPPRESS_ERR_HANDLING) return; @@ -460,27 +460,10 @@ static void _ma_check_print_msg(HA_CHECK *param, const LEX_CSTRING *msg_type, _ma_check_print(param, msg_type, msgbuf); return; } - length= (uint) (strxmov(name, param->db_name, ".", param->table_name, - NullS) - name); - /* - TODO: switch from protocol to push_warning here. The main reason we didn't - it yet is parallel repair, which threads have no THD object accessible via - current_thd. - - Also we likely need to lock mutex here (in both cases with protocol and - push_warning). - */ - protocol->prepare_for_resend(); - protocol->store(name, (uint)length, system_charset_info); - protocol->store(param->op_name, strlen(param->op_name), system_charset_info); - protocol->store(msg_type, system_charset_info); - protocol->store(msgbuf, msg_length, system_charset_info); - if (protocol->write()) - sql_print_error("Failed on my_net_write, writing to stderr instead: %s.%s: %s\n", - param->db_name, param->table_name, msgbuf); - else if (thd->variables.log_warnings > 2) + print_check_msg(thd, param->db_name, param->table_name, + param->op_name, msg_type->str, msgbuf, 0); + if (thd->variables.log_warnings > 2) _ma_check_print(param, msg_type, msgbuf); - return; } @@ -1952,41 +1935,46 @@ int ha_maria::preload_keys(THD * thd, HA_CHECK_OPT *check_opt) SYNOPSIS disable_indexes() - mode mode of operation: - HA_KEY_SWITCH_NONUNIQ disable all non-unique keys - HA_KEY_SWITCH_ALL disable all keys - HA_KEY_SWITCH_NONUNIQ_SAVE dis. non-uni. and make persistent - HA_KEY_SWITCH_ALL_SAVE dis. all keys and make persistent - IMPLEMENTATION - HA_KEY_SWITCH_NONUNIQ is not implemented. - HA_KEY_SWITCH_ALL_SAVE is not implemented. + DESCRIPTION + See handler::ha_disable_indexes() RETURN 0 ok HA_ERR_WRONG_COMMAND mode not implemented. */ -int ha_maria::disable_indexes(uint mode) +int ha_maria::disable_indexes(key_map map, bool persist) { int error; - if (mode == HA_KEY_SWITCH_ALL) + if (!persist) { /* call a storage engine function to switch the key map */ + DBUG_ASSERT(map.is_clear_all()); error= maria_disable_indexes(file); } - else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) + else { - maria_extra(file, HA_EXTRA_NO_KEYS, 0); + /* auto-inc key cannot be disabled */ + if (table->s->next_number_index < MAX_KEY) + DBUG_ASSERT(map.is_set(table->s->next_number_index)); + + /* unique keys cannot be disabled either */ + for (uint i=0; i < table->s->keys; i++) + DBUG_ASSERT(!(table->key_info[i].flags & HA_NOSAME) || map.is_set(i)); + + ulonglong ullmap= map.to_ulonglong(); + + /* make sure auto-inc key is enabled even if it's > 64 */ + if (map.length() > MARIA_KEYMAP_BITS && + table->s->next_number_index < MAX_KEY) + maria_set_key_active(ullmap, table->s->next_number_index); + + maria_extra(file, HA_EXTRA_NO_KEYS, &ullmap); info(HA_STATUS_CONST); // Read new key info error= 0; } - else - { - /* mode not implemented */ - error= HA_ERR_WRONG_COMMAND; - } return error; } @@ -1996,21 +1984,14 @@ int ha_maria::disable_indexes(uint mode) SYNOPSIS enable_indexes() - mode mode of operation: - HA_KEY_SWITCH_NONUNIQ enable all non-unique keys - HA_KEY_SWITCH_ALL enable all keys - HA_KEY_SWITCH_NONUNIQ_SAVE en. non-uni. and make persistent - HA_KEY_SWITCH_ALL_SAVE en. all keys and make persistent DESCRIPTION Enable indexes, which might have been disabled by disable_index() before. - The modes without _SAVE work only if both data and indexes are empty, - since the MARIA repair would enable them persistently. + If persist=false, it works only if both data and indexes are empty, + since the Aria repair would enable them persistently. To be sure in these cases, call handler::delete_all_rows() before. - IMPLEMENTATION - HA_KEY_SWITCH_NONUNIQ is not implemented. - HA_KEY_SWITCH_ALL_SAVE is not implemented. + See also handler::ha_enable_indexes() RETURN 0 ok @@ -2019,18 +2000,19 @@ int ha_maria::disable_indexes(uint mode) HA_ERR_WRONG_COMMAND mode not implemented. */ -int ha_maria::enable_indexes(uint mode) +int ha_maria::enable_indexes(key_map map, bool persist) { int error; ha_rows start_rows= file->state->records; - DBUG_PRINT("info", ("ha_maria::enable_indexes mode: %d", mode)); + DBUG_PRINT("info", ("ha_maria::enable_indexes mode: %d", persist)); if (maria_is_all_keys_active(file->s->state.key_map, file->s->base.keys)) { /* All indexes are enabled already. */ return 0; } - if (mode == HA_KEY_SWITCH_ALL) + DBUG_ASSERT(map.is_prefix(table->s->keys)); + if (!persist) { error= maria_enable_indexes(file); /* @@ -2039,7 +2021,7 @@ int ha_maria::enable_indexes(uint mode) but mode==HA_KEY_SWITCH_ALL forbids it. */ } - else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) + else { THD *thd= table->in_use; HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); @@ -2104,11 +2086,6 @@ int ha_maria::enable_indexes(uint mode) info(HA_STATUS_CONST); thd_proc_info(thd, save_proc_info); } - else - { - /* mode not implemented */ - error= HA_ERR_WRONG_COMMAND; - } DBUG_EXECUTE_IF("maria_flush_whole_log", { DBUG_PRINT("maria_flush_whole_log", ("now")); @@ -2311,7 +2288,7 @@ int ha_maria::end_bulk_insert() { int first_error, first_errno= 0, error; my_bool abort= file->s->deleting, empty_table= 0; - uint enable_index_mode= HA_KEY_SWITCH_NONUNIQ_SAVE; + bool enable_persistently= true; DBUG_ENTER("ha_maria::end_bulk_insert"); if ((first_error= maria_end_bulk_insert(file, abort))) @@ -2340,7 +2317,7 @@ int ha_maria::end_bulk_insert() first_error= 1; first_errno= my_errno; } - enable_index_mode= HA_KEY_SWITCH_ALL; + enable_persistently= false; empty_table= 1; /* Ignore all changed pages, required by _ma_renable_logging_for_table() @@ -2352,7 +2329,7 @@ int ha_maria::end_bulk_insert() if (!abort && can_enable_indexes) { - if ((error= enable_indexes(enable_index_mode))) + if ((error= enable_indexes(key_map(table->s->keys), enable_persistently))) { if (!first_error) { @@ -3361,6 +3338,8 @@ int ha_maria::create(const char *name, TABLE *table_arg, if (ha_create_info->tmp_table()) { create_flags|= HA_CREATE_TMP_TABLE | HA_CREATE_DELAY_KEY_WRITE; + if (ha_create_info->options & HA_LEX_CREATE_GLOBAL_TMP_TABLE) + create_flags|= HA_CREATE_GLOBAL_TMP_TABLE; create_info.transactional= 0; } if (ha_create_info->options & HA_CREATE_KEEP_FILES) @@ -3894,7 +3873,8 @@ static int ha_maria_init(void *p) if (!aria_readonly) res= maria_upgrade(); res= res || maria_init(); - tmp= ma_control_file_open(!aria_readonly, !aria_readonly, !aria_readonly); + tmp= ma_control_file_open(!aria_readonly, !aria_readonly, !aria_readonly, + control_file_open_flags); res= res || aria_readonly ? tmp == CONTROL_FILE_LOCKED : tmp != 0; res= res || ((force_start_after_recovery_failures != 0 && !aria_readonly) && diff --git a/storage/maria/ha_maria.h b/storage/maria/ha_maria.h index 2b8b5dc9..41f64436 100644 --- a/storage/maria/ha_maria.h +++ b/storage/maria/ha_maria.h @@ -122,8 +122,8 @@ public: int external_lock(THD * thd, int lock_type) override; int start_stmt(THD *thd, thr_lock_type lock_type) override final; int delete_all_rows(void) override final; - int disable_indexes(uint mode) override final; - int enable_indexes(uint mode) override final; + int disable_indexes(key_map map, bool persist) override final; + int enable_indexes(key_map map, bool persist) override final; int indexes_are_disabled(void) override final; void start_bulk_insert(ha_rows rows, uint flags) override final; int end_bulk_insert() override final; diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index 61fe4f9d..91f93755 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -232,7 +232,7 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, uint max_page_size; MARIA_FILE_BITMAP *bitmap= &share->bitmap; uint size= share->block_size; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); + myf flag= MY_WME | share->malloc_flag; pgcache_page_no_t first_bitmap_with_space; #ifndef DBUG_OFF /* We want to have a copy of the bitmap to be able to print differences */ diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c index 543ddcca..561cc324 100644 --- a/storage/maria/ma_blockrec.c +++ b/storage/maria/ma_blockrec.c @@ -488,7 +488,7 @@ my_bool _ma_init_block_record(MARIA_HA *info) { MARIA_ROW *row= &info->cur_row, *new_row= &info->new_row; MARIA_SHARE *share= info->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); + myf flag= MY_WME | share->malloc_flag; uint default_extents; DBUG_ENTER("_ma_init_block_record"); @@ -2654,7 +2654,6 @@ static my_bool write_block_record(MARIA_HA *info, LSN lsn; my_off_t position; uint save_my_errno; - myf myflag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); DBUG_ENTER("write_block_record"); head_block= bitmap_blocks->block; @@ -2721,7 +2720,7 @@ static my_bool write_block_record(MARIA_HA *info, for every data segment we want to store. */ if (_ma_alloc_buffer(&info->rec_buff, &info->rec_buff_size, - row->head_length, myflag)) + row->head_length, MY_WME | share->malloc_flag)) DBUG_RETURN(1); tmp_data_used= 0; /* Either 0 or last used uchar in 'data' */ @@ -4750,7 +4749,7 @@ int _ma_read_block_record2(MARIA_HA *info, uchar *record, MARIA_EXTENT_CURSOR extent; MARIA_COLUMNDEF *column, *end_column; MARIA_ROW *cur_row= &info->cur_row; - myf myflag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); + myf myflag= MY_WME | share->malloc_flag; DBUG_ENTER("_ma_read_block_record2"); start_of_data= data; @@ -5089,7 +5088,6 @@ static my_bool read_row_extent_info(MARIA_HA *info, uchar *buff, uint flag, row_extents, row_extents_size; uint field_lengths __attribute__ ((unused)); uchar *extents, *end; - myf myflag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); DBUG_ENTER("read_row_extent_info"); if (!(data= get_record_position(share, buff, @@ -5113,7 +5111,7 @@ static my_bool read_row_extent_info(MARIA_HA *info, uchar *buff, if (info->cur_row.extents_buffer_length < row_extents_size && _ma_alloc_buffer(&info->cur_row.extents, &info->cur_row.extents_buffer_length, - row_extents_size, myflag)) + row_extents_size, MY_WME | share->malloc_flag)) DBUG_RETURN(1); memcpy(info->cur_row.extents, data, ROW_EXTENT_SIZE); data+= ROW_EXTENT_SIZE; @@ -5283,7 +5281,7 @@ my_bool _ma_cmp_block_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def, my_bool _ma_scan_init_block_record(MARIA_HA *info) { MARIA_SHARE *share= info->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); + myf flag= MY_WME | share->malloc_flag; DBUG_ENTER("_ma_scan_init_block_record"); DBUG_ASSERT(info->dfile.file == share->bitmap.file.file); diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index 1f6aa2ee..e4ba0726 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -125,6 +125,7 @@ void maria_chk_init(HA_CHECK *param) param->max_stage= 1; param->stack_end_ptr= &my_thread_var->stack_ends_here; param->max_allowed_lsn= (LSN) ~0ULL; + /* Flag when initializing buffers possible used by parallel repair threads */ param->malloc_flags= MY_THREAD_SPECIFIC; } @@ -1305,7 +1306,6 @@ static int check_dynamic_record(HA_CHECK *param, MARIA_HA *info, int extend, ulong UNINIT_VAR(left_length); uint b_type; char llbuff[22],llbuff2[22],llbuff3[22]; - myf myflag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); DBUG_ENTER("check_dynamic_record"); pos= 0; @@ -1413,7 +1413,8 @@ static int check_dynamic_record(HA_CHECK *param, MARIA_HA *info, int extend, { if (_ma_alloc_buffer(&info->rec_buff, &info->rec_buff_size, block_info.rec_len + - share->base.extra_rec_buff_size, myflag)) + share->base.extra_rec_buff_size, + MY_WME | share->malloc_flag)) { _ma_check_print_error(param, @@ -2130,7 +2131,7 @@ int maria_chk_data_link(HA_CHECK *param, MARIA_HA *info, my_bool extend) if (!(record= (uchar*) my_malloc(PSI_INSTRUMENT_ME, share->base.default_rec_buff_size, - MYF(param->malloc_flags)))) + MYF(MY_THREAD_SPECIFIC)))) { _ma_check_print_error(param,"Not enough memory for record"); DBUG_RETURN(-1); @@ -2507,6 +2508,11 @@ static int initialize_variables_for_repair(HA_CHECK *param, maria_versioning(info, 0); /* remember original number of rows */ *info->state= info->s->state.state; + if (share->data_file_type == BLOCK_RECORD) + share->state.state.data_file_length= MY_ALIGN(sort_info->filelength, + share->block_size); + else + share->state.state.data_file_length= sort_info->filelength; return 0; } @@ -2743,7 +2749,7 @@ int maria_repair(HA_CHECK *param, register MARIA_HA *info, READ_CACHE, share->pack.header_length, 1, MYF(MY_WME))) goto err; } - if (sort_info.new_info->s->data_file_type != BLOCK_RECORD) + if (!block_record) { /* When writing to not block records, we need a write buffer */ if (!rep_quick) @@ -2756,7 +2762,7 @@ int maria_repair(HA_CHECK *param, register MARIA_HA *info, sort_info.new_info->opt_flag|=WRITE_CACHE_USED; } } - else if (block_record) + else { scan_inited= 1; if (maria_scan_init(sort_info.info)) @@ -2766,10 +2772,10 @@ int maria_repair(HA_CHECK *param, register MARIA_HA *info, if (!(sort_param.record= (uchar *) my_malloc(PSI_INSTRUMENT_ME, (uint) share->base.default_rec_buff_size, - MYF(param->malloc_flags))) || + MYF(MY_THREAD_SPECIFIC))) || _ma_alloc_buffer(&sort_param.rec_buff, &sort_param.rec_buff_size, share->base.default_rec_buff_size, - MYF(param->malloc_flags))) + MYF(MY_THREAD_SPECIFIC))) { _ma_check_print_error(param, "Not enough memory for extra record"); goto err; @@ -3389,7 +3395,7 @@ static int sort_one_index(HA_CHECK *param, MARIA_HA *info, length= page.size; bzero(buff+length,keyinfo->block_length-length); if (write_page(share, new_file, buff, keyinfo->block_length, - new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL))) + new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL) & param->myf_rw)) { _ma_check_print_error(param,"Can't write indexblock, error: %d",my_errno); goto err; @@ -3721,7 +3727,7 @@ int maria_filecopy(HA_CHECK *param, File to,File from,my_off_t start, buff_length=(ulong) MY_MIN(param->write_buffer_length,length); if (!(buff=my_malloc(PSI_INSTRUMENT_ME, buff_length, - MYF(param->malloc_flags)))) + MYF(MY_THREAD_SPECIFIC)))) { buff=tmp_buff; buff_length=IO_SIZE; } @@ -3867,10 +3873,10 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, if (!(sort_param.record= (uchar*) my_malloc(PSI_INSTRUMENT_ME, (size_t) share->base.default_rec_buff_size, - MYF(param->malloc_flags))) || + MYF(MY_THREAD_SPECIFIC))) || _ma_alloc_buffer(&sort_param.rec_buff, &sort_param.rec_buff_size, share->base.default_rec_buff_size, - MYF(param->malloc_flags))) + MYF(MY_THREAD_SPECIFIC))) { _ma_check_print_error(param, "Not enough memory for extra record"); goto err; @@ -3889,7 +3895,7 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, sort_param.wordlist=NULL; init_alloc_root(PSI_INSTRUMENT_ME, &sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0, - MYF(param->malloc_flags)); + MYF(MY_THREAD_SPECIFIC)); sort_param.key_cmp=sort_key_cmp; sort_param.lock_in_memory=maria_lock_memory; @@ -4109,6 +4115,9 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, _ma_check_print_error(param, "Couldn't change to new data file"); goto err; } + /* Inform sort_delete_record that we are using the new file */ + sort_info.new_info->dfile.file= info->rec_cache.file= info->dfile.file; + if (param->testflag & T_UNPACK) restore_data_file_type(share); @@ -4457,7 +4466,7 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info, if (!(sort_param=(MARIA_SORT_PARAM *) my_malloc(PSI_INSTRUMENT_ME, (uint) share->base.keys * (sizeof(MARIA_SORT_PARAM) + share->base.pack_reclength), - MYF(MY_ZEROFILL | param->malloc_flags)))) + MYF(MY_ZEROFILL | MY_THREAD_SPECIFIC)))) { _ma_check_print_error(param,"Not enough memory for key!"); goto err; @@ -4515,9 +4524,10 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info, sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+ (share->base.pack_reclength * i)); + /* These buffers are per thread */ if (_ma_alloc_buffer(&sort_param[i].rec_buff, &sort_param[i].rec_buff_size, share->base.default_rec_buff_size, - MYF(param->malloc_flags))) + MYF(0))) { _ma_check_print_error(param,"Not enough memory!"); goto err; @@ -4546,7 +4556,7 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info, sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN; init_alloc_root(PSI_INSTRUMENT_ME, &sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0, - MYF(param->malloc_flags)); + MYF(MY_THREAD_SPECIFIC)); } } sort_info.total_keys=i; @@ -6105,7 +6115,7 @@ static MA_SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *param, uint blocks, if (!(block= (MA_SORT_KEY_BLOCKS*) my_malloc(PSI_INSTRUMENT_ME, (sizeof(MA_SORT_KEY_BLOCKS)+buffer_length+IO_SIZE)*blocks, - MYF(param->malloc_flags)))) + MYF(MY_THREAD_SPECIFIC)))) { _ma_check_print_error(param,"Not enough memory for sort-key-blocks"); return(0); diff --git a/storage/maria/ma_control_file.c b/storage/maria/ma_control_file.c index 21befb70..03513c46 100644 --- a/storage/maria/ma_control_file.c +++ b/storage/maria/ma_control_file.c @@ -272,7 +272,8 @@ static int lock_control_file(const char *name, my_bool do_retry) CONTROL_FILE_ERROR ma_control_file_open(my_bool create_if_missing, my_bool print_error, - my_bool wait_for_lock) + my_bool wait_for_lock, + int open_flags) { uchar buffer[CF_MAX_SIZE]; char name[FN_REFLEN], errmsg_buff[256]; @@ -280,7 +281,6 @@ CONTROL_FILE_ERROR ma_control_file_open(my_bool create_if_missing, " file is probably in use by another process"; uint new_cf_create_time_size, new_cf_changeable_size, new_block_size; my_off_t file_size; - int open_flags= O_BINARY | /*O_DIRECT |*/ O_RDWR | O_CLOEXEC; int error= CONTROL_FILE_UNKNOWN_ERROR; DBUG_ENTER("ma_control_file_open"); @@ -460,6 +460,15 @@ err: DBUG_RETURN(error); } +/* + The most common way to open the control file when writing tests +*/ + +CONTROL_FILE_ERROR ma_control_file_open_or_create() +{ + return ma_control_file_open(TRUE, TRUE, TRUE, + control_file_open_flags); +} /* Write information durably to the control file; stores this information into @@ -630,7 +639,7 @@ my_bool print_aria_log_control() int error= CONTROL_FILE_UNKNOWN_ERROR; uint recovery_fails; File file; - DBUG_ENTER("ma_control_file_open"); + DBUG_ENTER("print_aria_log_control"); if (fn_format(name, CONTROL_FILE_BASE_NAME, maria_data_root, "", MYF(MY_WME)) == NullS) diff --git a/storage/maria/ma_control_file.h b/storage/maria/ma_control_file.h index 40428f66..b2b95e36 100644 --- a/storage/maria/ma_control_file.h +++ b/storage/maria/ma_control_file.h @@ -68,10 +68,13 @@ typedef enum enum_control_file_error { CONTROL_FILE_ERROR ma_control_file_open(my_bool create_if_missing, my_bool print_error, - my_bool wait_for_lock); + my_bool wait_for_lock, + int open_flags); int ma_control_file_write_and_force(LSN last_checkpoint_lsn_arg, uint32 last_logno_arg, TrID max_trid_arg, uint8 recovery_failures_arg); +/* For simple programs that creates Aria files*/ +CONTROL_FILE_ERROR ma_control_file_open_or_create(); int ma_control_file_end(void); my_bool ma_control_file_inited(void); my_bool print_aria_log_control(void); diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index 7fd739d1..9ce48ae9 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -101,7 +101,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, DBUG_ASSERT(maria_inited); - if (flags & HA_CREATE_TMP_TABLE) + if ((flags & HA_CREATE_TMP_TABLE) && !(flags & HA_CREATE_GLOBAL_TMP_TABLE)) common_flag|= MY_THREAD_SPECIFIC; if (!ci) diff --git a/storage/maria/ma_dynrec.c b/storage/maria/ma_dynrec.c index 33f238d9..fed1bf41 100644 --- a/storage/maria/ma_dynrec.c +++ b/storage/maria/ma_dynrec.c @@ -1488,7 +1488,6 @@ int _ma_read_dynamic_record(MARIA_HA *info, uchar *buf, uchar *UNINIT_VAR(to); uint UNINIT_VAR(left_length); MARIA_SHARE *share= info->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); DBUG_ENTER("_ma_read_dynamic_record"); if (filepos == HA_OFFSET_ERROR) @@ -1525,7 +1524,8 @@ int _ma_read_dynamic_record(MARIA_HA *info, uchar *buf, { if (_ma_alloc_buffer(&info->rec_buff, &info->rec_buff_size, block_info.rec_len + - share->base.extra_rec_buff_size, flag)) + share->base.extra_rec_buff_size, + MY_WME | share->malloc_flag)) goto err; } to= info->rec_buff; @@ -1784,7 +1784,6 @@ int _ma_read_rnd_dynamic_record(MARIA_HA *info, uchar *UNINIT_VAR(to); MARIA_BLOCK_INFO block_info; MARIA_SHARE *share= info->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); DBUG_ENTER("_ma_read_rnd_dynamic_record"); #ifdef MARIA_EXTERNAL_LOCKING @@ -1875,7 +1874,8 @@ int _ma_read_rnd_dynamic_record(MARIA_HA *info, { if (_ma_alloc_buffer(&info->rec_buff, &info->rec_buff_size, block_info.rec_len + - share->base.extra_rec_buff_size, flag)) + share->base.extra_rec_buff_size, + MY_WME | share->malloc_flag)) goto err; } to= info->rec_buff; diff --git a/storage/maria/ma_extra.c b/storage/maria/ma_extra.c index 425cb421..9ffc551c 100644 --- a/storage/maria/ma_extra.c +++ b/storage/maria/ma_extra.c @@ -239,25 +239,17 @@ int maria_extra(MARIA_HA *info, enum ha_extra_function function, break; /* we're going to modify pieces of the state, stall Checkpoint */ - mysql_mutex_lock(&share->intern_lock); if (info->lock_type == F_UNLCK) { - mysql_mutex_unlock(&share->intern_lock); error= 1; /* Not possibly if not lock */ break; } + mysql_mutex_lock(&share->intern_lock); if (maria_is_any_key_active(share->state.key_map)) { - MARIA_KEYDEF *key= share->keyinfo; - uint i; - for (i =0 ; i < share->base.keys ; i++,key++) - { - if (!(key->flag & HA_NOSAME) && info->s->base.auto_key != i+1) - { - maria_clear_key_active(share->state.key_map, i); - info->update|= HA_STATE_CHANGED; - } - } + if (share->state.key_map != *(ulonglong*)extra_arg) + info->update|= HA_STATE_CHANGED; + share->state.key_map= *(ulonglong*)extra_arg; if (!share->changed) { @@ -542,7 +534,7 @@ int maria_reset(MARIA_HA *info) { int error= 0; MARIA_SHARE *share= info->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); + myf flag= MY_WME | share->malloc_flag; DBUG_ENTER("maria_reset"); /* Free buffers and reset the following flags: @@ -601,6 +593,20 @@ uint _ma_file_callback_to_id(void *callback_data) return share ? share->id : 0; } +/* + Disable MY_WAIT_IF_FULL flag for temporary tables + + Temporary tables does not have MY_WAIT_IF_FULL in share->write_flags +*/ + +uint _ma_write_flags_callback(void *callback_data, myf flags) +{ + MARIA_SHARE *share= (MARIA_SHARE*) callback_data; + if (share) + flags&= ~(~share->write_flag & MY_WAIT_IF_FULL); + return flags; +} + /** @brief flushes the data and/or index file of a table diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index bfeb5e0c..05347b2d 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -1099,10 +1099,6 @@ static TRANSLOG_FILE *get_current_logfile() uchar maria_trans_file_magic[]= { (uchar) 254, (uchar) 254, (uchar) 11, '\001', 'M', 'A', 'R', 'I', 'A', 'L', 'O', 'G' }; -#define LOG_HEADER_DATA_SIZE (sizeof(maria_trans_file_magic) + \ - 8 + 4 + 4 + 4 + 2 + 3 + \ - LSN_STORE_SIZE) - /* Write log file page header in the just opened new log file @@ -3605,6 +3601,9 @@ static my_bool translog_is_LSN_chunk(uchar type) @retval 1 Error */ +/* Stack size 26120 from clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + my_bool translog_init_with_table(const char *directory, uint32 log_file_max_size, uint32 server_version, @@ -4238,6 +4237,7 @@ err: ma_message_no_user(0, "log initialization failed"); DBUG_RETURN(1); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /* diff --git a/storage/maria/ma_loghandler.h b/storage/maria/ma_loghandler.h index 3e5c58a8..db592adf 100644 --- a/storage/maria/ma_loghandler.h +++ b/storage/maria/ma_loghandler.h @@ -534,5 +534,13 @@ typedef enum } enum_maria_sync_log_dir; extern ulong sync_log_dir; +/* sizeof(maria_trans_file_magic) */ +#define LOG_MAGIC_SIZE 12 +#define LOG_HEADER_DATA_SIZE (LOG_MAGIC_SIZE + \ + 8 + 4 + 4 + 4 + 2 + 3 + \ + LSN_STORE_SIZE) +/* Flags when creating aria_log_control */ +#define control_file_open_flags (O_BINARY | /*O_DIRECT |*/ O_RDWR | O_CLOEXEC) + C_MODE_END #endif diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index ad98a534..35578877 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -94,7 +94,7 @@ static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, uint errpos; MARIA_HA info,*m_info; my_bitmap_map *changed_fields_bitmap; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); + myf flag= MY_WME | share->malloc_flag; DBUG_ENTER("maria_clone_internal"); errpos= 0; @@ -171,7 +171,6 @@ static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, mysql_mutex_lock(&share->intern_lock); info.read_record= share->read_record; share->reopen++; - share->write_flag=MYF(MY_NABP | MY_WAIT_IF_FULL); if (share->options & HA_OPTION_READ_ONLY_DATA) { info.lock_type=F_RDLCK; @@ -266,7 +265,9 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, uint i,j,len,errpos,head_length,base_pos,keys, realpath_err, key_parts,base_key_parts,unique_key_parts,fulltext_keys,uniques; uint internal_table= MY_TEST(open_flags & HA_OPEN_INTERNAL_TABLE); - myf common_flag= open_flags & HA_OPEN_TMP_TABLE ? MY_THREAD_SPECIFIC : 0; + myf common_flag= (((open_flags & HA_OPEN_TMP_TABLE) && + !(open_flags & HA_OPEN_GLOBAL_TMP_TABLE)) ? + MY_THREAD_SPECIFIC : 0); uint file_version; size_t info_length; char name_buff[FN_REFLEN], org_name[FN_REFLEN], index_name[FN_REFLEN], @@ -984,11 +985,13 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags, share->options|= HA_OPTION_READ_ONLY_DATA; share->is_log_table= FALSE; + share->write_flag=MYF(MY_NABP | MY_WAIT_IF_FULL); if (open_flags & HA_OPEN_TMP_TABLE || share->options & HA_OPTION_TMP_TABLE) { - common_flag|= MY_THREAD_SPECIFIC; share->options|= HA_OPTION_TMP_TABLE; share->temporary= share->delay_key_write= 1; + share->malloc_flag= + (open_flags & HA_OPEN_GLOBAL_TMP_TABLE) ? 0 : MY_THREAD_SPECIFIC; share->write_flag=MYF(MY_NABP); share->w_locks++; /* We don't have to update status */ share->tot_locks++; @@ -1555,6 +1558,9 @@ uint _ma_state_info_write(MARIA_SHARE *share, uint pWrite) @retval 1 Error */ +/* Stack size 26376 from clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + uint _ma_state_info_write_sub(File file, MARIA_STATE_INFO *state, uint pWrite) { uchar buff[MARIA_STATE_INFO_SIZE + MARIA_STATE_EXTRA_SIZE]; @@ -1629,6 +1635,7 @@ uint _ma_state_info_write_sub(File file, MARIA_STATE_INFO *state, uint pWrite) MYF(MY_NABP)); DBUG_RETURN(res != 0); } +PRAGMA_REENABLE_CHECK_STACK_FRAME static uchar *_ma_state_info_read(uchar *ptr, MARIA_STATE_INFO *state, myf flag) @@ -2046,9 +2053,8 @@ void _ma_set_index_pagecache_callbacks(PAGECACHE_FILE *file, int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share) { - myf flags= (share->mode & O_NOFOLLOW) ? MY_NOSYMLINKS | MY_WME : MY_WME; - if (share->temporary) - flags|= MY_THREAD_SPECIFIC; + myf flags= ((share->mode & O_NOFOLLOW) ? MY_NOSYMLINKS | MY_WME : MY_WME) | + share->malloc_flag; DEBUG_SYNC_C("mi_open_datafile"); info->dfile.file= share->bitmap.file.file= mysql_file_open(key_file_dfile, share->data_file_name.str, diff --git a/storage/maria/ma_packrec.c b/storage/maria/ma_packrec.c index 19783423..57926ee4 100644 --- a/storage/maria/ma_packrec.c +++ b/storage/maria/ma_packrec.c @@ -1417,7 +1417,6 @@ uint _ma_pack_get_block_info(MARIA_HA *maria, MARIA_BIT_BUFF *bit_buff, uchar *header= info->header; uint head_length,UNINIT_VAR(ref_length); MARIA_SHARE *share= maria->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); if (file >= 0) { @@ -1444,7 +1443,8 @@ uint _ma_pack_get_block_info(MARIA_HA *maria, MARIA_BIT_BUFF *bit_buff, */ if (_ma_alloc_buffer(rec_buff_p, rec_buff_size_p, info->rec_len + info->blob_len + - share->base.extra_rec_buff_size, flag)) + share->base.extra_rec_buff_size, + MY_WME | share->malloc_flag)) return BLOCK_FATAL_ERROR; /* not enough memory */ bit_buff->blob_pos= *rec_buff_p + info->rec_len; bit_buff->blob_end= bit_buff->blob_pos + info->blob_len; @@ -1586,7 +1586,6 @@ _ma_mempack_get_block_info(MARIA_HA *maria, uchar *header) { MARIA_SHARE *share= maria->s; - myf flag= MY_WME | (share->temporary ? MY_THREAD_SPECIFIC : 0); header+= read_pack_length((uint) share->pack.version, header, &info->rec_len); @@ -1596,7 +1595,8 @@ _ma_mempack_get_block_info(MARIA_HA *maria, &info->blob_len); /* _ma_alloc_rec_buff sets my_errno on error */ if (_ma_alloc_buffer(rec_buff_p, rec_buff_size_p, - info->blob_len + share->base.extra_rec_buff_size, flag)) + info->blob_len + share->base.extra_rec_buff_size, + MY_WME | share->malloc_flag)) return 0; /* not enough memory */ bit_buff->blob_pos= *rec_buff_p; bit_buff->blob_end= *rec_buff_p + info->blob_len; diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index 144b10a8..49981e79 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -687,6 +687,8 @@ static my_bool pagecache_fwrite(PAGECACHE *pagecache, /* FIXME: ENGINE=Aria occasionally writes uninitialized data */ __msan_unpoison(args.page, pagecache->block_size); #endif + /* Reset MY_WAIT_IF_FULL for temporary tables */ + flags= _ma_write_flags_callback(filedesc->callback_data, flags); res= (int)my_pwrite(filedesc->file, args.page, pagecache->block_size, ((my_off_t) pageno << pagecache->shift), flags); (*filedesc->post_write_hook)(res, &args); diff --git a/storage/maria/ma_recovery.c b/storage/maria/ma_recovery.c index 006c8bef..df9b2024 100644 --- a/storage/maria/ma_recovery.c +++ b/storage/maria/ma_recovery.c @@ -1163,11 +1163,12 @@ end: /* The record may come from REPAIR, ALTER TABLE ENABLE KEYS, OPTIMIZE. */ + prototype_redo_exec_hook(REDO_REPAIR_TABLE) { int error= 1; MARIA_HA *info; - HA_CHECK param; + HA_CHECK *param; char *name; my_bool quick_repair; DBUG_ENTER("exec_REDO_LOGREC_REDO_REPAIR_TABLE"); @@ -1199,35 +1200,39 @@ prototype_redo_exec_hook(REDO_REPAIR_TABLE) */ tprint(tracef, " repairing...\n"); - maria_chk_init(¶m); - param.isam_file_name= name= info->s->open_file_name.str; - param.testflag= uint8korr(rec->header + FILEID_STORE_SIZE); - param.tmpdir= maria_tmpdir; - param.max_trid= max_long_trid; + if (!(param= my_malloc(PSI_INSTRUMENT_ME, sizeof(*param), MYF(MY_WME)))) + DBUG_RETURN(0); + + maria_chk_init(param); + param->isam_file_name= name= info->s->open_file_name.str; + param->testflag= uint8korr(rec->header + FILEID_STORE_SIZE); + param->tmpdir= maria_tmpdir; + param->max_trid= max_long_trid; DBUG_ASSERT(maria_tmpdir); info->s->state.key_map= uint8korr(rec->header + FILEID_STORE_SIZE + 8); - quick_repair= MY_TEST(param.testflag & T_QUICK); + quick_repair= MY_TEST(param->testflag & T_QUICK); - if (param.testflag & T_REP_PARALLEL) + if (param->testflag & T_REP_PARALLEL) { - if (maria_repair_parallel(¶m, info, name, quick_repair)) + if (maria_repair_parallel(param, info, name, quick_repair)) goto end; } - else if (param.testflag & T_REP_BY_SORT) + else if (param->testflag & T_REP_BY_SORT) { - if (maria_repair_by_sort(¶m, info, name, quick_repair)) + if (maria_repair_by_sort(param, info, name, quick_repair)) goto end; } - else if (maria_repair(¶m, info, name, quick_repair)) + else if (maria_repair(param, info, name, quick_repair)) goto end; if (_ma_update_state_lsns(info->s, rec->lsn, trnman_get_min_safe_trid(), - TRUE, !(param.testflag & T_NO_CREATE_RENAME_LSN))) + TRUE, !(param->testflag & T_NO_CREATE_RENAME_LSN))) goto end; error= 0; end: + my_free(param); DBUG_RETURN(error); } @@ -2579,6 +2584,8 @@ prototype_undo_exec_hook(UNDO_BULK_INSERT) return error; } +/* Stack size 18776 in clang. Ok as this is during recover */ +PRAGMA_DISABLE_CHECK_STACK_FRAME static int run_redo_phase(LSN lsn, LSN lsn_end, enum maria_apply_log_way apply) { @@ -2822,6 +2829,7 @@ err: translog_free_record_header(&rec); DBUG_RETURN(1); } +PRAGMA_REENABLE_CHECK_STACK_FRAME /** diff --git a/storage/maria/ma_rt_test.c b/storage/maria/ma_rt_test.c index 3af7d938..17145231 100644 --- a/storage/maria/ma_rt_test.c +++ b/storage/maria/ma_rt_test.c @@ -101,7 +101,7 @@ int main(int argc, char *argv[]) if (maria_init() || (init_pagecache(maria_pagecache, maria_block_size * 16, 0, 0, maria_block_size, 0, MY_WME) == 0) || - ma_control_file_open(TRUE, TRUE, TRUE) || + ma_control_file_open_or_create() || (init_pagecache(maria_log_pagecache, TRANSLOG_PAGECACHE_SIZE, 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0) || diff --git a/storage/maria/ma_test1.c b/storage/maria/ma_test1.c index 22f80ca2..a14679d3 100644 --- a/storage/maria/ma_test1.c +++ b/storage/maria/ma_test1.c @@ -81,7 +81,7 @@ int main(int argc,char *argv[]) if (maria_init() || (init_pagecache(maria_pagecache, maria_block_size * 16, 0, 0, maria_block_size, 0, MY_WME) == 0) || - ma_control_file_open(TRUE, TRUE, TRUE) || + ma_control_file_open_or_create() || (init_pagecache(maria_log_pagecache, TRANSLOG_PAGECACHE_SIZE, 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0) || diff --git a/storage/maria/ma_test2.c b/storage/maria/ma_test2.c index 66284653..400e6193 100644 --- a/storage/maria/ma_test2.c +++ b/storage/maria/ma_test2.c @@ -90,7 +90,7 @@ int main(int argc, char *argv[]) if (maria_init() || (init_pagecache(maria_pagecache, pagecache_size, 0, 0, maria_block_size, 0, MY_WME) == 0) || - ma_control_file_open(TRUE, TRUE, TRUE) || + ma_control_file_open_or_create() || (init_pagecache(maria_log_pagecache, TRANSLOG_PAGECACHE_SIZE, 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0) || diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index 91e1b10a..b12f21e3 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -754,6 +754,11 @@ typedef struct st_maria_share ulong max_pack_length; ulong state_diff_length; uint rec_reflength; /* rec_reflength in use now */ + /* + Extra flag to use for my_malloc(); set to MY_THREAD_SPECIFIC for temporary + tables whose memory allocation should be accounted to the current THD. + */ + uint malloc_flag; uint keypage_header; uint32 ftkeys; /* Number of distinct full-text keys + 1 */ @@ -1740,6 +1745,7 @@ extern my_bool ma_yield_and_check_if_killed(MARIA_HA *info, int inx); extern my_bool ma_killed_standalone(MARIA_HA *); extern uint _ma_file_callback_to_id(void *callback_data); +extern uint _ma_write_flags_callback(void *callback_data, myf flags); extern void free_maria_share(MARIA_SHARE *share); static inline void unmap_file(MARIA_HA *info __attribute__((unused))) diff --git a/storage/maria/test_ma_backup.c b/storage/maria/test_ma_backup.c index c57ec6ec..5cb2b074 100644 --- a/storage/maria/test_ma_backup.c +++ b/storage/maria/test_ma_backup.c @@ -47,7 +47,7 @@ int main(int argc __attribute__((unused)), char *argv[]) if (maria_init() || (init_pagecache(maria_pagecache, maria_block_size * 2000, 0, 0, maria_block_size, 0, MY_WME) == 0) || - ma_control_file_open(TRUE, TRUE, TRUE) || + ma_control_file_open_or_create() || (init_pagecache(maria_log_pagecache, TRANSLOG_PAGECACHE_SIZE, 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0) || diff --git a/storage/maria/unittest/CMakeLists.txt b/storage/maria/unittest/CMakeLists.txt index a2da1507..a7b34410 100644 --- a/storage/maria/unittest/CMakeLists.txt +++ b/storage/maria/unittest/CMakeLists.txt @@ -15,7 +15,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/zlib ${CMAKE_SOURCE_DIR}/unittest/mytap) -LINK_LIBRARIES(aria myisam mytap mysys dbug strings ${ZLIB_LIBRARY}) +LINK_LIBRARIES(aria myisam mytap mysys dbug strings ${ZLIB_LIBRARIES}) MY_ADD_TESTS(ma_control_file trnman) diff --git a/storage/maria/unittest/ma_control_file-t.c b/storage/maria/unittest/ma_control_file-t.c index 859d5514..fdbe86de 100644 --- a/storage/maria/unittest/ma_control_file-t.c +++ b/storage/maria/unittest/ma_control_file-t.c @@ -114,7 +114,7 @@ static CONTROL_FILE_ERROR local_ma_control_file_open(void) { CONTROL_FILE_ERROR error; error_handler_hook= my_ignore_message; - error= ma_control_file_open(TRUE, TRUE, TRUE); + error= ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags); error_handler_hook= default_error_handler_hook; return error; } diff --git a/storage/maria/unittest/ma_test_loghandler-t.c b/storage/maria/unittest/ma_test_loghandler-t.c index ccda66af..0a1d396e 100644 --- a/storage/maria/unittest/ma_test_loghandler-t.c +++ b/storage/maria/unittest/ma_test_loghandler-t.c @@ -143,6 +143,8 @@ static my_bool read_and_check_content(TRANSLOG_HEADER_BUFFER *rec, } +PRAGMA_DISABLE_CHECK_STACK_FRAME + int main(int argc __attribute__((unused)), char *argv[]) { uint32 i; @@ -197,7 +199,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); @@ -664,5 +666,6 @@ err: my_end(0); return(MY_TEST(exit_status())); } +PRAGMA_REENABLE_CHECK_STACK_FRAME #include "../ma_check_standalone.h" diff --git a/storage/maria/unittest/ma_test_loghandler_first_lsn-t.c b/storage/maria/unittest/ma_test_loghandler_first_lsn-t.c index 21f6b7d7..5c4045cc 100644 --- a/storage/maria/unittest/ma_test_loghandler_first_lsn-t.c +++ b/storage/maria/unittest/ma_test_loghandler_first_lsn-t.c @@ -66,7 +66,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE,TRUE)) + if (ma_control_file_open(TRUE, TRUE,TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_max_lsn-t.c b/storage/maria/unittest/ma_test_loghandler_max_lsn-t.c index 391d7851..eedeb9d7 100644 --- a/storage/maria/unittest/ma_test_loghandler_max_lsn-t.c +++ b/storage/maria/unittest/ma_test_loghandler_max_lsn-t.c @@ -64,7 +64,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_multigroup-t.c b/storage/maria/unittest/ma_test_loghandler_multigroup-t.c index e8e114dd..b0e1be8f 100644 --- a/storage/maria/unittest/ma_test_loghandler_multigroup-t.c +++ b/storage/maria/unittest/ma_test_loghandler_multigroup-t.c @@ -280,7 +280,7 @@ int main(int argc __attribute__((unused)), char *argv[]) bzero(long_tr_id, 6); - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); @@ -443,7 +443,7 @@ int main(int argc __attribute__((unused)), char *argv[]) end_pagecache(&pagecache, 1); ma_control_file_end(); - if (ma_control_file_open(TRUE,TRUE,TRUE)) + if (ma_control_file_open(TRUE,TRUE,TRUE, control_file_open_flags)) { fprintf(stderr, "pass2: Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_multithread-t.c b/storage/maria/unittest/ma_test_loghandler_multithread-t.c index be6046ab..9b7e239b 100644 --- a/storage/maria/unittest/ma_test_loghandler_multithread-t.c +++ b/storage/maria/unittest/ma_test_loghandler_multithread-t.c @@ -331,7 +331,7 @@ int main(int argc __attribute__((unused)), exit(1); } - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_noflush-t.c b/storage/maria/unittest/ma_test_loghandler_noflush-t.c index 46b3a8e7..28cac2ee 100644 --- a/storage/maria/unittest/ma_test_loghandler_noflush-t.c +++ b/storage/maria/unittest/ma_test_loghandler_noflush-t.c @@ -65,7 +65,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_nologs-t.c b/storage/maria/unittest/ma_test_loghandler_nologs-t.c index b95d8bee..a7c526b2 100644 --- a/storage/maria/unittest/ma_test_loghandler_nologs-t.c +++ b/storage/maria/unittest/ma_test_loghandler_nologs-t.c @@ -66,7 +66,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); @@ -139,7 +139,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } } - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_pagecache-t.c b/storage/maria/unittest/ma_test_loghandler_pagecache-t.c index 892a773b..4ecc8f1f 100644 --- a/storage/maria/unittest/ma_test_loghandler_pagecache-t.c +++ b/storage/maria/unittest/ma_test_loghandler_pagecache-t.c @@ -69,7 +69,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/maria/unittest/ma_test_loghandler_purge-t.c b/storage/maria/unittest/ma_test_loghandler_purge-t.c index 07b50f19..b26a2f99 100644 --- a/storage/maria/unittest/ma_test_loghandler_purge-t.c +++ b/storage/maria/unittest/ma_test_loghandler_purge-t.c @@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char *argv[]) } #endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) + if (ma_control_file_open(TRUE, TRUE, TRUE, control_file_open_flags)) { fprintf(stderr, "Can't init control file (%d)\n", errno); exit(1); diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt index bea0eecc..4553fd43 100644 --- a/storage/mroonga/CMakeLists.txt +++ b/storage/mroonga/CMakeLists.txt @@ -57,6 +57,11 @@ if(MRN_BUNDLED) "${PLUGIN_MROONGA}" STREQUAL "NO") return() endif() + if(WITHOUT_DYNAMIC_PLUGINS) + if(NOT (PLUGIN_MROONGA STREQUAL STATIC)) + return() + endif() + endif() endif() set(MRN_BUNDLED_GROONGA_RELATIVE_DIR "vendor/groonga") diff --git a/storage/mroonga/ha_mroonga.cpp b/storage/mroonga/ha_mroonga.cpp index a176ccf3..d5f150de 100644 --- a/storage/mroonga/ha_mroonga.cpp +++ b/storage/mroonga/ha_mroonga.cpp @@ -4351,9 +4351,9 @@ int ha_mroonga::wrapper_open(const char *name, int mode, uint open_options) // TODO: implemented by "reindex" instead of "remove and recreate". // Because "remove and recreate" invalidates opened indexes by // other threads. - error = wrapper_disable_indexes_mroonga(HA_KEY_SWITCH_ALL); + error = wrapper_disable_indexes_mroonga(key_map(table->s->keys), false); if (!error) { - error = wrapper_enable_indexes_mroonga(HA_KEY_SWITCH_ALL); + error = wrapper_enable_indexes_mroonga(key_map(table->s->keys), false); } } } @@ -13639,197 +13639,184 @@ int ha_mroonga::generic_disable_index(int i, KEY *key_info) DBUG_RETURN(error); } -int ha_mroonga::wrapper_disable_indexes_mroonga(uint mode) +int ha_mroonga::wrapper_disable_indexes_mroonga(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); - if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE || mode == HA_KEY_SWITCH_ALL) { - uint i; - for (i = 0; i < table_share->keys; i++) { - if (i == table->s->primary_key) { - continue; - } - if (share->wrap_key_nr[i] < MAX_KEY) { - continue; - } - if (!grn_index_tables[i]) { - DBUG_PRINT("info", ("mroonga: keys are disabled already %u", i)); - DBUG_RETURN(0); - } + uint i; + for (i = 0; i < table_share->keys; i++) { + if (i == table->s->primary_key) { + continue; + } + if (share->wrap_key_nr[i] < MAX_KEY) { + continue; + } + if (!grn_index_tables[i]) { + DBUG_PRINT("info", ("mroonga: keys are disabled already %u", i)); + DBUG_RETURN(0); + } + } + KEY *key_info = table_share->key_info; + for (i = 0; i < table_share->keys; i++) { + if (!(key_info[i].flags & HA_FULLTEXT) && + !mrn_is_geo_key(&key_info[i])) { + continue; } - KEY *key_info = table_share->key_info; - for (i = 0; i < table_share->keys; i++) { - if (!(key_info[i].flags & HA_FULLTEXT) && - !mrn_is_geo_key(&key_info[i])) { - continue; - } - int sub_error = generic_disable_index(i, key_info); - if (error != 0 && sub_error != 0) { - error = sub_error; - } + int sub_error = generic_disable_index(i, key_info); + if (error != 0 && sub_error != 0) { + error = sub_error; } - } else { - error = HA_ERR_WRONG_COMMAND; } DBUG_RETURN(error); } -int ha_mroonga::wrapper_disable_indexes(uint mode) +int ha_mroonga::wrapper_disable_indexes(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); MRN_SET_WRAP_SHARE_KEY(share, table->s); MRN_SET_WRAP_TABLE_KEY(this, table); - error = wrap_handler->ha_disable_indexes(mode); + error = wrap_handler->ha_disable_indexes(map, persist); MRN_SET_BASE_SHARE_KEY(share, table->s); MRN_SET_BASE_TABLE_KEY(this, table); if (error == HA_ERR_WRONG_COMMAND) { error = 0; } if (!error) { - error = wrapper_disable_indexes_mroonga(mode); + error = wrapper_disable_indexes_mroonga(map, persist); } DBUG_RETURN(error); } -int ha_mroonga::storage_disable_indexes(uint mode) +int ha_mroonga::storage_disable_indexes(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); - if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE || mode == HA_KEY_SWITCH_ALL) { - uint i; - for (i = 0; i < table_share->keys; i++) { - if (i == table->s->primary_key) { - continue; - } - if (!grn_index_tables[i]) { - DBUG_PRINT("info", ("mroonga: keys are disabled already %u", i)); - DBUG_RETURN(0); - } + uint i; + for (i = 0; i < table_share->keys; i++) { + if (i == table->s->primary_key) { + continue; + } + if (!grn_index_tables[i]) { + DBUG_PRINT("info", ("mroonga: keys are disabled already %u", i)); + DBUG_RETURN(0); + } + } + KEY *key_info = table_share->key_info; + for (i = 0; i < table_share->keys; i++) { + if (i == table->s->primary_key) { + continue; + } + if (map.is_set(i)) { + continue; } - KEY *key_info = table_share->key_info; - for (i = 0; i < table_share->keys; i++) { - if (i == table->s->primary_key) { - continue; - } - if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE && - (key_info[i].flags & HA_NOSAME)) { - continue; - } - int sub_error = generic_disable_index(i, key_info); - if (error != 0 && sub_error != 0) { - error = sub_error; - } + int sub_error = generic_disable_index(i, key_info); + if (error != 0 && sub_error != 0) { + error = sub_error; } - } else { - DBUG_RETURN(HA_ERR_WRONG_COMMAND); } DBUG_RETURN(error); } -int ha_mroonga::disable_indexes(uint mode) +int ha_mroonga::disable_indexes(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); if (share->wrapper_mode) { - error = wrapper_disable_indexes(mode); + error = wrapper_disable_indexes(map, persist); } else { - error = storage_disable_indexes(mode); + error = storage_disable_indexes(map, persist); } DBUG_RETURN(error); } -int ha_mroonga::wrapper_enable_indexes_mroonga(uint mode) +int ha_mroonga::wrapper_enable_indexes_mroonga(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); - if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE || mode == HA_KEY_SWITCH_ALL) { - uint i, j; - for (i = 0; i < table_share->keys; i++) { - if (i == table->s->primary_key) { - continue; - } - if (share->wrap_key_nr[i] < MAX_KEY) { - continue; - } - if (!grn_index_columns[i]) { - break; - } + uint i, j; + for (i = 0; i < table_share->keys; i++) { + if (i == table->s->primary_key) { + continue; } - if (i == table_share->keys) { - DBUG_PRINT("info", ("mroonga: keys are enabled already")); - DBUG_RETURN(0); + if (share->wrap_key_nr[i] < MAX_KEY) { + continue; + } + if (!grn_index_columns[i]) { + break; + } + } + if (i == table_share->keys) { + DBUG_PRINT("info", ("mroonga: keys are enabled already")); + DBUG_RETURN(0); + } + KEY *p_key_info = &table->key_info[table_share->primary_key]; + KEY *key_info = table_share->key_info; + uint n_keys = table_share->keys; + MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(grn_obj *, index_tables, n_keys); + MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(grn_obj *, index_columns, n_keys); + bitmap_clear_all(table->read_set); + mrn_set_bitmap_by_key(table->read_set, p_key_info); + mrn::PathMapper mapper(share->table_name); + for (i = 0, j = 0; i < n_keys; i++) { + if (!(key_info[i].flags & HA_FULLTEXT) && + !mrn_is_geo_key(&key_info[i])) { + j++; + continue; } - KEY *p_key_info = &table->key_info[table_share->primary_key]; - KEY *key_info = table_share->key_info; - uint n_keys = table_share->keys; - MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(grn_obj *, index_tables, n_keys); - MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(grn_obj *, index_columns, n_keys); - bitmap_clear_all(table->read_set); - mrn_set_bitmap_by_key(table->read_set, p_key_info); - mrn::PathMapper mapper(share->table_name); - for (i = 0, j = 0; i < n_keys; i++) { - if (!(key_info[i].flags & HA_FULLTEXT) && - !mrn_is_geo_key(&key_info[i])) { - j++; - continue; - } - if ((error = mrn_add_index_param(share, &key_info[i], i))) - { + if ((error = mrn_add_index_param(share, &key_info[i], i))) + { + break; + } + index_tables[i] = NULL; + index_columns[i] = NULL; + if (!grn_index_columns[i]) { + if ( + (key_info[i].flags & HA_FULLTEXT) && + (error = wrapper_create_index_fulltext(mapper.table_name(), + i, &key_info[i], + index_tables, index_columns, + share)) + ) { + break; + } else if ( + mrn_is_geo_key(&key_info[i]) && + (error = wrapper_create_index_geo(mapper.table_name(), + i, &key_info[i], + index_tables, index_columns, + share)) + ) { break; } - index_tables[i] = NULL; - index_columns[i] = NULL; - if (!grn_index_columns[i]) { - if ( - (key_info[i].flags & HA_FULLTEXT) && - (error = wrapper_create_index_fulltext(mapper.table_name(), - i, &key_info[i], - index_tables, index_columns, - share)) - ) { - break; - } else if ( - mrn_is_geo_key(&key_info[i]) && - (error = wrapper_create_index_geo(mapper.table_name(), - i, &key_info[i], - index_tables, index_columns, - share)) - ) { - break; - } - grn_index_columns[i] = index_columns[i]; - } - mrn_set_bitmap_by_key(table->read_set, &key_info[i]); + grn_index_columns[i] = index_columns[i]; } - if (!error && i > j) - { - error = wrapper_fill_indexes(ha_thd(), table->key_info, index_columns, - n_keys); - } - bitmap_set_all(table->read_set); - MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); - MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); - } else { - error = HA_ERR_WRONG_COMMAND; + mrn_set_bitmap_by_key(table->read_set, &key_info[i]); } + if (!error && i > j) + { + error = wrapper_fill_indexes(ha_thd(), table->key_info, index_columns, + n_keys); + } + bitmap_set_all(table->read_set); + MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); + MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); DBUG_RETURN(error); } -int ha_mroonga::wrapper_enable_indexes(uint mode) +int ha_mroonga::wrapper_enable_indexes(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); - int mroonga_error = wrapper_enable_indexes_mroonga(mode); + int mroonga_error = wrapper_enable_indexes_mroonga(map, persist); MRN_SET_WRAP_SHARE_KEY(share, table->s); MRN_SET_WRAP_TABLE_KEY(this, table); - error = wrap_handler->ha_enable_indexes(mode); + error = wrap_handler->ha_enable_indexes(map, persist); MRN_SET_BASE_SHARE_KEY(share, table->s); MRN_SET_BASE_TABLE_KEY(this, table); if (error == HA_ERR_WRONG_COMMAND) { @@ -13838,95 +13825,86 @@ int ha_mroonga::wrapper_enable_indexes(uint mode) DBUG_RETURN(error); } -int ha_mroonga::storage_enable_indexes(uint mode) +int ha_mroonga::storage_enable_indexes(key_map map, bool persist) { int error = 0; uint n_keys = table_share->keys; MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(grn_obj *, index_tables, n_keys); MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(grn_obj *, index_columns, n_keys); bool have_multiple_column_index = false; - bool skip_unique_key = (mode == HA_KEY_SWITCH_NONUNIQ_SAVE); MRN_DBUG_ENTER_METHOD(); - if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE || mode == HA_KEY_SWITCH_ALL) { - uint i; - for (i = 0; i < table_share->keys; i++) { - if (i == table->s->primary_key) { - continue; - } - if (!grn_index_columns[i]) { - break; - } + uint i; + for (i = 0; i < n_keys; i++) { + if (i == table->s->primary_key) { + continue; } - if (i == table_share->keys) { - DBUG_PRINT("info", ("mroonga: keys are enabled already")); - MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); - MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); - DBUG_RETURN(0); + if (!grn_index_columns[i]) { + break; + } + } + if (i == n_keys) { + DBUG_PRINT("info", ("mroonga: keys are enabled already")); + MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); + MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); + DBUG_RETURN(0); + } + KEY *key_info = table->key_info; + bitmap_clear_all(table->read_set); + mrn::PathMapper mapper(share->table_name); + for (; i < n_keys; i++) { + if (!map.is_set(i)) { + continue; } - KEY *key_info = table->key_info; - bitmap_clear_all(table->read_set); - mrn::PathMapper mapper(share->table_name); - for (i = 0; i < n_keys; i++) { - if (i == table->s->primary_key) { - continue; - } - if (skip_unique_key && (key_info[i].flags & HA_NOSAME)) { - continue; - } - if ((error = mrn_add_index_param(share, &key_info[i], i))) + if ((error = mrn_add_index_param(share, &key_info[i], i))) + { + break; + } + index_tables[i] = NULL; + if (!grn_index_columns[i]) { + if ((error = storage_create_index(table, mapper.table_name(), grn_table, + share, &key_info[i], index_tables, + index_columns, i))) { break; } - index_tables[i] = NULL; - if (!grn_index_columns[i]) { - if ((error = storage_create_index(table, mapper.table_name(), grn_table, - share, &key_info[i], index_tables, - index_columns, i))) - { - break; - } - if ( - KEY_N_KEY_PARTS(&(key_info[i])) != 1 && - !(key_info[i].flags & HA_FULLTEXT) - ) { - mrn_set_bitmap_by_key(table->read_set, &key_info[i]); - have_multiple_column_index = true; - } - grn_index_tables[i] = index_tables[i]; - grn_index_columns[i] = index_columns[i]; - } else { - index_columns[i] = NULL; + if ( + KEY_N_KEY_PARTS(&(key_info[i])) != 1 && + !(key_info[i].flags & HA_FULLTEXT) + ) { + mrn_set_bitmap_by_key(table->read_set, &key_info[i]); + have_multiple_column_index = true; } + grn_index_tables[i] = index_tables[i]; + grn_index_columns[i] = index_columns[i]; + } else { + index_columns[i] = NULL; } - if (!error && have_multiple_column_index) - { - error = storage_add_index_multiple_columns(key_info, n_keys, - index_tables, - index_columns, - skip_unique_key); - } - bitmap_set_all(table->read_set); - } else { - MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); - MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); - DBUG_RETURN(HA_ERR_WRONG_COMMAND); } + if (!error && have_multiple_column_index) + { + bool skip_unique_key= !table->s->keys_in_use.is_clear_all(); + error = storage_add_index_multiple_columns(key_info, n_keys, + index_tables, + index_columns, + skip_unique_key); + } + bitmap_set_all(table->read_set); MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); DBUG_RETURN(error); } -int ha_mroonga::enable_indexes(uint mode) +int ha_mroonga::enable_indexes(key_map map, bool persist) { int error = 0; MRN_DBUG_ENTER_METHOD(); share->disable_keys = false; if (share->wrapper_mode) { - error = wrapper_enable_indexes(mode); + error = wrapper_enable_indexes(map, persist); } else { - error = storage_enable_indexes(mode); + error = storage_enable_indexes(map, persist); } DBUG_RETURN(error); } diff --git a/storage/mroonga/ha_mroonga.hpp b/storage/mroonga/ha_mroonga.hpp index 66767899..6bf971aa 100644 --- a/storage/mroonga/ha_mroonga.hpp +++ b/storage/mroonga/ha_mroonga.hpp @@ -542,8 +542,8 @@ public: bool is_crashed() const mrn_override; bool auto_repair(int error) const mrn_override; bool auto_repair() const; - int disable_indexes(uint mode) mrn_override; - int enable_indexes(uint mode) mrn_override; + int disable_indexes(key_map map, bool persist) mrn_override; + int enable_indexes(key_map map, bool persist) mrn_override; int check(THD* thd, HA_CHECK_OPT* check_opt) mrn_override; int repair(THD* thd, HA_CHECK_OPT* check_opt) mrn_override; bool check_and_repair(THD *thd) mrn_override; @@ -1140,12 +1140,12 @@ private: bool wrapper_auto_repair(int error) const; bool storage_auto_repair(int error) const; int generic_disable_index(int i, KEY *key_info); - int wrapper_disable_indexes_mroonga(uint mode); - int wrapper_disable_indexes(uint mode); - int storage_disable_indexes(uint mode); - int wrapper_enable_indexes_mroonga(uint mode); - int wrapper_enable_indexes(uint mode); - int storage_enable_indexes(uint mode); + int wrapper_disable_indexes_mroonga(key_map map, bool persist); + int wrapper_disable_indexes(key_map map, bool persist); + int storage_disable_indexes(key_map map, bool persist); + int wrapper_enable_indexes_mroonga(key_map map, bool persist); + int wrapper_enable_indexes(key_map map, bool persist); + int storage_enable_indexes(key_map map, bool persist); int wrapper_check(THD* thd, HA_CHECK_OPT* check_opt); int storage_check(THD* thd, HA_CHECK_OPT* check_opt); int wrapper_fill_indexes(THD *thd, KEY *key_info, diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index 5b25ada4..cdbfb8f1 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -200,6 +200,7 @@ endif() include_directories( BEFORE + ${CMAKE_SOURCE_DIR}/include ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/lib diff --git a/storage/mroonga/vendor/groonga/lib/db.c b/storage/mroonga/vendor/groonga/lib/db.c index c3bbb64f..65463bda 100644 --- a/storage/mroonga/vendor/groonga/lib/db.c +++ b/storage/mroonga/vendor/groonga/lib/db.c @@ -38,6 +38,7 @@ #include "grn_util.h" #include "grn_cache.h" #include "grn_window_functions.h" +#include <my_attribute.h> #include <string.h> #include <math.h> @@ -1060,6 +1061,8 @@ grn_table_create_validate(grn_ctx *ctx, const char *name, unsigned int name_size return ctx->rc; } +PRAGMA_DISABLE_CHECK_STACK_FRAME + static grn_obj * grn_table_create_with_max_n_subrecs(grn_ctx *ctx, const char *name, unsigned int name_size, const char *path, @@ -1238,6 +1241,7 @@ grn_table_create_with_max_n_subrecs(grn_ctx *ctx, const char *name, } return res; } +PRAGMA_REENABLE_CHECK_STACK_FRAME grn_obj * grn_table_create(grn_ctx *ctx, const char *name, unsigned int name_size, @@ -4776,6 +4780,9 @@ _grn_table_key(grn_ctx *ctx, grn_obj *table, grn_id id, uint32_t *key_size) /* column */ + +PRAGMA_DISABLE_CHECK_STACK_FRAME + grn_obj * grn_column_create(grn_ctx *ctx, grn_obj *table, const char *name, unsigned int name_size, @@ -4978,6 +4985,7 @@ exit : if (!res && id) { grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); } GRN_API_RETURN(res); } +PRAGMA_REENABLE_CHECK_STACK_FRAME grn_obj * grn_column_open(grn_ctx *ctx, grn_obj *table, @@ -8540,6 +8548,8 @@ grn_obj_spec_save(grn_ctx *ctx, grn_db_obj *obj) grn_obj_close(ctx, &v); } +PRAGMA_DISABLE_CHECK_STACK_FRAME + inline static void grn_obj_set_info_source_invalid_lexicon_error(grn_ctx *ctx, const char *message, @@ -8590,6 +8600,8 @@ grn_obj_set_info_source_invalid_lexicon_error(grn_ctx *ctx, source_name_size, source_name); } +PRAGMA_REENABLE_CHECK_STACK_FRAME + inline static grn_rc grn_obj_set_info_source_validate(grn_ctx *ctx, grn_obj *obj, grn_obj *value) { @@ -8597,7 +8609,7 @@ grn_obj_set_info_source_validate(grn_ctx *ctx, grn_obj *obj, grn_obj *value) grn_obj *lexicon = NULL; grn_id lexicon_domain_id; grn_obj *lexicon_domain = NULL; - grn_bool lexicon_domain_is_table; + grn_bool lexicon_domain_is_table __attribute__((unused)); grn_bool lexicon_have_tokenizer; grn_id *source_ids; int i, n_source_ids; @@ -9330,7 +9342,7 @@ remove_reference_tables(grn_ctx *ctx, grn_obj *table, grn_obj *db) grn_bool is_close_opened_object_mode = GRN_FALSE; grn_id table_id; char table_name[GRN_TABLE_MAX_KEY_SIZE]; - int table_name_size; + int table_name_size __attribute__((unused)); grn_table_cursor *cursor; if (grn_thread_get_limit() == 1) { @@ -10317,12 +10329,10 @@ grn_db_spec_unpack(grn_ctx *ctx, const char *error_message_tag) { grn_obj *db; - grn_db *db_raw; grn_rc rc; uint32_t spec_size; db = ctx->impl->db; - db_raw = (grn_db *)db; rc = grn_vector_decode(ctx, decoded_spec, diff --git a/storage/mroonga/vendor/groonga/lib/load.c b/storage/mroonga/vendor/groonga/lib/load.c index eb77f2b3..25d621e7 100644 --- a/storage/mroonga/vendor/groonga/lib/load.c +++ b/storage/mroonga/vendor/groonga/lib/load.c @@ -20,6 +20,9 @@ #include "grn_ctx_impl.h" #include "grn_db.h" #include "grn_util.h" +#include <my_attribute.h> + +PRAGMA_DISABLE_CHECK_STACK_FRAME static void grn_loader_save_error(grn_ctx *ctx, grn_loader *loader) @@ -1228,3 +1231,5 @@ grn_load(grn_ctx *ctx, grn_content_type input_type, } GRN_API_RETURN(ctx->rc); } + +PRAGMA_REENABLE_CHECK_STACK_FRAME diff --git a/storage/mroonga/vendor/groonga/lib/operator.c b/storage/mroonga/vendor/groonga/lib/operator.c index 1e1f2cf7..940c0b14 100644 --- a/storage/mroonga/vendor/groonga/lib/operator.c +++ b/storage/mroonga/vendor/groonga/lib/operator.c @@ -20,6 +20,7 @@ #include "grn_db.h" #include "grn_str.h" #include "grn_normalizer.h" +#include <my_attribute.h> #include <string.h> @@ -31,6 +32,8 @@ # include <onigmo.h> #endif +PRAGMA_DISABLE_CHECK_STACK_FRAME + static const char *operator_names[] = { "push", "pop", @@ -1360,3 +1363,5 @@ grn_operator_exec_regexp(grn_ctx *ctx, grn_obj *target, grn_obj *pattern) } GRN_API_RETURN(matched); } + +PRAGMA_REENABLE_CHECK_STACK_FRAME diff --git a/storage/mroonga/vendor/groonga/lib/proc/proc_object_list.c b/storage/mroonga/vendor/groonga/lib/proc/proc_object_list.c index adb4c91b..eaf5504d 100644 --- a/storage/mroonga/vendor/groonga/lib/proc/proc_object_list.c +++ b/storage/mroonga/vendor/groonga/lib/proc/proc_object_list.c @@ -18,6 +18,7 @@ #include "../grn_proc.h" #include "../grn_db.h" +#include <my_attribute.h> #include <groonga/plugin.h> @@ -73,6 +74,8 @@ command_object_list_dump_flags(grn_ctx *ctx, grn_obj_spec *spec) GRN_OBJ_FIN(ctx, &flags); } +PRAGMA_DISABLE_CHECK_STACK_FRAME + static grn_obj * command_object_list(grn_ctx *ctx, int nargs, @@ -401,6 +404,7 @@ command_object_list(grn_ctx *ctx, return NULL; } +PRAGMA_REENABLE_CHECK_STACK_FRAME void grn_proc_init_object_list(grn_ctx *ctx) diff --git a/storage/mroonga/vendor/groonga/lib/proc/proc_schema.c b/storage/mroonga/vendor/groonga/lib/proc/proc_schema.c index 061c145a..7c632f45 100644 --- a/storage/mroonga/vendor/groonga/lib/proc/proc_schema.c +++ b/storage/mroonga/vendor/groonga/lib/proc/proc_schema.c @@ -17,9 +17,8 @@ */ #include "../grn_proc.h" - #include "../grn_db.h" - +#include <my_attribute.h> #include <groonga/plugin.h> typedef struct { @@ -572,6 +571,8 @@ command_schema_table_output_token_filters(grn_ctx *ctx, grn_obj *table) GRN_OBJ_FIN(ctx, &token_filters); } +PRAGMA_DISABLE_CHECK_STACK_FRAME + static void command_schema_table_command_collect_arguments(grn_ctx *ctx, grn_obj *table, @@ -692,6 +693,7 @@ command_schema_table_command_collect_arguments(grn_ctx *ctx, #undef ADD_OBJECT_NAME #undef ADD } +PRAGMA_REENABLE_CHECK_STACK_FRAME static void command_schema_table_output_command(grn_ctx *ctx, grn_obj *table) @@ -875,6 +877,8 @@ command_schema_output_indexes(grn_ctx *ctx, grn_obj *object) } } +PRAGMA_DISABLE_CHECK_STACK_FRAME + static void command_schema_column_command_collect_arguments(grn_ctx *ctx, grn_obj *table, @@ -973,6 +977,7 @@ command_schema_column_command_collect_arguments(grn_ctx *ctx, #undef ADD_OBJECT_NAME #undef ADD } +PRAGMA_REENABLE_CHECK_STACK_FRAME static void command_schema_column_output_command(grn_ctx *ctx, diff --git a/storage/mroonga/vendor/groonga/lib/proc/proc_select.c b/storage/mroonga/vendor/groonga/lib/proc/proc_select.c index a665b1cc..7588b18a 100644 --- a/storage/mroonga/vendor/groonga/lib/proc/proc_select.c +++ b/storage/mroonga/vendor/groonga/lib/proc/proc_select.c @@ -24,6 +24,7 @@ #include "../grn_util.h" #include "../grn_cache.h" #include "../grn_ii.h" +#include <my_attribute.h> #include "../grn_ts.h" @@ -2912,7 +2913,7 @@ grn_select(grn_ctx *ctx, grn_select_data *data) uint32_t nhits; grn_obj *outbuf = ctx->impl->output.buf; grn_content_type output_type = ctx->impl->output.type; - char cache_key[GRN_CACHE_MAX_KEY_SIZE]; + char *cache_key_buffer= 0; uint32_t cache_key_size; long long int threshold, original_threshold = 0; grn_cache *cache_obj = grn_cache_current_get(ctx); @@ -2985,8 +2986,9 @@ grn_select(grn_ctx *ctx, grn_select_data *data) } GRN_HASH_EACH_END(ctx, cursor); } #undef DRILLDOWN_CACHE_SIZE - if (cache_key_size <= GRN_CACHE_MAX_KEY_SIZE) { - char *cp = cache_key; + if (cache_key_size <= GRN_CACHE_MAX_KEY_SIZE && + (cache_key_buffer= (char*) malloc(cache_key_size+1))) { + char *cp = cache_key_buffer; #define PUT_CACHE_KEY(string) \ if ((string).value) \ @@ -3066,11 +3068,12 @@ grn_select(grn_ctx *ctx, grn_select_data *data) { grn_rc rc; - rc = grn_cache_fetch(ctx, cache_obj, cache_key, cache_key_size, outbuf); + rc = grn_cache_fetch(ctx, cache_obj, cache_key_buffer, cache_key_size, outbuf); if (rc == GRN_SUCCESS) { GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_CACHE, ":", "cache(%" GRN_FMT_LLD ")", (long long int)GRN_TEXT_LEN(outbuf)); + free(cache_key_buffer); return ctx->rc; } } @@ -3119,7 +3122,7 @@ grn_select(grn_ctx *ctx, grn_select_data *data) data->cache.length != 2 || data->cache.value[0] != 'n' || data->cache.value[1] != 'o')) { - grn_cache_update(ctx, cache_obj, cache_key, cache_key_size, outbuf); + grn_cache_update(ctx, cache_obj, cache_key_buffer, cache_key_size, outbuf); } goto exit; } @@ -3186,7 +3189,7 @@ grn_select(grn_ctx *ctx, grn_select_data *data) data->cache.length != 2 || data->cache.value[0] != 'n' || data->cache.value[1] != 'o')) { - grn_cache_update(ctx, cache_obj, cache_key, cache_key_size, outbuf); + grn_cache_update(ctx, cache_obj, cache_key_buffer, cache_key_size, outbuf); } if (data->taintable > 0) { grn_db_touch(ctx, DB_OBJ(data->tables.target)->db); @@ -3200,6 +3203,7 @@ exit : /* GRN_LOG(ctx, GRN_LOG_NONE, "%d", ctx->seqno); */ + free(cache_key_buffer); return ctx->rc; } @@ -3424,6 +3428,9 @@ grn_select_data_fill_drilldown_columns(grn_ctx *ctx, strlen(prefix)); } + +PRAGMA_DISABLE_CHECK_STACK_FRAME + static grn_bool grn_select_data_fill_drilldowns(grn_ctx *ctx, grn_user_data *user_data, @@ -3562,6 +3569,7 @@ grn_select_data_fill_drilldowns(grn_ctx *ctx, return succeeded; } } +PRAGMA_REENABLE_CHECK_STACK_FRAME static grn_obj * command_select(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c index f69f1869..3d95fffa 100644 --- a/storage/myisam/ft_boolean_search.c +++ b/storage/myisam/ft_boolean_search.c @@ -287,6 +287,8 @@ static int ftb_parse_query_internal(MYSQL_FTPARSER_PARAM *param, uchar *end= (uchar*) query + len; FT_WORD w; + w.pos= NULL; + w.len= 0; info.prev= ' '; info.quot= 0; while (ft_get_word(cs, start, end, &w, &info)) diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 14311c87..deeb1f2e 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -1570,41 +1570,38 @@ int ha_myisam::preload_keys(THD* thd, HA_CHECK_OPT *check_opt) SYNOPSIS disable_indexes() - mode mode of operation: - HA_KEY_SWITCH_NONUNIQ disable all non-unique keys - HA_KEY_SWITCH_ALL disable all keys - HA_KEY_SWITCH_NONUNIQ_SAVE dis. non-uni. and make persistent - HA_KEY_SWITCH_ALL_SAVE dis. all keys and make persistent - IMPLEMENTATION - HA_KEY_SWITCH_NONUNIQ is not implemented. - HA_KEY_SWITCH_ALL_SAVE is not implemented. + DESCRIPTION + See handler::ha_disable_indexes() RETURN 0 ok HA_ERR_WRONG_COMMAND mode not implemented. */ -int ha_myisam::disable_indexes(uint mode) +int ha_myisam::disable_indexes(key_map map, bool persist) { int error; - if (mode == HA_KEY_SWITCH_ALL) + if (!persist) { /* call a storage engine function to switch the key map */ + DBUG_ASSERT(map.is_clear_all()); error= mi_disable_indexes(file); } - else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) + else { - mi_extra(file, HA_EXTRA_NO_KEYS, 0); + ulonglong ullmap= map.to_ulonglong(); + + /* make sure auto-inc key is enabled even if it's > 64 */ + if (map.length() > MI_KEYMAP_BITS && + table->s->next_number_index < MAX_KEY) + mi_set_key_active(ullmap, table->s->next_number_index); + + mi_extra(file, HA_EXTRA_NO_KEYS, &ullmap); info(HA_STATUS_CONST); // Read new key info error= 0; } - else - { - /* mode not implemented */ - error= HA_ERR_WRONG_COMMAND; - } return error; } @@ -1614,21 +1611,14 @@ int ha_myisam::disable_indexes(uint mode) SYNOPSIS enable_indexes() - mode mode of operation: - HA_KEY_SWITCH_NONUNIQ enable all non-unique keys - HA_KEY_SWITCH_ALL enable all keys - HA_KEY_SWITCH_NONUNIQ_SAVE en. non-uni. and make persistent - HA_KEY_SWITCH_ALL_SAVE en. all keys and make persistent DESCRIPTION Enable indexes, which might have been disabled by disable_index() before. - The modes without _SAVE work only if both data and indexes are empty, + If persist=false, it works only if both data and indexes are empty, since the MyISAM repair would enable them persistently. To be sure in these cases, call handler::delete_all_rows() before. - IMPLEMENTATION - HA_KEY_SWITCH_NONUNIQ is not implemented. - HA_KEY_SWITCH_ALL_SAVE is not implemented. + See also handler::ha_enable_indexes() RETURN 0 ok @@ -1637,7 +1627,7 @@ int ha_myisam::disable_indexes(uint mode) HA_ERR_WRONG_COMMAND mode not implemented. */ -int ha_myisam::enable_indexes(uint mode) +int ha_myisam::enable_indexes(key_map map, bool persist) { int error; DBUG_ENTER("ha_myisam::enable_indexes"); @@ -1651,7 +1641,8 @@ int ha_myisam::enable_indexes(uint mode) DBUG_RETURN(0); } - if (mode == HA_KEY_SWITCH_ALL) + DBUG_ASSERT(map.is_prefix(table->s->keys)); + if (!persist) { error= mi_enable_indexes(file); /* @@ -1660,7 +1651,7 @@ int ha_myisam::enable_indexes(uint mode) but mode==HA_KEY_SWITCH_ALL forbids it. */ } - else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) + else { THD *thd= table->in_use; int was_error= thd->is_error(); @@ -1720,11 +1711,6 @@ int ha_myisam::enable_indexes(uint mode) restore_vcos_after_repair(); } - else - { - /* mode not implemented */ - error= HA_ERR_WRONG_COMMAND; - } DBUG_RETURN(error); } @@ -1887,7 +1873,7 @@ int ha_myisam::end_bulk_insert() setting the indexes as active and trying to recreate them. */ - if (((first_error= enable_indexes(HA_KEY_SWITCH_NONUNIQ_SAVE)) != 0) && + if (((first_error= enable_indexes(key_map(table->s->keys), true))) && table->in_use->killed) { delete_all_rows(); diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h index c4c46a63..e0121dbe 100644 --- a/storage/myisam/ha_myisam.h +++ b/storage/myisam/ha_myisam.h @@ -109,8 +109,8 @@ class ha_myisam final : public handler int external_lock(THD *thd, int lock_type); int delete_all_rows(void); int reset_auto_increment(ulonglong value); - int disable_indexes(uint mode); - int enable_indexes(uint mode); + int disable_indexes(key_map map, bool persist); + int enable_indexes(key_map map, bool persist); int indexes_are_disabled(void); void start_bulk_insert(ha_rows rows, uint flags); int end_bulk_insert(); diff --git a/storage/myisam/mi_extra.c b/storage/myisam/mi_extra.c index 66238745..68a19c7b 100644 --- a/storage/myisam/mi_extra.c +++ b/storage/myisam/mi_extra.c @@ -225,16 +225,9 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function, void *extra_arg) } if (mi_is_any_key_active(share->state.key_map)) { - MI_KEYDEF *key=share->keyinfo; - uint i; - for (i=0 ; i < share->base.keys ; i++,key++) - { - if (!(key->flag & HA_NOSAME) && info->s->base.auto_key != i+1) - { - mi_clear_key_active(share->state.key_map, i); - info->update|= HA_STATE_CHANGED; - } - } + if (share->state.key_map != *(ulonglong*)extra_arg) + info->update|= HA_STATE_CHANGED; + share->state.key_map= *(ulonglong*)extra_arg; if (!share->changed) { diff --git a/storage/perfschema/pfs_buffer_container.h b/storage/perfschema/pfs_buffer_container.h index d5745e76..b5506fe0 100644 --- a/storage/perfschema/pfs_buffer_container.h +++ b/storage/perfschema/pfs_buffer_container.h @@ -1084,8 +1084,7 @@ template <class T> class PFS_buffer_processor { public: - virtual ~PFS_buffer_processor<T> () - {} + virtual ~PFS_buffer_processor()= default; virtual void operator()(T *element) = 0; }; diff --git a/storage/perfschema/pfs_instr_class.cc b/storage/perfschema/pfs_instr_class.cc index 2b1a80d3..fa85d861 100644 --- a/storage/perfschema/pfs_instr_class.cc +++ b/storage/perfschema/pfs_instr_class.cc @@ -55,7 +55,7 @@ Indicate if the performance schema is enabled. This flag is set at startup, and never changes. */ -my_bool pfs_enabled= TRUE; +my_bool pfs_enabled= FALSE; /** PFS_INSTRUMENT option settings array diff --git a/storage/perfschema/table_replication_applier_status.cc b/storage/perfschema/table_replication_applier_status.cc index fda3c927..94981482 100644 --- a/storage/perfschema/table_replication_applier_status.cc +++ b/storage/perfschema/table_replication_applier_status.cc @@ -166,7 +166,7 @@ void table_replication_applier_status::make_row(Master_info *mi) m_row.service_state= PS_RPL_NO; m_row.remaining_delay= 0; - if (slave_sql_running_state == Relay_log_info::state_delaying_string) + if (slave_sql_running_state == stage_sql_thd_waiting_until_delay.m_name) { time_t t= my_time(0), sql_delay_end= mi->rli.get_sql_delay_end(); m_row.remaining_delay= (uint)(t < sql_delay_end ? diff --git a/storage/perfschema/table_replication_applier_status.h b/storage/perfschema/table_replication_applier_status.h index 4da2087a..5e97dba5 100644 --- a/storage/perfschema/table_replication_applier_status.h +++ b/storage/perfschema/table_replication_applier_status.h @@ -59,7 +59,7 @@ struct st_row_applier_status { enum_rpl_yes_no service_state; uint remaining_delay; bool remaining_delay_is_set; - ulong count_transactions_retries; + ulonglong count_transactions_retries; }; /** Table PERFORMANCE_SCHEMA.replication_applier_status */ diff --git a/storage/perfschema/unittest/pfs_instr-t.cc b/storage/perfschema/unittest/pfs_instr-t.cc index 9667d7ff..55c4f619 100644 --- a/storage/perfschema/unittest/pfs_instr-t.cc +++ b/storage/perfschema/unittest/pfs_instr-t.cc @@ -86,6 +86,8 @@ void test_no_instruments() cleanup_instruments(); } +PRAGMA_DISABLE_CHECK_STACK_FRAME + void test_no_instances() { int rc; @@ -245,6 +247,7 @@ void test_no_instances() cleanup_file_hash(); cleanup_instruments(); } +PRAGMA_REENABLE_CHECK_STACK_FRAME void test_with_instances() { diff --git a/storage/perfschema/unittest/pfs_instr_class-t.cc b/storage/perfschema/unittest/pfs_instr_class-t.cc index 7651898d..76e3668d 100644 --- a/storage/perfschema/unittest/pfs_instr_class-t.cc +++ b/storage/perfschema/unittest/pfs_instr_class-t.cc @@ -743,6 +743,7 @@ void do_all_tests() int main(int argc, char **argv) { plan(209); + pfs_enabled= 1; MY_INIT(argv[0]); do_all_tests(); my_end(0); diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index 50ff1df7..2308d69f 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -202,7 +202,7 @@ ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib ADD_DEPENDENCIES(rocksdb_aux_lib GenError) # MARIAROCKS-TODO: how to properly depend on -lrt ? -TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARY}) +TARGET_LINK_LIBRARIES(rocksdb_aux_lib rocksdblib ${ZLIB_LIBRARIES}) if (UNIX AND NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt) endif() diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index 27a8616c..e89dbfc0 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -79,7 +79,7 @@ check_lib(Snappy) check_lib(ZSTD ZDICT_trainFromBuffer) add_definitions(-DZLIB) -list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARY}) +list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES}) ADD_FEATURE_INFO(ROCKSDB_ZLIB "ON" "zlib Compression in the RocksDB storage engine") if(CMAKE_SYSTEM_NAME MATCHES "Cygwin") diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index f847ee25..ea91b9b3 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -397,7 +397,7 @@ class ha_rocksdb : public my_core::handler { current lookup to be covered. If the bitmap field is null, that means this index does not cover the current lookup for any record. */ - MY_BITMAP m_lookup_bitmap = {nullptr, nullptr, 0, 0}; + MY_BITMAP m_lookup_bitmap = {nullptr, nullptr, 0, 0, 0}; int alloc_key_buffers(const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg, diff --git a/storage/rocksdb/mysql-test/rocksdb/r/group_min_max.result b/storage/rocksdb/mysql-test/rocksdb/r/group_min_max.result index 5a1350fe..dbf4d7d3 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/group_min_max.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/group_min_max.result @@ -3517,7 +3517,7 @@ SHOW SESSION STATUS LIKE 'Handler_read%'; Variable_name Value Handler_read_first 0 Handler_read_key 3 -Handler_read_last 1 +Handler_read_last 0 Handler_read_next 0 Handler_read_prev 0 Handler_read_retry 0 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result index 8031d4c0..d99c9b11 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result @@ -68,7 +68,6 @@ buffer_flush_n_to_flush_by_age buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NUL buffer_flush_adaptive_avg_time buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Avg time (ms) spent for adaptive flushing recently. buffer_flush_adaptive_avg_pass buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of adaptive flushes passed during the recent Avg period. buffer_LRU_get_free_loops buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Total loops in LRU get free. -buffer_LRU_get_free_waits buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Total sleep waits in LRU get free. buffer_flush_avg_page_rate buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Average number of pages at which flushing is happening buffer_flush_lsn_avg_rate buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Average redo generation rate buffer_flush_pct_for_dirty buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Percent of IO capacity used to avoid max dirty page limit @@ -88,7 +87,6 @@ buffer_LRU_batch_num_scan buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NUL buffer_LRU_batch_scanned_per_call buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 set_member Pages scanned per LRU batch call buffer_LRU_batch_flush_total_pages buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 status_counter Total pages flushed as part of LRU batches buffer_LRU_batch_evict_total_pages buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 status_counter Total pages evicted as part of LRU batches -buffer_LRU_single_flush_failure_count Buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of times attempt to flush a single page from LRU failed buffer_LRU_get_free_search Buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of searches performed for a clean page buffer_LRU_search_scanned buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 set_owner Total pages scanned as part of LRU search buffer_LRU_search_num_scan buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 set_member Number of times LRU search is performed diff --git a/storage/rocksdb/mysql-test/rocksdb/r/partition.result b/storage/rocksdb/mysql-test/rocksdb/r/partition.result index a7f2a611..1ba966e9 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/partition.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/partition.result @@ -46,7 +46,6 @@ CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i)) ENGINE = ROCKSDB PARTITIO Table Op Msg_type Msg_text test.t1 optimize status OK Table Op Msg_type Msg_text -test.t1 analyze status Engine-independent statistics collected test.t1 analyze status OK Table Op Msg_type Msg_text test.t1 repair status OK diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index 627d7da4..c16c1021 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -87,6 +87,10 @@ compact_deletes: MDEV-12663 : rocksdb.compact_deletes times out and causes other blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api test fails information_schema: MDEV-14372: unstable testcase +bloomfilter2: MDEV-33789: rocksdb.bloomfilter2 failed on amd64-debian-12-rocksdb +write_sync: MDEV-33866: rocksdb.write_sync fails on amd64-windows-packages +locking_issues_case5_rc: MDEV-33781: rocksdb.locking_issues_case5_rc fails on amd64-windows-packages + ## ## Tests that fail for some other reason ## diff --git a/storage/rocksdb/rdb_source_revision.h b/storage/rocksdb/rdb_source_revision.h new file mode 100644 index 00000000..d0d81aa1 --- /dev/null +++ b/storage/rocksdb/rdb_source_revision.h @@ -0,0 +1 @@ +#define ROCKSDB_GIT_HASH "bba5e7bc21093d7cfa765e1280a7c4fdcd284288" diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index 5b85b3a2..c998fd18 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -269,19 +269,19 @@ int ha_spider::open( spider_bulk_malloc(spider_current_trx, 16, MYF(MY_WME | MY_ZEROFILL), &wide_handler, sizeof(SPIDER_WIDE_HANDLER), &searched_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &ft_discard_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &position_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &idx_read_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &idx_write_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &rnd_read_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &rnd_write_bitmap, - (uint) sizeof(uchar) * no_bytes_in_map(table->read_set), + (uint) sizeof(uchar) * my_bitmap_buffer_size(table->read_set), &partition_handler, (uint) sizeof(SPIDER_PARTITION_HANDLER), NullS) @@ -305,9 +305,9 @@ int ha_spider::open( wide_handler->top_share = table->s; owner->wide_handler_owner = TRUE; memset(wide_handler->ft_discard_bitmap, 0xFF, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); memset(wide_handler->searched_bitmap, 0, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); wide_handler_alloc = TRUE; if (!share && !spider_get_share(name, table, thd, this, &error_num)) @@ -347,12 +347,23 @@ int ha_spider::open( result_list.last = NULL; result_list.current = NULL; result_list.record_num = 0; - if ( - !(result_list.sqls = new spider_string[share->link_count]) || - !(result_list.insert_sqls = new spider_string[share->link_count]) || - !(result_list.update_sqls = new spider_string[share->link_count]) || - !(result_list.tmp_sqls = new spider_string[share->link_count]) - ) { + if (!(result_list.sqls = new spider_string[share->link_count])) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error_init_result_list; + } + if (!(result_list.insert_sqls = new spider_string[share->link_count])) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error_init_result_list; + } + if (!(result_list.update_sqls = new spider_string[share->link_count])) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error_init_result_list; + } + if (!(result_list.tmp_sqls = new spider_string[share->link_count])) + { error_num = HA_ERR_OUT_OF_MEM; goto error_init_result_list; } @@ -1000,9 +1011,9 @@ int ha_spider::reset() if (!is_clone) { memset(wide_handler->ft_discard_bitmap, 0xFF, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); memset(wide_handler->searched_bitmap, 0, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); } while (wide_handler->condition) { @@ -1230,7 +1241,7 @@ int ha_spider::index_init( bitmap_set_all(table->read_set); if (is_clone) memset(wide_handler->searched_bitmap, 0xFF, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); } } @@ -3125,7 +3136,7 @@ ha_rows ha_spider::multi_range_read_info_const( bitmap_set_all(table->read_set); if (is_clone) memset(wide_handler->searched_bitmap, 0xFF, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); } } @@ -3178,7 +3189,7 @@ ha_rows ha_spider::multi_range_read_info( bitmap_set_all(table->read_set); if (is_clone) memset(wide_handler->searched_bitmap, 0xFF, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); } } @@ -5437,7 +5448,7 @@ int ha_spider::rnd_init( bitmap_set_all(table->read_set); if (is_clone) memset(wide_handler->searched_bitmap, 0xFF, - no_bytes_in_map(table->read_set)); + my_bitmap_buffer_size(table->read_set)); } set_select_column_mode(); @@ -9023,27 +9034,35 @@ bool ha_spider::auto_repair() const } int ha_spider::disable_indexes( - uint mode + key_map map, bool persist ) { int error_num; backup_error_status(); DBUG_ENTER("ha_spider::disable_indexes"); DBUG_PRINT("info",("spider this=%p", this)); - if ((error_num = spider_db_disable_keys(this))) - DBUG_RETURN(check_error_mode(error_num)); - DBUG_RETURN(0); + if (persist) + { + if ((error_num = spider_db_disable_keys(this))) + DBUG_RETURN(check_error_mode(error_num)); + DBUG_RETURN(0); + } + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } int ha_spider::enable_indexes( - uint mode + key_map map, bool persist ) { int error_num; backup_error_status(); DBUG_ENTER("ha_spider::enable_indexes"); DBUG_PRINT("info",("spider this=%p", this)); - if ((error_num = spider_db_enable_keys(this))) - DBUG_RETURN(check_error_mode(error_num)); - DBUG_RETURN(0); + if (persist) + { + if ((error_num = spider_db_enable_keys(this))) + DBUG_RETURN(check_error_mode(error_num)); + DBUG_RETURN(0); + } + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } diff --git a/storage/spider/ha_spider.h b/storage/spider/ha_spider.h index 4dffdf78..4c178a33 100644 --- a/storage/spider/ha_spider.h +++ b/storage/spider/ha_spider.h @@ -483,10 +483,10 @@ public: bool auto_repair() const; #endif int disable_indexes( - uint mode + key_map map, bool persist ); int enable_indexes( - uint mode + key_map map, bool persist ); int check( THD* thd, @@ -792,3 +792,26 @@ public: int lock_tables(); int dml_init(); }; + + +/* This is a hack for ASAN + * Libraries such as libxml2 and libodbc do not like being unloaded before + * exit and will show as a leak in ASAN with no stack trace (as the plugin + * has been unloaded from memory). + * + * The below is designed to trick the compiler into adding a "UNIQUE" symbol + * which can be seen using: + * readelf -s storage/spider/ha_spider.so | grep UNIQUE + * + * Having this symbol means that the plugin remains in memory after dlclose() + * has been called. Thereby letting the libraries clean up properly. + */ +#if defined(__SANITIZE_ADDRESS__) +__attribute__((__used__)) +inline int dummy(void) +{ + static int d; + d++; + return d; +} +#endif diff --git a/storage/spider/mysql-test/spider/bg/r/spider3_fixes.result b/storage/spider/mysql-test/spider/bg/r/spider3_fixes.result index aa734573..e3510042 100644 --- a/storage/spider/mysql-test/spider/bg/r/spider3_fixes.result +++ b/storage/spider/mysql-test/spider/bg/r/spider3_fixes.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/bg/r/spider3_fixes_part.result b/storage/spider/mysql-test/spider/bg/r/spider3_fixes_part.result index b793346d..361fd336 100644 --- a/storage/spider/mysql-test/spider/bg/r/spider3_fixes_part.result +++ b/storage/spider/mysql-test/spider/bg/r/spider3_fixes_part.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/bg/r/spider_fixes.result b/storage/spider/mysql-test/spider/bg/r/spider_fixes.result index 51045e0b..a751d803 100644 --- a/storage/spider/mysql-test/spider/bg/r/spider_fixes.result +++ b/storage/spider/mysql-test/spider/bg/r/spider_fixes.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/bg/r/spider_fixes_part.result b/storage/spider/mysql-test/spider/bg/r/spider_fixes_part.result index faf0b6ef..3ce8dfbf 100644 --- a/storage/spider/mysql-test/spider/bg/r/spider_fixes_part.result +++ b/storage/spider/mysql-test/spider/bg/r/spider_fixes_part.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/bg/t/slave_test_init.inc b/storage/spider/mysql-test/spider/bg/t/slave_test_init.inc index 739d8897..84de2c7d 100644 --- a/storage/spider/mysql-test/spider/bg/t/slave_test_init.inc +++ b/storage/spider/mysql-test/spider/bg/t/slave_test_init.inc @@ -19,7 +19,7 @@ if (!$SLAVE1_1_SLAVE_STATUS) MASTER_PORT = $MASTER_1_MYPORT ; } -START SLAVE; +--source include/start_slave.inc --connection master_1 call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); --connection slave1_1 diff --git a/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_default.result b/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_default.result index cb1b190f..8b193975 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_default.result +++ b/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_default.result @@ -5,6 +5,7 @@ child2_2 child2_3 for child3 for slave1_1 +include/start_slave.inc connection slave1_1; connection master_1; set @old_binlog_format= @@binlog_format; diff --git a/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mariadb.result b/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mariadb.result index cb1b190f..8b193975 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mariadb.result +++ b/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mariadb.result @@ -5,6 +5,7 @@ child2_2 child2_3 for child3 for slave1_1 +include/start_slave.inc connection slave1_1; connection master_1; set @old_binlog_format= @@binlog_format; diff --git a/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mysql.result b/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mysql.result index 11310212..1e5d7d31 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mysql.result +++ b/storage/spider/mysql-test/spider/bugfix/r/delete_with_float_column_mysql.result @@ -5,6 +5,7 @@ child2_2 child2_3 for child3 for slave1_1 +include/start_slave.inc connection slave1_1; connection master_1; set @old_binlog_format= @@binlog_format; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result index 4386b4cb..dbf0f54c 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_19866.result @@ -72,7 +72,6 @@ SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argum argument select `pkey`,`val` from `auto_test_remote`.`tbl_a` select `pkey`,`val` from `auto_test_remote`.`tbl_a` where `pkey` = 1 -select 1 from (select 1) t0 select `pkey`,`val` from `auto_test_remote`.`tbl_a` select `pkey`,`val` from `auto_test_remote`.`tbl_a` SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' @@ -86,7 +85,6 @@ argument select `pkey`,`val` from `auto_test_remote2`.`tbl_a` select `pkey`,`val` from `auto_test_remote2`.`tbl_a` select `pkey`,`val` from `auto_test_remote2`.`tbl_a` where `pkey` = 2 -select 1 from (select 1) t0 select `pkey`,`val` from `auto_test_remote2`.`tbl_a` SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT pkey, val FROM tbl_a ORDER BY pkey; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_27172.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_27172.result index 531e48b8..60ef8e2d 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_27172.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_27172.result @@ -66,9 +66,6 @@ id greeting connection child2_1; SELECT argument FROM mysql.general_log WHERE argument LIKE 'select `id`,`greeting` from %'; argument -select `id`,`greeting` from `auto_test_remote`.`tbl_a` where `greeting` = 'Aloha!' and ((`greeting` = 'Aloha!')) -select `id`,`greeting` from `auto_test_remote`.`tbl_b` where `greeting` like 'Aloha%' and ((`greeting` = 'Aloha!')) -select `id`,`greeting` from `auto_test_remote`.`tbl_c` where `greeting` like 'Aloha%' and ((`greeting` = 'Aloha!')) connection child2_1; SET @@global.general_log = @general_log_backup; SET @@global.log_output = @log_output_backup; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result index 7e4fd3cd..5fd412b6 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result @@ -5,6 +5,8 @@ for master_1 for child2 for child3 set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # testing monitoring_* @@ -173,6 +175,7 @@ max(d) 93 drop table t1, t2; drop server srv; +set global spider_same_server_link=@old_spider_same_server_link; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30727.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30727.result new file mode 100644 index 00000000..79a383fe --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30727.result @@ -0,0 +1,24 @@ +CREATE FUNCTION spider_direct_sql RETURNS INT SONAME 'ha_spider.so'; +SELECT spider_direct_sql ('SELECT * FROM s','a','srv "b"'); +ERROR HY000: Can't initialize function 'spider_direct_sql'; Plugin 'SPIDER' is not loaded +CREATE FUNCTION spider_bg_direct_sql RETURNS INT SONAME 'ha_spider.so'; +SELECT spider_bg_direct_sql ('SELECT * FROM s','a','srv "b"'); +ERROR HY000: Can't initialize function 'spider_bg_direct_sql'; Plugin 'SPIDER' is not loaded +CREATE FUNCTION spider_copy_tables RETURNS INT SONAME 'ha_spider.so'; +SELECT spider_copy_tables ('t', '0', '0'); +ERROR HY000: Can't initialize function 'spider_copy_tables'; Plugin 'SPIDER' is not loaded +CREATE FUNCTION spider_flush_table_mon_cache RETURNS INT SONAME 'ha_spider.so'; +SELECT spider_flush_table_mon_cache (); +spider_flush_table_mon_cache () +1 +install soname 'ha_spider'; +SELECT spider_direct_sql ('SELECT * FROM s','a','srv "b"'); +ERROR HY000: The foreign server name you are trying to reference does not exist. Data source error: b +call mtr.add_suppression(".*\\[Error\\] (mysqld|mariadbd): Can't find record in 'spider_tables'"); +SELECT spider_copy_tables ('t', '0', '0'); +ERROR HY000: Can't find record in 'spider_tables' +SELECT spider_flush_table_mon_cache (); +spider_flush_table_mon_cache () +1 +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33242.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33242.result new file mode 100644 index 00000000..bbbc4499 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33242.result @@ -0,0 +1,4 @@ +set @old_old_mode=@@global.old_mode; +set global old_mode=4; +INSTALL SONAME 'ha_spider.so'; +set global old_mode=@old_old_mode; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33434.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33434.result new file mode 100644 index 00000000..2cbcff38 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33434.result @@ -0,0 +1,12 @@ +# +# MDEV-33434 MDEV-33434 UBSAN null pointer passed as argument 2, which is declared to never be null in spider_udf_direct_sql_create_conn +# +INSTALL SONAME 'ha_spider'; +SET character_set_connection=ucs2; +SELECT SPIDER_DIRECT_SQL('SELECT SLEEP(1)', '', 'srv "dummy", port "3307"'); +ERROR HY000: Unable to connect to foreign data source: localhost +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown +# +# end of test mdev_33434 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33441.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33441.result new file mode 100644 index 00000000..b19194f1 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33441.result @@ -0,0 +1,7 @@ +# +# MDEV-33441 No spider variables available is Spider is loaded upon server startup +# +set spider_same_server_link=0; +# +# end of test mdev_33441 +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33441_fail.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33441_fail.result new file mode 100644 index 00000000..e6123ed4 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33441_fail.result @@ -0,0 +1,10 @@ +# +# MDEV-33441 No spider variables available is Spider is loaded upon server startup +# +select * from mysql.plugin; +name dl +show variables like 'spider%'; +Variable_name Value +# +# end of test mdev_33441_fail +# diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33494.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33494.result new file mode 100644 index 00000000..3db28c0f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33494.result @@ -0,0 +1,4 @@ +set @old_sql_mode=@@global.sql_mode; +set global sql_mode=(SELECT CONCAT (@@sql_mode,',no_zero_date')); +install soname 'ha_spider'; +set global sql_mode=@old_sql_mode; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33538.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33538.result new file mode 100644 index 00000000..a5ab3c1d --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33538.result @@ -0,0 +1,12 @@ +show create table information_schema.SPIDER_ALLOC_MEM; +Table Create Table +SPIDER_ALLOC_MEM CREATE TEMPORARY TABLE `SPIDER_ALLOC_MEM` ( + `ID` int(10) unsigned NOT NULL, + `FUNC_NAME` varchar(64), + `FILE_NAME` varchar(64), + `LINE_NO` int(10) unsigned, + `TOTAL_ALLOC_MEM` bigint(20) unsigned, + `CURRENT_ALLOC_MEM` bigint(20), + `ALLOC_MEM_COUNT` bigint(20) unsigned, + `FREE_MEM_COUNT` bigint(20) unsigned +) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33538_fail_init.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33538_fail_init.result new file mode 100644 index 00000000..7535da83 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33538_fail_init.result @@ -0,0 +1,10 @@ +call mtr.add_suppression("\\[ERROR\\] SPIDER plugin initialization failed"); +call mtr.add_suppression(".*\\[ERROR\\] Plugin 'SPIDER' registration as a STORAGE ENGINE failed."); +call mtr.add_suppression(".*\\[ERROR\\] Plugin 'SPIDER_WRAPPER_PROTOCOLS' registration as a INFORMATION SCHEMA failed."); +call mtr.add_suppression(".*\\[ERROR\\] Plugin 'SPIDER_ALLOC_MEM' registration as a INFORMATION SCHEMA failed."); +create table mysql.spider_tables (c int); +# restart: --plugin-load-add=ha_spider +SELECT * FROM information_schema.SPIDER_ALLOC_MEM; +ID FUNC_NAME FILE_NAME LINE_NO TOTAL_ALLOC_MEM CURRENT_ALLOC_MEM ALLOC_MEM_COUNT FREE_MEM_COUNT +SELECT * FROM information_schema.SPIDER_WRAPPER_PROTOCOLS; +WRAPPER_NAME WRAPPER_VERSION WRAPPER_DESCRIPTION WRAPPER_MATURITY diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33584.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33584.result new file mode 100644 index 00000000..796c75cc --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33584.result @@ -0,0 +1,4 @@ +set @old_sql_mode=@@global.sql_mode; +set global sql_mode='traditional'; +install soname 'ha_spider'; +set global sql_mode=@old_sql_mode; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33679.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33679.result new file mode 100644 index 00000000..c021bc70 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33679.result @@ -0,0 +1,25 @@ +# +# MDEV-33679 spider returns parsing failure on valid left join select by translating the on expression to () +# +for master_1 +for child2 +for child3 +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +CREATE TABLE `t1` (`c` INT(10) UNSIGNED NOT NULL, `b` VARCHAR(255) NOT NULL , PRIMARY KEY (`c`) USING BTREE ) ENGINE=MYISAM; +CREATE TABLE `t2` (`a` INT(10) UNSIGNED NOT NULL, `c` INT(10) UNSIGNED NOT NULL ) ENGINE=MYISAM; +SET spider_same_server_link= on; +CREATE TABLE `t1_spider` (`c` INT(10) UNSIGNED NOT NULL, `b` VARCHAR(255) NOT NULL , PRIMARY KEY (`c`) USING BTREE ) COMMENT='wrapper "mysql",srv "srv", table "t1"' ENGINE=SPIDER; +CREATE TABLE `t2_spider` (`a` INT(10) UNSIGNED NOT NULL, `c` INT(10) UNSIGNED NOT NULL +, PRIMARY KEY (`a`) USING BTREE +) COMMENT='wrapper "mysql",srv "srv",table "t2"' ENGINE=SPIDER; +INSERT INTO t1_spider VALUES(1,'oooo'); +INSERT INTO t2_spider VALUES(1,1); +SELECT t2_spider.a,t1_spider.c FRoM t2_spider LEFT join t1_spider ON (t2_spider.c = t1_spider.c) WHERE t2_spider.a = 1; +a c +1 1 +drop table t1, t2, t1_spider, t2_spider; +drop server srv; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33731.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33731.result new file mode 100644 index 00000000..a63830f1 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33731.result @@ -0,0 +1,10 @@ +for master_1 +for child2 +for child3 +CREATE TABLE t (a INT) ENGINE=Spider PARTITION BY LIST (a) PARTITIONS 2 (PARTITION p1 VALUES IN (0,1),PARTITION p2 VALUES IN (2,3)); +DELETE FROM t PARTITION (p2); +ERROR HY000: Unable to connect to foreign data source: localhost +drop table t; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_34003.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_34003.result new file mode 100644 index 00000000..b78f18d1 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_34003.result @@ -0,0 +1,18 @@ +for master_1 +for child2 +for child3 +set spider_same_server_link= 1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c INT KEY,c1 BLOB,c2 TEXT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (0,1,0),(1,0,0),(2,0,0); +create table t1 (c INT KEY,c1 BLOB,c2 TEXT) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +SELECT * FROM t1 WHERE c=0; +c c1 c2 +0 1 0 +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/plugin_load_add_all.result b/storage/spider/mysql-test/spider/bugfix/r/plugin_load_add_all.result index 04e5ed6d..4d161a2f 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/plugin_load_add_all.result +++ b/storage/spider/mysql-test/spider/bugfix/r/plugin_load_add_all.result @@ -3,5 +3,9 @@ # select * from mysql.plugin; name dl +select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA = "information_schema" and TABLE_NAME like "SPIDER_%"; +TABLE_NAME +SPIDER_ALLOC_MEM +SPIDER_WRAPPER_PROTOCOLS create table t (c int) Engine=SPIDER; drop table t; diff --git a/storage/spider/mysql-test/spider/bugfix/r/slave_transaction_retry_errors_5digit.result b/storage/spider/mysql-test/spider/bugfix/r/slave_transaction_retry_errors_5digit.result index f2cab6b0..83dab07f 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/slave_transaction_retry_errors_5digit.result +++ b/storage/spider/mysql-test/spider/bugfix/r/slave_transaction_retry_errors_5digit.result @@ -5,11 +5,12 @@ child2_2 child2_3 for child3 for slave1_1 +include/start_slave.inc connection slave1_1; SHOW VARIABLES LIKE 'slave_transaction_retry_errors'; Variable_name Value -slave_transaction_retry_errors 1158,1159,1160,1161,1205,1213,1429,2013,12701,10000,20000,30000 +slave_transaction_retry_errors 1158,1159,1160,1161,1205,1213,1020,1429,2013,12701,10000,20000,30000 connection slave1_1; for slave1_1 for master_1 diff --git a/storage/spider/mysql-test/spider/bugfix/r/slave_trx_isolation.result b/storage/spider/mysql-test/spider/bugfix/r/slave_trx_isolation.result index e84d42bb..239402a5 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/slave_trx_isolation.result +++ b/storage/spider/mysql-test/spider/bugfix/r/slave_trx_isolation.result @@ -5,6 +5,7 @@ child2_2 child2_3 for child3 for slave1_1 +include/start_slave.inc connection slave1_1; set @old_spider_slave_trx_isolation= @@spider_slave_trx_isolation; set global spider_slave_trx_isolation= 1; diff --git a/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result b/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result index 045ddc43..b84f60a6 120000..100644 --- a/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result +++ b/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result @@ -1 +1,43 @@ -udf_mysql_func_early.result
\ No newline at end of file +# +# Test that udf created by inserting into mysql_func works as expected +# +CREATE SERVER s_1 FOREIGN DATA WRAPPER mysql OPTIONS ( +HOST 'localhost', +DATABASE 'auto_test_local', +USER 'root', +PASSWORD '', +SOCKET '$MASTER_1_MYSOCK' + ); +CREATE SERVER s_2_1 FOREIGN DATA WRAPPER mysql OPTIONS ( +HOST 'localhost', +DATABASE 'auto_test_remote', +USER 'root', +PASSWORD '', +SOCKET '$CHILD2_1_MYSOCK' + ); +connect master_1, localhost, root, , , $MASTER_1_MYPORT, $MASTER_1_MYSOCK; +connect child2_1, localhost, root, , , $CHILD2_1_MYPORT, $CHILD2_1_MYSOCK; +connection child2_1; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +CREATE TABLE tbl_a ( +a INT +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +insert into tbl_a values (42); +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +CREATE TABLE tbl_a ( +a INT +) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='table "tbl_a", srv "s_2_1"'; +create temporary table results (a int); +SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"'); +SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"') +1 +select * from results; +a +42 +connection master_1; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +DROP DATABASE IF EXISTS auto_test_remote; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test index a1642f7a..9ccfc378 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test @@ -10,6 +10,9 @@ # This test covers some table params under consideration for inclusion # in the engine-defined options to be implemented in MDEV-28856. set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; + evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); @@ -153,6 +156,7 @@ select max(d) from t1; drop table t1, t2; drop server srv; +set global spider_same_server_link=@old_spider_same_server_link; --disable_query_log --disable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test new file mode 100644 index 00000000..ebd08edc --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test @@ -0,0 +1,30 @@ +CREATE FUNCTION spider_direct_sql RETURNS INT SONAME 'ha_spider.so'; +--error ER_CANT_INITIALIZE_UDF +SELECT spider_direct_sql ('SELECT * FROM s','a','srv "b"'); + +CREATE FUNCTION spider_bg_direct_sql RETURNS INT SONAME 'ha_spider.so'; +--error ER_CANT_INITIALIZE_UDF +SELECT spider_bg_direct_sql ('SELECT * FROM s','a','srv "b"'); + +CREATE FUNCTION spider_copy_tables RETURNS INT SONAME 'ha_spider.so'; +--error ER_CANT_INITIALIZE_UDF +SELECT spider_copy_tables ('t', '0', '0'); + +# spider_flush_table_mon_cache does not require spider init to function +CREATE FUNCTION spider_flush_table_mon_cache RETURNS INT SONAME 'ha_spider.so'; +SELECT spider_flush_table_mon_cache (); + +# The function functions properly after the plugin is installed +install soname 'ha_spider'; + +--error ER_FOREIGN_SERVER_DOESNT_EXIST +SELECT spider_direct_sql ('SELECT * FROM s','a','srv "b"'); + +call mtr.add_suppression(".*\\[Error\\] (mysqld|mariadbd): Can't find record in 'spider_tables'"); +--error ER_KEY_NOT_FOUND +SELECT spider_copy_tables ('t', '0', '0'); + +SELECT spider_flush_table_mon_cache (); + +--disable_query_log +--source ../../include/clean_up_spider.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33242.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33242.test new file mode 100644 index 00000000..215dab25 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33242.test @@ -0,0 +1,6 @@ +set @old_old_mode=@@global.old_mode; +set global old_mode=4; +INSTALL SONAME 'ha_spider.so'; +set global old_mode=@old_old_mode; +--disable_query_log +--source ../../include/clean_up_spider.inc diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33434.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33434.test new file mode 100644 index 00000000..dd9f882f --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33434.test @@ -0,0 +1,15 @@ +--echo # +--echo # MDEV-33434 MDEV-33434 UBSAN null pointer passed as argument 2, which is declared to never be null in spider_udf_direct_sql_create_conn +--echo # + +INSTALL SONAME 'ha_spider'; +SET character_set_connection=ucs2; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +SELECT SPIDER_DIRECT_SQL('SELECT SLEEP(1)', '', 'srv "dummy", port "3307"'); +--disable_query_log +--source ../../include/clean_up_spider.inc +--enable_query_log + +--echo # +--echo # end of test mdev_33434 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33441.opt b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441.opt new file mode 100644 index 00000000..924ea4e3 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441.opt @@ -0,0 +1 @@ +--plugin-load-add=ha_spider diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33441.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441.test new file mode 100644 index 00000000..a2e0ddaf --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441.test @@ -0,0 +1,10 @@ +--echo # +--echo # MDEV-33441 No spider variables available is Spider is loaded upon server startup +--echo # + +# We test that at least one spider variable exists. +set spider_same_server_link=0; + +--echo # +--echo # end of test mdev_33441 +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33441_fail.opt b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441_fail.opt new file mode 100644 index 00000000..28f1ce48 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441_fail.opt @@ -0,0 +1,2 @@ +--plugin-load-add=ha_spider +--debug-dbug=d,fail_spider_init_retry diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33441_fail.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441_fail.test new file mode 100644 index 00000000..6734b477 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33441_fail.test @@ -0,0 +1,10 @@ +--source include/have_debug.inc +--echo # +--echo # MDEV-33441 No spider variables available is Spider is loaded upon server startup +--echo # +# We test that when retry fails, spider variables are deleted. +select * from mysql.plugin; +show variables like 'spider%'; +--echo # +--echo # end of test mdev_33441_fail +--echo # diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33494.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33494.test new file mode 100644 index 00000000..30beca77 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33494.test @@ -0,0 +1,11 @@ +# This test tests spider init with global no_zero_date sql mode +set @old_sql_mode=@@global.sql_mode; +set global sql_mode=(SELECT CONCAT (@@sql_mode,',no_zero_date')); +install soname 'ha_spider'; +set global sql_mode=@old_sql_mode; + +--disable_query_log +--disable_result_log +--source ../../include/clean_up_spider.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33538.opt b/storage/spider/mysql-test/spider/bugfix/t/mdev_33538.opt new file mode 100644 index 00000000..a90c3a39 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33538.opt @@ -0,0 +1,2 @@ +--plugin-load-add=ha_spider +--transaction-read-only=on diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33538.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33538.test new file mode 100644 index 00000000..e2d4d8b4 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33538.test @@ -0,0 +1,2 @@ +# we check that information_schema.SPIDER_ALLOC_MEM exists +show create table information_schema.SPIDER_ALLOC_MEM; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33538_fail_init.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33538_fail_init.test new file mode 100644 index 00000000..fafc2fdb --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33538_fail_init.test @@ -0,0 +1,17 @@ +call mtr.add_suppression("\\[ERROR\\] SPIDER plugin initialization failed"); +call mtr.add_suppression(".*\\[ERROR\\] Plugin 'SPIDER' registration as a STORAGE ENGINE failed."); +call mtr.add_suppression(".*\\[ERROR\\] Plugin 'SPIDER_WRAPPER_PROTOCOLS' registration as a INFORMATION SCHEMA failed."); +call mtr.add_suppression(".*\\[ERROR\\] Plugin 'SPIDER_ALLOC_MEM' registration as a INFORMATION SCHEMA failed."); +# We create a table with identical name of the spider system table, to +# fail the spider init query ([ERROR] SPIDER plugin initialization +# failed at 'alter table mysql.spider_tables add column if not exists +# link_id int not null default 0 after table_name, drop primary key, +# add primary key (db_name, table_name, link_id), algorithm=copy, +# lock=shared;' by 'Unknown column 'table_name' in 'spider_tables'') +# This will cause the init of spider_alloc_mem to fail because it +# depends on the main spider plugin. +create table mysql.spider_tables (c int); +--let $restart_parameters= --plugin-load-add=ha_spider +--source include/restart_mysqld.inc +SELECT * FROM information_schema.SPIDER_ALLOC_MEM; +SELECT * FROM information_schema.SPIDER_WRAPPER_PROTOCOLS; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33584.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33584.test new file mode 100644 index 00000000..88644971 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33584.test @@ -0,0 +1,11 @@ +# This test tests spider init with global no_zero_date sql mode +set @old_sql_mode=@@global.sql_mode; +set global sql_mode='traditional'; +install soname 'ha_spider'; +set global sql_mode=@old_sql_mode; + +--disable_query_log +--disable_result_log +--source ../../include/clean_up_spider.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33679.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33679.test new file mode 100644 index 00000000..eee47a21 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33679.test @@ -0,0 +1,29 @@ +--echo # +--echo # MDEV-33679 spider returns parsing failure on valid left join select by translating the on expression to () +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +CREATE TABLE `t1` (`c` INT(10) UNSIGNED NOT NULL, `b` VARCHAR(255) NOT NULL , PRIMARY KEY (`c`) USING BTREE ) ENGINE=MYISAM; +CREATE TABLE `t2` (`a` INT(10) UNSIGNED NOT NULL, `c` INT(10) UNSIGNED NOT NULL ) ENGINE=MYISAM; +SET spider_same_server_link= on; +CREATE TABLE `t1_spider` (`c` INT(10) UNSIGNED NOT NULL, `b` VARCHAR(255) NOT NULL , PRIMARY KEY (`c`) USING BTREE ) COMMENT='wrapper "mysql",srv "srv", table "t1"' ENGINE=SPIDER; +CREATE TABLE `t2_spider` (`a` INT(10) UNSIGNED NOT NULL, `c` INT(10) UNSIGNED NOT NULL +, PRIMARY KEY (`a`) USING BTREE +) COMMENT='wrapper "mysql",srv "srv",table "t2"' ENGINE=SPIDER; +INSERT INTO t1_spider VALUES(1,'oooo'); +INSERT INTO t2_spider VALUES(1,1); +SELECT t2_spider.a,t1_spider.c FRoM t2_spider LEFT join t1_spider ON (t2_spider.c = t1_spider.c) WHERE t2_spider.a = 1; + +drop table t1, t2, t1_spider, t2_spider; +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33731.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33731.test new file mode 100644 index 00000000..b98c9620 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33731.test @@ -0,0 +1,16 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +CREATE TABLE t (a INT) ENGINE=Spider PARTITION BY LIST (a) PARTITIONS 2 (PARTITION p1 VALUES IN (0,1),PARTITION p2 VALUES IN (2,3)); +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +DELETE FROM t PARTITION (p2); +drop table t; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_34003.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_34003.test new file mode 100644 index 00000000..f81259b9 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_34003.test @@ -0,0 +1,20 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set spider_same_server_link= 1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c INT KEY,c1 BLOB,c2 TEXT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (0,1,0),(1,0,0),(2,0,0); +create table t1 (c INT KEY,c1 BLOB,c2 TEXT) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +SELECT * FROM t1 WHERE c=0; +drop table t1, t2; +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/plugin_load_add_all.test b/storage/spider/mysql-test/spider/bugfix/t/plugin_load_add_all.test index 396145fb..35242b20 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/plugin_load_add_all.test +++ b/storage/spider/mysql-test/spider/bugfix/t/plugin_load_add_all.test @@ -3,5 +3,6 @@ --echo # # A simple test that tests plugin-load-add=ha_spider select * from mysql.plugin; +select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA = "information_schema" and TABLE_NAME like "SPIDER_%"; create table t (c int) Engine=SPIDER; drop table t; diff --git a/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test index 4263560b..de75f0ee 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test +++ b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test @@ -19,8 +19,10 @@ eval create table t0 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv", eval alter table t2 ENGINE=Spider COMMENT='WRAPPER "mysql", srv "$srv",TABLE "t0"'; --error 12719 select * from t0; +--replace_result test.t1 test.t0 test.t2 test.t0 --error 12719 select * from t1; +--replace_result test.t1 test.t0 test.t2 test.t0 --error 12719 select * from t2; drop table t0, t1, t2; diff --git a/storage/spider/mysql-test/spider/feature/r/pushdown_case.result b/storage/spider/mysql-test/spider/feature/r/pushdown_case.result new file mode 100644 index 00000000..613ce377 --- /dev/null +++ b/storage/spider/mysql-test/spider/feature/r/pushdown_case.result @@ -0,0 +1,57 @@ +# +# MDEV-28993 Spider: Push down CASE statement +# +for master_1 +for child2 +for child3 +set spider_same_server_link= 1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +insert into t1 values (42), (3), (848), (100); +explain select case c when 3 then "three" when 42 then "answer" else "other" end from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select case c when 3 then "three" when 42 then "answer" else "other" end from t1; +case c when 3 then "three" when 42 then "answer" else "other" end +answer +three +other +other +explain select case c when 3 then "three" when 42 then "answer" end from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select case c when 3 then "three" when 42 then "answer" end from t1; +case c when 3 then "three" when 42 then "answer" end +answer +three +NULL +NULL +explain select case when c = 3 then "three" when c = 42 then "answer" else "other" end from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select case when c = 3 then "three" when c = 42 then "answer" else "other" end from t1; +case when c = 3 then "three" when c = 42 then "answer" else "other" end +answer +three +other +other +explain select case when c = 3 then "three" when c = 42 then "answer" end from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select case when c = 3 then "three" when c = 42 then "answer" end from t1; +case when c = 3 then "three" when c = 42 then "answer" end +answer +three +NULL +NULL +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 +# +# end of test pushdown_case +# diff --git a/storage/spider/mysql-test/spider/feature/r/pushdown_timestamp_diff.result b/storage/spider/mysql-test/spider/feature/r/pushdown_timestamp_diff.result new file mode 100644 index 00000000..3acd8912 --- /dev/null +++ b/storage/spider/mysql-test/spider/feature/r/pushdown_timestamp_diff.result @@ -0,0 +1,111 @@ +# +# MDEV-28992 Spider: Push down TIMESTAMPDIFF function +# +for master_1 +for child2 +for child3 +set spider_same_server_link= 1; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 ( +a INT, +b CHAR(1), +c DATETIME, +PRIMARY KEY(a) +); +CREATE TABLE t1 ( +a INT, +b CHAR(1), +c DATETIME, +PRIMARY KEY(a) +) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +INSERT INTO t1 (a, b, c) VALUES +(1, 'a', '2018-11-01 10:21:39'), +(2, 'b', '2015-06-30 23:59:59'), +(3, 'c', '2013-11-01 01:01:01'); +interval year +explain select a, b, timestampdiff(year, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(year, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(year, '2000-01-01 00:00:00', c) +1 a 18 +2 b 15 +3 c 13 +interval quarter +explain select a, b, timestampdiff(quarter, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(quarter, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(quarter, '2000-01-01 00:00:00', c) +1 a 75 +2 b 61 +3 c 55 +interval month +explain select a, b, timestampdiff(month, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(month, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(month, '2000-01-01 00:00:00', c) +1 a 226 +2 b 185 +3 c 166 +interval week +explain select a, b, timestampdiff(week, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(week, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(week, '2000-01-01 00:00:00', c) +1 a 982 +2 b 808 +3 c 721 +interval day +explain select a, b, timestampdiff(day, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(day, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(day, '2000-01-01 00:00:00', c) +1 a 6879 +2 b 5659 +3 c 5053 +internal hour +explain select a, b, timestampdiff(hour, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(hour, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(hour, '2000-01-01 00:00:00', c) +1 a 165106 +2 b 135839 +3 c 121273 +internal minute +explain select a, b, timestampdiff(minute, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(minute, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(minute, '2000-01-01 00:00:00', c) +1 a 9906381 +2 b 8150399 +3 c 7276381 +internal second +explain select a, b, timestampdiff(second, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(second, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(second, '2000-01-01 00:00:00', c) +1 a 594382899 +2 b 489023999 +3 c 436582861 +internal microsecond +explain select a, b, timestampdiff(microsecond, '2000-01-01 00:00:00', c) from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +select a, b, timestampdiff(microsecond, '2000-01-01 00:00:00', c) from t1; +a b timestampdiff(microsecond, '2000-01-01 00:00:00', c) +1 a 594382899000000 +2 b 489023999000000 +3 c 436582861000000 +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/feature/r/slave_transaction_retry_errors.result b/storage/spider/mysql-test/spider/feature/r/slave_transaction_retry_errors.result index 0a147c03..f7d4341c 100644 --- a/storage/spider/mysql-test/spider/feature/r/slave_transaction_retry_errors.result +++ b/storage/spider/mysql-test/spider/feature/r/slave_transaction_retry_errors.result @@ -5,11 +5,12 @@ child2_2 child2_3 for child3 for slave1_1 +include/start_slave.inc connection slave1_1; SHOW VARIABLES LIKE 'slave_transaction_retry_errors'; Variable_name Value -slave_transaction_retry_errors 1158,1159,1160,1161,1205,1213,1429,2013,12701 +slave_transaction_retry_errors 1158,1159,1160,1161,1205,1213,1020,1429,2013,12701 connection slave1_1; for slave1_1 for master_1 diff --git a/storage/spider/mysql-test/spider/feature/t/pushdown_case.test b/storage/spider/mysql-test/spider/feature/t/pushdown_case.test new file mode 100644 index 00000000..b86edceb --- /dev/null +++ b/storage/spider/mysql-test/spider/feature/t/pushdown_case.test @@ -0,0 +1,50 @@ +--echo # +--echo # MDEV-28993 Spider: Push down CASE statement +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set spider_same_server_link= 1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +insert into t1 values (42), (3), (848), (100); + +# everything +let $query= +select case c when 3 then "three" when 42 then "answer" else "other" end from t1; +eval explain $query; +eval $query; + +# no else +let $query= +select case c when 3 then "three" when 42 then "answer" end from t1; +eval explain $query; +eval $query; + +# no value +let $query= +select case when c = 3 then "three" when c = 42 then "answer" else "other" end from t1; +eval explain $query; +eval $query; + +# neither +let $query= +select case when c = 3 then "three" when c = 42 then "answer" end from t1; +eval explain $query; +eval $query; + +drop table t1, t2; +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log +--echo # +--echo # end of test pushdown_case +--echo # diff --git a/storage/spider/mysql-test/spider/feature/t/pushdown_timestamp_diff.test b/storage/spider/mysql-test/spider/feature/t/pushdown_timestamp_diff.test new file mode 100644 index 00000000..81251860 --- /dev/null +++ b/storage/spider/mysql-test/spider/feature/t/pushdown_timestamp_diff.test @@ -0,0 +1,93 @@ +--echo # +--echo # MDEV-28992 Spider: Push down TIMESTAMPDIFF function +--echo # +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +set spider_same_server_link= 1; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +create table t2 ( + a INT, + b CHAR(1), + c DATETIME, + PRIMARY KEY(a) +); +CREATE TABLE t1 ( + a INT, + b CHAR(1), + c DATETIME, + PRIMARY KEY(a) +) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; + +INSERT INTO t1 (a, b, c) VALUES + (1, 'a', '2018-11-01 10:21:39'), + (2, 'b', '2015-06-30 23:59:59'), + (3, 'c', '2013-11-01 01:01:01'); + +--echo interval year +let $query= +select a, b, timestampdiff(year, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo interval quarter +let $query= +select a, b, timestampdiff(quarter, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo interval month +let $query= +select a, b, timestampdiff(month, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo interval week +let $query= +select a, b, timestampdiff(week, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo interval day +let $query= +select a, b, timestampdiff(day, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo internal hour +let $query= +select a, b, timestampdiff(hour, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo internal minute +let $query= +select a, b, timestampdiff(minute, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo internal second +let $query= +select a, b, timestampdiff(second, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +--echo internal microsecond +let $query= +select a, b, timestampdiff(microsecond, '2000-01-01 00:00:00', c) from t1; +eval explain $query; +eval $query; + +drop table t1, t2; +drop server srv; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/mysql-test/spider/include/clean_up_spider.inc b/storage/spider/mysql-test/spider/include/clean_up_spider.inc index 1c977bfb..249606ec 100644 --- a/storage/spider/mysql-test/spider/include/clean_up_spider.inc +++ b/storage/spider/mysql-test/spider/include/clean_up_spider.inc @@ -3,7 +3,6 @@ DROP FUNCTION spider_copy_tables; DROP FUNCTION spider_ping_table; DROP FUNCTION spider_bg_direct_sql; DROP FUNCTION spider_direct_sql; ---replace_regex /\.dll/.so/ UNINSTALL SONAME IF EXISTS 'ha_spider'; DROP TABLE IF EXISTS mysql.spider_xa; DROP TABLE IF EXISTS mysql.spider_xa_member; diff --git a/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result b/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result index 850546fa..3eed8df2 100644 --- a/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result +++ b/storage/spider/mysql-test/spider/r/partition_join_pushdown_for_single_partition.result @@ -86,10 +86,8 @@ SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argum argument select `value` from `auto_test_remote2`.`tbl_a` where `value` = 5 select `value2` from `auto_test_remote2`.`tbl_b` where `value2` = 5 -select sum('5') `sum(a.value)`,count('5') `count(b.value2)` from (select 1) t0 join (select 1) t1 select `value` from `auto_test_remote2`.`tbl_a` where `value` = 5 select `value2` from `auto_test_remote2`.`tbl_b` where `value2` = 5 -select sum('5') `sum(a.value)`,count('5') `count(b.value2)` from (select 1) t0 join (select 1) t1 SELECT argument FROM mysql.general_log WHERE command_type != 'Execute' AND argument LIKE '%select %' SELECT value FROM tbl_a ORDER BY value; value diff --git a/storage/spider/mysql-test/spider/r/slave_trx_isolation.result b/storage/spider/mysql-test/spider/r/slave_trx_isolation.result index 28aaf74f..2f850b26 100644 --- a/storage/spider/mysql-test/spider/r/slave_trx_isolation.result +++ b/storage/spider/mysql-test/spider/r/slave_trx_isolation.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc connection slave1_1; set @old_spider_slave_trx_isolation= @@spider_slave_trx_isolation; set global spider_slave_trx_isolation= 1; diff --git a/storage/spider/mysql-test/spider/r/spider3_fixes.result b/storage/spider/mysql-test/spider/r/spider3_fixes.result index 7a80e102..614920be 100644 --- a/storage/spider/mysql-test/spider/r/spider3_fixes.result +++ b/storage/spider/mysql-test/spider/r/spider3_fixes.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/r/spider3_fixes_part.result b/storage/spider/mysql-test/spider/r/spider3_fixes_part.result index 937f222f..94390ff7 100644 --- a/storage/spider/mysql-test/spider/r/spider3_fixes_part.result +++ b/storage/spider/mysql-test/spider/r/spider3_fixes_part.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/r/spider_fixes.result b/storage/spider/mysql-test/spider/r/spider_fixes.result index 273f37a8..9083ad70 100644 --- a/storage/spider/mysql-test/spider/r/spider_fixes.result +++ b/storage/spider/mysql-test/spider/r/spider_fixes.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; diff --git a/storage/spider/mysql-test/spider/r/spider_fixes_part.result b/storage/spider/mysql-test/spider/r/spider_fixes_part.result index 571af94c..a59d2d2b 100644 --- a/storage/spider/mysql-test/spider/r/spider_fixes_part.result +++ b/storage/spider/mysql-test/spider/r/spider_fixes_part.result @@ -8,6 +8,7 @@ child3_1 child3_2 child3_3 for slave1_1 +include/start_slave.inc drop and create databases connection master_1; @@ -109,6 +110,7 @@ a b c 2.26 auto_increment with partition connection master_1; +include/save_master_pos.inc connection slave1_1; connection master_1; DROP TABLE IF EXISTS t1; diff --git a/storage/spider/mysql-test/spider/t/slave_test_init.inc b/storage/spider/mysql-test/spider/t/slave_test_init.inc index 739d8897..84de2c7d 100644 --- a/storage/spider/mysql-test/spider/t/slave_test_init.inc +++ b/storage/spider/mysql-test/spider/t/slave_test_init.inc @@ -19,7 +19,7 @@ if (!$SLAVE1_1_SLAVE_STATUS) MASTER_PORT = $MASTER_1_MYPORT ; } -START SLAVE; +--source include/start_slave.inc --connection master_1 call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); --connection slave1_1 diff --git a/storage/spider/mysql-test/spider/t/spider_fixes_part.opt b/storage/spider/mysql-test/spider/t/spider_fixes_part.opt new file mode 100644 index 00000000..1f24df45 --- /dev/null +++ b/storage/spider/mysql-test/spider/t/spider_fixes_part.opt @@ -0,0 +1 @@ +--log-warnings=3 diff --git a/storage/spider/mysql-test/spider/t/spider_fixes_part.test b/storage/spider/mysql-test/spider/t/spider_fixes_part.test index bff2e574..9604f83b 100644 --- a/storage/spider/mysql-test/spider/t/spider_fixes_part.test +++ b/storage/spider/mysql-test/spider/t/spider_fixes_part.test @@ -488,7 +488,23 @@ if ($HAVE_PARTITION) if ($USE_REPLICATION) { save_master_pos; + --source include/save_master_pos.inc --connection slave1_1 + --let $rc= `select master_pos_wait('$_master_file', $_master_pos, 300, '')` + if (`select $rc is NULL OR $rc < 0`) + { + --vertical_results + show slave status; + --horizontal_results + show global status; + show global variables; + --let $MYSQLD_DATADIR= `select @@datadir` + --exec $MYSQL_BINLOG -v $MYSQLD_DATADIR/mysqld-relay-bin.000001; + # Check that the relay-log file is fully on disk. + --exec ls -l $MYSQLD_DATADIR; + # After that try to restart the slave SQL thread + start slave sql_thread; + } sync_with_master; --connection master_1 --disable_query_log diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc index 92775896..68c5362d 100644 --- a/storage/spider/spd_conn.cc +++ b/storage/spider/spd_conn.cc @@ -56,7 +56,7 @@ extern struct charset_info_st *spd_charset_utf8mb3_bin; extern LEX_CSTRING spider_unique_id; pthread_mutex_t spider_conn_id_mutex; pthread_mutex_t spider_ipport_conn_mutex; -ulonglong spider_conn_id = 1; +ulonglong spider_conn_id; extern pthread_attr_t spider_pt_attr; @@ -93,7 +93,7 @@ extern sql_mode_t pushdown_sql_mode; HASH spider_open_connections; uint spider_open_connections_id; HASH spider_ipport_conns; -long spider_conn_mutex_id = 0; +long spider_conn_mutex_id; const char *spider_open_connections_func_name; const char *spider_open_connections_file_name; diff --git a/storage/spider/spd_copy_tables.cc b/storage/spider/spd_copy_tables.cc index 70ca17f8..4b608721 100644 --- a/storage/spider/spd_copy_tables.cc +++ b/storage/spider/spd_copy_tables.cc @@ -298,7 +298,7 @@ int spider_udf_get_copy_tgt_tables( if ( (error_num = spider_get_sys_tables_connect_info( - table_tables, tmp_share, 0, mem_root)) || + table_tables, tmp_share, mem_root)) || (error_num = spider_get_sys_tables_link_status( table_tables, tmp_share, 0, mem_root)) || (error_num = spider_get_sys_tables_link_idx( @@ -968,7 +968,12 @@ long long spider_copy_tables_body( all_link_cnt = copy_tables->link_idx_count[0] + copy_tables->link_idx_count[1]; if ( - !(tmp_sql = new spider_string[all_link_cnt]) || + !(tmp_sql = new spider_string[all_link_cnt]) + ) { + my_error(ER_OUT_OF_RESOURCES, MYF(0), HA_ERR_OUT_OF_MEM); + goto error; + } + if ( !(spider = new ha_spider[all_link_cnt]) ) { my_error(ER_OUT_OF_RESOURCES, MYF(0), HA_ERR_OUT_OF_MEM); @@ -997,13 +1002,6 @@ long long spider_copy_tables_body( tmp_spider->share = table_conn->share; tmp_spider->wide_handler = wide_handler; wide_handler->trx = copy_tables->trx; -/* - if (spider_db_append_set_names(table_conn->share)) - { - my_error(ER_OUT_OF_RESOURCES, MYF(0), HA_ERR_OUT_OF_MEM); - goto error_append_set_names; - } -*/ tmp_spider->conns = &table_conn->conn; tmp_sql[roop_count].init_calc_mem(SPD_MID_COPY_TABLES_BODY_3); tmp_sql[roop_count].set_charset(copy_tables->access_charset); @@ -1043,13 +1041,6 @@ long long spider_copy_tables_body( tmp_spider->share = table_conn->share; tmp_spider->wide_handler = wide_handler; wide_handler->trx = copy_tables->trx; -/* - if (spider_db_append_set_names(table_conn->share)) - { - my_error(ER_OUT_OF_RESOURCES, MYF(0), HA_ERR_OUT_OF_MEM); - goto error_append_set_names; - } -*/ tmp_spider->conns = &table_conn->conn; tmp_sql[roop_count].init_calc_mem(SPD_MID_COPY_TABLES_BODY_5); tmp_sql[roop_count].set_charset(copy_tables->access_charset); @@ -1076,14 +1067,6 @@ long long spider_copy_tables_body( bulk_insert_rows))) goto error_db_udf_copy_tables; -/* - for (table_conn = copy_tables->table_conn[0]; - table_conn; table_conn = table_conn->next) - spider_db_free_set_names(table_conn->share); - for (table_conn = copy_tables->table_conn[1]; - table_conn; table_conn = table_conn->next) - spider_db_free_set_names(table_conn->share); -*/ if (table_list->table) { (thd->is_error() ? trans_rollback_stmt(thd) : trans_commit_stmt(thd)); @@ -1104,8 +1087,7 @@ long long spider_copy_tables_body( } delete [] spider; } - if (tmp_sql) - delete [] tmp_sql; + delete [] tmp_sql; spider_udf_free_copy_tables_alloc(copy_tables); DBUG_RETURN(1); @@ -1113,17 +1095,6 @@ long long spider_copy_tables_body( error_db_udf_copy_tables: error_create_dbton_handler: error_init_dbton_handler: -/* -error_append_set_names: -*/ -/* - for (table_conn = copy_tables->table_conn[0]; - table_conn; table_conn = table_conn->next) - spider_db_free_set_names(table_conn->share); - for (table_conn = copy_tables->table_conn[1]; - table_conn; table_conn = table_conn->next) - spider_db_free_set_names(table_conn->share); -*/ error: if (spider) { @@ -1181,6 +1152,11 @@ my_bool spider_copy_tables_init_body( char *message ) { DBUG_ENTER("spider_copy_tables_init_body"); + if (!spider_hton_ptr) + { + strcpy(message, "Plugin 'SPIDER' is not loaded"); + goto error; + } if (args->arg_count != 3 && args->arg_count != 4) { strcpy(message, "spider_copy_tables() requires 3 or 4 arguments"); diff --git a/storage/spider/spd_db_conn.cc b/storage/spider/spd_db_conn.cc index 16d5b836..7c56ede2 100644 --- a/storage/spider/spd_db_conn.cc +++ b/storage/spider/spd_db_conn.cc @@ -63,7 +63,7 @@ pthread_mutex_t spider_open_conn_mutex; const char spider_dig_upper[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; /* UTC time zone for timestamp columns */ -Time_zone *UTC = 0; +Time_zone *UTC; int spider_db_connect( const SPIDER_SHARE *share, @@ -344,7 +344,6 @@ int spider_db_conn_queue_action( ) || ( conn->loop_check_queue.records && - conn->db_conn->set_loop_check_in_bulk_sql() && (error_num = spider_dbton[conn->dbton_id].db_util-> append_loop_check(&sql_str, conn)) ) || @@ -444,13 +443,6 @@ int spider_db_conn_queue_action( DBUG_RETURN(error_num); } if ( - conn->loop_check_queue.records && - !conn->db_conn->set_loop_check_in_bulk_sql() && - (error_num = conn->db_conn->set_loop_check((int *) conn->need_mon)) - ) { - DBUG_RETURN(error_num); - } - if ( conn->queued_trx_isolation && !conn->queued_semi_trx_isolation && conn->queued_trx_isolation_val != conn->trx_isolation && @@ -9138,6 +9130,9 @@ int spider_db_udf_ping_table_append_select( DBUG_RETURN(0); } +/* Stack size 33032 with clang */ +PRAGMA_DISABLE_CHECK_STACK_FRAME + int spider_db_udf_ping_table_mon_next( THD *thd, SPIDER_TABLE_MON *table_mon, @@ -9284,6 +9279,7 @@ int spider_db_udf_ping_table_mon_next( delete res; DBUG_RETURN(error_num); } +PRAGMA_REENABLE_CHECK_STACK_FRAME int spider_db_udf_copy_key_row( spider_string *str, diff --git a/storage/spider/spd_db_include.cc b/storage/spider/spd_db_include.cc index a4d7d4b4..514470a2 100644 --- a/storage/spider/spd_db_include.cc +++ b/storage/spider/spd_db_include.cc @@ -64,22 +64,6 @@ spider_db_conn::spider_db_conn( DBUG_VOID_RETURN; } -bool spider_db_conn::set_loop_check_in_bulk_sql() -{ - DBUG_ENTER("spider_db_conn::set_loop_check_in_bulk_sql"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_RETURN(FALSE); -} - -int spider_db_conn::set_loop_check( - int *need_mon -) { - DBUG_ENTER("spider_db_conn::set_loop_check"); - DBUG_PRINT("info",("spider this=%p", this)); - /* nothing to do */ - DBUG_RETURN(0); -} - int spider_db_conn::fin_loop_check() { st_spider_conn_loop_check *lcptr; diff --git a/storage/spider/spd_db_include.h b/storage/spider/spd_db_include.h index c2cbe5f2..4e9d6b1a 100644 --- a/storage/spider/spd_db_include.h +++ b/storage/spider/spd_db_include.h @@ -79,7 +79,6 @@ typedef st_spider_result SPIDER_RESULT; #define SPIDER_SQL_HS_LTEQUAL_STR "<=" #define SPIDER_SQL_HS_LTEQUAL_LEN (sizeof(SPIDER_SQL_HS_LTEQUAL_STR) - 1) -#ifdef ITEM_FUNC_CASE_PARAMS_ARE_PUBLIC #define SPIDER_SQL_CASE_STR "case " #define SPIDER_SQL_CASE_LEN (sizeof(SPIDER_SQL_CASE_STR) - 1) #define SPIDER_SQL_WHEN_STR " when " @@ -90,7 +89,6 @@ typedef st_spider_result SPIDER_RESULT; #define SPIDER_SQL_ELSE_LEN (sizeof(SPIDER_SQL_ELSE_STR) - 1) #define SPIDER_SQL_END_STR " end" #define SPIDER_SQL_END_LEN (sizeof(SPIDER_SQL_END_STR) - 1) -#endif #define SPIDER_SQL_USING_STR " using " #define SPIDER_SQL_USING_LEN (sizeof(SPIDER_SQL_USING_STR) - 1) @@ -1024,10 +1022,6 @@ public: Time_zone *time_zone, int *need_mon ) = 0; - virtual bool set_loop_check_in_bulk_sql(); - virtual int set_loop_check( - int *need_mon - ); virtual int fin_loop_check(); virtual int show_master_status( SPIDER_TRX *trx, diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index c2f0c6b8..450e1a02 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -1191,7 +1191,7 @@ int spider_db_mbase_result::fetch_table_cardinality( uint num_fields = this->num_fields(); if (num_fields < 12 || num_fields > 14) { - DBUG_PRINT("info",("spider num_fields < 12 || num_fields > 13")); + DBUG_PRINT("info",("spider num_fields < 12 || num_fields > 14")); DBUG_RETURN(ER_SPIDER_INVALID_REMOTE_TABLE_INFO_NUM); } @@ -3266,110 +3266,6 @@ int spider_db_mbase::set_time_zone( DBUG_RETURN(0); } -bool spider_db_mbase::set_loop_check_in_bulk_sql() -{ - DBUG_ENTER("spider_db_mbase::set_loop_check_in_bulk_sql"); - DBUG_PRINT("info",("spider this=%p", this)); - DBUG_RETURN(TRUE); -} - -int spider_db_mbase::set_loop_check( - int *need_mon -) { - SPIDER_CONN_LOOP_CHECK *lcptr; - char sql_buf[MAX_FIELD_WIDTH]; - spider_string sql_str(sql_buf, sizeof(sql_buf), &my_charset_bin); - DBUG_ENTER("spider_db_mbase::set_loop_check"); - DBUG_PRINT("info",("spider this=%p", this)); - sql_str.init_calc_mem(SPD_MID_DB_MBASE_SET_LOOP_CHECK_1); - while ((lcptr = (SPIDER_CONN_LOOP_CHECK *) my_hash_element( - &conn->loop_check_queue, 0))) - { - sql_str.length(0); - if (sql_str.reserve(SPIDER_SQL_SET_USER_VAL_LEN + - SPIDER_SQL_LOP_CHK_PRM_PRF_LEN + lcptr->to_name.length + - SPIDER_SQL_NAME_QUOTE_LEN + SPIDER_SQL_EQUAL_LEN + - SPIDER_SQL_VALUE_QUOTE_LEN + - lcptr->merged_value.length + SPIDER_SQL_VALUE_QUOTE_LEN)) - { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - sql_str.q_append(SPIDER_SQL_SET_USER_VAL_STR, SPIDER_SQL_SET_USER_VAL_LEN); - sql_str.q_append(SPIDER_SQL_LOP_CHK_PRM_PRF_STR, - SPIDER_SQL_LOP_CHK_PRM_PRF_LEN); - sql_str.q_append(lcptr->to_name.str, lcptr->to_name.length); - sql_str.q_append(SPIDER_SQL_NAME_QUOTE_STR, SPIDER_SQL_NAME_QUOTE_LEN); - sql_str.q_append(SPIDER_SQL_EQUAL_STR, SPIDER_SQL_EQUAL_LEN); - sql_str.q_append(SPIDER_SQL_VALUE_QUOTE_STR, SPIDER_SQL_VALUE_QUOTE_LEN); - sql_str.q_append(lcptr->merged_value.str, lcptr->merged_value.length); - sql_str.q_append(SPIDER_SQL_VALUE_QUOTE_STR, SPIDER_SQL_VALUE_QUOTE_LEN); - - pthread_mutex_assert_not_owner(&conn->mta_conn_mutex); - pthread_mutex_lock(&conn->mta_conn_mutex); - SPIDER_SET_FILE_POS(&conn->mta_conn_mutex_file_pos); - conn->need_mon = need_mon; - DBUG_ASSERT(!conn->mta_conn_mutex_lock_already); - DBUG_ASSERT(!conn->mta_conn_mutex_unlock_later); - conn->mta_conn_mutex_lock_already = TRUE; - conn->mta_conn_mutex_unlock_later = TRUE; - if (spider_db_query( - conn, - sql_str.ptr(), - sql_str.length(), - -1, - need_mon) - ) { - DBUG_ASSERT(conn->mta_conn_mutex_lock_already); - DBUG_ASSERT(conn->mta_conn_mutex_unlock_later); - conn->mta_conn_mutex_lock_already = FALSE; - conn->mta_conn_mutex_unlock_later = FALSE; - DBUG_RETURN(spider_db_errorno(conn)); - } - DBUG_ASSERT(conn->mta_conn_mutex_lock_already); - DBUG_ASSERT(conn->mta_conn_mutex_unlock_later); - conn->mta_conn_mutex_lock_already = FALSE; - conn->mta_conn_mutex_unlock_later = FALSE; - SPIDER_CLEAR_FILE_POS(&conn->mta_conn_mutex_file_pos); - pthread_mutex_unlock(&conn->mta_conn_mutex); - - my_hash_delete(&conn->loop_check_queue, (uchar*) lcptr); - } - DBUG_RETURN(0); -} - -int spider_db_mbase::fin_loop_check() -{ - st_spider_conn_loop_check *lcptr; - DBUG_ENTER("spider_db_mbase::fin_loop_check"); - DBUG_PRINT("info",("spider this=%p", this)); - if (conn->loop_check_queue.records) - { - uint l = 0; - while ((lcptr = (SPIDER_CONN_LOOP_CHECK *) my_hash_element( - &conn->loop_check_queue, l))) - { - lcptr->flag = 0; - ++l; - } - my_hash_reset(&conn->loop_check_queue); - } - lcptr = conn->loop_check_ignored_first; - while (lcptr) - { - lcptr->flag = 0; - lcptr = lcptr->next; - } - conn->loop_check_ignored_first = NULL; - lcptr = conn->loop_check_meraged_first; - while (lcptr) - { - lcptr->flag = 0; - lcptr = lcptr->next; - } - conn->loop_check_meraged_first = NULL; - DBUG_RETURN(0); -} - int spider_db_mbase::exec_simple_sql_with_result( SPIDER_TRX *trx, SPIDER_SHARE *share, @@ -5503,14 +5399,6 @@ int spider_db_mbase_util::open_item_func( alias_length, use_fields, fields)); } -static bool item_func_is_timestampdiff( - const char *func_name, - int func_name_length -) { - return func_name_length == 13 && - !strncasecmp("timestampdiff", func_name, func_name_length); -} - static bool not_func_should_be_skipped( Item_func *item_func ){ @@ -5580,16 +5468,10 @@ int spider_db_mbase_util::check_item_func( Item_func::Functype func_type = item_func->functype(); DBUG_PRINT("info",("spider functype = %d", func_type)); - const char *func_name = (char*) item_func->func_name(); - int func_name_length = strlen(func_name); - DBUG_PRINT("info",("spider func_name = %s", func_name)); - /* The blacklist of the functions that cannot be pushed down */ switch (func_type) { case Item_func::TRIG_COND_FUNC: - case Item_func::CASE_SEARCHED_FUNC: - case Item_func::CASE_SIMPLE_FUNC: DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); case Item_func::NOT_FUNC: /* Why the following check is necessary? */ @@ -5598,13 +5480,6 @@ int spider_db_mbase_util::check_item_func( break; case Item_func::FUNC_SP: case Item_func::UDF_FUNC: - /* Notes on merging regarding MDEV-29447: please refer to the - following commits for build error or merge conflicts: - 10.6: 1ed20b993b0dd4e95450cab2e8347e5bf4617a69 - 10.9: dd316b6e20265cfd832bb5585cb4c96e716387c8 - 10.10-11: 3f67f110ba1b23a89c5ede0fbeeb203cf5e164f4 - 11.0-1: 17ba6748afa8834df5658361088e6c8e65aca16f - Please remove this comment after merging. */ use_pushdown_udf= spider_param_use_pushdown_udf( spider->wide_handler->trx->thd, spider->share->use_pushdown_udf); if (!use_pushdown_udf) @@ -5614,12 +5489,18 @@ int spider_db_mbase_util::check_item_func( if (spider_db_check_ft_idx(item_func, spider) == MAX_KEY) DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); break; -#ifndef ITEM_FUNC_TIMESTAMPDIFF_ARE_PUBLIC - case Item_func::UNKNOWN_FUNC: - if (item_func_is_timestampdiff(func_name, func_name_length)) - DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); - break; -#endif + case Item_func::MULT_EQUAL_FUNC: + /* If there is still Item_equal by the time of + JOIN::make_aggr_tables_info() where the spider group by handler + is created, it indicates a bug in the optimizer, because there + shouldn't be any. */ + push_warning_printf( + spider->wide_handler->trx->thd, SPIDER_WARN_LEVEL_WARN, + ER_INTERNAL_ERROR, + ER_THD(spider->wide_handler->trx->thd, ER_INTERNAL_ERROR), + "Spider group by handler: Encountered multiple equalities, likely " + "an optimizer bug"); + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); default: break; } @@ -5661,7 +5542,7 @@ int spider_db_mbase_util::print_item_func( Item *item, **item_list = item_func->arguments(); Field *field; spider_string tmp_str; - uint roop_count, item_count = item_func->argument_count(), start_item = 0; + uint i, item_count = item_func->argument_count(), start_item = 0; LEX_CSTRING org_func_name= {SPIDER_SQL_NULL_CHAR_STR, SPIDER_SQL_NULL_CHAR_LEN}; const char *func_name = SPIDER_SQL_NULL_CHAR_STR, @@ -5670,8 +5551,8 @@ int spider_db_mbase_util::print_item_func( int func_name_length = SPIDER_SQL_NULL_CHAR_LEN, separator_str_length = SPIDER_SQL_NULL_CHAR_LEN, last_str_length = SPIDER_SQL_NULL_CHAR_LEN; - int use_pushdown_udf; - bool merge_func = FALSE; + int use_pushdown_udf, case_when_start, case_when_count; + bool merge_func = FALSE, case_with_else; DBUG_ENTER("spider_db_mbase_util::print_item_func"); DBUG_ASSERT(!check_item_func(item_func, spider, alias, alias_length, use_fields, fields)); @@ -5989,12 +5870,11 @@ int spider_db_mbase_util::print_item_func( alias, alias_length, dbton_id, use_fields, fields)); } else if (!strncasecmp("timestampdiff", func_name, func_name_length)) { -#ifdef ITEM_FUNC_TIMESTAMPDIFF_ARE_PUBLIC Item_func_timestamp_diff *item_func_timestamp_diff = (Item_func_timestamp_diff *) item_func; const char *interval_str; uint interval_len; - switch (item_func_timestamp_diff->int_type) + switch (item_func_timestamp_diff->get_int_type()) { case INTERVAL_YEAR: interval_str = SPIDER_SQL_YEAR_STR; @@ -6045,7 +5925,7 @@ int spider_db_mbase_util::print_item_func( str->q_append(SPIDER_SQL_OPEN_PAREN_STR, SPIDER_SQL_OPEN_PAREN_LEN); str->q_append(interval_str, interval_len); str->q_append(SPIDER_SQL_COMMA_STR, SPIDER_SQL_COMMA_LEN); - + if ((error_num = spider_db_print_item_type(item_list[0], NULL, spider, str, alias, alias_length, dbton_id, use_fields, fields))) DBUG_RETURN(error_num); @@ -6066,9 +5946,6 @@ int spider_db_mbase_util::print_item_func( SPIDER_SQL_CLOSE_PAREN_LEN); } DBUG_RETURN(0); -#else - DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); -#endif } } else if (func_name_length == 14) { @@ -6562,7 +6439,83 @@ int spider_db_mbase_util::print_item_func( DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); case Item_func::CASE_SEARCHED_FUNC: case Item_func::CASE_SIMPLE_FUNC: - DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); + /* + Arrangement of arguments: + - Item_func_case_searched: + when1 when2 ... whenk then1 then2 .. thenk [else] + - Item_func_case_simple: + value when1 when2 ... whenk then1 then2 .. thenk [else] + */ + if (item_func->functype() == Item_func::CASE_SEARCHED_FUNC) + { + case_when_start= 0; + case_when_count= item_count / 2; + case_with_else= item_count % 2; + } + else + { + case_when_start= 1; + case_when_count= (item_count - 1) / 2; + case_with_else= item_count % 2 == 0; + } + if (str) + { + if (str->reserve(SPIDER_SQL_CASE_LEN)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + str->q_append(SPIDER_SQL_CASE_STR, SPIDER_SQL_CASE_LEN); + } + if (case_when_start > 0) + { + if ((error_num = spider_db_print_item_type( + item_list[0], NULL, spider, str, + alias, alias_length, dbton_id, use_fields, fields))) + DBUG_RETURN(error_num); + } + for (i = 0; i < (uint) case_when_count; i++) + { + if (str) + { + if (str->reserve(SPIDER_SQL_WHEN_LEN)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + str->q_append(SPIDER_SQL_WHEN_STR, SPIDER_SQL_WHEN_LEN); + } + if ((error_num = spider_db_print_item_type( + item_list[i + case_when_start], NULL, spider, str, + alias, alias_length, dbton_id, use_fields, fields))) + DBUG_RETURN(error_num); + if (str) + { + if (str->reserve(SPIDER_SQL_THEN_LEN)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + str->q_append(SPIDER_SQL_THEN_STR, SPIDER_SQL_THEN_LEN); + } + if ((error_num = spider_db_print_item_type( + item_list[i + case_when_start + case_when_count], NULL, spider, str, + alias, alias_length, dbton_id, use_fields, fields))) + DBUG_RETURN(error_num); + } + if (case_with_else) + { + if (str) + { + if (str->reserve(SPIDER_SQL_ELSE_LEN)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + str->q_append(SPIDER_SQL_ELSE_STR, SPIDER_SQL_ELSE_LEN); + } + if ((error_num = spider_db_print_item_type( + item_list[item_count - 1], NULL, spider, str, + alias, alias_length, dbton_id, use_fields, fields))) + DBUG_RETURN(error_num); + } + if (str) + { + if (str->reserve(SPIDER_SQL_END_LEN + SPIDER_SQL_CLOSE_PAREN_LEN)) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + str->q_append(SPIDER_SQL_END_STR, SPIDER_SQL_END_LEN); + str->q_append(SPIDER_SQL_CLOSE_PAREN_STR, + SPIDER_SQL_CLOSE_PAREN_LEN); + } + DBUG_RETURN(0); case Item_func::JSON_EXTRACT_FUNC: func_name = (char*) item_func->func_name(); func_name_length = strlen(func_name); @@ -6577,6 +6530,18 @@ int spider_db_mbase_util::print_item_func( last_str = SPIDER_SQL_CLOSE_PAREN_STR; last_str_length = SPIDER_SQL_CLOSE_PAREN_LEN; break; + case Item_func::MULT_EQUAL_FUNC: + /* If there is still Item_equal by the time of + JOIN::make_aggr_tables_info() where the spider group by handler + is created, it indicates a bug in the optimizer, because there + shouldn't be any. */ + push_warning_printf( + spider->wide_handler->trx->thd, + SPIDER_WARN_LEVEL_WARN, ER_INTERNAL_ERROR, + ER_THD(spider->wide_handler->trx->thd, ER_INTERNAL_ERROR), + "Spider group by handler: Encountered multiple equalities, likely " + "an optimizer bug"); + DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM); default: THD *thd = spider->wide_handler->trx->thd; SPIDER_SHARE *share = spider->share; @@ -6606,13 +6571,13 @@ int spider_db_mbase_util::print_item_func( Loop through the items of the current function expression to print its portion of the statement */ - for (roop_count = start_item; roop_count < item_count; roop_count++) + for (i = start_item; i < item_count; i++) { - item = item_list[roop_count]; + item = item_list[i]; if ((error_num = spider_db_print_item_type(item, field, spider, str, alias, alias_length, dbton_id, use_fields, fields))) DBUG_RETURN(error_num); - if (roop_count == 1) + if (i == 1) { /* Remaining operands need to be preceded by the separator */ func_name = separator_str; @@ -6626,7 +6591,7 @@ int spider_db_mbase_util::print_item_func( } /* Print the last operand value */ - item = item_list[roop_count]; + item = item_list[i]; if ((error_num = spider_db_print_item_type(item, field, spider, str, alias, alias_length, dbton_id, use_fields, fields))) DBUG_RETURN(error_num); @@ -7259,11 +7224,9 @@ int spider_mbase_share::init() DBUG_RETURN(HA_ERR_OUT_OF_MEM); } - if (keys > 0 && - !(key_hint = new spider_string[keys]) - ) { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } + if (keys > 0) + if (!(key_hint = new spider_string[keys])) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); for (roop_count = 0; roop_count < keys; roop_count++) { key_hint[roop_count].init_calc_mem(SPD_MID_MBASE_SHARE_INIT_2); @@ -7271,12 +7234,12 @@ int spider_mbase_share::init() } DBUG_PRINT("info",("spider key_hint=%p", key_hint)); - if ( - !(table_select = new spider_string[1]) || - (keys > 0 && - !(key_select = new spider_string[keys]) - ) || - (error_num = create_table_names_str()) || + if (!(table_select = new spider_string[1])) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + if (keys > 0) + if (!(key_select = new spider_string[keys])) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + if ((error_num = create_table_names_str()) || (table_share && ( (error_num = create_column_name_str()) || @@ -7427,11 +7390,18 @@ int spider_mbase_share::create_table_names_str() table_names_str = NULL; db_names_str = NULL; db_table_str = NULL; - if ( - !(table_names_str = new spider_string[spider_share->all_link_count]) || - !(db_names_str = new spider_string[spider_share->all_link_count]) || - !(db_table_str = new spider_string[spider_share->all_link_count]) - ) { + if (!(table_names_str = new spider_string[spider_share->all_link_count])) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } + if (!(db_names_str = new spider_string[spider_share->all_link_count])) + { + error_num = HA_ERR_OUT_OF_MEM; + goto error; + } + if (!(db_table_str = new spider_string[spider_share->all_link_count])) + { error_num = HA_ERR_OUT_OF_MEM; goto error; } @@ -7580,11 +7550,9 @@ int spider_mbase_share::create_column_name_str() Field **field; TABLE_SHARE *table_share = spider_share->table_share; DBUG_ENTER("spider_mbase_share::create_column_name_str"); - if ( - table_share->fields && - !(column_name_str = new spider_string[table_share->fields]) - ) - DBUG_RETURN(HA_ERR_OUT_OF_MEM); + if (table_share->fields) + if (!(column_name_str = new spider_string[table_share->fields])) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); for (field = table_share->field, str = column_name_str; *field; field++, str++) { @@ -8360,7 +8328,7 @@ int spider_mbase_handler::init() &link_for_hash, sizeof(SPIDER_LINK_FOR_HASH) * share->link_count, &minimum_select_bitmap, - table ? sizeof(uchar) * no_bytes_in_map(table->read_set) : 0, + table ? sizeof(uchar) * my_bitmap_buffer_size(table->read_set) : 0, NullS)) ) { DBUG_RETURN(HA_ERR_OUT_OF_MEM); @@ -14829,7 +14797,7 @@ void spider_mbase_handler::minimum_select_bitmap_create() Field **field_p; DBUG_ENTER("spider_mbase_handler::minimum_select_bitmap_create"); DBUG_PRINT("info",("spider this=%p", this)); - memset(minimum_select_bitmap, 0, no_bytes_in_map(table->read_set)); + memset(minimum_select_bitmap, 0, my_bitmap_buffer_size(table->read_set)); if ( spider->use_index_merge || spider->is_clone @@ -14840,7 +14808,7 @@ void spider_mbase_handler::minimum_select_bitmap_create() table_share->primary_key == MAX_KEY ) { /* need all columns */ - memset(minimum_select_bitmap, 0xFF, no_bytes_in_map(table->read_set)); + memset(minimum_select_bitmap, 0xFF, my_bitmap_buffer_size(table->read_set)); DBUG_VOID_RETURN; } else { /* need primary key columns */ diff --git a/storage/spider/spd_db_mysql.h b/storage/spider/spd_db_mysql.h index 9ccc0e33..50b21117 100644 --- a/storage/spider/spd_db_mysql.h +++ b/storage/spider/spd_db_mysql.h @@ -519,11 +519,6 @@ public: Time_zone *time_zone, int *need_mon ); - bool set_loop_check_in_bulk_sql(); - int set_loop_check( - int *need_mon - ); - int fin_loop_check(); int exec_simple_sql_with_result( SPIDER_TRX *trx, SPIDER_SHARE *share, diff --git a/storage/spider/spd_direct_sql.cc b/storage/spider/spd_direct_sql.cc index d345bb04..89c872ee 100644 --- a/storage/spider/spd_direct_sql.cc +++ b/storage/spider/spd_direct_sql.cc @@ -364,6 +364,23 @@ int spider_udf_direct_sql_create_conn_key( DBUG_RETURN(0); } +static inline void spider_maybe_memcpy_string( + char **dest, + char *src, + char *tmp, + uint *dest_len, + uint src_len) +{ + *dest_len= src_len; + if (src_len) + { + *dest= tmp; + memcpy(*dest, src, src_len); + } else + *dest= NULL; +} + + SPIDER_CONN *spider_udf_direct_sql_create_conn( const SPIDER_DIRECT_SQL *direct_sql, int *error_num @@ -433,105 +450,49 @@ SPIDER_CONN *spider_udf_direct_sql_create_conn( conn->tgt_host = tmp_host; memcpy(conn->tgt_host, direct_sql->tgt_host, direct_sql->tgt_host_length); conn->tgt_port = direct_sql->tgt_port; - conn->tgt_socket_length = direct_sql->tgt_socket_length; - conn->tgt_socket = tmp_socket; - memcpy(conn->tgt_socket, direct_sql->tgt_socket, - direct_sql->tgt_socket_length); + spider_maybe_memcpy_string( + &conn->tgt_socket, direct_sql->tgt_socket, tmp_socket, + &conn->tgt_socket_length, direct_sql->tgt_socket_length); if (!tables_on_different_db_are_joinable) - { - conn->tgt_db_length = direct_sql->tgt_default_db_name_length; - conn->tgt_db = tmp_db; - memcpy(conn->tgt_db, direct_sql->tgt_default_db_name, - direct_sql->tgt_default_db_name_length); - } - conn->tgt_username_length = direct_sql->tgt_username_length; - conn->tgt_username = tmp_username; - memcpy(conn->tgt_username, direct_sql->tgt_username, - direct_sql->tgt_username_length); - conn->tgt_password_length = direct_sql->tgt_password_length; - conn->tgt_password = tmp_password; - memcpy(conn->tgt_password, direct_sql->tgt_password, - direct_sql->tgt_password_length); - conn->tgt_ssl_ca_length = direct_sql->tgt_ssl_ca_length; - if (conn->tgt_ssl_ca_length) - { - conn->tgt_ssl_ca = tmp_ssl_ca; - memcpy(conn->tgt_ssl_ca, direct_sql->tgt_ssl_ca, - direct_sql->tgt_ssl_ca_length); - } else - conn->tgt_ssl_ca = NULL; - conn->tgt_ssl_capath_length = direct_sql->tgt_ssl_capath_length; - if (conn->tgt_ssl_capath_length) - { - conn->tgt_ssl_capath = tmp_ssl_capath; - memcpy(conn->tgt_ssl_capath, direct_sql->tgt_ssl_capath, - direct_sql->tgt_ssl_capath_length); - } else - conn->tgt_ssl_capath = NULL; - conn->tgt_ssl_cert_length = direct_sql->tgt_ssl_cert_length; - if (conn->tgt_ssl_cert_length) - { - conn->tgt_ssl_cert = tmp_ssl_cert; - memcpy(conn->tgt_ssl_cert, direct_sql->tgt_ssl_cert, - direct_sql->tgt_ssl_cert_length); - } else - conn->tgt_ssl_cert = NULL; - conn->tgt_ssl_cipher_length = direct_sql->tgt_ssl_cipher_length; - if (conn->tgt_ssl_cipher_length) - { - conn->tgt_ssl_cipher = tmp_ssl_cipher; - memcpy(conn->tgt_ssl_cipher, direct_sql->tgt_ssl_cipher, - direct_sql->tgt_ssl_cipher_length); - } else - conn->tgt_ssl_cipher = NULL; - conn->tgt_ssl_key_length = direct_sql->tgt_ssl_key_length; - if (conn->tgt_ssl_key_length) - { - conn->tgt_ssl_key = tmp_ssl_key; - memcpy(conn->tgt_ssl_key, direct_sql->tgt_ssl_key, - direct_sql->tgt_ssl_key_length); - } else - conn->tgt_ssl_key = NULL; - conn->tgt_default_file_length = direct_sql->tgt_default_file_length; - if (conn->tgt_default_file_length) - { - conn->tgt_default_file = tmp_default_file; - memcpy(conn->tgt_default_file, direct_sql->tgt_default_file, - direct_sql->tgt_default_file_length); - } else - conn->tgt_default_file = NULL; - conn->tgt_default_group_length = direct_sql->tgt_default_group_length; - if (conn->tgt_default_group_length) - { - conn->tgt_default_group = tmp_default_group; - memcpy(conn->tgt_default_group, direct_sql->tgt_default_group, - direct_sql->tgt_default_group_length); - } else - conn->tgt_default_group = NULL; - conn->tgt_dsn_length = direct_sql->tgt_dsn_length; - if (conn->tgt_dsn_length) - { - conn->tgt_dsn = tmp_dsn; - memcpy(conn->tgt_dsn, direct_sql->tgt_dsn, - direct_sql->tgt_dsn_length); - } else - conn->tgt_dsn = NULL; - conn->tgt_filedsn_length = direct_sql->tgt_filedsn_length; - if (conn->tgt_filedsn_length) - { - conn->tgt_filedsn = tmp_filedsn; - memcpy(conn->tgt_filedsn, direct_sql->tgt_filedsn, - direct_sql->tgt_filedsn_length); - } else - conn->tgt_filedsn = NULL; - conn->tgt_driver_length = direct_sql->tgt_driver_length; - if (conn->tgt_driver_length) - { - conn->tgt_driver = tmp_driver; - memcpy(conn->tgt_driver, direct_sql->tgt_driver, - direct_sql->tgt_driver_length); - } else - conn->tgt_driver = NULL; + spider_maybe_memcpy_string( + &conn->tgt_db, direct_sql->tgt_default_db_name, tmp_db, + &conn->tgt_db_length, direct_sql->tgt_default_db_name_length); + spider_maybe_memcpy_string( + &conn->tgt_username, direct_sql->tgt_username, tmp_username, + &conn->tgt_username_length, direct_sql->tgt_username_length); + spider_maybe_memcpy_string( + &conn->tgt_password, direct_sql->tgt_password, tmp_password, + &conn->tgt_password_length, direct_sql->tgt_password_length); + spider_maybe_memcpy_string( + &conn->tgt_ssl_ca, direct_sql->tgt_ssl_ca, tmp_ssl_ca, + &conn->tgt_ssl_ca_length, direct_sql->tgt_ssl_ca_length); + spider_maybe_memcpy_string( + &conn->tgt_ssl_capath, direct_sql->tgt_ssl_capath, tmp_ssl_capath, + &conn->tgt_ssl_capath_length, direct_sql->tgt_ssl_capath_length); + spider_maybe_memcpy_string( + &conn->tgt_ssl_cert, direct_sql->tgt_ssl_cert, tmp_ssl_cert, + &conn->tgt_ssl_cert_length, direct_sql->tgt_ssl_cert_length); + spider_maybe_memcpy_string( + &conn->tgt_ssl_cipher, direct_sql->tgt_ssl_cipher, tmp_ssl_cipher, + &conn->tgt_ssl_cipher_length, direct_sql->tgt_ssl_cipher_length); + spider_maybe_memcpy_string( + &conn->tgt_ssl_key, direct_sql->tgt_ssl_key, tmp_ssl_key, + &conn->tgt_ssl_key_length, direct_sql->tgt_ssl_key_length); + spider_maybe_memcpy_string( + &conn->tgt_default_file, direct_sql->tgt_default_file, tmp_default_file, + &conn->tgt_default_file_length, direct_sql->tgt_default_file_length); + spider_maybe_memcpy_string( + &conn->tgt_default_group, direct_sql->tgt_default_group, tmp_default_group, + &conn->tgt_default_group_length, direct_sql->tgt_default_group_length); + spider_maybe_memcpy_string( + &conn->tgt_dsn, direct_sql->tgt_dsn, tmp_dsn, + &conn->tgt_dsn_length, direct_sql->tgt_dsn_length); + spider_maybe_memcpy_string( + &conn->tgt_filedsn, direct_sql->tgt_filedsn, tmp_filedsn, + &conn->tgt_filedsn_length, direct_sql->tgt_filedsn_length); + spider_maybe_memcpy_string( + &conn->tgt_driver, direct_sql->tgt_driver, tmp_driver, + &conn->tgt_driver_length, direct_sql->tgt_driver_length); conn->tgt_ssl_vsc = direct_sql->tgt_ssl_vsc; conn->dbton_id = direct_sql->dbton_id; conn->conn_need_mon = need_mon; @@ -1658,6 +1619,11 @@ my_bool spider_direct_sql_init_body( ) { SPIDER_BG_DIRECT_SQL *bg_direct_sql; DBUG_ENTER("spider_direct_sql_init_body"); + if (!spider_hton_ptr) + { + strcpy(message, "Plugin 'SPIDER' is not loaded"); + goto error; + } if (args->arg_count != 3) { strcpy(message, "spider_(bg)_direct_sql() requires 3 arguments"); diff --git a/storage/spider/spd_i_s.cc b/storage/spider/spd_i_s.cc index 7db95371..8ae88102 100644 --- a/storage/spider/spd_i_s.cc +++ b/storage/spider/spd_i_s.cc @@ -28,6 +28,7 @@ #include "spd_table.h" extern pthread_mutex_t spider_mem_calc_mutex; +extern handlerton *spider_hton_ptr; extern const char *spider_alloc_func_name[SPIDER_MEM_CALC_LIST_NUM]; extern const char *spider_alloc_file_name[SPIDER_MEM_CALC_LIST_NUM]; @@ -63,6 +64,8 @@ static int spider_i_s_alloc_mem_fill_table( uint roop_count; TABLE *table = tables->table; DBUG_ENTER("spider_i_s_alloc_mem_fill_table"); + if (!spider_hton_ptr) + DBUG_RETURN(0); for (roop_count = 0; roop_count < SPIDER_MEM_CALC_LIST_NUM; roop_count++) { table->field[0]->store(roop_count, TRUE); @@ -177,6 +180,8 @@ static int spider_i_s_wrapper_protocols_fill_table( SPIDER_DBTON *dbton; TABLE *table = tables->table; DBUG_ENTER("spider_i_s_wrapper_protocols_fill_table"); + if (!spider_hton_ptr) + DBUG_RETURN(0); for (roop_count = 0; roop_count < SPIDER_DBTON_SIZE; roop_count++) { dbton = &spider_dbton[roop_count]; diff --git a/storage/spider/spd_include.h b/storage/spider/spd_include.h index 0f61984e..3828f19b 100644 --- a/storage/spider/spd_include.h +++ b/storage/spider/spd_include.h @@ -250,7 +250,6 @@ enum spider_malloc_id { SPD_MID_DB_MBASE_RESULT_FETCH_ROW_FROM_TMP_TABLE_3, SPD_MID_DB_MBASE_ROW_APPEND_ESCAPED_TO_STR_1, SPD_MID_DB_MBASE_ROW_CLONE_1, - SPD_MID_DB_MBASE_SET_LOOP_CHECK_1, SPD_MID_DB_MBASE_SET_SQL_MODE_1, SPD_MID_DB_MBASE_SET_TIME_ZONE_1, SPD_MID_DB_MBASE_SET_WAIT_TIMEOUT_1, @@ -1417,6 +1416,7 @@ typedef struct st_spider_table_mon st_spider_table_mon *next; } SPIDER_TABLE_MON; +/* List of `SPIDER_TABLE_MON's */ typedef struct st_spider_table_mon_list { char *key; diff --git a/storage/spider/spd_init_query.h b/storage/spider/spd_init_query.h index 35a250c7..ced68e03 100644 --- a/storage/spider/spd_init_query.h +++ b/storage/spider/spd_init_query.h @@ -20,8 +20,16 @@ */ static LEX_STRING spider_init_queries[] = { + /* Use the default SQL_MODE for this connection. */ {C_STRING_WITH_LEN( - "SET @@SQL_MODE = REPLACE(@@SQL_MODE, 'ORACLE', '');" + "SET @@SQL_MODE = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO," + "NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';" + )}, + {C_STRING_WITH_LEN( + "SET @@OLD_MODE = CONCAT(@@OLD_MODE, ',UTF8_IS_UTF8MB3');" + )}, + {C_STRING_WITH_LEN( + "SET tx_read_only = off;" )}, {C_STRING_WITH_LEN( "create table if not exists mysql.spider_xa(" diff --git a/storage/spider/spd_ping_table.cc b/storage/spider/spd_ping_table.cc index 722b4cca..80b40c62 100644 --- a/storage/spider/spd_ping_table.cc +++ b/storage/spider/spd_ping_table.cc @@ -50,6 +50,8 @@ extern PSI_mutex_key spd_key_mutex_mon_list_update_status; extern PSI_mutex_key spd_key_mutex_mon_table_cache; #endif +/* Array (of size `spider_udf_table_mon_mutex_count') of hashes of +`SPIDER_TABLE_MON_LIST'. */ HASH *spider_udf_table_mon_list_hash; uint spider_udf_table_mon_list_hash_id; const char *spider_udf_table_mon_list_hash_func_name; @@ -59,23 +61,43 @@ pthread_mutex_t *spider_udf_table_mon_mutexes; pthread_cond_t *spider_udf_table_mon_conds; pthread_mutex_t spider_mon_table_cache_mutex; +/* A cache to store distinct SPIDER_MON_KEYs with db name, table name +and link id read from mysql.spider_link_mon_servers table. Initialised +and populated in spider_init_ping_table_mon_cache(), and used in +spider_ping_table_cache_compare(). The udf +spider_flush_table_mon_cache is used to flag a initialisation. */ DYNAMIC_ARRAY spider_mon_table_cache; uint spider_mon_table_cache_id; const char *spider_mon_table_cache_func_name; const char *spider_mon_table_cache_file_name; ulong spider_mon_table_cache_line_no; -volatile ulonglong spider_mon_table_cache_version = 0; -volatile ulonglong spider_mon_table_cache_version_req = 1; - +/* The mon table cache version, initialised at 0, and always no +greater than spider_mon_table_cache_version_req. When the inequality +is strict, an initialisation of spider_mon_table_cache will be +triggered. */ +volatile ulonglong spider_mon_table_cache_version; +/* The required mon table cache version, incremented by one by the +udf spider_flush_table_mon_cache */ +volatile ulonglong spider_mon_table_cache_version_req; + + /* Get or create a `SPIDER_TABLE_MON_LIST' for a key `str' */ SPIDER_TABLE_MON_LIST *spider_get_ping_table_mon_list( SPIDER_TRX *trx, THD *thd, - spider_string *str, + spider_string *str, /* The key to search in + `spider_udf_table_mon_list_hash', + usually in the format of + "./$db_name/$table_name000000000$link_idx" */ uint conv_name_length, int link_idx, char *static_link_id, uint static_link_id_length, - uint32 server_id, + uint32 server_id, /* The server id of the monitor + server, used for creating a new + table mon list having a + `SPIDER_TABLE_MON' corresponding to + the server id as the `current' + field */ bool need_lock, int *error_num ) { @@ -85,6 +107,7 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_mon_list( ulonglong mon_table_cache_version; my_hash_value_type hash_value; DBUG_ENTER("spider_get_ping_table_mon_list"); + /* Reset the cache if the version does not match the requirement */ if (spider_mon_table_cache_version != spider_mon_table_cache_version_req) { SPD_INIT_ALLOC_ROOT(&mem_root, 4096, 0, MYF(MY_WME)); @@ -97,6 +120,9 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_mon_list( free_root(&mem_root, MYF(0)); } + /* Search for the table mon list in the hash, if one is not found or + if it is found but has the wrong cache version, create and + initialise a new one. */ mutex_hash= spider_udf_calc_hash(str->c_ptr(), spider_udf_table_mon_mutex_count); DBUG_PRINT("info",("spider hash key=%s", str->c_ptr())); @@ -113,12 +139,15 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_mon_list( table_mon_list->mon_table_cache_version != mon_table_cache_version ) { + /* If table_mon_list is found but the cache version does not + match, remove it from the hash and free it. */ if ( table_mon_list && table_mon_list->mon_table_cache_version != mon_table_cache_version ) spider_release_ping_table_mon_list_loop(mutex_hash, table_mon_list); - + /* create and initialise `table_mon_list' and insert it into the + hash */ if (!(table_mon_list = spider_get_ping_table_tgt(thd, str->c_ptr(), conv_name_length, link_idx, static_link_id, static_link_id_length, server_id, str, need_lock, error_num))) @@ -240,6 +269,14 @@ int spider_release_ping_table_mon_list( DBUG_RETURN(0); } +/* + Look for a `SPIDER_MON_KEY` in `spider_mon_table_cache' whose db and + table name and link_idx matching `name' and `link_idx' with wild + card matching. If a match is found, create `SPIDER_TABLE_MON's from + all rows in mysql.spider_link_mon_servers that match the info in the + `SPIDER_MON_KEY' and populate the `table_mon_list' with these + `SPIDER_TABLE_MON's. +*/ int spider_get_ping_table_mon( THD *thd, SPIDER_TABLE_MON_LIST *table_mon_list, @@ -316,6 +353,8 @@ int spider_get_ping_table_mon( goto error; create_table_mon: + /* Find the first row in mysql.spider_link_mon_servers matching the + db name, table name and link_idx */ if ((error_num = spider_get_sys_table_by_idx(table_link_mon, table_key, table_link_mon->s->primary_key, 3))) { @@ -323,6 +362,9 @@ create_table_mon: goto error; } + /* create one `SPIDER_TABLE_MON' per row in + mysql.spider_link_mon_servers with matching db name, table name and + link_idx, and add it to `table_mon_list'. */ do { if (!(table_mon = (SPIDER_TABLE_MON *) spider_bulk_malloc(spider_current_trx, SPD_MID_GET_PING_TABLE_MON_1, MYF(MY_WME | MY_ZEROFILL), @@ -356,7 +398,7 @@ create_table_mon: (error_num = spider_get_sys_link_mon_server_id( table_link_mon, &table_mon->server_id, mem_root)) || (error_num = spider_get_sys_link_mon_connect_info( - table_link_mon, tmp_share, 0, mem_root)) + table_link_mon, tmp_share, mem_root)) ) { table_link_mon->file->print_error(error_num, MYF(0)); spider_sys_index_end(table_link_mon); @@ -418,15 +460,21 @@ error: DBUG_RETURN(error_num); } +/* + creates and return table_mon_list associated with table with `name' + and `link_idx'th link. +*/ SPIDER_TABLE_MON_LIST *spider_get_ping_table_tgt( THD *thd, - char *name, + char *name, /* The table name, usually fully qualified */ uint name_length, int link_idx, char *static_link_id, uint static_link_id_length, - uint32 server_id, - spider_string *str, + uint32 server_id, /* The server_id will determine the + `current' field of the returned + `SPIDER_TABLE_MON_LIST'. */ + spider_string *str, /* str->c_ptr() == name */ bool need_lock, int *error_num ) { @@ -471,6 +519,7 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_tgt( memcpy(key_str, str->ptr(), table_mon_list->key_length); tmp_share->access_charset = thd->variables.character_set_client; + /* Open mysql.spider_tables */ if ( !(table_tables = spider_open_sys_table( thd, SPIDER_SYS_TABLES_TABLE_NAME_STR, @@ -480,6 +529,8 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_tgt( my_error(*error_num, MYF(0)); goto error; } + /* store db and table names and link idx in mysql.spider_tables for + reading */ spider_store_tables_name(table_tables, name, name_length); if (static_link_id) { @@ -503,9 +554,10 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_tgt( goto error; } } + /* Populate tmp_share with info read from mysql.spider_tables */ if ( (*error_num = spider_get_sys_tables_connect_info( - table_tables, tmp_share, 0, &mem_root)) || + table_tables, tmp_share, &mem_root)) || (*error_num = spider_get_sys_tables_link_status( table_tables, tmp_share, 0, &mem_root)) ) { @@ -527,9 +579,8 @@ SPIDER_TABLE_MON_LIST *spider_get_ping_table_tgt( tmp_share, name, name_length )) || (*error_num = spider_create_conn_keys(tmp_share)) || -/* - (*error_num = spider_db_create_table_names_str(tmp_share)) || -*/ + /* Pinally, populate `table_mon_list' with newly created + `SPIDER_TABLE_MON's */ (*error_num = spider_get_ping_table_mon( thd, table_mon_list, name, name_length, link_idx, server_id, &mem_root, need_lock)) @@ -819,6 +870,11 @@ error_open_table_tables: DBUG_RETURN(error_num); } +/* + Initialise `spider_mon_table_cache' by scanning the + mysql.spider_link_mon_servers table, creating distinct + `SPIDER_MON_KEY's with the info and inserting them into the cache. +*/ int spider_init_ping_table_mon_cache( THD *thd, MEM_ROOT *mem_root, @@ -847,6 +903,7 @@ int spider_init_ping_table_mon_cache( /* reset */ spider_mon_table_cache.elements = 0; + /* start at the first row */ if ((error_num = spider_sys_index_first(table_link_mon, table_link_mon->s->primary_key))) { @@ -863,10 +920,16 @@ int spider_init_ping_table_mon_cache( mon_key.table_name_length = SPIDER_SYS_LINK_MON_TABLE_TABLE_NAME_SIZE + 1; mon_key.link_id_length = SPIDER_SYS_LINK_MON_TABLE_LINK_ID_SIZE + 1; do { + /* update content of `mon_key' */ if ((error_num = spider_get_sys_link_mon_key(table_link_mon, &mon_key, mem_root, &same))) goto error_get_sys_link_mon_key; + /* `mon_key' has changed content. since + mysql.spider_link_mon_servers is indexed by db_name, + table_name, link_idx, and server_id, it is possible that + different server_ids share the same mon_key which only has + db_name, table_name, link_idx */ if (!same) { mon_key.sort = spider_calc_for_sort(3, mon_key.db_name, @@ -937,6 +1000,13 @@ error_open_sys_table: DBUG_RETURN(error_num); } +/* + Read from msyql.spider_link_mon_servers table fields the db name, + table name and link_id and search for them with wild card matching + in `spider_mon_table_cache'. store the db name, table name, and + link_id of the matching `SPIDER_MON_KEY' back to the table field on + success. +*/ int spider_ping_table_cache_compare( TABLE *table, MEM_ROOT *mem_root @@ -1213,9 +1283,6 @@ long long spider_ping_table_body( DBUG_PRINT("info",("spider mon_table_result->result_status=SPIDER_LINK_MON_NG 2")); if (table_mon_list->mon_status != SPIDER_LINK_MON_NG) { -/* - pthread_mutex_lock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_lock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (table_mon_list->mon_status != SPIDER_LINK_MON_NG) { @@ -1230,9 +1297,6 @@ long long spider_ping_table_body( conv_name.c_ptr(), conv_name_length, link_idx, TRUE); status_changed_to_ng = TRUE; } -/* - pthread_mutex_unlock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_unlock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (status_changed_to_ng) { @@ -1286,9 +1350,6 @@ long long spider_ping_table_body( DBUG_PRINT("info",("spider mon_table_result->result_status=SPIDER_LINK_MON_NG 3")); if (table_mon_list->mon_status != SPIDER_LINK_MON_NG) { -/* - pthread_mutex_lock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_lock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (table_mon_list->mon_status != SPIDER_LINK_MON_NG) { @@ -1303,9 +1364,6 @@ long long spider_ping_table_body( conv_name.c_ptr(), conv_name_length, link_idx, TRUE); status_changed_to_ng = TRUE; } -/* - pthread_mutex_unlock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_unlock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (status_changed_to_ng) { @@ -1349,9 +1407,6 @@ long long spider_ping_table_body( mon_table_result->result_status == SPIDER_LINK_MON_NG && table_mon_list->mon_status != SPIDER_LINK_MON_NG ) { -/* - pthread_mutex_lock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_lock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (table_mon_list->mon_status != SPIDER_LINK_MON_NG) { @@ -1366,9 +1421,6 @@ long long spider_ping_table_body( conv_name.c_ptr(), conv_name_length, link_idx, TRUE); status_changed_to_ng = TRUE; } -/* - pthread_mutex_unlock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_unlock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (status_changed_to_ng) { @@ -1539,9 +1591,9 @@ int spider_ping_table_mon_from_table( SPIDER_SHARE *share, int base_link_idx, uint32 server_id, - char *conv_name, + char *conv_name, /* Usually fully qualified table name */ uint conv_name_length, - int link_idx, + int link_idx, /* The link id to ping */ char *where_clause, uint where_clause_length, long monitoring_kind, @@ -1551,9 +1603,6 @@ int spider_ping_table_mon_from_table( ) { int error_num = 0, current_mon_count, flags; uint32 first_sid; -/* - THD *thd = trx->thd; -*/ SPIDER_TABLE_MON_LIST *table_mon_list; SPIDER_TABLE_MON *table_mon; SPIDER_MON_TABLE_RESULT mon_table_result; @@ -1622,6 +1671,7 @@ int spider_ping_table_mon_from_table( if (monitoring_flag & 1) flags |= SPIDER_UDF_PING_TABLE_USE_ALL_MONITORING_NODES; + /* Get or create `table_mon_list' for `conv_name_str'. */ if (!(table_mon_list = spider_get_ping_table_mon_list(trx, thd, &conv_name_str, conv_name_length, link_idx, share->static_link_ids[link_idx], @@ -1653,6 +1703,8 @@ int spider_ping_table_mon_from_table( table_mon = table_mon_list->current; first_sid = table_mon->server_id; current_mon_count = 1; + /* Call spider_ping_table on each table_mon of `table_mon_list', + until one succeeds */ while (TRUE) { DBUG_PRINT("info",("spider thd->killed=%s", @@ -1697,16 +1749,13 @@ int spider_ping_table_mon_from_table( if (!spider_db_udf_ping_table_mon_next( thd, table_mon, mon_conn, &mon_table_result, conv_name, conv_name_length, link_idx, - where_clause, where_clause_length, -1, table_mon_list->list_size, + where_clause, where_clause_length, /*first_sid=*/-1, table_mon_list->list_size, 0, 0, 0, flags, monitoring_limit)) { if ( mon_table_result.result_status == SPIDER_LINK_MON_NG && table_mon_list->mon_status != SPIDER_LINK_MON_NG ) { -/* - pthread_mutex_lock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_lock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); if (table_mon_list->mon_status != SPIDER_LINK_MON_NG) { @@ -1721,9 +1770,6 @@ int spider_ping_table_mon_from_table( spider_sys_log_tables_link_failed(thd, conv_name, conv_name_length, link_idx, need_lock); } -/* - pthread_mutex_unlock(&table_mon_list->update_status_mutex); -*/ pthread_mutex_unlock(&spider_udf_table_mon_mutexes[table_mon_list->mutex_hash]); } table_mon_list->last_caller_result = mon_table_result.result_status; diff --git a/storage/spider/spd_sys_table.cc b/storage/spider/spd_sys_table.cc index 912e02a7..bd3e7a6f 100644 --- a/storage/spider/spd_sys_table.cc +++ b/storage/spider/spd_sys_table.cc @@ -574,6 +574,11 @@ int spider_check_sys_table_for_update_all_columns( HA_WHOLE_KEY, HA_READ_KEY_EXACT)); } +/* + Creates a key (`table_key') consisting of `col_count' key parts of + `idx'th index of the table, then positions an index cursor to that + key. +*/ int spider_get_sys_table_by_idx( TABLE *table, char *table_key, @@ -629,7 +634,7 @@ int spider_sys_index_next_same( int spider_sys_index_first( TABLE *table, - const int idx + const int idx /* which index to use */ ) { int error_num; DBUG_ENTER("spider_sys_index_first"); @@ -871,6 +876,10 @@ void spider_store_xa_member_info( DBUG_VOID_RETURN; } +/* + Store db and table names from `name' to `table's corresponding + fields +*/ void spider_store_tables_name( TABLE *table, const char *name, @@ -2149,14 +2158,16 @@ int spider_get_sys_tables( DBUG_RETURN(0); } +/* Read table info from mysql.spider_tables into a `SPIDER_SHARE' */ int spider_get_sys_tables_connect_info( - TABLE *table, - SPIDER_SHARE *share, - int link_idx, + TABLE *table, /* The mysql.spider_tables table */ + SPIDER_SHARE *share, /* The `SPIDER_SHARE' to + update info */ MEM_ROOT *mem_root ) { char *ptr; int error_num = 0; + const int link_idx= 0; DBUG_ENTER("spider_get_sys_tables_connect_info"); DBUG_PRINT("info",("spider link_idx:%d", link_idx)); if ((ptr = get_field(mem_root, table->field[SPIDER_TABLES_PRIORITY_POS]))) @@ -2431,9 +2442,14 @@ int spider_get_sys_tables_monitoring_binlog_pos_at_failing( DBUG_RETURN(error_num); } +/* + Read the link status from mysql.spider_tables into a `SPIDER_SHARE' + with default value 1 (`SPIDER_LINK_STATUS_OK') +*/ int spider_get_sys_tables_link_status( - TABLE *table, - SPIDER_SHARE *share, + TABLE *table, /* The mysql.spider_tables table */ + SPIDER_SHARE *share, /* The share to read link + status into */ int link_idx, MEM_ROOT *mem_root ) { @@ -2607,11 +2623,17 @@ error: DBUG_RETURN(error_num); } +/* Populate `mon_key' from the current row in `table' */ int spider_get_sys_link_mon_key( - TABLE *table, - SPIDER_MON_KEY *mon_key, + TABLE *table, /* the mysql.spider_link_mon_servers + table */ + SPIDER_MON_KEY *mon_key, /* output, to be populated in this + function */ MEM_ROOT *mem_root, - int *same + int *same /* output, true if the data from the + current row in the table agrees with + existing data in `mon_key' and false + otherwise */ ) { char *db_name, *table_name, *link_id; uint db_name_length, table_name_length, link_id_length; @@ -2627,6 +2649,7 @@ int spider_get_sys_link_mon_key( DBUG_RETURN(ER_SPIDER_SYS_TABLE_VERSION_NUM); } + /* get data for `mon_key' from the table record */ if (!(db_name= get_field(mem_root, table->field[SPIDER_LINK_MON_SERVERS_DB_NAME_POS]))) @@ -2678,9 +2701,12 @@ int spider_get_sys_link_mon_key( DBUG_RETURN(0); } +/* Get the server id from the spider_link_mon_servers table field */ int spider_get_sys_link_mon_server_id( - TABLE *table, - uint32 *server_id, + TABLE *table, /* the + mysql.spider_link_mon_servers + table */ + uint32 *server_id, /* output to server_id */ MEM_ROOT *mem_root ) { char *ptr; @@ -2694,14 +2720,17 @@ int spider_get_sys_link_mon_server_id( DBUG_RETURN(error_num); } +/* Get connect info from the spider_link_mon_servers table fields */ int spider_get_sys_link_mon_connect_info( - TABLE *table, - SPIDER_SHARE *share, - int link_idx, + TABLE *table, /* The + mysql.spider_link_mon_servers + table */ + SPIDER_SHARE *share, /* The output spider_share */ MEM_ROOT *mem_root ) { char *ptr; int error_num = 0; + const int link_idx= 0; DBUG_ENTER("spider_get_sys_link_mon_connect_info"); if ( !table->field[SPIDER_LINK_MON_SERVERS_SERVER_POS]->is_null() && @@ -2930,9 +2959,6 @@ int spider_get_link_statuses( if ( (error_num == HA_ERR_KEY_NOT_FOUND || error_num == HA_ERR_END_OF_FILE) ) { -/* - table->file->print_error(error_num, MYF(0)); -*/ DBUG_RETURN(error_num); } } else if ((error_num = diff --git a/storage/spider/spd_sys_table.h b/storage/spider/spd_sys_table.h index f1983323..ce3897c7 100644 --- a/storage/spider/spd_sys_table.h +++ b/storage/spider/spd_sys_table.h @@ -56,6 +56,7 @@ #define SPIDER_SYS_LINK_MON_TABLE_TABLE_NAME_SIZE 64 #define SPIDER_SYS_LINK_MON_TABLE_LINK_ID_SIZE 64 +/* For insertion into `spider_mon_table_cache'. */ class SPIDER_MON_KEY: public SPIDER_SORT { public: @@ -401,7 +402,6 @@ int spider_get_sys_tables( int spider_get_sys_tables_connect_info( TABLE *table, SPIDER_SHARE *share, - int link_idx, MEM_ROOT *mem_root ); @@ -478,7 +478,6 @@ int spider_get_sys_link_mon_server_id( int spider_get_sys_link_mon_connect_info( TABLE *table, SPIDER_SHARE *share, - int link_idx, MEM_ROOT *mem_root ); diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index 208b804f..4b0690be 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -124,6 +124,11 @@ extern SPIDER_DBTON spider_dbton_mysql; extern SPIDER_DBTON spider_dbton_mariadb; SPIDER_THREAD *spider_table_sts_threads; SPIDER_THREAD *spider_table_crd_threads; +extern volatile ulonglong spider_mon_table_cache_version; +extern volatile ulonglong spider_mon_table_cache_version_req; +extern ulonglong spider_conn_id; +extern Time_zone *UTC; +extern ulonglong spider_thread_id; #ifdef HAVE_PSI_INTERFACE PSI_mutex_key spd_key_mutex_tbl; @@ -3468,11 +3473,13 @@ error_alloc_conn_string: DBUG_RETURN(error_num); } +/* Set default connect info of a SPIDER_SHARE if needed */ int spider_set_connect_info_default( - SPIDER_SHARE *share, - partition_element *part_elem, - partition_element *sub_elem, - TABLE_SHARE *table_share + SPIDER_SHARE *share, /* The `SPIDER_SHARE' to set + default connect info */ + partition_element *part_elem, /* partition info used as input */ + partition_element *sub_elem, /* subpartition info used as input */ + TABLE_SHARE *table_share /* table share info used as input */ ) { bool check_socket; bool check_database; @@ -3706,22 +3713,6 @@ int spider_set_connect_info_default( } } -/* - if (!share->static_link_ids[roop_count]) - { - DBUG_PRINT("info",("spider create default static_link_ids")); - share->static_link_ids_lengths[roop_count] = - SPIDER_DB_STATIC_LINK_ID_LEN; - if ( - !(share->static_link_ids[roop_count] = spider_create_string( - SPIDER_DB_STATIC_LINK_ID_STR, - share->static_link_ids_lengths[roop_count])) - ) { - DBUG_RETURN(HA_ERR_OUT_OF_MEM); - } - } -*/ - if (port_has_default_value) { share->tgt_ports[roop_count] = MYSQL_PORT; @@ -3811,7 +3802,11 @@ int spider_set_connect_info_default( DBUG_RETURN(0); } - +/* + This function is a no-op if all share->tgt_dbs and + share->tgt_table_names are non-null, otherwise it may assign them + with db_name and table_name +*/ int spider_set_connect_info_default_db_table( SPIDER_SHARE *share, const char *db_name, @@ -3897,6 +3892,11 @@ int spider_set_connect_info_default_db_table( DBUG_RETURN(0); } +/* + Parse `dbtable_name' into db name and table name, and call + spider_set_connect_info_default_db_table() to set the db/table name + values of `share' if needed +*/ int spider_set_connect_info_default_dbtable( SPIDER_SHARE *share, const char *dbtable_name, @@ -4256,12 +4256,12 @@ SPIDER_SHARE *spider_create_share( share->table.key_info = table_share->key_info; share->table.read_set = &table_share->all_set; - if (table_share->keys > 0 && - !(share->key_hint = new spider_string[table_share->keys]) - ) { - *error_num = HA_ERR_OUT_OF_MEM; - goto error_init_hint_string; - } + if (table_share->keys > 0) + if (!(share->key_hint = new spider_string[table_share->keys])) + { + *error_num = HA_ERR_OUT_OF_MEM; + goto error_init_hint_string; + } for (roop_count = 0; roop_count < (int) table_share->keys; roop_count++) share->key_hint[roop_count].init_calc_mem(SPD_MID_CREATE_SHARE_2); DBUG_PRINT("info",("spider share->key_hint=%p", share->key_hint)); @@ -5797,7 +5797,7 @@ int spider_open_all_tables( (error_num = spider_get_sys_tables( table_tables, &db_name, &table_name, &mem_root)) || (error_num = spider_get_sys_tables_connect_info( - table_tables, &tmp_share, 0, &mem_root)) || + table_tables, &tmp_share, &mem_root)) || (error_num = spider_set_connect_info_default( &tmp_share, NULL, @@ -6084,7 +6084,11 @@ int spider_close_connection( } spider_rollback(spider_hton_ptr, thd, TRUE); + + Dummy_error_handler deh; // suppress network errors at this stage + thd->push_internal_handler(&deh); spider_free_trx(trx, TRUE, false); + thd->pop_internal_handler(); DBUG_RETURN(0); } @@ -6255,6 +6259,7 @@ int spider_db_done( pthread_mutex_destroy(&spider_tbl_mutex); pthread_attr_destroy(&spider_pt_attr); + spider_hton_ptr= NULL; for (roop_count = 0; roop_count < SPIDER_MEM_CALC_LIST_NUM; roop_count++) { if (spider_alloc_func_name[roop_count]) @@ -6346,6 +6351,12 @@ int spider_db_init( handlerton *spider_hton = (handlerton *)p; DBUG_ENTER("spider_db_init"); + spider_mon_table_cache_version= 0; + spider_mon_table_cache_version_req= 1; + spider_conn_id= 1; + spider_conn_mutex_id= 0; + UTC = 0; + spider_thread_id = 1; const LEX_CSTRING aria_name={STRING_WITH_LEN("Aria")}; if (!plugin_is_ready(&aria_name, MYSQL_STORAGE_ENGINE_PLUGIN)) DBUG_RETURN(HA_ERR_RETRY_INIT); @@ -6579,11 +6590,6 @@ int spider_db_init( spider_udf_table_mon_list_hash[roop_count].array.size_of_element); } - if (spider_init_system_tables()) - { - goto error_system_table_creation; - } - if (!(spider_table_sts_threads = (SPIDER_THREAD *) spider_bulk_malloc(NULL, SPD_MID_DB_INIT_12, MYF(MY_WME | MY_ZEROFILL), &spider_table_sts_threads, (uint) (sizeof(SPIDER_THREAD) * @@ -6592,7 +6598,7 @@ int spider_db_init( spider_param_table_crd_thread_count()), NullS)) ) - goto error_alloc_mon_mutxes; + goto error_alloc_table_sts_crd_threads; for (roop_count = 0; roop_count < (int) spider_param_table_sts_thread_count(); @@ -6653,6 +6659,7 @@ error_init_table_sts_threads: { spider_free_sts_threads(&spider_table_sts_threads[roop_count]); } +error_alloc_table_sts_crd_threads: spider_free(NULL, spider_table_sts_threads, MYF(0)); roop_count= spider_udf_table_mon_mutex_count - 1; error_init_udf_table_mon_list_hash: @@ -6668,7 +6675,6 @@ error_init_udf_table_mon_list_hash: error_init_udf_table_mon_cond: for (; roop_count >= 0; roop_count--) pthread_cond_destroy(&spider_udf_table_mon_conds[roop_count]); -error_system_table_creation: roop_count= spider_udf_table_mon_mutex_count - 1; error_init_udf_table_mon_mutex: for (; roop_count >= 0; roop_count--) diff --git a/storage/spider/spd_trx.cc b/storage/spider/spd_trx.cc index ebf9a777..b7a92286 100644 --- a/storage/spider/spd_trx.cc +++ b/storage/spider/spd_trx.cc @@ -41,7 +41,7 @@ extern struct charset_info_st *spd_charset_utf8mb3_bin; extern handlerton *spider_hton_ptr; extern SPIDER_DBTON spider_dbton[SPIDER_DBTON_SIZE]; pthread_mutex_t spider_thread_id_mutex; -ulonglong spider_thread_id = 1; +ulonglong spider_thread_id; #ifdef HAVE_PSI_INTERFACE extern PSI_mutex_key spd_key_mutex_udf_table; |