diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-04 18:04:16 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-04 18:04:16 +0000 |
commit | a68fb2d8219f6bccc573009600e9f23e89226a5e (patch) | |
tree | d742d35d14ae816e99293d2b01face30e9f3a46b /wsrep-lib | |
parent | Initial commit. (diff) | |
download | mariadb-10.6-upstream.tar.xz mariadb-10.6-upstream.zip |
Adding upstream version 1:10.6.11.upstream/1%10.6.11upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
174 files changed, 36331 insertions, 0 deletions
diff --git a/wsrep-lib/.clang-format b/wsrep-lib/.clang-format new file mode 100644 index 00000000..2ea756ab --- /dev/null +++ b/wsrep-lib/.clang-format @@ -0,0 +1,58 @@ +--- +Language: Cpp +# BasedOnStyle: WebKit +AccessModifierOffset: -4 +ConstructorInitializerIndentWidth: 4 +AlignEscapedNewlinesLeft: false +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AlwaysBreakTemplateDeclarations: false +AlwaysBreakBeforeMultilineStrings: false +BreakBeforeBinaryOperators: true +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: true +BinPackParameters: true +ColumnLimit: 80 +ConstructorInitializerAllOnOneLineOrOnePerLine: false +DerivePointerAlignment: false +ExperimentalAutoDetectBinPacking: false +IndentCaseLabels: false +IndentWrappedFunctionNames: false +IndentFunctionDeclarationAfterType: false +MaxEmptyLinesToKeep: 1 +KeepEmptyLinesAtTheStartOfBlocks: true +NamespaceIndentation: All +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakString: 1000 +PenaltyBreakFirstLessLess: 120 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Left +SpacesBeforeTrailingComments: 1 +Cpp11BracedListStyle: false +Standard: Cpp11 +IndentWidth: 4 +TabWidth: 8 +UseTab: Never +BreakBeforeBraces: Allman +SpacesInParentheses: false +SpacesInAngles: false +SpaceInEmptyParentheses: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: true +SpaceBeforeAssignmentOperators: true +ContinuationIndentWidth: 4 +CommentPragmas: '^ IWYU pragma:' +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +SpaceBeforeParens: ControlStatements +DisableFormat: false +... + diff --git a/wsrep-lib/.github/workflows/build.yml b/wsrep-lib/.github/workflows/build.yml new file mode 100644 index 00000000..7f317a75 --- /dev/null +++ b/wsrep-lib/.github/workflows/build.yml @@ -0,0 +1,186 @@ +name: Build + +on: + pull_request: + branches: + - master + +jobs: + build: + runs-on: ${{ matrix.config.os }} + strategy: + fail-fast: false + matrix: + config: + # GCC/G++ + - os: ubuntu-18.04 + CC: gcc + version: "4.8" + type: Debug + - os: ubuntu-18.04 + CC: gcc + version: "4.8" + type: Release + - os: ubuntu-18.04 + CC: gcc + version: "5" + type: Debug + - os: ubuntu-18.04 + CC: gcc + version: "5" + type: Release + - os: ubuntu-18.04 + CC: gcc + version: "6" + type: Debug + - os: ubuntu-18.04 + CC: gcc + version: "6" + type: Release + - os: ubuntu-18.04 + CC: gcc + version: "7" + type: Debug + - os: ubuntu-18.04 + CC: gcc + version: "7" + type: Release + - os: ubuntu-18.04 + CC: gcc + version: "8" + type: Debug + - os: ubuntu-18.04 + CC: gcc + version: "8" + type: Release + - os: ubuntu-20.04 + CC: gcc + version: "9" + type: Debug + - os: ubuntu-20.04 + CC: gcc + version: "9" + type: Release + - os: ubuntu-20.04 + CC: gcc + version: "10" + type: Debug + - os: ubuntu-20.04 + CC: gcc + version: "10" + type: Release + - os: ubuntu-20.04 + CC: gcc + version: "11" + type: Debug + - os: ubuntu-20.04 + CC: gcc + version: "11" + type: Release + # Clang + - os: ubuntu-18.04 + CC: clang + version: "6.0" + type: Debug + - os: ubuntu-18.04 + CC: clang + version: "6.0" + type: Release + - os: ubuntu-20.04 + CC: clang + version: "10" + type: Debug + - os: ubuntu-20.04 + CC: clang + version: "10" + type: Release + - os: ubuntu-20.04 + CC: clang + version: "11" + type: Debug + - os: ubuntu-20.04 + CC: clang + version: "11" + type: Release + - os: ubuntu-20.04 + CC: clang + version: "12" + type: Debug + - os: ubuntu-20.04 + CC: clang + version: "12" + type: Release + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: "recursive" + + - name: Install compiler + run: | + sudo apt-get install -y ${{ matrix.config.CC }}-${{ matrix.config.version }} + if [ ${{ matrix.config.CC }} == "gcc" ] + then + sudo apt-get install -y g++-${{ matrix.config.version }} + fi + + - name: Install build dependencies + run: sudo apt-get install -y libboost-filesystem-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev ccache + + # See https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: cmake -P {0} + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) + message("::set-output name=timestamp::${current_date}") + + - name: Configure ccache + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ${{ matrix.config.os }}-${{ matrix.config.CC }}-${{ matrix.config.version }}-${{ matrix.config.type }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: ${{ matrix.config.os }}-${{ matrix.config.CC }}-${{ matrix.config.version }}-${{ matrix.config.type }}-ccache- + + - name: Create Build Environment + run: cmake -E make_directory ${{runner.workspace}}/build + + - name: Configure CMake + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + working-directory: ${{runner.workspace}}/build + run: | + export CC="ccache ${{ matrix.config.CC }}-${{ matrix.config.version }}" + if [ ${{ matrix.config.CC }} == "gcc" ] + then + export CXX="ccache g++-${{ matrix.config.version }}" + else + export CXX="ccache clang++-${{ matrix.config.version }}" + fi + if [ ${{ matrix.config.version }} == "4.8" ] + then + STRICT=OFF + DBSIM=OFF + else + STRICT=ON + DBSIM=ON + fi + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.config.type }} \ + -DWSREP_LIB_MAINTAINER_MODE:BOOL=ON \ + -DWSREP_LIB_STRICT_BUILD_FLAGS:BOOL=$STRICT \ + -DWSREP_LIB_WITH_DBSIM:BOOL=$DBSIM \ + -DWSREP_LIB_WITH_ASAN:BOOL=ON . + + - name: Build + working-directory: ${{runner.workspace}}/build + shell: bash + run: | + make -j3 VERBOSE=1 + ccache -s + + - name: Test + working-directory: ${{runner.workspace}}/build + shell: bash + run: ctest -C ${{ matrix.config.type }} --output-on-failure diff --git a/wsrep-lib/.gitignore b/wsrep-lib/.gitignore new file mode 100644 index 00000000..1fa1627f --- /dev/null +++ b/wsrep-lib/.gitignore @@ -0,0 +1,15 @@ +# CMake +CMakeCache.txt +CMakeFiles/ +CTestTestfile.cmake +Makefile +cmake_install.cmake + +# Built binaries +dbsim/dbsim +src/libwsrep-lib.a +test/wsrep-lib_test +wsrep-API/libwsrep_api_v26.a + +# Gcov generated files +*.dgcov diff --git a/wsrep-lib/.gitmodules b/wsrep-lib/.gitmodules new file mode 100644 index 00000000..886054a3 --- /dev/null +++ b/wsrep-lib/.gitmodules @@ -0,0 +1,3 @@ +[submodule "wsrep-API/v26"] + path = wsrep-API/v26 + url = https://github.com/codership/wsrep-API.git diff --git a/wsrep-lib/CMakeLists.txt b/wsrep-lib/CMakeLists.txt new file mode 100644 index 00000000..d928fa53 --- /dev/null +++ b/wsrep-lib/CMakeLists.txt @@ -0,0 +1,211 @@ +# +# Copyright (C) 2021 Codership Oy <info@codership.com> +# + +cmake_minimum_required (VERSION 2.8) + +# Parse version from version header file and store it into +# WSREP_LIB_VERSION. +file (READ "include/wsrep/version.hpp" ver) +string(REGEX MATCH "WSREP_LIB_VERSION_MAJOR ([0-9]*)" _ ${ver}) +set(ver_major ${CMAKE_MATCH_1}) +file (READ "include/wsrep/version.hpp" ver) +string(REGEX MATCH "WSREP_LIB_VERSION_MINOR ([0-9]*)" _ ${ver}) +set(ver_minor ${CMAKE_MATCH_1}) +file (READ "include/wsrep/version.hpp" ver) +string(REGEX MATCH "WSREP_LIB_VERSION_PATCH ([0-9]*)" _ ${ver}) +set(ver_patch ${CMAKE_MATCH_1}) +set(WSREP_LIB_VERSION "${ver_major}.${ver_minor}.${ver_patch}") +message(STATUS "Wsrep-lib version: ${WSREP_LIB_VERSION}") + +if (POLICY CMP0048) + cmake_policy(SET CMP0048 NEW) + project(wsrep-lib VERSION ${WSREP_LIB_VERSION}) +else() + project(wsrep-lib) +endif() + +if (POLICY CMP0057) + cmake_policy(SET CMP0057 NEW) +endif() + +if (POLICY CMP0077) + cmake_policy(SET CMP0077 NEW) +endif() + +include(CheckIncludeFileCXX) +include(CheckLibraryExists) +include(CheckCXXCompilerFlag) + +# Options + +# Compile unit tests +option(WSREP_LIB_WITH_UNIT_TESTS "Compile unit tests" ON) +if (WSREP_LIB_WITH_UNIT_TESTS) + # Run tests automatically by default if compiled + option(WSREP_LIB_WITH_AUTO_TEST "Run unit tests automatically after build" OFF) + option(WSREP_LIB_WITH_UNIT_TESTS_EXTRA "Compile unit tests that may require additional software" OFF) +endif() + +# Build a sample program +option(WSREP_LIB_WITH_DBSIM "Compile sample dbsim program" ON) + +option(WSREP_LIB_WITH_ASAN "Enable address sanitizer" OFF) +option(WSREP_LIB_WITH_TSAN "Enable thread sanitizer" OFF) + +option(WSREP_LIB_WITH_DOCUMENTATION "Generate documentation" OFF) +option(WSREP_LIB_WITH_COVERAGE "Compile with coverage instrumentation" OFF) + +option(WSREP_LIB_STRICT_BUILD_FLAGS "Compile with strict build flags" OFF) +option(WSREP_LIB_MAINTAINER_MODE "Fail compilation on any warnings" OFF) + +# Compiler options + +# Set std to C++0x/C++11 if superproject has not set standard yet +if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) + string(FIND "${CMAKE_CXX_FLAGS}" "-std=" HAVE_STD) + if (HAVE_STD EQUAL -1) + if (CMAKE_VERSION VERSION_LESS "3.1") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND + CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8.1) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() + else() + set(CMAKE_CXX_STANDARD 11) + endif() + endif() +endif() + +# C flags +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wconversion") +# CXX flags +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Woverloaded-virtual -Wconversion -g") +check_cxx_compiler_flag("-Wsuggest-override" HAVE_SUGGEST_OVERRIDE) +if (HAVE_SUGGEST_OVERRIDE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsuggest-override") +endif() + +if (WSREP_LIB_STRICT_BUILD_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weffc++") +endif() +if (WSREP_LIB_MAINTAINER_MODE) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") +endif() + +# Enable extra libstdc++ assertions with debug build. +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions("-D_GLIBCXX_ASSERTIONS") +else() + # Make sure that NDEBUG is enabled in release builds even + # if the parent project does not define it. + add_definitions("-DNDEBUG") +endif() + +# Set up include directories +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include") +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/wsrep-API") + +# Find libraries. +CHECK_LIBRARY_EXISTS(dl dlopen "" WSREP_LIB_HAVE_LIBDL) +if (WSREP_LIB_HAVE_LIBDL) + set(WSREP_LIB_LIBDL "dl") +else() + set(WSREP_LIB_LIBDL "") +endif() + +set(MIN_BOOST_VERSION "1.54.0") +if (WSREP_LIB_WITH_UNIT_TESTS) + if (WSREP_LIB_WITH_UNIT_TESTS_EXTRA) + set(json_HEADER "boost/json/src.hpp") + # Extra tests may require packages from very recent boost which may be + # unavailable on the system. In such case download private boost distro. + check_include_file_cxx(${json_HEADER} system_json_FOUND) + if (NOT system_json_FOUND) + if (NOT WITH_BOOST) + set(WITH_BOOST "${CMAKE_SOURCE_DIR}/../boost") + endif() + set(DOWNLOAD_BOOST ON) + include (cmake/boost.cmake) + set(MIN_BOOST_VERSION "${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH}") + message(STATUS "Boost includes: ${BOOST_INCLUDE_DIR}, ver: ${MIN_BOOST_VERSION}") + find_package(Boost ${MIN_BOOST_VERSION} REQUIRED + COMPONENTS json headers + PATHS ${WITH_BOOST}/lib/cmake + NO_DEFAULT_PATH + ) + # Boost 1.76.0 comes very sloppy (and not only in JSON department) + # - need to disable some checks. + set(ADDITIONAL_CXX_FLAGS "-Wno-effc++ -Wno-conversion -Wno-suggest-override -Wno-overloaded-virtual") + set(ADDITIONAL_CXX_INCLUDES ${BOOST_INCLUDE_DIR}) + check_include_file_cxx(${json_HEADER} json_FOUND + "-I${ADDITIONAL_CXX_INCLUDES} ${ADDITIONAL_CXX_FLAGS}" + ) + if (NOT json_FOUND) + message(FATAL_ERROR "Required header 'boost/json.hpp' not found: ${json_FOUND}") + else() + include_directories(SYSTEM ${ADDITIONAL_CXX_INCLUDES}) + endif() + endif() + endif() + find_package(Boost ${MIN_BOOST_VERSION} REQUIRED + unit_test_framework + ) +endif() + +if (WSREP_LIB_WITH_DBSIM) + find_package(Boost ${MIN_BOOST_VERSION} REQUIRED + program_options + filesystem + thread + ) +endif() + +if (WSREP_LIB_WITH_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") +endif() +if (WSREP_LIB_WITH_TSAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") +endif() + +# Coverage +# +# To produce a coverage report, call cmake with -DWITH_COVERAGE=ON, +# run +# +# make +# make test +# make coverage_report +# +# The coverage report output will be in directory coverage_report/index.html +# +if (WSREP_LIB_WITH_COVERAGE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") +endif() + +add_custom_target(coverage_report + lcov --base-directory ${CMAKE_CURRENT_SOURCE_DIR} --capture --directory ${CMAKE_CURRENT_BINARY_DIR} --output lcov.info --no-external --quiet + COMMAND genhtml --output-directory coverage_report lcov.info) + + +if (WSREP_LIB_WITH_DOCUMENTATION) + find_package(Doxygen REQUIRED) + add_custom_target(doc ALL + COMMAND doxygen ${CMAKE_CURRENT_SOURCE_DIR}/doc/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/doc + COMMENT "Generating documentation with Doxygen" + VERBATIM) +endif() + +add_subdirectory(src) +add_subdirectory(wsrep-API) +if (WSREP_LIB_WITH_UNIT_TESTS) + enable_testing() + add_subdirectory(test) +endif() +if (WSREP_LIB_WITH_DBSIM) + add_subdirectory(dbsim) +endif() diff --git a/wsrep-lib/CONTRIBUTORS.txt b/wsrep-lib/CONTRIBUTORS.txt new file mode 100644 index 00000000..5b7072d7 --- /dev/null +++ b/wsrep-lib/CONTRIBUTORS.txt @@ -0,0 +1,30 @@ +All contributors are required to add their name and [Github username/email] +to this file in connection with their first contribution. If you are making +a contribution on behalf of a company, you should add the said company name. + +By adding your name and [Github username/email] to this file you agree that +your contribution is a contribution under a contributor agreement between +you and Codership Oy. To the extent that you are an employee of a company and +contribute in that role, you confirm that your contribution is a contribution +under the contribution license agreement between your employer and Codership +Oy; and that you have the authorization to give such confirmation. You confirm +that you have read, understood and signed the contributor license agreement +applicable to you. + +For the individual contributor agreement see file CONTRIBUTOR_AGREEMENT.txt +in the same directory as this file. + +Authors from Codership Oy: + + * Seppo Jaakola <seppo.jaakola@galeracluster.com>, Codership Oy + * Teemu Ollakka <teemu.ollakka@galeracluster.com>, Codership Oy + * Leandro Pacheco de Sousa <leandro.pacheco@galeracluster.com>, Codership Oy + * Alexey Yurchenko <alexey.yurchenko@galeracluster.com>, Codership Oy + * Mario Karuza <mario.karuza@galeracluster.com>, Codership Oy + * Daniele Sciascia <daniele.sciascia@galeracluster.com>, Codership Oy + [Codership employees, add name and email/username above this line, but leave this line intact] + +Other contributors: + + * Otto Kekäläinen <otto@kekalainen.net> + [add name and email/username above this line, but leave this line intact] diff --git a/wsrep-lib/CONTRIBUTOR_AGREEMENT.txt b/wsrep-lib/CONTRIBUTOR_AGREEMENT.txt new file mode 100644 index 00000000..8bdec2fd --- /dev/null +++ b/wsrep-lib/CONTRIBUTOR_AGREEMENT.txt @@ -0,0 +1,218 @@ + Codership + Contributor License Agreement + Codership CLA + +Thank you for your interest in contributing to Galera Cluster, a project +managed by Codership Oy, a legal entity established under Finnish laws, with +its principal address at Pohjolankatu 64 A, 00600 Helsinki Finland ("We", "Us" +or "Our"). + +This contributor agreement ("Agreement") documents the rights granted by +contributors to Us. To make this document effective, please either accept it +in an electronic service such as clahub.com or sign and scan it and send it to +Us by email. This is a legally binding document, so please read it carefully +before agreeing to it. This Agreement covers the Galera Cluster project: the +Galera library, the wsrep-lib library, the wsrep-API library, the Wsrep patch +for MySQL and other eventual patches to MySQL or other technologies. + +1. Definitions + +"You" means the individual who Submits a Contribution to Us or the Legal +Entity on behalf of whom a Contribution has been Submitted to Us. "Legal +Entity" means an entity which is not a natural person. "Affiliates" means +other Legal Entities that control, are controlled by, or under common control +with that Legal Entity. For the purposes of this definition, "control" means +(i) the power, direct or indirect, to cause the direction or management of +such Legal Entity, whether by contract or otherwise, (ii) ownership of fifty +percent (50%) or more of the outstanding shares or securities which vote to +elect the management or other persons who direct such Legal Entity or (iii) +beneficial ownership of such entity. + +"Contribution" means any work of authorship that is Submitted by You to Us in +which You own or assert ownership of the Copyright. If You do not own the +Copyright in the entire work of authorship, you need to have a separate +permission from Us. + +"Copyright" means all rights protecting works of authorship owned or +controlled by You, including copyright, moral and neighboring rights, as +appropriate, for the full term of their existence including any extensions by +You. + +"Material" means the work of authorship which is made available by Us to third +parties, i.e. the Galera library, the Wsrep patch for MySQL; other eventual +patches to MySQL; other eventual patches to other database technologies; all +these together with a database technology, such as MySQL, or its +derivatives. After You Submit the Contribution, it may be included in the +Material. + +"Submit" means any form of electronic, verbal, or written communication sent +to Us or our representatives, including but not limited to electronic mailing +lists, source code control systems, and issue tracking systems that are +managed by, or on behalf of, Us for the purpose of discussing and improving +the Material, provided that such communication is (i) conspicuously marked or +otherwise designated in writing by You or Your employee as a "Contribution" or +(ii) submitted in source code control system pursuant to Section 3 (e). + +"Submission Date" means the date on which You Submit a Contribution to Us. + +"Effective Date" means the date You execute this Agreement or the date You +first Submit a Contribution to Us, whichever is earlier. + +"Media" means any portion of a Contribution which is not software. + +2. Grant of Rights + +2.1 Copyright License + +(a) You retain ownership of the Copyright in Your Contribution and have the +same rights to use or license the Contribution which You would have had +without entering into the Agreement. In case we have in writing permitted +submitting a sublicense to licensed rights, You will not transfer the original +license, but grant us a sublicense in accordance with this Agreement. + +(b) To the maximum extent permitted by the relevant law, You grant to Us a +perpetual, worldwide, non-exclusive, transferable, royalty-free, irrevocable +license under the Copyright covering the Contribution, with the right to +sublicense such rights through multiple tiers of sublicensees, to reproduce, +modify, display, perform and distribute the Contribution as part of the +Material; provided that this license is conditioned upon compliance with +Section 2.3. + +2.2 Patent License + +For patent claims including, without limitation, method, process, and +apparatus claims which You, or in case You are a Legal Entity, You or Your +Affiliates, own, control or have the right to grant, now or in the future, You +grant to Us a perpetual, worldwide, non-exclusive, transferable, royalty-free, +irrevocable patent license, with the right to sublicense these rights to +multiple tiers of sublicensees, to make, have made, use, sell, offer for sale, +import and otherwise transfer the Contribution and the Contribution in +combination with the Material (and portions of such combination). This license +is granted only to the extent that the exercise of the licensed rights +infringes such patent claims; and provided that this license is conditioned +upon compliance with Section 2.3. + +2.3 Outbound License + +As a condition on the grant of rights in Sections 2.1 and 2.2, to the extent +we include Your Contribution or a part of it in a Material, we agree to +license the Contribution under the terms of the license or licenses which We +are using on the Submission Date for the Material or any licenses which are +approved by the Open Source Initiative ("OSI") on or after the Effective Date, +including both permissive and copyleft licenses, whether or not such licenses +are subsequently disapproved (including any right to adopt any future version +of a license if approved by the OSI). For clarity, this entitles us to license +Your Contribution also under a permissive open source license, such as the MIT +license, and include binaries created under the MIT license in a proprietary +licensed whole. + +In addition to above defined licenses, We may use the following licenses for +Media in the Contribution: Creative Commons BY 3.0 or Creative Commons BY-SA +3.0 (including the right to adopt any future version of a license). + +2.4 Moral Rights. + +If moral rights apply to the Contribution, to the maximum extent permitted by +law, You waive and agree not to assert such moral rights against Us or our +successors in interest, or any of our licensees, either direct or indirect. + +2.5 Enforcement. + +You, as a copyright holder to Your Contribution, hereby authorize us to +enforce the OSI approved license applied by Us to a Material, but only to the +extent Your Contribution has been included in a Material and always subject to +Our free discretion on whether such enforcement is necessary or not. + +2.6 Our Rights. + +You acknowledge that We are not obligated to use Your Contribution as part of +the Material and may decide to include any Contribution We consider +appropriate. + +2.7 Reservation of Rights. + +Any rights not expressly licensed under this section are expressly reserved by +You. + +3. Agreement + +You confirm that: + +(a) You have the legal authority to enter into this Agreement. + +(b) You or Your Affiliates, own the Copyright and patent claims covering the + Contribution which are required to grant the rights under Section 2. + +(c) The grant of rights under Section 2 does not violate any grant of rights + which You or Your Affiliates have made to third parties, including Your + employer. If You are an employee, You have had Your employer approve this + Agreement or sign the Entity version of this document. If You are less + than eighteen years old, please have Your parents or guardian sign the + Agreement. + +(d) You have not Submitted any Code You do not own without written permission + from US. + +(e) All pull or merge requests issued under usernames confirmed by You in + writing are issued by You; and all such pull or merge requests contain + Your Contributions under this Agreement. You will notify Us in writing in + the event of You no longer control such usernames. + +4. Disclaimer + +EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED +"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, +WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO +US. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH WARRANTY +IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW. + +5. Consequential Damage Waiver + +TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU BE +LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA, +INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT +OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT +OR OTHERWISE) UPON WHICH THE CLAIM IS BASED. + +THIS WAIVER DOES NOT APPLY TO GROSS NEGLIGENT OR MALICIOUS ACTS OR FRAUD. + +6. Miscellaneous + +6.1 This Agreement will be governed by and construed in accordance with the +laws of Finland excluding its conflicts of law provisions. Under certain +circumstances, the governing law in this section might be superseded by the +United Nations Convention on Contracts for the International Sale of Goods +("UN Convention") and the parties intend to avoid the application of the UN +Convention to this Agreement and, thus, exclude the application of the UN +Convention in its entirety to this Agreement. + +6.2 Any and all Submissions done by You prior to execution of this Agreement +shall be nonetheless covered by this Agreement. + +6.3 This Agreement sets out the entire agreement between You and Us for Your +Contributions to Us and overrides all other agreements or understandings. + +6.4 If You or We assign the rights or obligations received through this +Agreement to a third party, as a condition of the assignment, that third party +must agree in writing to abide by all the rights and obligations in the +Agreement. + +6.5 The failure of either party to require performance by the other party of +any provision of this Agreement in one situation shall not affect the right of +a party to require such performance at any time in the future. A waiver of +performance under a provision in one situation shall not be considered a +waiver of the performance of the provision in the future or a waiver of the +provision in its entirety. + +6.6 If any provision of this Agreement is found void and unenforceable, such +provision will be replaced to the extent possible with a provision that comes +closest to the meaning of the original provision and which is enforceable. +The terms and conditions set forth in this Agreement shall apply +notwithstanding any failure of essential purpose of this Agreement or any +limited remedy to the maximum extent possible under law. + +This document has been drafted based on Harmony Inividual Contributor License +Agreement (HA-CLA-I) Version 1.0 July 4, 2011. HA-CLA-I is available from +harmonyagreements.org and is licensed by under Creative Commons Attribution +3.0 Unported License. diff --git a/wsrep-lib/COPYING b/wsrep-lib/COPYING new file mode 100644 index 00000000..7deb2f88 --- /dev/null +++ b/wsrep-lib/COPYING @@ -0,0 +1,16 @@ +Copyright (C) 2018 Codership Oy <info@codership.com> + +This file is part of wsrep-lib. + +Wsrep-lib is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or +(at your option) any later version. + +Wsrep-lib is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. diff --git a/wsrep-lib/LICENSE b/wsrep-lib/LICENSE new file mode 100644 index 00000000..d159169d --- /dev/null +++ b/wsrep-lib/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/wsrep-lib/README.md b/wsrep-lib/README.md new file mode 100644 index 00000000..4ab164be --- /dev/null +++ b/wsrep-lib/README.md @@ -0,0 +1,46 @@ +# Introduction + +Project name: wsrep-lib - Integration library for WSREP API + +The purpose of this project is to implement C++ wrapper +for wsrep API with additional convenience for transaction +processing. + +This project will abstract away most of the transaction state +management required on DBMS side and will provide simple +C++ interface for key set population, data set population, +commit processing, write set applying etc. + +# Build Instructions + +In order to build the library, run + + cmake <cmake options> . + make + +## Build Requirements + +* C++ compiler (g++ 5.4 or later recommended) +* CMake version 2.8 or later +* The following Boost libraries are required if the unit tests and + the sample program is compiled + * Unit Test Framework + * Program Options + * Filesystem + * Thread + +## CMake Options + +* WSREP_LIB_WITH_UNIT_TESTS - Compile unit tests (default ON) +* WSREP_LIB_WITH_AUTO_TEST - Run unit tests automatically as a part + of compilation (default OFF) +* WSREP_LIB_WITH_DBSIM - Compile sample program (default ON) +* WSREP_LIB_WITH_ASAN - Enable address sanitizer instrumentation (default OFF) +* WSREP_LIB_WITH_TSAN - Enable thread sanitizer instrumentation (default OFF) +* WSREP_LIB_WITH_DOCUMENTATION - Generate documentation, requires Doxygen + (default OFF) +* WSREP_LIB_WITH_COVERAGE - Compile with coverage instrumentation (default OFF) +* WSREP_LIB_STRICT_BUILD_FLAGS - Compile with strict build flags, currently + enables -Weffc++ (default OFF) +* WSREP_LIB_MAINTAINER_MODE - Make every compiler warning to be treated + as error, enables -Werror compiler flag (default OFF) diff --git a/wsrep-lib/cmake/boost.cmake b/wsrep-lib/cmake/boost.cmake new file mode 100644 index 00000000..8c48fda6 --- /dev/null +++ b/wsrep-lib/cmake/boost.cmake @@ -0,0 +1,335 @@ +# Copyright (c) 2014, 2021, Oracle and/or its affiliates. +# Copyright (c) 2021, Codership OY. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License, version 2.0, +# as published by the Free Software Foundation. +# +# This program is also distributed with certain software (including +# but not limited to OpenSSL) that is licensed under separate terms, +# as designated in a particular file or component or in included license +# documentation. The authors of MySQL hereby grant you an additional +# permission to link the program and your derivative works with the +# separately licensed software that they have included with MySQL. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License, version 2.0, for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +# We need boost verision >= 1.75 to test JSON-related code. + +# Invoke with -DWITH_BOOST=<directory> or set WITH_BOOST in environment. +# If WITH_BOOST is *not* set, or is set to the special value "system", +# we assume that the correct version (see below) +# is installed on the compile host in the standard location. + +FUNCTION(GET_FILE_SIZE FILE_NAME OUTPUT_SIZE) + IF(WIN32) + FILE(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/cmake/filesize.bat" FILESIZE_BAT) + FILE(TO_NATIVE_PATH "${FILE_NAME}" NATIVE_FILE_NAME) + + EXECUTE_PROCESS( + COMMAND "${FILESIZE_BAT}" "${NATIVE_FILE_NAME}" + RESULT_VARIABLE COMMAND_RESULT + OUTPUT_VARIABLE RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE) + + ELSEIF(APPLE OR FREEBSD) + EXEC_PROGRAM(stat ARGS -f '%z' ${FILE_NAME} OUTPUT_VARIABLE RESULT) + ELSE() + EXEC_PROGRAM(stat ARGS -c '%s' ${FILE_NAME} OUTPUT_VARIABLE RESULT) + ENDIF() + SET(${OUTPUT_SIZE} ${RESULT} PARENT_SCOPE) +ENDFUNCTION() + +SET(BOOST_MAJOR "1") +SET(BOOST_MINOR "76") +SET(BOOST_PATCH "0") +SET(BOOST_PACKAGE_NAME "boost_${BOOST_MAJOR}_${BOOST_MINOR}_${BOOST_PATCH}") +SET(BOOST_TARBALL "${BOOST_PACKAGE_NAME}.tar.gz") +SET(BOOST_DOWNLOAD_URL + "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_MAJOR}.${BOOST_MINOR}.${BOOST_PATCH}/source/${BOOST_TARBALL}" + ) + +MACRO(RESET_BOOST_VARIABLES) + UNSET(BOOST_INCLUDE_DIR) + UNSET(BOOST_INCLUDE_DIR CACHE) + UNSET(LOCAL_BOOST_DIR) + UNSET(LOCAL_BOOST_DIR CACHE) + UNSET(LOCAL_BOOST_ZIP) + UNSET(LOCAL_BOOST_ZIP CACHE) +ENDMACRO() + +MACRO(ECHO_BOOST_VARIABLES) + MESSAGE(STATUS "BOOST_INCLUDE_DIR ${BOOST_INCLUDE_DIR}") + MESSAGE(STATUS "LOCAL_BOOST_DIR ${LOCAL_BOOST_DIR}") + MESSAGE(STATUS "LOCAL_BOOST_ZIP ${LOCAL_BOOST_ZIP}") +ENDMACRO() + +MACRO(COULD_NOT_FIND_BOOST) + ECHO_BOOST_VARIABLES() + RESET_BOOST_VARIABLES() + MESSAGE(STATUS "Could not find (the correct version of) boost.") + MESSAGE(STATUS "wsrep-lib currently requires ${BOOST_PACKAGE_NAME}\n") + MESSAGE(FATAL_ERROR + "You can download it with -DDOWNLOAD_BOOST=1 -DWITH_BOOST=<directory>\n" + "This CMake script will look for boost in <directory>. " + "If it is not there, it will download and unpack it " + "(in that directory) for you.\n" + "You can also download boost manually, from ${BOOST_DOWNLOAD_URL}\n" + "If you are inside a firewall, you may need to use an https proxy:\n" + "export https_proxy=http://example.com:80\n" + ) +ENDMACRO() + +# Pick value from environment if not set on command line. +IF(DEFINED ENV{WITH_BOOST} AND NOT DEFINED WITH_BOOST) + SET(WITH_BOOST "$ENV{WITH_BOOST}") +ENDIF() + +# Pick value from environment if not set on command line. +IF(DEFINED ENV{BOOST_ROOT} AND NOT DEFINED WITH_BOOST) + SET(WITH_BOOST "$ENV{BOOST_ROOT}") +ENDIF() + +IF(WITH_BOOST AND WITH_BOOST STREQUAL "system") + UNSET(WITH_BOOST) + UNSET(WITH_BOOST CACHE) +ENDIF() + +# Update the cache, to make it visible in cmake-gui. +SET(WITH_BOOST ${WITH_BOOST} CACHE PATH + "Path to boost sources: a directory, or a tarball to be unzipped.") + +# If the value of WITH_BOOST changes, we must unset all dependent variables: +IF(OLD_WITH_BOOST) + IF(NOT "${OLD_WITH_BOOST}" STREQUAL "${WITH_BOOST}") + RESET_BOOST_VARIABLES() + ENDIF() +ENDIF() + +SET(OLD_WITH_BOOST ${WITH_BOOST} CACHE INTERNAL + "Previous version of WITH_BOOST" FORCE) + +IF (WITH_BOOST) + ## Did we get a full path name, including file name? + IF (${WITH_BOOST} MATCHES ".*\\.tar.gz" OR ${WITH_BOOST} MATCHES ".*\\.zip") + GET_FILENAME_COMPONENT(BOOST_DIR ${WITH_BOOST} PATH) + GET_FILENAME_COMPONENT(BOOST_ZIP ${WITH_BOOST} NAME) + FIND_FILE(LOCAL_BOOST_ZIP + NAMES ${BOOST_ZIP} + PATHS ${BOOST_DIR} + NO_DEFAULT_PATH + ) + ENDIF() + ## Did we get a path name to the directory of the .tar.gz or .zip file? + FIND_FILE(LOCAL_BOOST_ZIP + NAMES "${BOOST_PACKAGE_NAME}.tar.gz" "${BOOST_PACKAGE_NAME}.zip" + PATHS "${WITH_BOOST}" + NO_DEFAULT_PATH + ) + IF(LOCAL_BOOST_ZIP) + MESSAGE(STATUS "Local boost zip: ${LOCAL_BOOST_ZIP}") + GET_FILE_SIZE(${LOCAL_BOOST_ZIP} LOCAL_BOOST_ZIP_SIZE) + IF(LOCAL_BOOST_ZIP_SIZE EQUAL 0) + # A previous failed download has left an empty file, most likely the + # user pressed Ctrl-C to kill a hanging connection due to missing vpn + # proxy. Remove it! + MESSAGE("${LOCAL_BOOST_ZIP} is zero length. Deleting it.") + FILE(REMOVE ${WITH_BOOST}/${BOOST_TARBALL}) + UNSET(LOCAL_BOOST_ZIP) + UNSET(LOCAL_BOOST_ZIP CACHE) + ENDIF() + UNSET(LOCAL_BOOST_ZIP_ZERO_LENGTH) + ENDIF() + + ## Did we get a path name to the directory of an unzipped version? + FIND_FILE(LOCAL_BOOST_SOURCE + NAMES "${BOOST_PACKAGE_NAME}" + PATHS "${WITH_BOOST}" + NO_DEFAULT_PATH + ) + ## Did we get a path name to an unzippped version? + FIND_PATH(LOCAL_BOOST_VER + NAMES "boost/version.hpp" + PATHS "${WITH_BOOST}" + NO_DEFAULT_PATH + ) + IF(LOCAL_BOOST_VER AND NOT LOCAL_BOOST_SOURCE) + SET(LOCAL_BOOST_SOURCE ${WITH_BOOST}) + UNSET(LOCAL_BOOST_VER) + UNSET(LOCAL_BOOST_VER CACHE) + ENDIF() + IF(LOCAL_BOOST_SOURCE) + MESSAGE(STATUS "Local boost source: ${LOCAL_BOOST_SOURCE}") + ENDIF() + + ## Did we get a path to compiled boost install? + FIND_PATH(LOCAL_BOOST_LIB + NAMES "lib" + PATHS "${WITH_BOOST}" + NO_DEFAULT_PATH + ) + IF(LOCAL_BOOST_LIB) + SET(LOCAL_BOOST_BUILD ${WITH_BOOST}) + UNSET(LOCAL_BOOST_LIB) + UNSET(LOCAL_BOOST_LIB CACHE) + ENDIF() +ENDIF() + +# There is a similar option in unittest/gunit. +# But the boost tarball is much bigger, so we have a separate option. +OPTION(DOWNLOAD_BOOST "Download boost from sourceforge." OFF) +SET(DOWNLOAD_BOOST_TIMEOUT 600 CACHE STRING + "Timeout in seconds when downloading boost.") + +# If we could not find it, then maybe download it. +IF(WITH_BOOST AND NOT LOCAL_BOOST_ZIP AND NOT LOCAL_BOOST_SOURCE AND NOT LOCAL_BOOST_BUILD) + IF(NOT DOWNLOAD_BOOST) + MESSAGE(STATUS "WITH_BOOST=${WITH_BOOST}") + COULD_NOT_FIND_BOOST() + ENDIF() + # Download the tarball + MESSAGE(STATUS "Downloading ${BOOST_TARBALL} to ${WITH_BOOST}") + FILE(DOWNLOAD ${BOOST_DOWNLOAD_URL} + ${WITH_BOOST}/${BOOST_TARBALL} + TIMEOUT ${DOWNLOAD_BOOST_TIMEOUT} + STATUS ERR + SHOW_PROGRESS + ) + IF(ERR EQUAL 0) + SET(LOCAL_BOOST_ZIP "${WITH_BOOST}/${BOOST_TARBALL}") + SET(LOCAL_BOOST_ZIP "${WITH_BOOST}/${BOOST_TARBALL}" CACHE INTERNAL "") + ELSE() + MESSAGE(STATUS "Download failed, error: ${ERR}") + # A failed DOWNLOAD leaves an empty file, remove it + FILE(REMOVE ${WITH_BOOST}/${BOOST_TARBALL}) + # STATUS returns a list of length 2 + LIST(GET ERR 0 NUMERIC_RETURN) + IF(NUMERIC_RETURN EQUAL 28) + MESSAGE(FATAL_ERROR + "You can try downloading ${BOOST_DOWNLOAD_URL} manually" + " using curl/wget or a similar tool," + " or increase the value of DOWNLOAD_BOOST_TIMEOUT" + " (which is now ${DOWNLOAD_BOOST_TIMEOUT} seconds)" + ) + ENDIF() + MESSAGE(FATAL_ERROR + "You can try downloading ${BOOST_DOWNLOAD_URL} manually" + " using curl/wget or a similar tool" + ) + ENDIF() +ENDIF() + +IF(LOCAL_BOOST_ZIP AND NOT LOCAL_BOOST_SOURCE AND NOT LOCAL_BOOST_BUILD) + GET_FILENAME_COMPONENT(LOCAL_BOOST_DIR ${LOCAL_BOOST_ZIP} PATH) + IF(NOT EXISTS "${LOCAL_BOOST_DIR}/${BOOST_PACKAGE_NAME}") + GET_FILENAME_COMPONENT(BOOST_TARBALL ${LOCAL_BOOST_ZIP} NAME) + MESSAGE(STATUS "cd ${LOCAL_BOOST_DIR}; tar xfz ${BOOST_TARBALL}") + EXECUTE_PROCESS( + COMMAND ${CMAKE_COMMAND} -E tar xfz "${BOOST_TARBALL}" + WORKING_DIRECTORY "${LOCAL_BOOST_DIR}" + RESULT_VARIABLE tar_result + ) + IF (tar_result MATCHES 0) + SET(BOOST_FOUND 1 CACHE INTERNAL "") + SET(LOCAL_BOOST_SOURCE "${LOCAL_BOOST_DIR}/${BOOST_PACKAGE_NAME}") + ELSE() + MESSAGE(STATUS "WITH_BOOST ${WITH_BOOST}.") + MESSAGE(STATUS "Failed to extract files.\n" + " Please try downloading and extracting yourself.\n" + " The url is: ${BOOST_DOWNLOAD_URL}") + MESSAGE(FATAL_ERROR "Giving up.") + ENDIF() + ENDIF() +ENDIF() + +IF (LOCAL_BOOST_SOURCE AND NOT LOCAL_BOOST_BUILD) + GET_FILENAME_COMPONENT(LOCAL_BOOST_BUILD ${LOCAL_BOOST_SOURCE} PATH) + SET(BOOST_TO_BUILD "thread,filesystem,program_options,test,json") + MESSAGE(STATUS "Executing Boost configure") + EXECUTE_PROCESS( + COMMAND ./bootstrap.sh --prefix=${LOCAL_BOOST_BUILD} --with-libraries=${BOOST_TO_BUILD} + WORKING_DIRECTORY "${LOCAL_BOOST_SOURCE}" + RESULT_VARIABLE RES + ERROR_VARIABLE ERR + OUTPUT_FILE "bootstrap.log" + ERROR_FILE "bootstrap.err" + COMMAND_ECHO STDOUT + ) + IF (NOT RES MATCHES 0) + MESSAGE(FATAL_ERROR "Boost configure failed: ${ERR}. Logs at ${LOCAL_BOOST_SOURCE}/bootstrap.log, ${LOCAL_BOOST_SOURCE}/bootstrap.err") + ENDIF() + MESSAGE(STATUS "Executing Boost build") + EXECUTE_PROCESS( + COMMAND ./b2 install -q + WORKING_DIRECTORY "${LOCAL_BOOST_SOURCE}" + RESULT_VARIABLE RES + ERROR_VARIABLE ERR + OUTPUT_FILE "b2.log" + ERROR_FILE "b2.err" + COMMAND_ECHO STDOUT + ) + IF (NOT RES MATCHES 0) + MESSAGE(FATAL_ERROR "Boost build failed: ${ERR}. Logs at ${LOCAL_BOOST_SOURCE}/b2.log, ${LOCAL_BOOST_SOURCE}/b2.err") + ENDIF() + # remove the source directory so save space + FILE(REMOVE_RECURSE ${LOCAL_BOOST_SOURCE}) +ENDIF() + +# Search for the version file, first in LOCAL_BOOST_DIR or WITH_BOOST +MESSAGE(STATUS + "Looking for boost/version.hpp in ${LOCAL_BOOST_BUILD}/include") +FIND_PATH(BOOST_INCLUDE_DIR + NAMES "boost/version.hpp" + NO_DEFAULT_PATH + PATHS ${LOCAL_BOOST_BUILD}/include +) +# Then search in standard places (if not found above). +FIND_PATH(BOOST_INCLUDE_DIR + NAMES "boost/version.hpp" +) + +IF(NOT BOOST_INCLUDE_DIR) + MESSAGE(STATUS + "Looked for boost/version.hpp in ${LOCAL_BOOST_BUILD}/include and default locations") + COULD_NOT_FIND_BOOST() +ELSE() + MESSAGE(STATUS "Found ${BOOST_INCLUDE_DIR}/boost/version.hpp ") +ENDIF() + +# Verify version number. Version information looks like: +# // BOOST_VERSION % 100 is the patch level +# // BOOST_VERSION / 100 % 1000 is the minor version +# // BOOST_VERSION / 100000 is the major version +# #define BOOST_VERSION 107300 +FILE(STRINGS "${BOOST_INCLUDE_DIR}/boost/version.hpp" + BOOST_VERSION_NUMBER + REGEX "^#define[\t ]+BOOST_VERSION[\t ][0-9]+.*" +) +STRING(REGEX REPLACE + "^.*BOOST_VERSION[\t ]([0-9][0-9])([0-9][0-9])([0-9][0-9]).*$" "\\1" + BOOST_MAJOR_VERSION "${BOOST_VERSION_NUMBER}") +STRING(REGEX REPLACE + "^.*BOOST_VERSION[\t ]([0-9][0-9])([0-9][0-9])([0-9][0-9]).*$" "\\2" + BOOST_MINOR_VERSION "${BOOST_VERSION_NUMBER}") + +MESSAGE(STATUS "BOOST_VERSION_NUMBER is ${BOOST_VERSION_NUMBER}") + +MESSAGE(STATUS "BOOST_INCLUDE_DIR ${BOOST_INCLUDE_DIR}") + +# Boost gets confused about language support with Clang 7 + MSVC 15.9 +IF(WIN32_CLANG) + ADD_DEFINITIONS(-DBOOST_NO_CXX17_HDR_STRING_VIEW) +ENDIF() + +IF(LOCAL_BOOST_BUILD OR LOCAL_BOOST_ZIP) + SET(USING_LOCAL_BOOST 1) +ELSE() + SET(USING_SYSTEM_BOOST 1) +ENDIF() diff --git a/wsrep-lib/dbsim/CMakeLists.txt b/wsrep-lib/dbsim/CMakeLists.txt new file mode 100644 index 00000000..aee964cd --- /dev/null +++ b/wsrep-lib/dbsim/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# Copyright (C) 2018 Codership Oy <info@codership.com> +# + +add_executable(dbsim + db_client.cpp + db_client_service.cpp + db_high_priority_service.cpp + db_params.cpp + db_server.cpp + db_server_service.cpp + db_server_state.cpp + db_simulator.cpp + db_storage_engine.cpp + db_threads.cpp + db_tls.cpp + dbsim.cpp +) + +target_link_libraries(dbsim wsrep-lib ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY}) +set_property(TARGET dbsim PROPERTY CXX_STANDARD 14) diff --git a/wsrep-lib/dbsim/db_client.cpp b/wsrep-lib/dbsim/db_client.cpp new file mode 100644 index 00000000..b5d56d37 --- /dev/null +++ b/wsrep-lib/dbsim/db_client.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_client.hpp" +#include "db_server.hpp" + +#include "wsrep/logger.hpp" + +db::client::client(db::server& server, + wsrep::client_id client_id, + enum wsrep::client_state::mode mode, + const db::params& params) + : mutex_() + , cond_() + , params_(params) + , server_(server) + , server_state_(server.server_state()) + , client_state_(mutex_, cond_, server_state_, client_service_, client_id, mode) + , client_service_(*this) + , se_trx_(server.storage_engine()) + , data_() + , random_device_() + , random_engine_(random_device_()) + , stats_() +{ + data_.resize(params.max_data_size); +} + +void db::client::start() +{ + client_state_.open(client_state_.id()); + for (size_t i(0); i < params_.n_transactions; ++i) + { + run_one_transaction(); + report_progress(i + 1); + } + client_state_.close(); + client_state_.cleanup(); +} + +bool db::client::bf_abort(wsrep::seqno seqno) +{ + return client_state_.bf_abort(seqno); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private // +//////////////////////////////////////////////////////////////////////////////// + +template <class F> +int db::client::client_command(F f) +{ + int err(client_state_.before_command()); + // wsrep::log_debug() << "before_command: " << err; + // If err != 0, transaction was BF aborted while client idle + if (err == 0) + { + err = client_state_.before_statement(); + if (err == 0) + { + err = f(); + } + client_state_.after_statement(); + } + client_state_.after_command_before_result(); + if (client_state_.current_error()) + { + // wsrep::log_info() << "Current error"; + assert(client_state_.transaction().state() == + wsrep::transaction::s_aborted); + err = 1; + } + client_state_.after_command_after_result(); + // wsrep::log_info() << "client_command(): " << err; + return err; +} + +void db::client::run_one_transaction() +{ + if (params_.sync_wait) + { + if (client_state_.sync_wait(5)) + { + throw wsrep::runtime_error("Sync wait failed"); + } + } + client_state_.reset_error(); + int err = client_command( + [&]() + { + // wsrep::log_debug() << "Start transaction"; + err = client_state_.start_transaction( + wsrep::transaction_id(server_.next_transaction_id())); + assert(err == 0); + se_trx_.start(this); + return err; + }); + + const wsrep::transaction& transaction( + client_state_.transaction()); + + err = err || client_command( + [&]() + { + // wsrep::log_debug() << "Generate write set"; + assert(transaction.active()); + assert(err == 0); + std::uniform_int_distribution<size_t> uniform_dist(0, params_.n_rows); + const size_t randkey(uniform_dist(random_engine_)); + ::memcpy(data_.data(), &randkey, + std::min(sizeof(randkey), data_.size())); + wsrep::key key(wsrep::key::exclusive); + key.append_key_part("dbms", 4); + unsigned long long client_key(client_state_.id().get()); + key.append_key_part(&client_key, sizeof(client_key)); + key.append_key_part(&randkey, sizeof(randkey)); + err = client_state_.append_key(key); + size_t bytes_to_append(data_.size()); + if (params_.random_data_size) + { + bytes_to_append = std::uniform_int_distribution<size_t>( + 1, data_.size())(random_engine_); + } + err = err || client_state_.append_data( + wsrep::const_buffer(data_.data(), bytes_to_append)); + return err; + }); + + err = err || client_command( + [&]() + { + // wsrep::log_debug() << "Commit"; + assert(err == 0); + if (do_2pc()) + { + err = err || client_state_.before_prepare(); + err = err || client_state_.after_prepare(); + } + err = err || client_state_.before_commit(); + if (err == 0) se_trx_.commit(transaction.ws_meta().gtid()); + err = err || client_state_.ordered_commit(); + err = err || client_state_.after_commit(); + if (err) + { + client_state_.before_rollback(); + se_trx_.rollback(); + client_state_.after_rollback(); + } + return err; + }); + + assert(err || + transaction.state() == wsrep::transaction::s_aborted || + transaction.state() == wsrep::transaction::s_committed); + assert(se_trx_.active() == false); + assert(transaction.active() == false); + + switch (transaction.state()) + { + case wsrep::transaction::s_committed: + ++stats_.commits; + break; + case wsrep::transaction::s_aborted: + ++stats_.rollbacks; + break; + default: + assert(0); + } +} + +void db::client::report_progress(size_t i) const +{ + if ((i % 1000) == 0) + { + wsrep::log_info() << "client: " << client_state_.id().get() + << " transactions: " << i + << " " << 100*double(i)/double(params_.n_transactions) << "%"; + } +} diff --git a/wsrep-lib/dbsim/db_client.hpp b/wsrep-lib/dbsim/db_client.hpp new file mode 100644 index 00000000..5536a449 --- /dev/null +++ b/wsrep-lib/dbsim/db_client.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file db_client.hpp */ + +#ifndef WSREP_DB_CLIENT_HPP +#define WSREP_DB_CLIENT_HPP + +#include "db_server_state.hpp" +#include "db_storage_engine.hpp" +#include "db_client_state.hpp" +#include "db_client_service.hpp" +#include "db_high_priority_service.hpp" + +#include <random> + +namespace db +{ + class server; + class client + { + public: + struct stats + { + long long commits; + long long rollbacks; + long long replays; + stats() + : commits(0) + , rollbacks(0) + , replays(0) + { } + }; + client(db::server&, + wsrep::client_id, + enum wsrep::client_state::mode, + const db::params&); + bool bf_abort(wsrep::seqno); + const struct stats stats() const { return stats_; } + void store_globals() + { + client_state_.store_globals(); + } + void reset_globals() + { } + void start(); + wsrep::client_state& client_state() { return client_state_; } + wsrep::client_service& client_service() { return client_service_; } + bool do_2pc() const { return false; } + private: + friend class db::server_state; + friend class db::client_service; + friend class db::high_priority_service; + template <class F> int client_command(F f); + void run_one_transaction(); + void reset_error(); + void report_progress(size_t) const; + wsrep::default_mutex mutex_; + wsrep::default_condition_variable cond_; + const db::params& params_; + db::server& server_; + db::server_state& server_state_; + db::client_state client_state_; + db::client_service client_service_; + db::storage_engine::transaction se_trx_; + wsrep::mutable_buffer data_; + std::random_device random_device_; + std::default_random_engine random_engine_; + struct stats stats_; + }; +} + +#endif // WSREP_DB_CLIENT_HPP diff --git a/wsrep-lib/dbsim/db_client_service.cpp b/wsrep-lib/dbsim/db_client_service.cpp new file mode 100644 index 00000000..edf7df8f --- /dev/null +++ b/wsrep-lib/dbsim/db_client_service.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_client_service.hpp" +#include "db_high_priority_service.hpp" +#include "db_client.hpp" + +db::client_service::client_service(db::client& client) + : wsrep::client_service() + , client_(client) + , client_state_(client_.client_state()) +{ } + +int db::client_service::bf_rollback() +{ + int ret(client_state_.before_rollback()); + assert(ret == 0); + client_.se_trx_.rollback(); + ret = client_state_.after_rollback(); + assert(ret == 0); + return ret; +} + +enum wsrep::provider::status +db::client_service::replay() +{ + wsrep::high_priority_context high_priority_context(client_state_); + db::replayer_service replayer_service( + client_.server_, client_); + auto ret(client_state_.provider().replay( + client_state_.transaction().ws_handle(), + &replayer_service)); + if (ret == wsrep::provider::success) + { + ++client_.stats_.replays; + } + return ret; +} diff --git a/wsrep-lib/dbsim/db_client_service.hpp b/wsrep-lib/dbsim/db_client_service.hpp new file mode 100644 index 00000000..be6f9ad8 --- /dev/null +++ b/wsrep-lib/dbsim/db_client_service.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_CLIENT_SERVICE_HPP +#define WSREP_DB_CLIENT_SERVICE_HPP + +#include "wsrep/client_service.hpp" +#include "wsrep/transaction.hpp" + +namespace db +{ + class client; + class client_state; + + class client_service : public wsrep::client_service + { + public: + client_service(db::client& client); + + bool interrupted(wsrep::unique_lock<wsrep::mutex>&) + const override + { return false; } + void reset_globals() override { } + void store_globals() override { } + int prepare_data_for_replication() override + { + return 0; + } + void cleanup_transaction() override { } + size_t bytes_generated() const override + { + return 0; + } + bool statement_allowed_for_streaming() const override + { + return true; + } + int prepare_fragment_for_replication(wsrep::mutable_buffer&, + size_t& position) override + { + position = 0; + return 0; + } + int remove_fragments() override { return 0; } + int bf_rollback() override; + void will_replay() override { } + void signal_replayed() override { } + void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) override { } + enum wsrep::provider::status replay() + override; + + enum wsrep::provider::status replay_unordered() override + { + return wsrep::provider::success; + } + + void emergency_shutdown() override { ::abort(); } + + enum wsrep::provider::status commit_by_xid() override + { + return wsrep::provider::success; + } + + bool is_explicit_xa() override + { + return false; + } + + bool is_xa_rollback() override + { + return false; + } + + void debug_sync(const char*) override { } + void debug_crash(const char*) override { } + private: + db::client& client_; + wsrep::client_state& client_state_; + }; +} + +#endif // WSREP_DB_CLIENT_SERVICE_HPP diff --git a/wsrep-lib/dbsim/db_client_state.hpp b/wsrep-lib/dbsim/db_client_state.hpp new file mode 100644 index 00000000..a5cf71b2 --- /dev/null +++ b/wsrep-lib/dbsim/db_client_state.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_CLIENT_CONTEXT_HPP +#define WSREP_DB_CLIENT_CONTEXT_HPP + +#include "wsrep/client_state.hpp" +#include "db_server_state.hpp" + +namespace db +{ + class client; + class client_state : public wsrep::client_state + { + public: + client_state(wsrep::mutex& mutex, + wsrep::condition_variable& cond, + db::server_state& server_state, + wsrep::client_service& client_service, + const wsrep::client_id& client_id, + enum wsrep::client_state::mode mode) + : wsrep::client_state(mutex, + cond, + server_state, + client_service, + client_id, + mode) + { } + + private: + client_state(const client_state&); + client_state& operator=(const client_state&); + }; +} + +#endif // WSREP_DB_CLIENT_CONTEXT diff --git a/wsrep-lib/dbsim/db_high_priority_service.cpp b/wsrep-lib/dbsim/db_high_priority_service.cpp new file mode 100644 index 00000000..669fe502 --- /dev/null +++ b/wsrep-lib/dbsim/db_high_priority_service.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_high_priority_service.hpp" +#include "db_server.hpp" +#include "db_client.hpp" + +db::high_priority_service::high_priority_service( + db::server& server, db::client& client) + : wsrep::high_priority_service(server.server_state()) + , server_(server) + , client_(client) +{ } + +int db::high_priority_service::start_transaction( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + return client_.client_state().start_transaction(ws_handle, ws_meta); +} + +int db::high_priority_service::next_fragment(const wsrep::ws_meta& ws_meta) +{ + return client_.client_state().next_fragment(ws_meta); +} + +const wsrep::transaction& db::high_priority_service::transaction() const +{ + return client_.client_state().transaction(); +} + +int db::high_priority_service::adopt_transaction(const wsrep::transaction&) +{ + throw wsrep::not_implemented_error(); +} + +int db::high_priority_service::apply_write_set( + const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) +{ + client_.se_trx_.start(&client_); + client_.se_trx_.apply(client_.client_state().transaction()); + return 0; +} + +int db::high_priority_service::apply_toi( + const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) +{ + throw wsrep::not_implemented_error(); +} + +int db::high_priority_service::apply_nbo_begin( + const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) +{ + throw wsrep::not_implemented_error(); +} + +int db::high_priority_service::commit(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + client_.client_state_.prepare_for_ordering(ws_handle, ws_meta, true); + int ret(client_.client_state_.before_commit()); + if (ret == 0) client_.se_trx_.commit(ws_meta.gtid()); + ret = ret || client_.client_state_.ordered_commit(); + ret = ret || client_.client_state_.after_commit(); + return ret; +} + +int db::high_priority_service::rollback(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + client_.client_state_.prepare_for_ordering(ws_handle, ws_meta, false); + int ret(client_.client_state_.before_rollback()); + assert(ret == 0); + client_.se_trx_.rollback(); + ret = client_.client_state_.after_rollback(); + assert(ret == 0); + return ret; +} + +void db::high_priority_service::adopt_apply_error(wsrep::mutable_buffer& err) +{ + client_.client_state_.adopt_apply_error(err); +} + +void db::high_priority_service::after_apply() +{ + client_.client_state_.after_applying(); +} + +int db::high_priority_service::log_dummy_write_set( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + wsrep::mutable_buffer& err) +{ + int ret(client_.client_state_.start_transaction(ws_handle, ws_meta)); + assert(ret == 0); + if (ws_meta.ordered()) + { + client_.client_state_.adopt_apply_error(err); + client_.client_state_.prepare_for_ordering(ws_handle, ws_meta, true); + ret = client_.client_state_.before_commit(); + assert(ret == 0); + ret = client_.client_state_.ordered_commit(); + assert(ret == 0); + ret = client_.client_state_.after_commit(); + assert(ret == 0); + } + client_.client_state_.after_applying(); + return ret; +} + +bool db::high_priority_service::is_replaying() const +{ + return false; +} diff --git a/wsrep-lib/dbsim/db_high_priority_service.hpp b/wsrep-lib/dbsim/db_high_priority_service.hpp new file mode 100644 index 00000000..d4a80f1b --- /dev/null +++ b/wsrep-lib/dbsim/db_high_priority_service.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_HIGH_PRIORITY_SERVICE_HPP +#define WSREP_DB_HIGH_PRIORITY_SERVICE_HPP + +#include "wsrep/high_priority_service.hpp" + +namespace db +{ + class server; + class client; + class high_priority_service : public wsrep::high_priority_service + { + public: + high_priority_service(db::server& server, db::client& client); + int start_transaction(const wsrep::ws_handle&, + const wsrep::ws_meta&) override; + int next_fragment(const wsrep::ws_meta&) override; + const wsrep::transaction& transaction() const override; + int adopt_transaction(const wsrep::transaction&) override; + int apply_write_set(const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) override; + int append_fragment_and_commit( + const wsrep::ws_handle&, + const wsrep::ws_meta&, + const wsrep::const_buffer&, + const wsrep::xid&) override + { return 0; } + int remove_fragments(const wsrep::ws_meta&) override + { return 0; } + int commit(const wsrep::ws_handle&, const wsrep::ws_meta&) override; + int rollback(const wsrep::ws_handle&, const wsrep::ws_meta&) override; + int apply_toi(const wsrep::ws_meta&, const wsrep::const_buffer&, + wsrep::mutable_buffer&) override; + int apply_nbo_begin(const wsrep::ws_meta&, const wsrep::const_buffer&, + wsrep::mutable_buffer&) + override; + void adopt_apply_error(wsrep::mutable_buffer&) override; + virtual void after_apply() override; + void store_globals() override { } + void reset_globals() override { } + void switch_execution_context(wsrep::high_priority_service&) override + { } + int log_dummy_write_set(const wsrep::ws_handle&, + const wsrep::ws_meta&, + wsrep::mutable_buffer&) override; + virtual bool is_replaying() const override; + void debug_crash(const char*) override { } + private: + high_priority_service(const high_priority_service&); + high_priority_service& operator=(const high_priority_service&); + db::server& server_; + db::client& client_; + }; + + class replayer_service : public db::high_priority_service + { + public: + replayer_service(db::server& server, db::client& client) + : db::high_priority_service(server, client) + { } + // After apply is empty for replayer to keep the transaction + // context available for the client session after replaying + // is over. + void after_apply() override {} + bool is_replaying() const override { return true; } + }; + +} + +#endif // WSREP_DB_HIGH_PRIORITY_SERVICE_HPP diff --git a/wsrep-lib/dbsim/db_params.cpp b/wsrep-lib/dbsim/db_params.cpp new file mode 100644 index 00000000..40433f6c --- /dev/null +++ b/wsrep-lib/dbsim/db_params.cpp @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_params.hpp" + +#include <boost/program_options.hpp> +#include <iostream> +#include <stdexcept> + +namespace +{ + void validate_params(const db::params& params) + { + std::ostringstream os; + if (params.n_servers != params.topology.size()) + { + if (params.topology.size() > 0) + { + os << "Error: --topology=" << params.topology << " does not " + << "match the number of server --servers=" + << params.n_servers << "\n"; + } + } + if (os.str().size()) + { + throw std::invalid_argument(os.str()); + } + } +} + +db::params db::parse_args(int argc, char** argv) +{ + namespace po = boost::program_options; + db::params params; + po::options_description desc("Allowed options"); + desc.add_options() + ("help", "produce help message") + ("wsrep-provider", + po::value<std::string>(¶ms.wsrep_provider)->required(), + "wsrep provider to load") + ("wsrep-provider-options", + po::value<std::string>(¶ms.wsrep_provider_options), + "wsrep provider options") + ("status-file", + po::value<std::string>(¶ms.status_file), + "status output file") + ("servers", po::value<size_t>(¶ms.n_servers)->required(), + "number of servers to start") + ("topology", po::value<std::string>(¶ms.topology), + "replication topology (e.g. mm for multi master, ms for master/slave") + ("clients", po::value<size_t>(¶ms.n_clients)->required(), + "number of clients to start per master") + ("transactions", po::value<size_t>(¶ms.n_transactions), + "number of transactions run by a client") + ("rows", po::value<size_t>(¶ms.n_rows), + "number of rows per table") + ("max-data-size", po::value<size_t>(¶ms.max_data_size), + "maximum size of data payload (default 8)") + ("random-data-size", po::value<bool>(¶ms.random_data_size), + "randomized payload data size (default 0)") + ("alg-freq", po::value<size_t>(¶ms.alg_freq), + "ALG frequency") + ("sync-wait", po::value<bool>(¶ms.sync_wait), + "Turn on sync wait for each transaction") + ("debug-log-level", po::value<int>(¶ms.debug_log_level), + "debug logging level: 0 - none, 1 - verbose") + ("fast-exit", po::value<int>(¶ms.fast_exit), + "exit from simulation without graceful shutdown") + ("ti", + po::value<int>(¶ms.thread_instrumentation), + "use instrumentation for threads/mutexes/condition variables" + "(0 default disabled, 1 total counts, 2 per object)") + ("ti-cond-checks", + po::value<bool>(¶ms.cond_checks), + "Enable checks for correct condition variable use. " + " Effective only if thread-instrumentation is enabled") + ("tls-service", + po::value<int>(¶ms.tls_service), + "Configure TLS service stubs.\n0 default disabled\n1 enabled\n" + "2 enabled with short read/write and renegotiation simulation\n" + "3 enabled with error simulation.") + ; + try + { + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + if (vm.count("help")) + { + std::cerr << desc << "\n"; + exit(0); + } + po::notify(vm); + validate_params(params); + } + catch (const po::error& e) + { + std::cerr << "Error parsing arguments: " << e.what() << "\n"; + std::cerr << desc << "\n"; + exit(1); + } + catch (...) + { + std::cerr << "Error parsing arguments\n"; + std::cerr << desc << "\n"; + exit(1); + } + return params; +} diff --git a/wsrep-lib/dbsim/db_params.hpp b/wsrep-lib/dbsim/db_params.hpp new file mode 100644 index 00000000..e5df8062 --- /dev/null +++ b/wsrep-lib/dbsim/db_params.hpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_PARAMS_HPP +#define WSREP_DB_PARAMS_HPP + +#include <cstddef> +#include <string> + +namespace db +{ + struct params + { + size_t n_servers; + size_t n_clients; + size_t n_transactions; + size_t n_rows; + size_t max_data_size; // Maximum size of write set data payload. + bool random_data_size; // If true, randomize data payload size. + size_t alg_freq; + bool sync_wait; + std::string topology; + std::string wsrep_provider; + std::string wsrep_provider_options; + std::string status_file; + int debug_log_level; + int fast_exit; + int thread_instrumentation; + bool cond_checks; + int tls_service; + params() + : n_servers(0) + , n_clients(0) + , n_transactions(0) + , n_rows(1000) + , max_data_size(8) + , random_data_size(false) + , alg_freq(0) + , sync_wait(false) + , topology() + , wsrep_provider() + , wsrep_provider_options() + , status_file("status.json") + , debug_log_level(0) + , fast_exit(0) + , thread_instrumentation() + , cond_checks() + , tls_service() + { } + }; + + params parse_args(int argc, char** argv); +} + +#endif // WSREP_DB_PARAMS_HPP diff --git a/wsrep-lib/dbsim/db_server.cpp b/wsrep-lib/dbsim/db_server.cpp new file mode 100644 index 00000000..a54610d1 --- /dev/null +++ b/wsrep-lib/dbsim/db_server.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_server.hpp" +#include "db_server_service.hpp" +#include "db_high_priority_service.hpp" +#include "db_client.hpp" +#include "db_simulator.hpp" + +#include "wsrep/logger.hpp" + +#include <ostream> +#include <cstdio> + +static wsrep::default_mutex logger_mtx; + +static void +logger_fn(wsrep::log::level l, const char* pfx, const char* msg) +{ + wsrep::unique_lock<wsrep::mutex> lock(logger_mtx); + + struct timespec time; + clock_gettime(CLOCK_REALTIME, &time); + + time_t const tt(time.tv_sec); + struct tm date; + localtime_r(&tt, &date); + + char date_str[85] = { '\0', }; + snprintf(date_str, sizeof(date_str) - 1, + "%04d-%02d-%02d %02d:%02d:%02d.%03d", + date.tm_year + 1900, date.tm_mon + 1, date.tm_mday, + date.tm_hour, date.tm_min, date.tm_sec, (int)time.tv_nsec/1000000); + + std::cerr << date_str << ' ' << pfx << wsrep::log::to_c_string(l) << ' ' + << msg << std::endl; +} + +db::server::server(simulator& simulator, + const std::string& name, + const std::string& address) + : simulator_(simulator) + , storage_engine_(simulator_.params()) + , mutex_() + , cond_() + , server_service_(*this) + , reporter_(mutex_, name + ".json", 4) + , server_state_(server_service_, + name, address, "dbsim_" + name + "_data") + , last_client_id_(0) + , last_transaction_id_(0) + , appliers_() + , clients_() + , client_threads_() +{ + wsrep::log::logger_fn(logger_fn); +} + +void db::server::applier_thread() +{ + wsrep::client_id client_id(last_client_id_.fetch_add(1) + 1); + db::client applier(*this, client_id, + wsrep::client_state::m_high_priority, + simulator_.params()); + wsrep::client_state* cc(static_cast<wsrep::client_state*>( + &applier.client_state())); + db::high_priority_service hps(*this, applier); + cc->open(cc->id()); + cc->before_command(); + enum wsrep::provider::status ret( + server_state_.provider().run_applier(&hps)); + wsrep::log_info() << "Applier thread exited with error code " << ret; + cc->after_command_before_result(); + cc->after_command_after_result(); + cc->close(); + cc->cleanup(); +} + +void db::server::start_applier() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + appliers_.push_back(boost::thread(&server::applier_thread, this)); +} + +void db::server::stop_applier() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + appliers_.front().join(); + appliers_.erase(appliers_.begin()); +} + + +void db::server::start_clients() +{ + size_t n_clients(simulator_.params().n_clients); + for (size_t i(0); i < n_clients; ++i) + { + start_client(i + 1); + } +} + +void db::server::stop_clients() +{ + for (auto& i : client_threads_) + { + i.join(); + } + for (const auto& i : clients_) + { + const struct db::client::stats& stats(i->stats()); + simulator_.stats_.commits += stats.commits; + simulator_.stats_.rollbacks += stats.rollbacks; + simulator_.stats_.replays += stats.replays; + } +} + +void db::server::client_thread(const std::shared_ptr<db::client>& client) +{ + client->start(); +} + +void db::server::start_client(size_t id) +{ + auto client(std::make_shared<db::client>( + *this, wsrep::client_id(id), + wsrep::client_state::m_local, + simulator_.params())); + clients_.push_back(client); + client_threads_.push_back( + boost::thread(&db::server::client_thread, this, client)); +} + +void db::server::donate_sst(const std::string& req, + const wsrep::gtid& gtid, + bool bypass) +{ + simulator_.sst(*this, req, gtid, bypass); +} + + +wsrep::high_priority_service* db::server::streaming_applier_service() +{ + throw wsrep::not_implemented_error(); +} + +void db::server::log_state_change(enum wsrep::server_state::state from, + enum wsrep::server_state::state to) +{ + wsrep::log_info() << "State changed " << from << " -> " << to; + reporter_.report_state(to); +} diff --git a/wsrep-lib/dbsim/db_server.hpp b/wsrep-lib/dbsim/db_server.hpp new file mode 100644 index 00000000..9f197be5 --- /dev/null +++ b/wsrep-lib/dbsim/db_server.hpp @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_SERVER_HPP +#define WSREP_DB_SERVER_HPP + +#include "wsrep/gtid.hpp" +#include "wsrep/client_state.hpp" +#include "wsrep/reporter.hpp" + +#include "db_storage_engine.hpp" +#include "db_server_state.hpp" +#include "db_server_service.hpp" + +#include <boost/thread.hpp> + +#include <string> +#include <memory> + +namespace db +{ + class simulator; + class client; + class server + { + public: + server(simulator& simulator, + const std::string& name, + const std::string& address); + void applier_thread(); + void start_applier(); + void stop_applier(); + void start_clients(); + void stop_clients(); + void client_thread(const std::shared_ptr<db::client>& client); + db::storage_engine& storage_engine() { return storage_engine_; } + db::server_state& server_state() { return server_state_; } + wsrep::transaction_id next_transaction_id() + { + return wsrep::transaction_id(last_transaction_id_.fetch_add(1) + 1); + } + void donate_sst(const std::string&, const wsrep::gtid&, bool); + wsrep::client_state* local_client_state(); + void release_client_state(wsrep::client_state*); + wsrep::high_priority_service* streaming_applier_service(); + void log_state_change(enum wsrep::server_state::state, + enum wsrep::server_state::state); + private: + void start_client(size_t id); + + db::simulator& simulator_; + db::storage_engine storage_engine_; + wsrep::default_mutex mutex_; + wsrep::default_condition_variable cond_; + db::server_service server_service_; + wsrep::reporter reporter_; + db::server_state server_state_; + std::atomic<size_t> last_client_id_; + std::atomic<size_t> last_transaction_id_; + std::vector<boost::thread> appliers_; + std::vector<std::shared_ptr<db::client>> clients_; + std::vector<boost::thread> client_threads_; + }; +}; + +#endif // WSREP_DB_SERVER_HPP diff --git a/wsrep-lib/dbsim/db_server_service.cpp b/wsrep-lib/dbsim/db_server_service.cpp new file mode 100644 index 00000000..8fc58dde --- /dev/null +++ b/wsrep-lib/dbsim/db_server_service.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_server_service.hpp" +#include "db_server.hpp" +#include "db_storage_service.hpp" + +#include "wsrep/logger.hpp" +#include "wsrep/high_priority_service.hpp" + +db::server_service::server_service(db::server& server) + : server_(server) +{ } + +wsrep::storage_service* db::server_service::storage_service( + wsrep::client_service&) +{ + return new db::storage_service(); +} + +wsrep::storage_service* db::server_service::storage_service( + wsrep::high_priority_service&) +{ + return new db::storage_service(); +} + +void db::server_service::release_storage_service( + wsrep::storage_service* storage_service) +{ + delete storage_service; +} + +wsrep::high_priority_service* db::server_service::streaming_applier_service( + wsrep::client_service&) +{ + return server_.streaming_applier_service(); +} + +wsrep::high_priority_service* db::server_service::streaming_applier_service( + wsrep::high_priority_service&) +{ + return server_.streaming_applier_service(); +} + +void db::server_service::release_high_priority_service( + wsrep::high_priority_service *high_priority_service) +{ + delete high_priority_service; +} + +bool db::server_service::sst_before_init() const +{ + return true; +} + +std::string db::server_service::sst_request() +{ + std::ostringstream os; + os << server_.server_state().name(); + wsrep::log_info() << "SST request: " + << server_.server_state().name(); + + return os.str(); +} + +int db::server_service::start_sst( + const std::string& request, const wsrep::gtid& gtid, bool bypass) +{ + server_.donate_sst(request, gtid, bypass); + return 0; +} + +void db::server_service::background_rollback(wsrep::client_state&) +{ +} + +void db::server_service::bootstrap() +{ +} + +void db::server_service::log_message(enum wsrep::log::level level, + const char* message) +{ + wsrep::log(level, server_.server_state().name().c_str()) << message; +} +void db::server_service::log_dummy_write_set( + wsrep::client_state&, const wsrep::ws_meta& meta) +{ + wsrep::log_info() << "Dummy write set: " << meta.seqno(); +} + +void db::server_service::log_view(wsrep::high_priority_service*, + const wsrep::view& v) +{ + wsrep::log_info() << "View:\n" << v; + server_.storage_engine().store_view(v); +} + +void db::server_service::recover_streaming_appliers( + wsrep::client_service&) +{ +} + +void db::server_service::recover_streaming_appliers( + wsrep::high_priority_service&) +{ +} + +wsrep::view db::server_service::get_view(wsrep::client_service&, + const wsrep::id& own_id) +{ + wsrep::view stored_view(server_.storage_engine().get_view()); + int const my_idx(stored_view.member_index(own_id)); + wsrep::view my_view( + stored_view.state_id(), + stored_view.view_seqno(), + stored_view.status(), + stored_view.capabilities(), + my_idx, + stored_view.protocol_version(), + stored_view.members() + ); + return my_view; +} + +wsrep::gtid db::server_service::get_position(wsrep::client_service&) +{ + return server_.storage_engine().get_position(); +} + +void db::server_service::set_position(wsrep::client_service&, + const wsrep::gtid& gtid) +{ + return server_.storage_engine().store_position(gtid); +} + +void db::server_service::log_state_change( + enum wsrep::server_state::state prev_state, + enum wsrep::server_state::state current_state) +{ + server_.log_state_change(prev_state, current_state); +} + +int db::server_service::wait_committing_transactions(int) +{ + throw wsrep::not_implemented_error(); +} + +void db::server_service::debug_sync(const char*) +{ + +} diff --git a/wsrep-lib/dbsim/db_server_service.hpp b/wsrep-lib/dbsim/db_server_service.hpp new file mode 100644 index 00000000..e5cd70d2 --- /dev/null +++ b/wsrep-lib/dbsim/db_server_service.hpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_SERVER_SERVICE_HPP +#define WSREP_DB_SERVER_SERVICE_HPP + +#include "wsrep/server_service.hpp" +#include <string> + +namespace db +{ + class server; + class server_service : public wsrep::server_service + { + public: + server_service(db::server& server); + wsrep::storage_service* storage_service(wsrep::client_service&) override; + wsrep::storage_service* storage_service(wsrep::high_priority_service&) override; + + void release_storage_service(wsrep::storage_service*) override; + wsrep::high_priority_service* streaming_applier_service(wsrep::client_service&) override; + wsrep::high_priority_service* streaming_applier_service(wsrep::high_priority_service&) override; + void release_high_priority_service(wsrep::high_priority_service*) override; + + bool sst_before_init() const override; + int start_sst(const std::string&, const wsrep::gtid&, bool) override; + std::string sst_request() override; + void background_rollback(wsrep::client_state&) override; + void bootstrap() override; + void log_message(enum wsrep::log::level, const char* message) override; + void log_dummy_write_set(wsrep::client_state&, const wsrep::ws_meta&) + override; + void log_view(wsrep::high_priority_service*, + const wsrep::view&) override; + void recover_streaming_appliers(wsrep::client_service&) override; + void recover_streaming_appliers(wsrep::high_priority_service&) override; + wsrep::view get_view(wsrep::client_service&, const wsrep::id&) + override; + wsrep::gtid get_position(wsrep::client_service&) override; + void set_position(wsrep::client_service&, const wsrep::gtid&) override; + void log_state_change(enum wsrep::server_state::state, + enum wsrep::server_state::state) override; + int wait_committing_transactions(int) override; + void debug_sync(const char*) override; + private: + db::server& server_; + }; +} + +#endif // WSREP_DB_SERVER_SERVICE_HPP diff --git a/wsrep-lib/dbsim/db_server_state.cpp b/wsrep-lib/dbsim/db_server_state.cpp new file mode 100644 index 00000000..2bc17d29 --- /dev/null +++ b/wsrep-lib/dbsim/db_server_state.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_server_state.hpp" +#include "db_server.hpp" + +#include "wsrep/logger.hpp" + diff --git a/wsrep-lib/dbsim/db_server_state.hpp b/wsrep-lib/dbsim/db_server_state.hpp new file mode 100644 index 00000000..49d4499e --- /dev/null +++ b/wsrep-lib/dbsim/db_server_state.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_SERVER_CONTEXT_HPP +#define WSREP_DB_SERVER_CONTEXT_HPP + +#include "wsrep/server_state.hpp" +#include "wsrep/server_service.hpp" +#include "wsrep/client_state.hpp" + +#include <atomic> + +namespace db +{ + class server; + class server_state : public wsrep::server_state + { + public: + server_state(wsrep::server_service& server_service, + const std::string& name, + const std::string& address, + const std::string& working_dir) + : wsrep::server_state( + mutex_, + cond_, + server_service, + nullptr, + name, + "", + address, + working_dir, + wsrep::gtid::undefined(), + 1, + wsrep::server_state::rm_async) + , mutex_() + , cond_() + { } + private: + wsrep::default_mutex mutex_; + wsrep::default_condition_variable cond_; + }; +} + +#endif // WSREP_DB_SERVER_CONTEXT_HPP diff --git a/wsrep-lib/dbsim/db_simulator.cpp b/wsrep-lib/dbsim/db_simulator.cpp new file mode 100644 index 00000000..0971b110 --- /dev/null +++ b/wsrep-lib/dbsim/db_simulator.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_simulator.hpp" +#include "db_client.hpp" +#include "db_threads.hpp" +#include "db_tls.hpp" + +#include "wsrep/logger.hpp" + +#include <boost/filesystem.hpp> +#include <sstream> + +static db::ti thread_instrumentation; +static db::tls tls_service; + +void db::simulator::run() +{ + start(); + stop(); + std::flush(std::cerr); + std::cout << "Results:\n"; + std::cout << stats() << std::endl; + std::cout << db::ti::stats() << std::endl; + std::cout << db::tls::stats() << std::endl; +} + +void db::simulator::sst(db::server& server, + const std::string& request, + const wsrep::gtid& gtid, + bool bypass) +{ + // The request may contain extra trailing '\0' after it goes + // through the provider, strip it first. + std::string name(request); + name.erase(std::find(name.begin(), name.end(), '\0'), name.end()); + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + auto i(servers_.find(name)); + wsrep::log_info() << "SST request '" << name << "'"; + if (i == servers_.end()) + { + wsrep::log_error() << "Server " << request << " not found"; + wsrep::log_info() << "servers:"; + for (const auto& s : servers_) + { + wsrep::log_info() << "server: " << s.first; + } + throw wsrep::runtime_error("Server " + request + " not found"); + } + if (bypass == false) + { + wsrep::log_info() << "SST " + << server.server_state().name() + << " -> " << request; + i->second->storage_engine().store_position(gtid); + i->second->storage_engine().store_view( + server.storage_engine().get_view()); + } + + db::client dummy(*(i->second), wsrep::client_id(-1), + wsrep::client_state::m_local, params()); + + i->second->server_state().sst_received(dummy.client_service(), 0); + server.server_state().sst_sent(gtid, 0); +} + +std::string db::simulator::stats() const +{ + auto duration(std::chrono::duration<double>( + clients_stop_ - clients_start_).count()); + long long transactions(stats_.commits + stats_.rollbacks); + long long bf_aborts(0); + for (const auto& s : servers_) + { + bf_aborts += s.second->storage_engine().bf_aborts(); + } + std::ostringstream os; + os << "Number of transactions: " << transactions + << "\n" + << "Seconds: " << duration + << " \n" + << "Transactions per second: " << double(transactions)/double(duration) + << "\n" + << "BF aborts: " + << bf_aborts + << "\n" + << "Client commits: " << stats_.commits + << "\n" + << "Client rollbacks: " << stats_.rollbacks + << "\n" + << "Client replays: " << stats_.replays; + return os.str(); +} + +//////////////////////////////////////////////////////////////////////////////// +// Private // +//////////////////////////////////////////////////////////////////////////////// + +void db::simulator::start() +{ + thread_instrumentation.level(params_.thread_instrumentation); + thread_instrumentation.cond_checks(params_.cond_checks); + tls_service.init(params_.tls_service); + wsrep::log_info() << "Provider: " << params_.wsrep_provider; + + std::string cluster_address(build_cluster_address()); + wsrep::log_info() << "Cluster address: " << cluster_address; + for (size_t i(0); i < params_.n_servers; ++i) + { + std::ostringstream name_os; + name_os << (i + 1); + std::ostringstream id_os; + id_os << (i + 1); + std::ostringstream address_os; + address_os << "127.0.0.1:" << server_port(i); + wsrep::id server_id(id_os.str()); + auto it(servers_.insert( + std::make_pair( + name_os.str(), + std::make_unique<db::server>( + *this, + name_os.str(), + address_os.str())))); + if (it.second == false) + { + throw wsrep::runtime_error("Failed to add server"); + } + boost::filesystem::path dir("dbsim_" + id_os.str() + "_data"); + boost::filesystem::create_directory(dir); + + db::server& server(*it.first->second); + server.server_state().debug_log_level(params_.debug_log_level); + std::string server_options(params_.wsrep_provider_options); + + wsrep::provider::services services; + services.thread_service = params_.thread_instrumentation + ? &thread_instrumentation + : nullptr; + services.tls_service = params_.tls_service + ? &tls_service + : nullptr; + if (server.server_state().load_provider(params_.wsrep_provider, + server_options, services)) + { + throw wsrep::runtime_error("Failed to load provider"); + } + if (server.server_state().connect("sim_cluster", cluster_address, "", + i == 0)) + { + throw wsrep::runtime_error("Failed to connect"); + } + wsrep::log_debug() << "main: Starting applier"; + server.start_applier(); + wsrep::log_debug() << "main: Waiting initializing state"; + server.server_state().wait_until_state(wsrep::server_state::s_initializing); + wsrep::log_debug() << "main: Calling initialized"; + server.server_state().initialized(); + wsrep::log_debug() << "main: Waiting for synced state"; + server.server_state().wait_until_state( + wsrep::server_state::s_synced); + wsrep::log_debug() << "main: Server synced"; + } + + // Start client threads + wsrep::log_info() << "####################### Starting client load"; + clients_start_ = std::chrono::steady_clock::now(); + size_t index(0); + for (auto& i : servers_) + { + if (params_.topology.size() == 0 || params_.topology[index] == 'm') + { + i.second->start_clients(); + } + ++index; + } +} + +void db::simulator::stop() +{ + for (auto& i : servers_) + { + db::server& server(*i.second); + server.stop_clients(); + } + clients_stop_ = std::chrono::steady_clock::now(); + wsrep::log_info() << "######## Stats ############"; + wsrep::log_info() << stats(); + std::cout << db::ti::stats() << std::endl; + wsrep::log_info() << "######## Stats ############"; + if (params_.fast_exit) + { + exit(0); + } + for (auto& i : servers_) + { + db::server& server(*i.second); + wsrep::log_info() << "Status for server: " + << server.server_state().id(); + auto status(server.server_state().provider().status()); + for_each(status.begin(), status.end(), + [](const wsrep::provider::status_variable& sv) + { + wsrep::log_info() << sv.name() << " = " << sv.value(); + }); + server.server_state().disconnect(); + server.server_state().wait_until_state( + wsrep::server_state::s_disconnected); + server.stop_applier(); + server.server_state().unload_provider(); + } +} + +std::string db::simulator::server_port(size_t i) const +{ + std::ostringstream os; + os << (10000 + (i + 1)*10); + return os.str(); +} + +std::string db::simulator::build_cluster_address() const +{ + std::string ret; + if (params_.wsrep_provider.find("galera_smm") != std::string::npos) + { + ret += "gcomm://"; + } + + for (size_t i(0); i < params_.n_servers; ++i) + { + std::ostringstream sa_os; + sa_os << "127.0.0.1:"; + sa_os << server_port(i); + ret += sa_os.str(); + if (i < params_.n_servers - 1) ret += ","; + } + return ret; +} diff --git a/wsrep-lib/dbsim/db_simulator.hpp b/wsrep-lib/dbsim/db_simulator.hpp new file mode 100644 index 00000000..693645e3 --- /dev/null +++ b/wsrep-lib/dbsim/db_simulator.hpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_SIMULATOR_HPP +#define WSREP_DB_SIMULATOR_HPP + +#include "wsrep/gtid.hpp" +#include "wsrep/mutex.hpp" +#include "wsrep/lock.hpp" + +#include "db_params.hpp" +#include "db_server.hpp" + +#include <memory> +#include <chrono> +#include <unordered_set> +#include <map> + +namespace db +{ + class server; + class simulator + { + public: + simulator(const params& params) + : mutex_() + , params_(params) + , servers_() + , clients_start_() + , clients_stop_() + , stats_() + { } + + void run(); + void sst(db::server&, + const std::string&, const wsrep::gtid&, bool); + const db::params& params() const + { return params_; } + std::string stats() const; + private: + void start(); + void stop(); + std::string server_port(size_t i) const; + std::string build_cluster_address() const; + + wsrep::default_mutex mutex_; + const db::params& params_; + std::map<std::string, std::unique_ptr<db::server>> servers_; + std::chrono::time_point<std::chrono::steady_clock> clients_start_; + std::chrono::time_point<std::chrono::steady_clock> clients_stop_; + public: + struct stats + { + long long commits; + long long rollbacks; + long long replays; + stats() + : commits(0) + , rollbacks(0) + , replays(0) + { } + } stats_; + }; +} +#endif // WSRE_DB_SIMULATOR_HPP diff --git a/wsrep-lib/dbsim/db_storage_engine.cpp b/wsrep-lib/dbsim/db_storage_engine.cpp new file mode 100644 index 00000000..c102783a --- /dev/null +++ b/wsrep-lib/dbsim/db_storage_engine.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_storage_engine.hpp" +#include "db_client.hpp" + +void db::storage_engine::transaction::start(db::client* cc) +{ + wsrep::unique_lock<wsrep::mutex> lock(se_.mutex_); + if (se_.transactions_.insert(cc).second == false) + { + ::abort(); + } + cc_ = cc; +} + +void db::storage_engine::transaction::apply( + const wsrep::transaction& transaction) +{ + assert(cc_); + se_.bf_abort_some(transaction); +} + +void db::storage_engine::transaction::commit(const wsrep::gtid& gtid) +{ + if (cc_) + { + wsrep::unique_lock<wsrep::mutex> lock(se_.mutex_); + se_.transactions_.erase(cc_); + se_.store_position(gtid); + } + cc_ = nullptr; +} + + +void db::storage_engine::transaction::rollback() +{ + if (cc_) + { + wsrep::unique_lock<wsrep::mutex> lock(se_.mutex_); + se_.transactions_.erase(cc_); + } + cc_ = nullptr; +} + +void db::storage_engine::bf_abort_some(const wsrep::transaction& txc) +{ + std::uniform_int_distribution<size_t> uniform_dist(0, alg_freq_); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + if (alg_freq_ && uniform_dist(random_engine_) == 0) + { + if (transactions_.empty() == false) + { + for (auto victim : transactions_) + { + wsrep::client_state& cc(victim->client_state()); + if (cc.mode() == wsrep::client_state::m_local) + { + if (victim->bf_abort(txc.seqno())) + { + ++bf_aborts_; + } + break; + } + } + } + } +} + +void db::storage_engine::store_position(const wsrep::gtid& gtid) +{ + validate_position(gtid); + position_ = gtid; +} + +wsrep::gtid db::storage_engine::get_position() const +{ + return position_; +} + +void db::storage_engine::store_view(const wsrep::view& view) +{ + view_ = view; +} + +wsrep::view db::storage_engine::get_view() const +{ + return view_; +} + +void db::storage_engine::validate_position(const wsrep::gtid& gtid) const +{ + using std::rel_ops::operator<=; + if (position_.id() == gtid.id() && gtid.seqno() <= position_.seqno()) + { + std::ostringstream os; + os << "Invalid position submitted, position seqno " + << position_.seqno() + << " is greater than submitted seqno " + << gtid.seqno(); + throw wsrep::runtime_error(os.str()); + } +} diff --git a/wsrep-lib/dbsim/db_storage_engine.hpp b/wsrep-lib/dbsim/db_storage_engine.hpp new file mode 100644 index 00000000..0721a2ad --- /dev/null +++ b/wsrep-lib/dbsim/db_storage_engine.hpp @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_STORAGE_ENGINE_HPP +#define WSREP_DB_STORAGE_ENGINE_HPP + +#include "db_params.hpp" + +#include "wsrep/mutex.hpp" +#include "wsrep/client_state.hpp" + +#include <atomic> +#include <unordered_set> +#include <random> + +namespace db +{ + class client; + class storage_engine + { + public: + storage_engine(const params& params) + : mutex_() + , transactions_() + , alg_freq_(params.alg_freq) + , bf_aborts_() + , position_() + , view_() + , random_device_() + , random_engine_(random_device_()) + { } + + class transaction + { + public: + transaction(storage_engine& se) + : se_(se) + , cc_() + { } + ~transaction() + { + rollback(); + } + bool active() const { return cc_ != nullptr; } + void start(client* cc); + void apply(const wsrep::transaction&); + void commit(const wsrep::gtid&); + void rollback(); + db::client* client() { return cc_; } + transaction(const transaction&) = delete; + transaction& operator=(const transaction&) = delete; + private: + db::storage_engine& se_; + db::client* cc_; + }; + void bf_abort_some(const wsrep::transaction& tc); + long long bf_aborts() const { return bf_aborts_; } + void store_position(const wsrep::gtid& gtid); + wsrep::gtid get_position() const; + void store_view(const wsrep::view& view); + wsrep::view get_view() const; + private: + void validate_position(const wsrep::gtid& gtid) const; + wsrep::default_mutex mutex_; + std::unordered_set<db::client*> transactions_; + size_t alg_freq_; + std::atomic<long long> bf_aborts_; + wsrep::gtid position_; + wsrep::view view_; + std::random_device random_device_; + std::default_random_engine random_engine_; + }; +} + +#endif // WSREP_DB_STORAGE_ENGINE_HPP diff --git a/wsrep-lib/dbsim/db_storage_service.hpp b/wsrep-lib/dbsim/db_storage_service.hpp new file mode 100644 index 00000000..839253db --- /dev/null +++ b/wsrep-lib/dbsim/db_storage_service.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_STORAGE_SERVICE_HPP +#define WSREP_DB_STORAGE_SERVICE_HPP + +#include "wsrep/storage_service.hpp" +#include "wsrep/exception.hpp" + +namespace db +{ + class storage_service : public wsrep::storage_service + { + int start_transaction(const wsrep::ws_handle&) override + { throw wsrep::not_implemented_error(); } + void adopt_transaction(const wsrep::transaction&) override + { throw wsrep::not_implemented_error(); } + int append_fragment(const wsrep::id&, + wsrep::transaction_id, + int, + const wsrep::const_buffer&, + const wsrep::xid&) override + { throw wsrep::not_implemented_error(); } + int update_fragment_meta(const wsrep::ws_meta&) override + { throw wsrep::not_implemented_error(); } + int remove_fragments() override + { throw wsrep::not_implemented_error(); } + int commit(const wsrep::ws_handle&, const wsrep::ws_meta&) override + { throw wsrep::not_implemented_error(); } + int rollback(const wsrep::ws_handle&, const wsrep::ws_meta&) + override + { throw wsrep::not_implemented_error(); } + void store_globals() override { } + void reset_globals() override { } + }; +} + +#endif // WSREP_DB_STORAGE_SERVICE_HPP diff --git a/wsrep-lib/dbsim/db_threads.cpp b/wsrep-lib/dbsim/db_threads.cpp new file mode 100644 index 00000000..d066c65a --- /dev/null +++ b/wsrep-lib/dbsim/db_threads.cpp @@ -0,0 +1,727 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_threads.hpp" +#include "wsrep/compiler.hpp" +#include "wsrep/logger.hpp" + +#include <cassert> +#include <pthread.h> + +#include <algorithm> +#include <atomic> +#include <chrono> +#include <map> +#include <mutex> +#include <ostream> +#include <sstream> +#include <thread> +#include <unordered_map> +#include <vector> + +extern "C" { static void* start_thread(void* args_ptr); } +namespace +{ + struct ti_obj + { + }; + enum ti_opcode + { + oc_thread_create, + oc_thread_destroy, + oc_mutex_create, + oc_mutex_destroy, + oc_mutex_lock, + oc_mutex_trylock, + oc_mutex_unlock, + oc_cond_create, + oc_cond_destroy, + oc_cond_wait, + oc_cond_timedwait, + oc_cond_signal, + oc_cond_broadcast, + oc_max // must be the last + }; + + static const char* ti_opstring(enum ti_opcode op) + { + switch (op) + { + case oc_thread_create: return "thread_create"; + case oc_thread_destroy: return "thread_destroy"; + case oc_mutex_create: return "mutex_create"; + case oc_mutex_destroy: return "mutex_destroy"; + case oc_mutex_lock: return "mutex_lock"; + case oc_mutex_trylock: return "mutex_trylock"; + case oc_mutex_unlock: return "mutex_unlock"; + case oc_cond_create: return "cond_create"; + case oc_cond_destroy: return "cond_destroy"; + case oc_cond_wait: return "cond_wait"; + case oc_cond_timedwait: return "cond_timedwait"; + case oc_cond_signal: return "cond_signal"; + case oc_cond_broadcast: return "cond_broadcast"; + default: return "unknown"; + } + } + + static std::vector<std::string> key_vec; + static std::atomic<int> key_cnt; + static std::vector<std::vector<size_t>> ops_map; + static std::vector<std::mutex*> ops_map_sync; + static struct ops_map_sync_deleter + { + ~ops_map_sync_deleter() + { + std::for_each(ops_map_sync.begin(), ops_map_sync.end(), + [](auto entry) { delete entry; }); + } + } ops_map_sync_deleter; + static std::array<std::atomic<size_t>, oc_max> total_ops; + static std::atomic<size_t> total_allocations; + static std::atomic<size_t> mutex_contention; + static std::unordered_map<std::string, size_t> mutex_contention_counts; + static int op_level; + // Check correct condition variable usage: + // - Associated mutex must be locked when waiting for cond + // - There must be at least one waiter when signalling for condition + static bool cond_checks; + static inline void cond_check(bool condition, const char* name, + const char* message) + { + if (cond_checks && !condition) + { + wsrep::log_error() << "Condition variable check failed for '" + << name << "': " << message; + ::abort(); + } + } + static inline int append_key(const char* name, const char* type) + { + + key_vec.push_back(std::string(name) + "_" + type); + wsrep::log_info() << "Register key " << name << "_" << type + << " with index " << (key_cnt + 1); + ops_map.push_back(std::vector<size_t>()); + ops_map_sync.push_back(new std::mutex()); + ops_map.back().resize(oc_max); + return ++key_cnt; + } + + template <class Key> static inline size_t get_key_index(const Key* key) + { + size_t index(reinterpret_cast<const size_t>(key) - 1); + assert(index < key_vec.size()); + return index; + } + + template <class Key> + static inline const char* get_key_name(const Key* key) + { + return key_vec[get_key_index(key)].c_str(); + } + + static inline const std::string& get_key_name_by_index(size_t index) + { + assert(index < key_vec.size()); + return key_vec[index]; + } + + // Note: Do not refer the obj pointer in this function, it may + // have been deleted before the call. + template <class Key> + static inline void update_ops(const ti_obj* obj, + const Key* key, + enum ti_opcode op) + { + if (op_level < 1) + return; + total_ops[op] += 1; + if (op_level < 2) + return; + if (false && op == oc_mutex_destroy) + { + wsrep::log_info() << "thread: " << std::this_thread::get_id() + << " object: " << obj + << ": name: " << get_key_name(key) + << " op: " << ti_opstring(op); + } + + std::lock_guard<std::mutex> lock(*ops_map_sync[get_key_index(key)]); + ops_map[get_key_index(key)][op] += 1; + } + + struct thread_args + { + void* this_thread; + void* (*fn)(void*); + void* args; + }; + + pthread_key_t this_thread_key; + struct this_thread_key_initializer + { + this_thread_key_initializer() + { + pthread_key_create(&this_thread_key, nullptr); + } + + ~this_thread_key_initializer() + { + pthread_key_delete(this_thread_key); + } + }; + + + class ti_thread : public ti_obj + { + public: + ti_thread(const wsrep::thread_service::thread_key* key) + : key_(key) + , th_() + , retval_() + , detached_() + { + update_ops(this, key_, oc_thread_create); + } + ~ti_thread() + { + update_ops(this, key_, oc_thread_destroy); + } + + ti_thread(const ti_thread&) = delete; + ti_thread& operator=(const ti_thread&) = delete; + int run(void* (*fn)(void *), void* args) + { + auto ta(new thread_args{this, fn, args}); + return pthread_create(&th_, nullptr, start_thread, ta); + } + + int detach() + { + detached_ = true; + return pthread_detach(th_); + } + + int join(void** retval) + { + return pthread_join(th_, retval); + } + + bool detached() const { return detached_; } + + void retval(void* retval) { retval_ = retval; } + + static ti_thread* self() + { + return reinterpret_cast<ti_thread*>( + pthread_getspecific(this_thread_key)); + } + + int setschedparam(int policy, const struct sched_param* param) + { + return pthread_setschedparam(th_, policy, param); + } + + int getschedparam(int* policy, struct sched_param* param) + { + return pthread_getschedparam(th_, policy, param); + } + + int equal(ti_thread* other) + { + return pthread_equal(th_, other->th_); + } + private: + const wsrep::thread_service::thread_key* key_; + pthread_t th_; + void* retval_; + bool detached_; + }; + + class ti_mutex : public ti_obj + { + public: + ti_mutex(const wsrep::thread_service::mutex_key* key, bool inplace) + : mutex_(PTHREAD_MUTEX_INITIALIZER) + , key_(key) + , inplace_(inplace) +#ifndef NDEBUG + , locked_() + , owner_() +#endif // ! NDEBUG + { + update_ops(this, key_, oc_mutex_create); + if (not inplace) total_allocations++; + } + + ~ti_mutex() { update_ops(this, key_, oc_mutex_destroy); } + + ti_mutex& operator=(const ti_mutex&) = delete; + ti_mutex(const ti_mutex&) = delete; + + int lock() + { + update_ops(this, key_, oc_mutex_lock); + int ret(pthread_mutex_trylock(&mutex_)); + if (ret == EBUSY) + { + mutex_contention++; + { + std::lock_guard<std::mutex> lock(*ops_map_sync[get_key_index(key_)]); + mutex_contention_counts[get_key_name(key_)] += 1; + } + ret = pthread_mutex_lock(&mutex_); + } +#ifndef NDEBUG + if (ret == 0) + { + assert(owner_ == std::thread::id()); + locked_ = true; + owner_ = std::this_thread::get_id(); + } +#endif // ! NDEBUG + return ret; + } + int trylock() + { + update_ops(this, key_, oc_mutex_trylock); + int ret(pthread_mutex_trylock(&mutex_)); +#ifndef NDEBUG + if (ret == 0) + { + assert(owner_ == std::thread::id()); + locked_ = true; + owner_ = std::this_thread::get_id(); + } +#endif // ! NDEBUG + return ret; + } + + int unlock() + { + assert(locked_); +#ifndef NDEBUG + assert(owner_ == std::this_thread::get_id()); + owner_ = std::thread::id(); +#endif // ! NDEBUG + // Use temporary object. After mutex is unlocked it may be + // destroyed before this update_ops() finishes. + auto key(key_); + int ret(pthread_mutex_unlock(&mutex_)); + update_ops(this, key, oc_mutex_unlock); + return ret; + } + + struct condwait_context + { +#ifndef NDEBUG + bool locked; + std::thread::id owner; +#endif // ! NDEBUG + }; + + condwait_context save_for_condwait() + { +#ifndef NDEBUG + return condwait_context{ locked_, owner_ }; +#else + return condwait_context{}; +#endif // ! NDEBUG + } + + void reset() + { +#ifndef NDEBUG + locked_ = false; + owner_ = std::thread::id(); +#endif // ! NDEBUG + } + + void restore_from_condwait(const condwait_context& ctx WSREP_UNUSED) + { +#ifndef NDEBUG + locked_ = ctx.locked; + owner_ = ctx.owner; +#endif // ! NDEBUG + } + + pthread_mutex_t* native_handle() { return &mutex_; } + const wsrep::thread_service::mutex_key* key() const { return key_; } + + bool inplace() const { return inplace_; } + private: + pthread_mutex_t mutex_; + const wsrep::thread_service::mutex_key* key_; + const bool inplace_; +#ifndef NDEBUG + bool locked_; + std::atomic<std::thread::id> owner_; +#endif // ! NDEBU + }; + + class ti_cond : public ti_obj + { + public: + ti_cond(const wsrep::thread_service::cond_key* key, bool inplace) + : cond_(PTHREAD_COND_INITIALIZER) + , key_(key) + , inplace_(inplace) + , waiter_() + { + update_ops(this, key_, oc_cond_create); + if (not inplace) total_allocations++; + } + + ~ti_cond() { update_ops(this, key_, oc_cond_destroy); } + + ti_cond& operator=(const ti_cond&) = delete; + ti_cond(const ti_cond&) = delete; + + int wait(ti_mutex& mutex) + { + cond_check(pthread_mutex_trylock(mutex.native_handle()), + get_key_name(key_), "Mutex not locked in cond wait"); + waiter_ = true; + update_ops(this, key_, oc_cond_wait); + // update_ops(&mutex, mutex.key(), oc_mutex_unlock); + auto condwait_ctx(mutex.save_for_condwait()); + mutex.reset(); + int ret(pthread_cond_wait(&cond_, mutex.native_handle())); + // update_ops(&mutex, mutex.key(), oc_mutex_lock); + mutex.restore_from_condwait(condwait_ctx); + waiter_ = false; + return ret; + } + + int timedwait(ti_mutex& mutex, const struct timespec* ts) + { + cond_check(pthread_mutex_trylock(mutex.native_handle()), + get_key_name(key_), "Mutex not locked in cond wait"); + waiter_ = true; + update_ops(this, key_, oc_cond_timedwait); + // update_ops(&mutex, mutex.key(), oc_mutex_unlock); + auto condwait_ctx(mutex.save_for_condwait()); + mutex.reset(); + int ret(pthread_cond_timedwait(&cond_, mutex.native_handle(), ts)); + // update_ops(&mutex, mutex.key(), oc_mutex_lock); + mutex.restore_from_condwait(condwait_ctx); + waiter_ = false; + return ret; + } + + int signal() + { + update_ops(this, key_, oc_cond_signal); + cond_check(waiter_, get_key_name(key_), + "Signalling condition variable without waiter"); + return pthread_cond_signal(&cond_); + } + + int broadcast() + { + update_ops(this, key_, oc_cond_broadcast); + return pthread_cond_broadcast(&cond_); + } + + bool inplace() const { return inplace_; } + private: + pthread_cond_t cond_; + const wsrep::thread_service::cond_key* key_; + const bool inplace_; + bool waiter_; + }; +} + +int db::ti::before_init() +{ + wsrep::log_info() << "db::ti::before_init()"; + return 0; +} + +int db::ti::after_init() +{ + wsrep::log_info() << "db::ti::after_init()"; + return 0; +} + +////////////////////////////////////////////////////////////////////////////// +// Thread // +////////////////////////////////////////////////////////////////////////////// + +extern "C" +{ +static void* start_thread(void* args_ptr) +{ + thread_args* ta(reinterpret_cast<thread_args*>(args_ptr)); + ti_thread* thread = reinterpret_cast<ti_thread*>(ta->this_thread); + pthread_setspecific(this_thread_key, thread); + void* (*fn)(void*) = ta->fn; + void* args = ta->args; + delete ta; + void* ret = (*fn)(args); + pthread_setspecific(this_thread_key, nullptr); + // If we end here the thread returned instead of calling + // pthread_exit() + if (thread->detached()) + delete thread; + return ret; +} + +WSREP_NORETURN +static void exit_thread(wsrep::thread_service::thread* thread, void* retval) +{ + pthread_setspecific(this_thread_key, nullptr); + ti_thread* th(reinterpret_cast<ti_thread*>(thread)); + th->retval(retval); + if (th->detached()) + delete th; + pthread_exit(retval); +} +} // extern "C" + +db::ti::ti() +{ + thread_service::exit = exit_thread; +} + +const wsrep::thread_service::thread_key* +db::ti::create_thread_key(const char* name) WSREP_NOEXCEPT +{ + assert(name); + return reinterpret_cast<const wsrep::thread_service::thread_key*>( + append_key(name, "thread")); +} + +int db::ti::create_thread(const wsrep::thread_service::thread_key* key, + wsrep::thread_service::thread** thread, + void* (*fn)(void*), void* args) WSREP_NOEXCEPT +{ + auto pit(new ti_thread(key)); + total_allocations++; + int ret; + if ((ret = pit->run(fn, args))) + { + delete pit; + } + else + { + *thread = reinterpret_cast<wsrep::thread_service::thread*>(pit); + } + return ret; +} + +int db::ti::detach(wsrep::thread_service::thread* thread) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_thread*>(thread)->detach(); +} + +int db::ti::equal(wsrep::thread_service::thread* thread_1, + wsrep::thread_service::thread* thread_2) WSREP_NOEXCEPT +{ + return (reinterpret_cast<ti_thread*>(thread_1)->equal( + reinterpret_cast<ti_thread*>(thread_2))); +} + +int db::ti::join(wsrep::thread_service::thread* thread, void** retval) WSREP_NOEXCEPT +{ + ti_thread* th(reinterpret_cast<ti_thread*>(thread)); + int ret(th->join(retval)); + if (not th->detached()) + { + delete th; + } + return ret; +} + +wsrep::thread_service::thread* db::ti::self() WSREP_NOEXCEPT +{ + return reinterpret_cast<wsrep::thread_service::thread*>(ti_thread::self()); +} + +int db::ti::setschedparam(wsrep::thread_service::thread* thread, + int policy, const struct sched_param* param) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_thread*>(thread)->setschedparam(policy, param); +} + +int db::ti::getschedparam(wsrep::thread_service::thread* thread, + int* policy, struct sched_param* param) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_thread*>(thread)->getschedparam(policy, param); +} + +////////////////////////////////////////////////////////////////////////////// +// Mutex // +////////////////////////////////////////////////////////////////////////////// + +const wsrep::thread_service::mutex_key* +db::ti::create_mutex_key(const char* name) WSREP_NOEXCEPT +{ + assert(name); + return reinterpret_cast<const wsrep::thread_service::mutex_key*>( + append_key(name, "mutex")); +} + +wsrep::thread_service::mutex* +db::ti::init_mutex(const wsrep::thread_service::mutex_key* key, void* memblock, + size_t memblock_size) WSREP_NOEXCEPT +{ + return reinterpret_cast<wsrep::thread_service::mutex*>( + memblock_size >= sizeof(ti_mutex) ? new (memblock) ti_mutex(key, true) + : new ti_mutex(key, false)); +} + +int db::ti::destroy(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT +{ + ti_mutex* m(reinterpret_cast<ti_mutex*>(mutex)); + if (m->inplace()) + { + m->~ti_mutex(); + } + else + { + delete m; + } + return 0; +} + +int db::ti::lock(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_mutex*>(mutex)->lock(); +} + +int db::ti::trylock(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_mutex*>(mutex)->trylock(); +} + +int db::ti::unlock(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_mutex*>(mutex)->unlock(); +} + +////////////////////////////////////////////////////////////////////////////// +// Cond // +////////////////////////////////////////////////////////////////////////////// + +const wsrep::thread_service::cond_key* db::ti::create_cond_key(const char* name) WSREP_NOEXCEPT +{ + assert(name); + return reinterpret_cast<const wsrep::thread_service::cond_key*>( + append_key(name, "cond")); +} + +wsrep::thread_service::cond* +db::ti::init_cond(const wsrep::thread_service::cond_key* key, void* memblock, + size_t memblock_size) WSREP_NOEXCEPT +{ + return reinterpret_cast<wsrep::thread_service::cond*>( + memblock_size >= sizeof(ti_cond) ? new (memblock) ti_cond(key, true) + : new ti_cond(key, false)); +} + +int db::ti::destroy(wsrep::thread_service::cond* cond) WSREP_NOEXCEPT +{ + ti_cond* c(reinterpret_cast<ti_cond*>(cond)); + if (c->inplace()) + { + c->~ti_cond(); + } + else + { + delete c; + } + return 0; +} + +int db::ti::wait(wsrep::thread_service::cond* cond, + wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_cond*>(cond)->wait( + *reinterpret_cast<ti_mutex*>(mutex)); +} + +int db::ti::timedwait(wsrep::thread_service::cond* cond, + wsrep::thread_service::mutex* mutex, + const struct timespec* ts) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_cond*>(cond)->timedwait( + *reinterpret_cast<ti_mutex*>(mutex), ts); +} + +int db::ti::signal(wsrep::thread_service::cond* cond) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_cond*>(cond)->signal(); +} + +int db::ti::broadcast(wsrep::thread_service::cond* cond) WSREP_NOEXCEPT +{ + return reinterpret_cast<ti_cond*>(cond)->broadcast(); +} + +void db::ti::level(int level) +{ + ::op_level = level; +} + +void db::ti::cond_checks(bool cond_checks) +{ + if (cond_checks) + wsrep::log_info() << "Enabling condition variable checking"; + ::cond_checks = cond_checks; +} + +std::string db::ti::stats() +{ + std::ostringstream os; + os << "Totals:\n"; + for (size_t i(0); i < total_ops.size(); ++i) + { + if (total_ops[i] > 0) + { + os << " " << ti_opstring(static_cast<enum ti_opcode>(i)) << ": " + << total_ops[i] << "\n"; + } + } + os << "Total allocations: " << total_allocations << "\n"; + os << "Mutex contention: " << mutex_contention << "\n"; + for (auto i : mutex_contention_counts) + { + os << " " << i.first << ": " << i.second << "\n"; + } + os << "Per key:\n"; + std::map<std::string, std::vector<size_t>> sorted; + for (size_t i(0); i < ops_map.size(); ++i) + { + sorted.insert(std::make_pair(get_key_name_by_index(i), ops_map[i])); + } + for (auto i : sorted) + { + for (size_t j(0); j < i.second.size(); ++j) + { + if (i.second[j]) + { + os << " " << i.first << ": " + << ti_opstring(static_cast<enum ti_opcode>(j)) << ": " + << i.second[j] << "\n"; + } + } + } + return os.str(); +} diff --git a/wsrep-lib/dbsim/db_threads.hpp b/wsrep-lib/dbsim/db_threads.hpp new file mode 100644 index 00000000..32d44c10 --- /dev/null +++ b/wsrep-lib/dbsim/db_threads.hpp @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_THREADS_HPP +#define WSREP_DB_THREADS_HPP + +#include "wsrep/thread_service.hpp" +#include <string> + +namespace db +{ + class ti : public wsrep::thread_service + { + public: + ti(); + int before_init() override; + int after_init() override; + + /* Thread */ + const wsrep::thread_service::thread_key* + create_thread_key(const char* name) WSREP_NOEXCEPT override; + int create_thread(const wsrep::thread_service::thread_key* key, + wsrep::thread_service::thread**, + void* (*fn)(void*), void*) WSREP_NOEXCEPT override; + int detach(wsrep::thread_service::thread*) WSREP_NOEXCEPT override; + int equal(wsrep::thread_service::thread*, + wsrep::thread_service::thread*) WSREP_NOEXCEPT override; + int join(wsrep::thread_service::thread*, void**) WSREP_NOEXCEPT override; + wsrep::thread_service::thread* self() WSREP_NOEXCEPT override; + int setschedparam(wsrep::thread_service::thread*, int, + const struct sched_param*) WSREP_NOEXCEPT override; + int getschedparam(wsrep::thread_service::thread*, int*, + struct sched_param*) WSREP_NOEXCEPT override; + + /* Mutex */ + const wsrep::thread_service::mutex_key* + create_mutex_key(const char* name) WSREP_NOEXCEPT override; + wsrep::thread_service::mutex* + init_mutex(const wsrep::thread_service::mutex_key* key, void* memblock, + size_t memblock_size) WSREP_NOEXCEPT override; + int destroy(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT override; + int lock(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT override; + int trylock(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT override; + int unlock(wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT override; + /* Cond */ + const wsrep::thread_service::cond_key* + create_cond_key(const char* name) WSREP_NOEXCEPT override; + wsrep::thread_service::cond* + init_cond(const wsrep::thread_service::cond_key* key, void* memblock, + size_t memblock_size) WSREP_NOEXCEPT override; + int destroy(wsrep::thread_service::cond* cond) WSREP_NOEXCEPT override; + int wait(wsrep::thread_service::cond* cond, + wsrep::thread_service::mutex* mutex) WSREP_NOEXCEPT override; + int timedwait(wsrep::thread_service::cond* cond, + wsrep::thread_service::mutex* mutex, + const struct timespec* ts) WSREP_NOEXCEPT override; + int signal(wsrep::thread_service::cond* cond) WSREP_NOEXCEPT override; + int broadcast(wsrep::thread_service::cond* cond) WSREP_NOEXCEPT override; + + static void level(int level); + static void cond_checks(bool cond_checks); + static std::string stats(); + }; + + +} + +#endif // WSREP_DB_THREADS_HPP diff --git a/wsrep-lib/dbsim/db_tls.cpp b/wsrep-lib/dbsim/db_tls.cpp new file mode 100644 index 00000000..c242668c --- /dev/null +++ b/wsrep-lib/dbsim/db_tls.cpp @@ -0,0 +1,451 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file db_tls.cpp + * + * This file demonstrates the use of TLS service. It does not implement + * real encryption, but may manipulate stream bytes for testing purposes. + */ + +#include "db_tls.hpp" + +#include "wsrep/logger.hpp" + +#include <unistd.h> // read() +#include <fcntl.h> +#include <sys/types.h> +#include <sys/socket.h> // send() +#include <cassert> +#include <cerrno> +#include <cstring> + +#include <mutex> +#include <string> + +namespace +{ + class db_stream : public wsrep::tls_stream + { + public: + db_stream(int fd, int mode) + : fd_(fd) + , state_(s_initialized) + , last_error_() + , mode_(mode) + , stats_() + , is_blocking_() + { + int val(fcntl(fd_, F_GETFL, 0)); + is_blocking_ = not (val & O_NONBLOCK); + } + struct stats + { + size_t bytes_read{0}; + size_t bytes_written{0}; + }; + + /* + * in idle --| + * |-> ch -| ^ | -> want_read --| + * |-> sh -| ---- |--> | -> want_write --| + * |----------------------| + */ + enum state + { + s_initialized, + s_client_handshake, + s_server_handshake, + s_idle, + s_want_read, + s_want_write + }; + + int get_error_number() const { return last_error_; } + const void* get_error_category() const + { + return reinterpret_cast<const void*>(1); + } + + static char* get_error_message(int value, const void*) + { + return ::strerror(value); + } + + enum wsrep::tls_service::status client_handshake(); + + enum wsrep::tls_service::status server_handshake(); + + wsrep::tls_service::op_result read(void*, size_t); + + wsrep::tls_service::op_result write(const void*, size_t); + + enum state state() const { return state_; } + + int fd() const { return fd_; } + void inc_reads(size_t val) { stats_.bytes_read += val; } + void inc_writes(size_t val) { stats_.bytes_written += val; } + const stats& get_stats() const { return stats_; } + private: + enum wsrep::tls_service::status handle_handshake_read(const char* expect); + size_t determine_read_count(size_t max_count) + { + if (is_blocking_ || mode_ < 2) return max_count; + else if (::rand() % 100 == 0) return std::min(size_t(42), max_count); + else return max_count; + } + size_t determine_write_count(size_t count) + { + if (is_blocking_ || mode_ < 2) return count; + else if (::rand() % 100 == 0) return std::min(size_t(43), count); + else return count; + } + + ssize_t do_read(void* buf, size_t max_count) + { + if (is_blocking_ || mode_ < 3 ) + return ::read(fd_, buf, max_count); + else if (::rand() % 1000 == 0) return EINTR; + else return ::read(fd_, buf, max_count); + } + + ssize_t do_write(const void* buf, size_t count) + { + if (is_blocking_ || mode_ < 3) + return ::send(fd_, buf, count, MSG_NOSIGNAL); + else if (::rand() % 1000 == 0) return EINTR; + else return ::send(fd_, buf, count, MSG_NOSIGNAL); + } + + wsrep::tls_service::op_result map_success(ssize_t result) + { + if (is_blocking_ || mode_ < 2) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, size_t(result)}; + } + else if (::rand() % 1000 == 0) + { + wsrep::log_info() << "Success want extra read"; + state_ = s_want_read; + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_read, size_t(result)}; + } + else if (::rand() % 1000 == 0) + { + wsrep::log_info() << "Success want extra write"; + state_ = s_want_write; + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_write, size_t(result)}; + } + else + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, size_t(result)}; + } + } + + wsrep::tls_service::op_result map_result(ssize_t result) + { + if (result > 0) + { + return map_success(result); + } + else if (result == 0) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::eof, 0}; + } + else if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_read, 0}; + } + else + { + last_error_ = errno; + return wsrep::tls_service::op_result{ + wsrep::tls_service::error, 0}; + } + } + + void clear_error() { last_error_ = 0; } + + int fd_; + enum state state_; + int last_error_; + // Operation mode: + // 1 - simulate handshake exchange + // 2 - simulate errors and short reads + int mode_; + stats stats_; + bool is_blocking_; + }; + + enum wsrep::tls_service::status db_stream::client_handshake() + { + clear_error(); + enum wsrep::tls_service::status ret; + assert(state_ == s_initialized || + state_ == s_client_handshake || + state_ == s_want_write); + if (state_ == s_initialized) + { + (void)::send(fd_, "clie", 4, MSG_NOSIGNAL); + ret = wsrep::tls_service::want_read; + state_ = s_client_handshake; + wsrep::log_info() << this << " client handshake sent"; + stats_.bytes_written += 4; + if (not is_blocking_) return ret; + } + + if (state_ == s_client_handshake) + { + if ((ret = handle_handshake_read("serv")) == + wsrep::tls_service::success) + { + state_ = s_want_write; + ret = wsrep::tls_service::want_write; + } + if (not is_blocking_) return ret; + } + + if (state_ == s_want_write) + { + state_ = s_idle; + ret = wsrep::tls_service::success; + if (not is_blocking_) return ret; + } + + if (not is_blocking_) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + return ret; + } + + + + enum wsrep::tls_service::status db_stream::server_handshake() + { + enum wsrep::tls_service::status ret; + assert(state_ == s_initialized || + state_ == s_server_handshake || + state_ == s_want_write); + + if (state_ == s_initialized) + { + ::send(fd_, "serv", 4, MSG_NOSIGNAL); + ret = wsrep::tls_service::want_read; + state_ = s_server_handshake; + stats_.bytes_written += 4; + if (not is_blocking_) return ret; + } + + if (state_ == s_server_handshake) + { + if ((ret = handle_handshake_read("clie")) == + wsrep::tls_service::success) + { + state_ = s_want_write; + ret = wsrep::tls_service::want_write; + } + if (not is_blocking_) return ret; + } + + if (state_ == s_want_write) + { + state_ = s_idle; + ret = wsrep::tls_service::success; + if (not is_blocking_) return ret; + } + + if (not is_blocking_) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + return ret; + } + + enum wsrep::tls_service::status db_stream::handle_handshake_read( + const char* expect) + { + assert(::strlen(expect) >= 4); + char buf[4] = { }; + ssize_t read_result(::read(fd_, buf, sizeof(buf))); + if (read_result > 0) stats_.bytes_read += size_t(read_result); + enum wsrep::tls_service::status ret; + if (read_result == -1 && + (errno == EWOULDBLOCK || errno == EAGAIN)) + { + ret = wsrep::tls_service::want_read; + } + else if (read_result == 0) + { + ret = wsrep::tls_service::eof; + } + else if (read_result != 4 || ::memcmp(buf, expect, 4)) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + else + { + wsrep::log_info() << "Handshake success: " << std::string(buf, 4); + ret = wsrep::tls_service::success; + } + return ret; + } + + wsrep::tls_service::op_result db_stream::read(void* buf, size_t max_count) + { + clear_error(); + if (state_ == s_want_read) + { + state_ = s_idle; + if (max_count == 0) + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, 0}; + } + max_count = determine_read_count(max_count); + ssize_t read_result(do_read(buf, max_count)); + if (read_result > 0) + { + inc_reads(size_t(read_result)); + } + return map_result(read_result); + } + + wsrep::tls_service::op_result db_stream::write( + const void* buf, size_t count) + { + clear_error(); + if (state_ == s_want_write) + { + state_ = s_idle; + if (count == 0) + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, 0}; + } + count = determine_write_count(count); + ssize_t write_result(do_write(buf, count)); + if (write_result > 0) + { + inc_writes(size_t(write_result)); + } + return map_result(write_result); + } +} + + +static db_stream::stats global_stats; +std::mutex global_stats_lock; +static int global_mode; + +static void merge_to_global_stats(const db_stream::stats& stats) +{ + std::lock_guard<std::mutex> lock(global_stats_lock); + global_stats.bytes_read += stats.bytes_read; + global_stats.bytes_written += stats.bytes_written; +} + +wsrep::tls_stream* db::tls::create_tls_stream(int fd) WSREP_NOEXCEPT +{ + auto ret(new db_stream(fd, global_mode)); + wsrep::log_debug() << "New DB stream: " << ret; + return ret; +} + +void db::tls::destroy(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + auto dbs(static_cast<db_stream*>(stream)); + merge_to_global_stats(dbs->get_stats()); + wsrep::log_debug() << "Stream destroy: " << dbs->get_stats().bytes_read + << " " << dbs->get_stats().bytes_written; + wsrep::log_debug() << "Stream destroy" << dbs; + delete dbs; +} + +int db::tls::get_error_number(const wsrep::tls_stream* stream) + const WSREP_NOEXCEPT +{ + return static_cast<const db_stream*>(stream)->get_error_number(); +} + +const void* db::tls::get_error_category(const wsrep::tls_stream* stream) + const WSREP_NOEXCEPT +{ + return static_cast<const db_stream*>(stream)->get_error_category(); +} + +const char* db::tls::get_error_message(const wsrep::tls_stream*, + int value, const void* category) + const WSREP_NOEXCEPT +{ + return db_stream::get_error_message(value, category); +} + +enum wsrep::tls_service::status +db::tls::client_handshake(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + return static_cast<db_stream*>(stream)->client_handshake(); +} + +enum wsrep::tls_service::status +db::tls::server_handshake(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + return static_cast<db_stream*>(stream)->server_handshake(); +} + +wsrep::tls_service::op_result db::tls::read( + wsrep::tls_stream* stream, + void* buf, size_t max_count) WSREP_NOEXCEPT +{ + return static_cast<db_stream*>(stream)->read(buf, max_count); +} + +wsrep::tls_service::op_result db::tls::write( + wsrep::tls_stream* stream, + const void* buf, size_t count) WSREP_NOEXCEPT +{ + return static_cast<db_stream*>(stream)->write(buf, count); +} + +wsrep::tls_service::status +db::tls::shutdown(wsrep::tls_stream*) WSREP_NOEXCEPT +{ + // @todo error simulation + return wsrep::tls_service::success; +} + + +void db::tls::init(int mode) +{ + global_mode = mode; +} + +std::string db::tls::stats() +{ + std::ostringstream oss; + oss << "Transport stats:\n" + << " bytes_read: " << global_stats.bytes_read << "\n" + << " bytes_written: " << global_stats.bytes_written << "\n"; + return oss.str(); +} diff --git a/wsrep-lib/dbsim/db_tls.hpp b/wsrep-lib/dbsim/db_tls.hpp new file mode 100644 index 00000000..97c4387e --- /dev/null +++ b/wsrep-lib/dbsim/db_tls.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_DB_TLS_HPP +#define WSREP_DB_TLS_HPP + +#include "wsrep/tls_service.hpp" + +#include <string> + +namespace db +{ + class tls : public wsrep::tls_service + { + public: + virtual wsrep::tls_stream* create_tls_stream(int) + WSREP_NOEXCEPT override; + virtual void destroy(wsrep::tls_stream*) WSREP_NOEXCEPT override; + virtual int get_error_number(const wsrep::tls_stream*) const + WSREP_NOEXCEPT override; + virtual const void* get_error_category(const wsrep::tls_stream*) const + WSREP_NOEXCEPT override; + virtual const char* get_error_message(const wsrep::tls_stream*, + int, const void*) const + WSREP_NOEXCEPT override; + virtual enum wsrep::tls_service::status + client_handshake(wsrep::tls_stream*) + WSREP_NOEXCEPT override; + virtual enum wsrep::tls_service::status + server_handshake(wsrep::tls_stream*) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::op_result + read(wsrep::tls_stream*, void* buf, size_t max_count) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::op_result + write(wsrep::tls_stream*, const void* buf, size_t count) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::status + shutdown(wsrep::tls_stream*) WSREP_NOEXCEPT override; + + static void init(int mode); + static std::string stats(); + }; +}; + +#endif // WSREP_DB_TLS_HPP diff --git a/wsrep-lib/dbsim/dbsim.cpp b/wsrep-lib/dbsim/dbsim.cpp new file mode 100644 index 00000000..070f83b4 --- /dev/null +++ b/wsrep-lib/dbsim/dbsim.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "db_params.hpp" +#include "db_simulator.hpp" + +int main(int argc, char** argv) +{ + try + { + db::simulator(db::parse_args(argc, argv)).run(); + } + catch (const std::exception& e) + { + std::cerr << e.what() << std::endl; + return 1; + } + return 0; +} diff --git a/wsrep-lib/doc/Doxyfile b/wsrep-lib/doc/Doxyfile new file mode 100644 index 00000000..bc6f15fa --- /dev/null +++ b/wsrep-lib/doc/Doxyfile @@ -0,0 +1,2429 @@ +# Doxyfile 1.8.11 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "wsrep-lib" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = +ALIASES = "startuml=\if DontIgnorePlantUMLCode" +ALIASES += "enduml=\endif" + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = YES + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if <section_label> ... \endif and \cond <section_label> +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = ../include/wsrep ../dbsim ../src + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl, +# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js. + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# <filter> <input-file> +# +# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = ../README.md + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the +# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the +# cost of reduced performance. This can be particularly helpful with template +# rich C++ code for which doxygen's built-in parser lacks the necessary type +# information. +# Note: The availability of this option depends on whether or not doxygen was +# generated with the -Duse-libclang=ON option for CMake. +# The default value is: NO. + +CLANG_ASSISTED_PARSING = NO + +# If clang assisted parsing is enabled you can provide the compiler with command +# line options that you would normally use when invoking the compiler. Note that +# the include paths will already be set by doxygen for the files and directories +# specified with INPUT and INCLUDE_PATH. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_OPTIONS = + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = NO + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use <access key> + S +# (what the <access key> is depends on the OS and browser, but it is typically +# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down +# key> to jump into the search results window, the results can be navigated +# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel +# the search. The filter options can be selected when the cursor is inside the +# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys> +# to select a filter and <Enter> or <escape> to activate or cancel the filter +# option. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a web server instead of a web client using Javascript. There +# are two flavors of web server based searching depending on the EXTERNAL_SEARCH +# setting. When disabled, doxygen will generate a PHP script for searching and +# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing +# and searching needs to be provided by external tools. See the section +# "External Indexing and Searching" for details. +# The default value is: NO. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SERVER_BASED_SEARCH = NO + +# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP +# script for searching. Instead the search results are written to an XML file +# which needs to be processed by an external indexer. Doxygen will invoke an +# external search engine pointed to by the SEARCHENGINE_URL option to obtain the +# search results. +# +# Doxygen ships with an example indexer (doxyindexer) and search engine +# (doxysearch.cgi) which are based on the open source search engine library +# Xapian (see: http://xapian.org/). +# +# See the section "External Indexing and Searching" for details. +# The default value is: NO. +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTERNAL_SEARCH = NO + +# The SEARCHENGINE_URL should point to a search engine hosted by a web server +# which will return the search results when EXTERNAL_SEARCH is enabled. +# +# Doxygen ships with an example indexer (doxyindexer) and search engine +# (doxysearch.cgi) which are based on the open source search engine library +# Xapian (see: http://xapian.org/). See the section "External Indexing and +# Searching" for details. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SEARCHENGINE_URL = + +# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed +# search data is written to a file for indexing by an external tool. With the +# SEARCHDATA_FILE tag the name of this file can be specified. +# The default file is: searchdata.xml. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SEARCHDATA_FILE = searchdata.xml + +# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the +# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is +# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple +# projects and redirect the results back to the right project. +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTERNAL_SEARCH_ID = + +# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen +# projects other than the one defined by this configuration file, but that are +# all added to the same external search index. Each project needs to have a +# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of +# to a relative location where the documentation can be found. The format is: +# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ... +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTRA_SEARCH_MAPPINGS = + +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = YES + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: latex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. +# +# Note that when enabling USE_PDFLATEX this option is only used for generating +# bitmaps for formulas in the HTML output, but not in the Makefile that is +# written to the output directory. +# The default file is: latex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate +# index for LaTeX. +# The default file is: makeindex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX +# documents. This may be useful for small projects and may help to save some +# trees in general. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used by the +# printer. +# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x +# 14 inches) and executive (7.25 x 10.5 inches). +# The default value is: a4. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PAPER_TYPE = a4 + +# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names +# that should be included in the LaTeX output. The package can be specified just +# by its name or with the correct syntax as to be used with the LaTeX +# \usepackage command. To get the times font for instance you can specify : +# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times} +# To use the option intlimits with the amsmath package you can specify: +# EXTRA_PACKAGES=[intlimits]{amsmath} +# If left blank no extra packages will be included. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the +# generated LaTeX document. The header should contain everything until the first +# chapter. If it is left blank doxygen will generate a standard header. See +# section "Doxygen usage" for information on how to let doxygen write the +# default header to a separate file. +# +# Note: Only use a user-defined header if you know what you are doing! The +# following commands have a special meaning inside the header: $title, +# $datetime, $date, $doxygenversion, $projectname, $projectnumber, +# $projectbrief, $projectlogo. Doxygen will replace $title with the empty +# string, for the replacement values of the other commands the user is referred +# to HTML_HEADER. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the +# generated LaTeX document. The footer should contain everything after the last +# chapter. If it is left blank doxygen will generate a standard footer. See +# LATEX_HEADER for more information on how to generate a default footer and what +# special commands can be used inside the footer. +# +# Note: Only use a user-defined footer if you know what you are doing! +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_FOOTER = + +# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# LaTeX style sheets that are included after the standard style sheets created +# by doxygen. Using this option one can overrule certain style aspects. Doxygen +# will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EXTRA_STYLESHEET = + +# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the LATEX_OUTPUT output +# directory. Note that the files will be copied as-is; there are no commands or +# markers available. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EXTRA_FILES = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is +# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will +# contain links (just like the HTML output) instead of page references. This +# makes the output suitable for online browsing using a PDF viewer. +# The default value is: YES. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate +# the PDF file directly from the LaTeX files. Set this option to YES, to get a +# higher quality PDF documentation. +# The default value is: YES. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode +# command to the generated LaTeX files. This will instruct LaTeX to keep running +# if errors occur, instead of asking the user for help. This option is also used +# when generating formulas in HTML. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_BATCHMODE = NO + +# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the +# index chapters (such as File Index, Compound Index, etc.) in the output. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_HIDE_INDICES = NO + +# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source +# code with syntax highlighting in the LaTeX output. +# +# Note that which sources are shown also depends on other settings such as +# SOURCE_BROWSER. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_SOURCE_CODE = NO + +# The LATEX_BIB_STYLE tag can be used to specify the style to use for the +# bibliography, e.g. plainnat, or ieeetr. See +# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# The default value is: plain. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_BIB_STYLE = plain + +# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated +# page will contain the date and time when the page was generated. Setting this +# to NO can help when comparing the output of multiple runs. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_TIMESTAMP = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The +# RTF output is optimized for Word 97 and may not look too pretty with other RTF +# readers/editors. +# The default value is: NO. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: rtf. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF +# documents. This may be useful for small projects and may help to save some +# trees in general. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will +# contain hyperlink fields. The RTF file will contain links (just like the HTML +# output) instead of page references. This makes the output suitable for online +# browsing using Word or some other Word compatible readers that support those +# fields. +# +# Note: WordPad (write) and others do not support links. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's config +# file, i.e. a series of assignments. You only have to provide replacements, +# missing definitions are set to their default value. +# +# See also section "Doxygen usage" for information on how to generate the +# default style sheet that doxygen normally uses. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an RTF document. Syntax is +# similar to doxygen's config file. A template extensions file can be generated +# using doxygen -e rtf extensionFile. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_EXTENSIONS_FILE = + +# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code +# with syntax highlighting in the RTF output. +# +# Note that which sources are shown also depends on other settings such as +# SOURCE_BROWSER. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for +# classes and files. +# The default value is: NO. + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. A directory man3 will be created inside the directory specified by +# MAN_OUTPUT. +# The default directory is: man. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to the generated +# man pages. In case the manual section does not start with a number, the number +# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is +# optional. +# The default value is: .3. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_EXTENSION = .3 + +# The MAN_SUBDIR tag determines the name of the directory created within +# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by +# MAN_EXTENSION with the initial . removed. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_SUBDIR = + +# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it +# will generate one additional man file for each entity documented in the real +# man page(s). These additional files only source the real man page, but without +# them the man command would be unable to find the correct page. +# The default value is: NO. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that +# captures the structure of the code including all documentation. +# The default value is: NO. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: xml. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_OUTPUT = xml + +# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program +# listings (including syntax highlighting and cross-referencing information) to +# the XML output. Note that enabling this will significantly increase the size +# of the XML output. +# The default value is: YES. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- + +# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files +# that can be used to generate PDF. +# The default value is: NO. + +GENERATE_DOCBOOK = NO + +# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in +# front of it. +# The default directory is: docbook. +# This tag requires that the tag GENERATE_DOCBOOK is set to YES. + +DOCBOOK_OUTPUT = docbook + +# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the +# program listings (including syntax highlighting and cross-referencing +# information) to the DOCBOOK output. Note that enabling this will significantly +# increase the size of the DOCBOOK output. +# The default value is: NO. +# This tag requires that the tag GENERATE_DOCBOOK is set to YES. + +DOCBOOK_PROGRAMLISTING = NO + +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an +# AutoGen Definitions (see http://autogen.sf.net) file that captures the +# structure of the code including all documentation. Note that this feature is +# still experimental and incomplete at the moment. +# The default value is: NO. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module +# file that captures the structure of the code including all documentation. +# +# Note that this feature is still experimental and incomplete at the moment. +# The default value is: NO. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary +# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI +# output from the Perl module output. +# The default value is: NO. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely +# formatted so it can be parsed by a human reader. This is useful if you want to +# understand what is going on. On the other hand, if this tag is set to NO, the +# size of the Perl module output will be much smaller and Perl will parse it +# just the same. +# The default value is: YES. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file are +# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful +# so different doxyrules.make files included by the same Makefile don't +# overwrite each other's variables. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all +# C-preprocessor directives found in the sources and include files. +# The default value is: YES. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES, the include files in the +# INCLUDE_PATH will be searched if a #include is found. +# The default value is: YES. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will be +# used. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that are +# defined before the preprocessor is started (similar to the -D option of e.g. +# gcc). The argument of the tag is a list of macros of the form: name or +# name=definition (no spaces). If the definition and the "=" are omitted, "=1" +# is assumed. To prevent a macro definition from being undefined via #undef or +# recursively expanded use the := operator instead of the = operator. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will +# remove all references to function-like macros that are alone on a line, have +# an all uppercase name, and do not end with a semicolon. Such function macros +# are typically used for boiler-plate code, and will confuse the parser if not +# removed. +# The default value is: YES. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES tag can be used to specify one or more tag files. For each tag +# file the location of the external documentation should be added. The format of +# a tag file without this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where loc1 and loc2 can be relative or absolute paths or URLs. See the +# section "Linking to external documentation" for more information about the use +# of tag files. +# Note: Each tag file must have a unique name (where the name does NOT include +# the path). If a tag file is not located in the directory in which doxygen is +# run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create a +# tag file that is based on the input files it reads. See section "Linking to +# external documentation" for more information about the usage of tag files. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES, all external class will be listed in +# the class index. If set to NO, only the inherited external classes will be +# listed. +# The default value is: NO. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will be +# listed. +# The default value is: YES. + +EXTERNAL_GROUPS = YES + +# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in +# the related pages index. If set to NO, only the current project's pages will +# be listed. +# The default value is: YES. + +EXTERNAL_PAGES = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of 'which perl'). +# The default file (with absolute path) is: /usr/bin/perl. + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram +# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to +# NO turns the diagrams off. Note that this option also works with HAVE_DOT +# disabled, but it is recommended to install and use dot, since it yields more +# powerful graphs. +# The default value is: YES. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see: +# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. + +DIA_PATH = + +# If set to YES the inheritance and collaboration graphs will hide inheritance +# and usage relations if the target is undocumented or is not a class. +# The default value is: YES. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz (see: +# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# Bell Labs. The other options in this section have no effect if this option is +# set to NO +# The default value is: YES. + +HAVE_DOT = YES + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed +# to run in parallel. When set to 0 doxygen will base this on the number of +# processors available in the system. You can set it explicitly to a value +# larger than 0 to get control over the balance between CPU load and processing +# speed. +# Minimum value: 0, maximum value: 32, default value: 0. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NUM_THREADS = 0 + +# When you want a differently looking font in the dot files that doxygen +# generates you can specify the font name using DOT_FONTNAME. You need to make +# sure dot is able to find the font, which can be done by putting it in a +# standard location or by setting the DOTFONTPATH environment variable or by +# setting DOT_FONTPATH to the directory containing the font. +# The default value is: Helvetica. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of +# dot graphs. +# Minimum value: 4, maximum value: 24, default value: 10. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the default font as specified with +# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set +# the path where dot can find it using this tag. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTPATH = + +# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for +# each documented class showing the direct and indirect inheritance relations. +# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a +# graph for each documented class showing the direct and indirect implementation +# dependencies (inheritance, containment, and class references variables) of the +# class with other documented classes. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for +# groups, showing the direct groups dependencies. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +UML_LOOK = NO + +# If the UML_LOOK tag is enabled, the fields and methods are shown inside the +# class node. If there are many fields or methods and many nodes the graph may +# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the +# number of items for each type to make the size more manageable. Set this to 0 +# for no limit. Note that the threshold may be exceeded by 50% before the limit +# is enforced. So when you set the threshold to 10, up to 15 fields may appear, +# but if the number exceeds 15, the total amount of fields shown is limited to +# 10. +# Minimum value: 0, maximum value: 100, default value: 10. +# This tag requires that the tag HAVE_DOT is set to YES. + +UML_LIMIT_NUM_FIELDS = 10 + +# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and +# collaboration graphs will show the relations between templates and their +# instances. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +TEMPLATE_RELATIONS = NO + +# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to +# YES then doxygen will generate a graph for each documented file showing the +# direct and indirect include dependencies of the file with other documented +# files. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +INCLUDE_GRAPH = YES + +# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are +# set to YES then doxygen will generate a graph for each documented file showing +# the direct and indirect include dependencies of the file with other documented +# files. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH tag is set to YES then doxygen will generate a call +# dependency graph for every global function or class method. +# +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. Disabling a call graph can be +# accomplished by means of the command \hidecallgraph. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller +# dependency graph for every global function or class method. +# +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. Disabling a caller graph can be +# accomplished by means of the command \hidecallergraph. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical +# hierarchy of all classes instead of a textual one. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the +# dependencies a directory has on other directories in a graphical way. The +# dependency relations are determined by the #include relations between the +# files in the directories. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. For an explanation of the image formats see the section +# output formats in the documentation of the dot tool (Graphviz (see: +# http://www.graphviz.org/)). +# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order +# to make the SVG files visible in IE 9+ (other browsers do not have this +# requirement). +# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd, +# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo, +# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo, +# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and +# png:gdiplus:gdiplus. +# The default value is: png. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_IMAGE_FORMAT = png + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# +# Note that this requires a modern browser other than Internet Explorer. Tested +# and working are Firefox, Chrome, Safari, and Opera. +# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make +# the SVG files visible. Older versions of IE do not have SVG support. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +INTERACTIVE_SVG = NO + +# The DOT_PATH tag can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the \dotfile +# command). +# This tag requires that the tag HAVE_DOT is set to YES. + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = + +# The DIAFILE_DIRS tag can be used to specify one or more directories that +# contain dia files that are included in the documentation (see the \diafile +# command). + +DIAFILE_DIRS = + +# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the +# path where java can find the plantuml.jar file. If left blank, it is assumed +# PlantUML is not used or called during a preprocessing step. Doxygen will +# generate a warning when it encounters a \startuml command in this case and +# will not generate output for the diagram. + +PLANTUML_JAR_PATH = + +# When using plantuml, the specified paths are searched for files specified by +# the !include statement in a plantuml block. + +PLANTUML_INCLUDE_PATH = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes +# that will be shown in the graph. If the number of nodes in a graph becomes +# larger than this value, doxygen will truncate the graph, which is visualized +# by representing a node as a red box. Note that doxygen if the number of direct +# children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that +# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. +# Minimum value: 0, maximum value: 10000, default value: 50. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs +# generated by dot. A depth value of 3 means that only nodes reachable from the +# root by following a path via at most 3 edges will be shown. Nodes that lay +# further from the root node will be omitted. Note that setting this option to 1 +# or 2 may greatly reduce the computation time needed for large code bases. Also +# note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. +# Minimum value: 0, maximum value: 1000, default value: 0. +# This tag requires that the tag HAVE_DOT is set to YES. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not seem +# to support this out of the box. +# +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) support +# this, this feature is disabled by default. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page +# explaining the meaning of the various boxes and arrows in the dot generated +# graphs. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# files that are used to generate the various graphs. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_CLEANUP = YES diff --git a/wsrep-lib/include/wsrep/allowlist_service.hpp b/wsrep-lib/include/wsrep/allowlist_service.hpp new file mode 100644 index 00000000..1e619ce8 --- /dev/null +++ b/wsrep-lib/include/wsrep/allowlist_service.hpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + + +/** @file allowlist_service.hpp + * + * Service interface for interacting with DBMS provided + * allowlist callback. + */ + +#ifndef WSREP_ALLOWLIST_SERVICE_HPP +#define WSREP_ALLOWLIST_SERVICE_HPP + +#include "compiler.hpp" +#include "wsrep/buffer.hpp" + +#include <sys/types.h> // ssize_t + +namespace wsrep +{ + class allowlist_service + { + public: + enum allowlist_key + { + /** IP allowlist check */ + allowlist_ip, + /** SSL allowlist check */ + allowlist_ssl + }; + + virtual ~allowlist_service() { } + + /** + * Allowlist callback. + */ + virtual bool allowlist_cb(allowlist_key key, + const wsrep::const_buffer& value) WSREP_NOEXCEPT = 0; + }; +} + +#endif // WSREP_ALLOWLIST_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/atomic.hpp b/wsrep-lib/include/wsrep/atomic.hpp new file mode 100644 index 00000000..6d92c167 --- /dev/null +++ b/wsrep-lib/include/wsrep/atomic.hpp @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_ATOMIC_HPP +#define WSREP_ATOMIC_HPP + +#if defined(__GNUG__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4) +#include <cstdatomic> +#else +#include <atomic> +#endif // defined(__GNUG__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4) + +#endif // WSREP_ATOMIC_HPP diff --git a/wsrep-lib/include/wsrep/buffer.hpp b/wsrep-lib/include/wsrep/buffer.hpp new file mode 100644 index 00000000..1c1ac599 --- /dev/null +++ b/wsrep-lib/include/wsrep/buffer.hpp @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_BUFFER_HPP +#define WSREP_BUFFER_HPP + +#include <cstddef> +#include <vector> + +namespace wsrep +{ + class const_buffer + { + public: + const_buffer() + : ptr_() + , size_() + { } + + const_buffer(const void* ptr, size_t size) + : ptr_(ptr) + , size_(size) + { } + + const_buffer(const const_buffer& b) + : ptr_(b.ptr()) + , size_(b.size()) + { } + + const void* ptr() const { return ptr_; } + const char* data() const { return static_cast<const char*>(ptr_); } + size_t size() const { return size_; } + + const_buffer& operator=(const const_buffer& b) + { + ptr_ = b.ptr(); + size_ = b.size(); + return *this; + } + private: + const void* ptr_; + size_t size_; + }; + + + class mutable_buffer + { + public: + mutable_buffer() + : buffer_() + { } + + mutable_buffer(const mutable_buffer& b) + : buffer_(b.buffer_) + { } + + void resize(size_t s) { buffer_.resize(s); } + + void clear() + { + // using swap to ensure deallocation + std::vector<char>().swap(buffer_); + } + + void push_back(const char* begin, const char* end) + { + buffer_.insert(buffer_.end(), begin, end); + } + + template <class C> void push_back(const C& c) + { + std::copy(c.begin(), c.end(), std::back_inserter(buffer_)); + } + + size_t size() const { return buffer_.size(); } + + /** + * Return pointer to underlying data array. The returned pointer + * may or may not be null in case of empty buffer, it is up to + * user to check the size of the array before dereferencing the + * pointer. + * + * @return Pointer to underlying data array. + */ + char* data() { return buffer_.data(); } + + /** + * Return const pointer to underlying data array. The returned pointer + * may or may not be null in case of empty buffer, it is up to + * user to check the size of the array before dereferencing the + * pointer. + * + * @return Const pointer to underlying data array. + */ + const char* data() const { return buffer_.data(); } + + mutable_buffer& operator= (const mutable_buffer& other) + { + buffer_ = other.buffer_; + return *this; + } + + bool operator==(const mutable_buffer& other) const + { + return buffer_ == other.buffer_; + } + private: + std::vector<char> buffer_; + }; +} + +#endif // WSREP_BUFFER_HPP diff --git a/wsrep-lib/include/wsrep/chrono.hpp b/wsrep-lib/include/wsrep/chrono.hpp new file mode 100644 index 00000000..14961d77 --- /dev/null +++ b/wsrep-lib/include/wsrep/chrono.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file chrono.hpp + * + * Type definitions to work around GCC 4.4 incompatibilities with + * C++11 chrono. + */ + +#ifndef WSREP_CHRONO_HPP +#define WSREP_CHRONO_HPP + +#include <chrono> + +namespace wsrep +{ + /* wsrep::clock - clock type compatible with std::chrono::steady_clock. */ +#if defined(__GNUG__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4) + typedef std::chrono::monotonic_clock clock; +#else + using clock = std::chrono::steady_clock; +#endif // defined(__GNUG__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4) + +} + +#endif // WSREP_CHRONO_HPP diff --git a/wsrep-lib/include/wsrep/client_id.hpp b/wsrep-lib/include/wsrep/client_id.hpp new file mode 100644 index 00000000..f7597c88 --- /dev/null +++ b/wsrep-lib/include/wsrep/client_id.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_CLIENT_ID_HPP +#define WSREP_CLIENT_ID_HPP + +#include <ostream> +#include <limits> + +namespace wsrep +{ + class client_id + { + public: + typedef unsigned long long type; + client_id() + : id_(std::numeric_limits<type>::max()) + { } + template <typename I> + explicit client_id(I id) + : id_(static_cast<type>(id)) + { } + type get() const { return id_; } + static type undefined() { return std::numeric_limits<type>::max(); } + bool operator<(const client_id& other) const + { + return (id_ < other.id_); + } + bool operator==(const client_id& other) const + { + return (id_ == other.id_); + } + private: + type id_; + }; + static inline std::ostream& operator<<( + std::ostream& os, const wsrep::client_id& client_id) + { + return (os << client_id.get()); + } +} + + +#endif // WSREP_CLIENT_ID_HPP diff --git a/wsrep-lib/include/wsrep/client_service.hpp b/wsrep-lib/include/wsrep/client_service.hpp new file mode 100644 index 00000000..d47396df --- /dev/null +++ b/wsrep-lib/include/wsrep/client_service.hpp @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file client_service.hpp + * + * This file will define a `callback` abstract interface for a + * DBMS client session service. The interface will define methods + * which will be called by the wsrep-lib. + */ + +#ifndef WSREP_CLIENT_SERVICE_HPP +#define WSREP_CLIENT_SERVICE_HPP + +#include "buffer.hpp" +#include "provider.hpp" +#include "mutex.hpp" +#include "lock.hpp" + +namespace wsrep +{ + class client_service + { + public: + client_service() { } + virtual ~client_service() { } + + /** + * Return true if the current transaction has been interrupted + * by the DBMS. The lock which is passed to interrupted call + * will always have underlying mutex locked. + * + * @param lock Lock object grabbed by the client_state + */ + virtual bool interrupted(wsrep::unique_lock<wsrep::mutex>& lock) const = 0; + + /** + * Reset possible global or thread local parameters associated + * to the thread. + */ + virtual void reset_globals() = 0; + + /** + * Store possible global or thread local parameters associated + * to the thread. + */ + virtual void store_globals() = 0; + + /** + * Set up a data for replication. + */ + virtual int prepare_data_for_replication() = 0; + + /** + * Clean up after transcation has been terminated. + */ + virtual void cleanup_transaction() = 0; + + // + // Streaming + // + /** + * Return true if current statement is allowed for streaming, + * otherwise false. + */ + virtual bool statement_allowed_for_streaming() const = 0; + + /** + * Return the total number of bytes generated by the transaction + * context. + */ + virtual size_t bytes_generated() const = 0; + + /** + * Prepare a buffer containing data for the next fragment to replicate. + * The caller may set log_position to record the database specific + * position corresponding to changes contained in the buffer. + * When the call returns, the log_position will be available to read + * from streaming_context::log_position(). + * + * @return Zero in case of success, non-zero on failure. + * If there is no data to replicate, the method shall return + * zero and leave the buffer empty. + */ + virtual int prepare_fragment_for_replication(wsrep::mutable_buffer& buffer, + size_t& log_position) = 0; + + /** + * Remove fragments from the storage within current transaction. + * Fragment removal will be committed once the current transaction + * commits. + * + * @return Zero in case of success, non-zero on failure. + */ + virtual int remove_fragments() = 0; + + // + // Rollback + // + /** + * Perform brute force rollback. + * + * This method may be called from two contexts, either from + * client state methods when the BF abort condition is detected, + * or from the background rollbacker thread. The task for this + * method is to release all reasources held by the client + * after BF abort so that the high priority thread can continue + * applying. + */ + virtual int bf_rollback() = 0; + + // + // Interface to global server state + // + /** + * Forcefully shut down the DBMS process or replication system. + * This may be called in situations where + * the process may encounter a situation where data integrity + * may not be guaranteed or other unrecoverable condition is + * encontered. + */ + virtual void emergency_shutdown() = 0; + + // Replaying + /** + * Notify that the client will replay. + * + * @todo This should not be visible to DBMS level, should be + * handled internally by wsrep-lib. + */ + virtual void will_replay() = 0; + + /** + * Signal that replay is done. + */ + virtual void signal_replayed() = 0; + + /** + * Replay the current transaction. The implementation must put + * the caller Client Context into applying mode and call + * client_state::replay(). + * + * @todo This should not be visible to DBMS level, should be + * handled internally by wsrep-lib. + */ + virtual enum wsrep::provider::status replay() = 0; + + /** + * Replay the current transaction. This is used for replaying + * prepared XA transactions, which are BF aborted but not + * while orderding commit / rollback. + */ + virtual enum wsrep::provider::status replay_unordered() = 0; + + /** + * Wait until all replaying transactions have been finished + * replaying. + * + * @todo This should not be visible to DBMS level, should be + * handled internally by wsrep-lib. + */ + virtual void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) = 0; + + // + // XA + // + /** + * Send a commit by xid + */ + virtual enum wsrep::provider::status commit_by_xid() = 0; + + /** + * Returns true if the client has an ongoing XA transaction. + * This method is used to determine when to cleanup the + * corresponding wsrep-lib transaction object. + * This method should return false when the XA transaction + * is over, and the wsrep-lib transaction object can be + * cleaned up. + */ + virtual bool is_explicit_xa() = 0; + + /** + * Returns true if the currently executing command is + * a rollback for XA. This is used to avoid setting a + * a deadlock error rollback as it may be unexpected + * by the DBMS. + */ + virtual bool is_xa_rollback() = 0; + + // + // Debug interface + // + /** + * Enter debug sync point. + * + * @params sync_point Name of the debug sync point. + */ + virtual void debug_sync(const char* sync_point) = 0; + + /** + * Forcefully kill the process if the crash_point has + * been enabled. + */ + virtual void debug_crash(const char* crash_point) = 0; + }; +} + +#endif // WSREP_CLIENT_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/client_state.hpp b/wsrep-lib/include/wsrep/client_state.hpp new file mode 100644 index 00000000..138bf5f0 --- /dev/null +++ b/wsrep-lib/include/wsrep/client_state.hpp @@ -0,0 +1,1209 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file client_state.hpp + * + * + * Return value conventions: + * + * The calls which may alter either client_state or associated + * transaction state will generally return zero on success and + * non-zero on failure. More detailed error information is stored + * into client state and persisted there until explicitly cleared. + */ + +#ifndef WSREP_CLIENT_STATE_HPP +#define WSREP_CLIENT_STATE_HPP + +#include "server_state.hpp" +#include "server_service.hpp" +#include "provider.hpp" +#include "transaction.hpp" +#include "client_id.hpp" +#include "client_service.hpp" +#include "mutex.hpp" +#include "lock.hpp" +#include "buffer.hpp" +#include "thread.hpp" +#include "xid.hpp" +#include "chrono.hpp" + +namespace wsrep +{ + class server_state; + class provider; + + enum client_error + { + e_success, + e_error_during_commit, + e_deadlock_error, + e_interrupted_error, + e_size_exceeded_error, + e_append_fragment_error, + e_not_supported_error, + e_timeout_error + }; + + static inline const char* to_c_string(enum client_error error) + { + switch (error) + { + case e_success: return "success"; + case e_error_during_commit: return "commit_error"; + case e_deadlock_error: return "deadlock_error"; + case e_interrupted_error: return "interrupted_error"; + case e_size_exceeded_error: return "size_exceeded_error"; + case e_append_fragment_error: return "append_fragment_error"; + case e_not_supported_error: return "not_supported_error"; + case e_timeout_error: return "timeout_error"; + } + return "unknown"; + } + + static inline std::string to_string(enum client_error error) + { + return to_c_string(error); + } + /** + * Client State + */ + class client_state + { + public: + /** + * Client mode enumeration. + */ + enum mode + { + /** undefined mode */ + m_undefined, + /** Locally operating client session. */ + m_local, + /** High priority mode */ + m_high_priority, + /** Client is in total order isolation mode */ + m_toi, + /** Client is executing rolling schema upgrade */ + m_rsu, + /** Client is executing NBO */ + m_nbo + }; + + static const int n_modes_ = m_nbo + 1; + /** + * Client state enumeration. + * + */ + enum state + { + /** + * Client session has not been initialized yet. + */ + s_none, + /** + * Client is idle, the control is in the application which + * uses the DBMS system. + */ + s_idle, + /** + * The control of the client processing is inside the DBMS + * system. + */ + s_exec, + /** + * Client handler is sending result to client. + */ + s_result, + /** + * The client session is terminating. + */ + s_quitting + }; + + static const int state_max_ = s_quitting + 1; + + /** + * Aqcuire ownership on the thread. + * + * This method should be called every time the thread + * operating the client state changes. This method is called + * implicitly from before_command() and + * wait_rollback_complete_and_acquire_ownership(). + */ + void acquire_ownership() + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + do_acquire_ownership(lock); + } + + /** + * @deprecated Use acquire_ownership() instead. + */ + void store_globals() + { + acquire_ownership(); + } + + /** + * Destructor. + */ + virtual ~client_state() + { + assert(transaction_.active() == false); + } + + /** @name Client session handling */ + /** @{ */ + /** + * This method should be called when opening the client session. + * + * Initializes client id and changes the state to s_idle. + */ + void open(wsrep::client_id); + + /** + * This method should be called before closing the client session. + * + * The state is changed to s_quitting and any open transactions + * are rolled back. + */ + void close(); + + /** + * This method should be called after closing the client session + * to clean up. + * + * The state is changed to s_none. + */ + void cleanup(); + + /** + * Overload of cleanup() method which takes lock as argument. + * This method does not release the lock during execution, but + * the lock is needed for debug build sanity checks. + */ + void cleanup(wsrep::unique_lock<wsrep::mutex>& lock); + /** @} */ + + /** @name Client command handling */ + /** @{ */ + /** + * This mehod should be called before the processing of command + * received from DBMS client starts. + * + * This method will wait until the possible synchronous + * rollback for associated transaction has finished unless + * wait_rollback_complete_and_acquire_ownership() has been + * called before. + * + * The method has a side effect of changing the client + * context state to executing. + * + * The value set by keep_command_error has an effect on + * how before_command() behaves when it is entered after + * background rollback has been processed: + * + * - If keep_command_error is set true, the current error + * is set and success will be returned. + * - If keep_command_error is set false, the transaction is + * cleaned up and the return value will be non-zero to + * indicate error. + * + * @param keep_command_error Make client state to preserve error + * state in command hooks. + * This is needed if a current command is not supposed to + * return an error status to the client and the protocol must + * advance until the next client command to return error status. + * + * @return Zero in case of success, non-zero in case of the + * associated transaction was BF aborted. + */ + int before_command(bool keep_command_error); + + int before_command() + { + return before_command(false); + } + + /** + * This method should be called before returning + * a result to DBMS client. + * + * The method will check if the transaction associated to + * the connection has been aborted. Rollback is performed + * if needed. After the call, current_error() will return an error + * code associated to the client state. If the error code is + * not success, the transaction associated to the client state + * has been aborted and rolled back. + */ + void after_command_before_result(); + + /** + * Method which should be called after returning the + * control back to DBMS client.. + * + * The method will do the check if the transaction associated + * to the connection has been aborted. If so, rollback is + * performed and the transaction is left to aborted state. + * The next call to before_command() will return an error and + * the error state can be examined after after_command_before_resul() + * is called. + * + * This method has a side effect of changing state to + * idle. + */ + void after_command_after_result(); + /** @} */ + + /** @name Statement level operations */ + /** @{ */ + /** + * Before statement execution operations. + * + * Check if server is synced and if dirty reads are allowed. + * + * @return Zero in case of success, non-zero if the statement + * is not allowed to be executed due to read or write + * isolation requirements. + */ + int before_statement(); + + /** + * After statement execution operations. + * + * * Check for must_replay state + * * Do rollback if requested + */ + int after_statement(); + /** @} */ + + /** + * Perform cleanup after applying a transaction. + * + * @param err Applying error (empty for no error) + */ + void after_applying() + { + assert(mode_ == m_high_priority); + transaction_.after_applying(); + } + + /** @name Replication interface */ + /** @{ */ + /** + * Start a new transaction with a transaction id. + * + * @todo This method should + * - Register the transaction on server level for bookkeeping + * - Isolation levels? Or part of the transaction? + */ + int start_transaction(const wsrep::transaction_id& id) + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(state_ == s_exec); + return transaction_.start_transaction(id); + } + + /** + * Establish read view ID of the transaction. + * + * This method should be preferably called immediately before any + * first read or write operation in the transaction is performed, + * Then it can be called with default NULL parameter and will use + * the current last committed GTID. + * Alternatively it can be called at any time before commit with an + * explicit GTID that corresponds to transaction read view. + * + * @param gtid optional explicit GTID of the transaction read view. + */ + int assign_read_view(const wsrep::gtid* const gtid = NULL) + { + assert(mode_ == m_local); + assert(state_ == s_exec); + return transaction_.assign_read_view(gtid); + } + + /** + * Append a key into transaction write set. + * + * @param key Key to be appended + * + * @return Zero on success, non-zero on failure. + */ + int append_key(const wsrep::key& key) + { + assert(mode_ == m_local); + assert(state_ == s_exec); + return transaction_.append_key(key); + } + + /** + * Append keys in key_array into transaction write set. + * + * @param keys Array of keys to be appended + * + * @return Zero in case of success, non-zero on failure. + */ + int append_keys(const wsrep::key_array& keys) + { + assert(mode_ == m_local || mode_ == m_toi); + assert(state_ == s_exec); + for (auto i(keys.begin()); i != keys.end(); ++i) + { + if (transaction_.append_key(*i)) + { + return 1; + } + } + return 0; + } + + /** + * Append data into transaction write set. + */ + int append_data(const wsrep::const_buffer& data) + { + assert(mode_ == m_local); + assert(state_ == s_exec); + return transaction_.append_data(data); + } + + /** @} */ + + /** @name Streaming replication interface */ + /** @{ */ + /** + * This method should be called after every row operation. + */ + int after_row() + { + assert(mode_ == m_local); + assert(state_ == s_exec); + return (transaction_.streaming_context().fragment_size() ? + transaction_.after_row() : 0); + } + + /** + * Set streaming parameters. + * + * @param fragment_unit Desired fragment unit + * @param fragment_size Desired fragment size + */ + void streaming_params(enum wsrep::streaming_context::fragment_unit + fragment_unit, + size_t fragment_size); + + /** + * Enable streaming replication. + * + * Currently it is not possible to change the fragment unit + * for active streaming transaction. + * + * @param fragment_unit Desired fragment unit + * @param fragment_size Desired fragment size + * + * @return Zero on success, non-zero if the streaming cannot be + * enabled. + */ + int enable_streaming( + enum wsrep::streaming_context::fragment_unit + fragment_unit, + size_t fragment_size); + + /** + * Disable streaming for context. + */ + void disable_streaming(); + + void fragment_applied(wsrep::seqno seqno) + { + assert(mode_ == m_high_priority); + transaction_.fragment_applied(seqno); + } + + /** + * Prepare write set meta data for ordering. + * This method should be called before ordered commit or + * rollback if the commit time meta data was not known + * at the time of the start of the transaction. + * This mostly applies to streaming replication. + * + * @param ws_handle Write set handle + * @param ws_meta Write set meta data + * @param is_commit Boolean to denote whether the operation + * is commit or rollback. + */ + int prepare_for_ordering(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + bool is_commit) + { + assert(state_ == s_exec); + return transaction_.prepare_for_ordering( + ws_handle, ws_meta, is_commit); + } + /** @} */ + + /** @name Applying interface */ + /** @{ */ + int start_transaction(const wsrep::ws_handle& wsh, + const wsrep::ws_meta& meta) + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(mode_ == m_high_priority); + return transaction_.start_transaction(wsh, meta); + } + + int next_fragment(const wsrep::ws_meta& meta) + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(mode_ == m_high_priority); + return transaction_.next_fragment(meta); + } + + /** @name Commit ordering interface */ + /** @{ */ + int before_prepare() + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_exec); + return transaction_.before_prepare(lock); + } + + int after_prepare() + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_exec); + return transaction_.after_prepare(lock); + } + + int before_commit() + { + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_exec || mode_ == m_local); + return transaction_.before_commit(); + } + + int ordered_commit() + { + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_exec || mode_ == m_local); + return transaction_.ordered_commit(); + } + + int after_commit() + { + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_exec || mode_ == m_local); + return transaction_.after_commit(); + } + /** @} */ + int before_rollback() + { + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_idle || + state_ == s_exec || + state_ == s_result || + state_ == s_quitting); + return transaction_.before_rollback(); + } + + int after_rollback() + { + assert(owning_thread_id_ == wsrep::this_thread::get_id()); + assert(state_ == s_idle || + state_ == s_exec || + state_ == s_result || + state_ == s_quitting); + return transaction_.after_rollback(); + } + + /** + * This method should be called by the background rollbacker + * thread after the rollback is complete. This will allow + * the client to proceed through before_command() and + * wait_rollback_complete_and_acquire_ownership(). + */ + void sync_rollback_complete(); + + /** + * Wait for background rollback to complete. This method can + * be called before before_command() to verify that the + * background rollback has been finished. After the call returns, + * it is guaranteed that BF abort does not launch background + * rollback process before after_command_after_result() is called. + * This method is idempotent, it can be called many times + * by the same thread before before_command() is called. + */ + void wait_rollback_complete_and_acquire_ownership(); + /** @} */ + + // + // XA + // + /** + * Assign transaction external id. + * + * Other than storing the xid, the transaction is marked as XA. + * This should be called when XA transaction is started. + * + * @param xid transaction id + */ + void assign_xid(const wsrep::xid& xid) + { + transaction_.assign_xid(xid); + } + + /** + * Restores the client's transaction to prepared state + * + * The purpose of this method is to restore transaction state + * during recovery of a prepared XA transaction. + */ + int restore_xid(const wsrep::xid& xid) + { + return transaction_.restore_to_prepared_state(xid); + } + + /** + * Commit transaction with the given xid + * + * Sends a commit fragment to terminate the transaction with + * the given xid. For the fragment to be sent, a streaming + * applier for the transaction must exist, and the transaction + * must be in prepared state. + * + * @param xid the xid of the the transaction to commit + * + * @return Zero on success, non-zero on error. In case of error + * the client_state's current_error is set + */ + int commit_by_xid(const wsrep::xid& xid) + { + return transaction_.commit_or_rollback_by_xid(xid, true); + } + + /** + * Rollback transaction with the given xid + * + * Sends a rollback fragment to terminate the transaction with + * the given xid. For the fragment to be sent, a streaming + * applier for the transaction must exist, and the transaction + * must be in prepared state. + * + * @param xid the xid of the the transaction to commit + * + * @return Zero on success, non-zero on error. In case of error + * the client_state's current_error is set + */ + int rollback_by_xid(const wsrep::xid& xid) + { + return transaction_.commit_or_rollback_by_xid(xid, false); + } + + /** + * Detach a prepared XA transaction + * + * This method cleans up a local XA transaction in prepared state + * and converts it to high priority mode. + * This can be used to handle the case where the client of a XA + * transaction disconnects, and the transaction must not rollback. + * After this call, a different client may later attempt to terminate + * the transaction by calling method commit_by_xid() or rollback_by_xid(). + */ + void xa_detach() + { + assert(mode_ == m_local); + assert(state_ == s_none || state_ == s_exec || state_ == s_quitting); + transaction_.xa_detach(); + } + + /** + * Replay a XA transaction + * + * Replay a XA transaction that is in s_idle state. + * This may happen if the transaction is BF aborted + * between prepare and commit. + * Since the victim is idle, this method can be called + * by the BF aborter or the backround rollbacker. + */ + void xa_replay() + { + assert(mode_ == m_local); + assert(state_ == s_idle); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + transaction_.xa_replay(lock); + } + + // + // BF aborting + // + /** + * Brute force abort a transaction. This method should be + * called by a transaction which needs to BF abort a conflicting + * locally processing transaction. + */ + int bf_abort(wsrep::seqno bf_seqno) + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(mode_ == m_local || transaction_.is_streaming()); + return transaction_.bf_abort(lock, bf_seqno); + } + /** + * Brute force abort a transaction in total order. This method + * should be called by the TOI operation which needs to + * BF abort a transaction. + */ + int total_order_bf_abort(wsrep::seqno bf_seqno) + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(mode_ == m_local || transaction_.is_streaming()); + return transaction_.total_order_bf_abort(lock, bf_seqno); + } + + /** + * Adopt a streaming transaction state. This is must be + * called from high_priority_service::adopt_transaction() + * during streaming transaction rollback. The call will + * set up enough context for handling the rollback + * fragment. + */ + void adopt_transaction(const wsrep::transaction& transaction) + { + assert(mode_ == m_high_priority); + transaction_.adopt(transaction); + } + + /** + * Adopt (store) transaction applying error for further processing. + */ + void adopt_apply_error(wsrep::mutable_buffer& err) + { + assert(mode_ == m_high_priority); + transaction_.adopt_apply_error(err); + } + + /** + * Clone enough state from another transaction so that replaing will + * be possible with a transaction contained in this client state. + * + * @param transaction Transaction which is to be replied in this + * client state + */ + void clone_transaction_for_replay(const wsrep::transaction& transaction) + { + // assert(mode_ == m_high_priority); + transaction_.clone_for_replay(transaction); + } + + /** @name Non-transactional operations */ + /** @{*/ + + /** + * Enter total order isolation critical section. If the wait_until + * is given non-default value, the operation is retried until + * successful, the given time point is reached or the client is + * interrupted. + * + * @param key_array Array of keys + * @param buffer Buffer containing the action to execute inside + * total order isolation section + * @param flags Provider flags for TOI operation + * @param wait_until Time point to wait until for successful + * certification. + * + * @return Zero on success, non-zero otherwise. + */ + int enter_toi_local( + const wsrep::key_array& key_array, + const wsrep::const_buffer& buffer, + std::chrono::time_point<wsrep::clock> + wait_until = + std::chrono::time_point<wsrep::clock>()); + /** + * Enter applier TOI mode + * + * @param ws_meta Write set meta data + */ + void enter_toi_mode(const wsrep::ws_meta& ws_meta); + + /** + * Return true if the client_state is under TOI operation. + */ + bool in_toi() const + { + return (toi_meta_.seqno().is_undefined() == false); + } + + /** + * Return the mode where client entered into TOI mode. + * The return value can be either m_local or + * m_high_priority. + */ + enum mode toi_mode() const + { + return toi_mode_; + } + + /** + * Leave total order isolation critical section. + * (for local mode clients) + * + * @param err definition of the error that happened during the + * execution of TOI operation (empty for no error) + */ + int leave_toi_local(const wsrep::mutable_buffer& err); + + /** + * Leave applier TOI mode. + */ + void leave_toi_mode(); + + /** + * Begin rolling schema upgrade operation. + * + * @param timeout Timeout in seconds to wait for committing + * connections to finish. + */ + int begin_rsu(int timeout); + + /** + * End rolling schema upgrade operation. + */ + int end_rsu(); + + /** + * Begin non-blocking operation. + * + * The NBO operation is started by grabbing TOI critical + * section. The keys and buffer are certifed as in TOI + * operation. If the call fails due to error returned by + * the provider, the provider error code can be retrieved + * by current_error_status() call. + * + * If the wait_until is given non-default value, the operation is + * retried until successful, the given time point is reached or the + * client is interrupted. + * + * @param keys Array of keys for NBO operation. + * @param buffer NBO write set + * @param wait_until Time point to wait until for successful certification. + * @return Zero in case of success, non-zero in case of failure. + */ + int begin_nbo_phase_one( + const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + std::chrono::time_point<wsrep::clock> + wait_until = + std::chrono::time_point<wsrep::clock>()); + + /** + * End non-blocking operation phase after aquiring required + * resources for operation. + * + * @param err definition of the error that happened during the + * execution of phase one (empty for no error) + */ + int end_nbo_phase_one(const wsrep::mutable_buffer& err); + + /** + * Enter in NBO mode. This method should be called when the + * applier launches the asynchronous process to perform the + * operation. The purpose of the call is to adjust + * the state and set write set meta data. + * + * @param ws_meta Write set meta data. + * + * @return Zero in case of success, non-zero on failure. + */ + int enter_nbo_mode(const wsrep::ws_meta& ws_meta); + + /** + * Begin non-blocking operation phase two. The keys argument + * passed to this call must contain the same keys which were + * passed to begin_nbo_phase_one(). + * + * If the wait_until is given non-default value, the operation is + * retried until successful, the given time point is reached or the + * client is interrupted. + * + * @param keys Key array. + * @param wait_until Time point to wait until for entering TOI for + * phase two. + */ + int begin_nbo_phase_two(const wsrep::key_array& keys, + std::chrono::time_point<wsrep::clock> + wait_until = + std::chrono::time_point<wsrep::clock>()); + + /** + * End non-blocking operation phase two. This call will + * release TOI critical section and set the mode to m_local. + * + * @param err definition of the error that happened during the + * execution of phase two (empty for no error) + */ + int end_nbo_phase_two(const wsrep::mutable_buffer& err); + + /** + * Get reference to the client mutex. + * + * @return Reference to the client mutex. + */ + wsrep::mutex& mutex() { return mutex_; } + + /** + * Get server context associated the the client session. + * + * @return Reference to server context. + */ + wsrep::server_state& server_state() const + { return server_state_; } + + wsrep::client_service& client_service() const + { return client_service_; } + /** + * Get reference to the Provider which is associated + * with the client context. + * + * @return Reference to the provider. + * @throw wsrep::runtime_error if no providers are associated + * with the client context. + * + * @todo Should be removed. + */ + wsrep::provider& provider() const; + + /** + * Get Client identifier. + * + * @return Client Identifier + */ + client_id id() const { return id_; } + + /** + * Get Client mode. + * + * @todo Enforce mutex protection if called from other threads. + * + * @return Client mode. + */ + enum mode mode() const { return mode_; } + + /** + * Get Client state. + * + * @todo Enforce mutex protection if called from other threads. + * + * @return Client state + */ + enum state state() const { return state_; } + + /** + * Return a const reference to the transaction associated + * with the client state. + */ + const wsrep::transaction& transaction() const + { + return transaction_; + } + + /** + * Mark the transaction associated with the client state + * (if any), as unsafe for parallel applying + * + * @return Zero on success, non-zero on error. + */ + int mark_transaction_pa_unsafe() + { + if (transaction_.active()) + { + transaction_.pa_unsafe(true); + return 0; + } + return 1; + } + + const wsrep::ws_meta& toi_meta() const + { + return toi_meta_; + } + + /** + * Do sync wait operation. If the method fails, current_error() + * can be inspected about the reason of error. + * + * @param Sync wait timeout in seconds. + * + * @return Zero on success, non-zero on error. + */ + int sync_wait(int timeout); + + /** + * Return the current sync wait GTID. + * + * Sync wait GTID is updated on each sync_wait() call and + * reset to wsrep::gtid::undefined() in after_command_after_result() + * method. The variable can thus be used to check if a sync wait + * has been performend for the current client command. + */ + const wsrep::gtid& sync_wait_gtid() const + { + return sync_wait_gtid_; + } + /** + * Return the last written GTID. + */ + const wsrep::gtid& last_written_gtid() const + { + return last_written_gtid_; + } + + /** + * Set debug logging level. + * + * Levels: + * 0 - Debug logging is disabled + * 1..n - Debug logging with increasing verbosity. + */ + void debug_log_level(int level) { debug_log_level_ = level; } + + /** + * Return current debug logging level. The return value + * is a maximum of client state and server state debug log + * levels. + * + * @return Current debug log level. + */ + int debug_log_level() const + { + return std::max(debug_log_level_, + wsrep::log::debug_log_level()); + } + + // + // Error handling + // + + /** + * Reset the current error state. + * + * @todo There should be some protection about when this can + * be done. + */ + void reset_error() + { + current_error_ = wsrep::e_success; + } + + /** + * Return current error code. + * + * @return Current error code. + */ + enum wsrep::client_error current_error() const + { + return current_error_; + } + + enum wsrep::provider::status current_error_status() const + { + return current_error_status_; + } + protected: + /** + * Client context constuctor. This is protected so that it + * can be called from derived class constructors only. + */ + client_state(wsrep::mutex& mutex, + wsrep::condition_variable& cond, + wsrep::server_state& server_state, + wsrep::client_service& client_service, + const client_id& id, + enum mode mode) + : owning_thread_id_(wsrep::this_thread::get_id()) + , rollbacker_active_(false) + , mutex_(mutex) + , cond_(cond) + , server_state_(server_state) + , client_service_(client_service) + , id_(id) + , mode_(mode) + , toi_mode_(m_undefined) + , state_(s_none) + , state_hist_() + , transaction_(*this) + , toi_meta_() + , nbo_meta_() + , allow_dirty_reads_() + , sync_wait_gtid_() + , last_written_gtid_() + , debug_log_level_(0) + , current_error_(wsrep::e_success) + , current_error_status_(wsrep::provider::success) + , keep_command_error_() + { } + + private: + client_state(const client_state&); + client_state& operator=(client_state&); + + friend class client_state_switch; + friend class high_priority_context; + friend class transaction; + + void do_acquire_ownership(wsrep::unique_lock<wsrep::mutex>& lock); + // Wait for sync rollbacker to finish, with lock. Changes state + // to exec. + void do_wait_rollback_complete_and_acquire_ownership( + wsrep::unique_lock<wsrep::mutex>& lock); + void update_last_written_gtid(const wsrep::gtid&); + void debug_log_state(const char*) const; + void debug_log_keys(const wsrep::key_array& keys) const; + void state(wsrep::unique_lock<wsrep::mutex>& lock, enum state state); + void mode(wsrep::unique_lock<wsrep::mutex>& lock, enum mode mode); + + // Override current client error status. Optionally provide + // an error status from the provider if the error was caused + // by the provider call. + void override_error(enum wsrep::client_error error, + enum wsrep::provider::status status = + wsrep::provider::success); + + // Poll provider::enter_toi() until return status from provider + // does not indicate certification failure, timeout expires + // or client is interrupted. + enum wsrep::provider::status + poll_enter_toi(wsrep::unique_lock<wsrep::mutex>& lock, + const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + wsrep::ws_meta& meta, + int flags, + std::chrono::time_point<wsrep::clock> wait_until, + bool& timed_out); + void enter_toi_common(wsrep::unique_lock<wsrep::mutex>&); + void leave_toi_common(); + + wsrep::thread::id owning_thread_id_; + bool rollbacker_active_; + wsrep::mutex& mutex_; + wsrep::condition_variable& cond_; + wsrep::server_state& server_state_; + wsrep::client_service& client_service_; + wsrep::client_id id_; + enum mode mode_; + enum mode toi_mode_; + enum state state_; + std::vector<enum state> state_hist_; + wsrep::transaction transaction_; + wsrep::ws_meta toi_meta_; + wsrep::ws_meta nbo_meta_; + bool allow_dirty_reads_; + wsrep::gtid sync_wait_gtid_; + wsrep::gtid last_written_gtid_; + int debug_log_level_; + enum wsrep::client_error current_error_; + enum wsrep::provider::status current_error_status_; + bool keep_command_error_; + + /** + * Marks external rollbacker thread for the client + * as active. This will block client in before_command(), until + * rolbacker has released the client. + */ + void set_rollbacker_active(bool value) + { + rollbacker_active_ = value; + } + + bool is_rollbacker_active() + { + return rollbacker_active_; + } + }; + + static inline const char* to_c_string( + enum wsrep::client_state::state state) + { + switch (state) + { + case wsrep::client_state::s_none: return "none"; + case wsrep::client_state::s_idle: return "idle"; + case wsrep::client_state::s_exec: return "exec"; + case wsrep::client_state::s_result: return "result"; + case wsrep::client_state::s_quitting: return "quit"; + } + return "unknown"; + } + + static inline std::string to_string(enum wsrep::client_state::state state) + { + return to_c_string(state); + } + + static inline const char* to_c_string(enum wsrep::client_state::mode mode) + { + switch (mode) + { + case wsrep::client_state::m_undefined: return "undefined"; + case wsrep::client_state::m_local: return "local"; + case wsrep::client_state::m_high_priority: return "high priority"; + case wsrep::client_state::m_toi: return "toi"; + case wsrep::client_state::m_rsu: return "rsu"; + case wsrep::client_state::m_nbo: return "nbo"; + } + return "unknown"; + } + + static inline std::string to_string(enum wsrep::client_state::mode mode) + { + return to_c_string(mode); + } + + /** + * Utility class to switch the client state to high priority + * mode. The client is switched back to the original mode + * when the high priority context goes out of scope. + */ + class high_priority_context + { + public: + high_priority_context(wsrep::client_state& client) + : client_(client) + , orig_mode_(client.mode_) + { + wsrep::unique_lock<wsrep::mutex> lock(client.mutex_); + client.mode(lock, wsrep::client_state::m_high_priority); + } + virtual ~high_priority_context() + { + wsrep::unique_lock<wsrep::mutex> lock(client_.mutex_); + assert(client_.mode() == wsrep::client_state::m_high_priority); + client_.mode(lock, orig_mode_); + } + private: + wsrep::client_state& client_; + enum wsrep::client_state::mode orig_mode_; + }; +} + +#endif // WSREP_CLIENT_STATE_HPP diff --git a/wsrep-lib/include/wsrep/compiler.hpp b/wsrep-lib/include/wsrep/compiler.hpp new file mode 100644 index 00000000..5f8e1ce0 --- /dev/null +++ b/wsrep-lib/include/wsrep/compiler.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + + +/** @file compiler.hpp + * + * Compiler specific macro definitions. + * + * WSREP_NOEXCEPT - Specifies that the method/function does not throw. If + * and exception is thrown inside, std::terminate is called + * without propagating the exception. + * Set to "noexcept" if the compiler supports it, otherwise + * left empty. + * WSREP_NORETURN - Indicates that the method/function does not return. + * Set to attribute "[[noreturn]]" if the compiler supports, + * it, otherwise "__attribute__((noreturn))". + * WSREP_OVERRIDE - Set to "override" if the compiler supports it, otherwise + * left empty. + * WSREP_UNUSED - Can be used to mark variables which may be present in + * debug builds but not in release builds. + * WSREP_FALLTHROUGH - Silence implicit fallthrough warning. + */ + +#ifndef WSREP_LIB_COMPILER_HPP +#define WSREP_LIB_COMPILER_HPP + +#if __cplusplus >= 201103L && !(__GNUC__ == 4 && __GNUG_MINOR__ < 8) +#define WSREP_NORETURN [[noreturn]] +#else +#define WSREP_NORETURN __attribute__((noreturn)) +#endif // __cplusplus >= 201103L && !(__GNUC__ == 4 && __GNUG_MINOR__ < 8) + +#if __cplusplus >= 201103L +#define WSREP_NOEXCEPT noexcept +#define WSREP_OVERRIDE override +#else +#define WSREP_NOEXCEPT +#define WSREP_OVERRIDE +#endif // __cplusplus >= 201103L +#define WSREP_UNUSED __attribute__((unused)) + +#if __GNUC__ >= 7 +#define WSREP_FALLTHROUGH __attribute__((fallthrough)) +#elif defined(__clang__) +# if defined(__has_warning) +# if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +# define WSREP_FALLTHROUGH [[clang::fallthrough]] +# endif +# endif +#else // __clang __ +#define WSREP_FALLTHROUGH ((void)0) +#endif // __GNUC__ >= 7 || (__clang__ && __clang_major__ >= 10) + +#endif // WSREP_LIB_COMPILER_HPP diff --git a/wsrep-lib/include/wsrep/condition_variable.hpp b/wsrep-lib/include/wsrep/condition_variable.hpp new file mode 100644 index 00000000..25a0e16e --- /dev/null +++ b/wsrep-lib/include/wsrep/condition_variable.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_CONDITION_VARIABLE_HPP +#define WSREP_CONDITION_VARIABLE_HPP + +#include "compiler.hpp" +#include "lock.hpp" + +#include <cstdlib> + +namespace wsrep +{ + class condition_variable + { + public: + condition_variable() { } + virtual ~condition_variable() { } + virtual void notify_one() = 0; + virtual void notify_all() = 0; + virtual void wait(wsrep::unique_lock<wsrep::mutex>& lock) = 0; + private: + condition_variable(const condition_variable&); + condition_variable& operator=(const condition_variable&); + }; + + // Default pthreads based condition variable implementation + class default_condition_variable : public condition_variable + { + public: + default_condition_variable() + : cond_() + { + if (pthread_cond_init(&cond_, 0)) + { + throw wsrep::runtime_error("Failed to initialized condvar"); + } + } + + ~default_condition_variable() + { + if (pthread_cond_destroy(&cond_)) + { + ::abort(); + } + } + void notify_one() WSREP_OVERRIDE + { + (void)pthread_cond_signal(&cond_); + } + + void notify_all() WSREP_OVERRIDE + { + (void)pthread_cond_broadcast(&cond_); + } + + void wait(wsrep::unique_lock<wsrep::mutex>& lock) WSREP_OVERRIDE + { + if (pthread_cond_wait( + &cond_, + reinterpret_cast<pthread_mutex_t*>(lock.mutex()->native()))) + { + throw wsrep::runtime_error("Cond wait failed"); + } + } + + private: + pthread_cond_t cond_; + }; + +} + +#endif // WSREP_CONDITION_VARIABLE_HPP diff --git a/wsrep-lib/include/wsrep/encryption_service.hpp b/wsrep-lib/include/wsrep/encryption_service.hpp new file mode 100644 index 00000000..0efcde06 --- /dev/null +++ b/wsrep-lib/include/wsrep/encryption_service.hpp @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_ENCRYPTION_SERVICE_HPP +#define WSREP_ENCRYPTION_SERVICE_HPP + +#include "buffer.hpp" + +namespace wsrep +{ + /** + * Encryption service. + */ + class encryption_service + { + public: + + virtual ~encryption_service() { } + + /** + * Encryption/decryption callback. Can be NULL for no encryption. + * + * @param ctx Encryption context + * @param key Key used in encryption/decryption + * @param iv IV vector + * @param input Input data buffer + * @param output An output buffer, must be at least the size of the input + * data plus unwritten bytes from the previous call(s). + * @param encrypt Flag used to either encrypt or decrypt data + * @param last true if this is the last buffer to encrypt/decrypt + * + * @return Number of bytes written to output or a negative error code. + */ + virtual int do_crypt(void** ctx, + wsrep::const_buffer& key, + const char (*iv)[32], + wsrep::const_buffer& input, + void* output, + bool encrypt, + bool last) = 0; + + /** + * Is encryption enabled on server. + * + * @return True if encryption is enabled. False otherwise + */ + virtual bool encryption_enabled() = 0; + }; +} + +#endif // WSREP_ENCRYPTION_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/event_service.hpp b/wsrep-lib/include/wsrep/event_service.hpp new file mode 100644 index 00000000..9cc8465f --- /dev/null +++ b/wsrep-lib/include/wsrep/event_service.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + + +/** @file event_service.hpp + * + * Service interface for providing events to DBMS. + */ + +#ifndef WSREP_EVENT_SERVICE_HPP +#define WSREP_EVENT_SERVICE_HPP + +#include <string> + +namespace wsrep +{ + + /** @class event_service + * + * Event service interface. This provides an interface corresponding + * to wsrep-API event service. For details see + * wsrep-API/wsrep_event_service.h + */ + class event_service + { + public: + virtual ~event_service() { } + + /** + * Process event with name name and value value. + */ + virtual void process_event(const std::string& name, + const std::string& value) = 0; + }; +} + +#endif // WSREP_EVENT_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/exception.hpp b/wsrep-lib/include/wsrep/exception.hpp new file mode 100644 index 00000000..ef19de9a --- /dev/null +++ b/wsrep-lib/include/wsrep/exception.hpp @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_EXCEPTION_HPP +#define WSREP_EXCEPTION_HPP + +#include <stdexcept> +#include <cstdlib> + +namespace wsrep +{ + extern bool abort_on_exception; + + class runtime_error : public std::runtime_error + { + public: + runtime_error(const char* msg) + : std::runtime_error(msg) + { + if (abort_on_exception) + { + ::abort(); + } + } + + runtime_error(const std::string& msg) + : std::runtime_error(msg) + { + if (abort_on_exception) + { + ::abort(); + } + } + }; + + class not_implemented_error : public std::exception + { + public: + not_implemented_error() + : std::exception() + { + ::abort(); + } + }; + + class fatal_error : public std::exception + { + }; +} + + +#endif // WSREP_EXCEPTION_HPP diff --git a/wsrep-lib/include/wsrep/gtid.hpp b/wsrep-lib/include/wsrep/gtid.hpp new file mode 100644 index 00000000..0d49c58d --- /dev/null +++ b/wsrep-lib/include/wsrep/gtid.hpp @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_GTID_HPP +#define WSREP_GTID_HPP + +#include "id.hpp" +#include "seqno.hpp" +#include "compiler.hpp" + +#include <iosfwd> + +/** + * Minimum number of bytes guaratneed to store GTID string representation, + * terminating '\0' not included (36 + 1 + 20). + */ +#define WSREP_LIB_GTID_C_STR_LEN 57 + +namespace wsrep +{ + class gtid + { + public: + gtid() + : id_() + , seqno_() + { } + gtid(const wsrep::id& id, wsrep::seqno seqno) + : id_(id) + , seqno_(seqno) + { } + const wsrep::id& id() const { return id_; } + wsrep::seqno seqno() const { return seqno_ ; } + bool is_undefined() const + { + return (seqno_.is_undefined() && id_.is_undefined()); + } + static const wsrep::gtid& undefined() + { + return undefined_; + } + bool operator==(const gtid& other) const + { + return ( + seqno_ == other.seqno_ && + id_ == other.id_ + ); + } + private: + static const wsrep::gtid undefined_; + wsrep::id id_; + wsrep::seqno seqno_; + }; + + /** + * Scan a GTID from C string. + * + * @param buf Buffer containing the string + * @param len Length of buffer + * @param[out] gtid Gtid to be printed to + * + * @return Number of bytes scanned, negative value on error. + */ + ssize_t scan_from_c_str(const char* buf, size_t buf_len, + wsrep::gtid& gtid); + + /* + * Deprecated version of the above for backwards compatibility. + * Will be removed when all the superprojects have been updated. + */ + static inline ssize_t gtid_scan_from_c_str(const char* buf, size_t buf_len, + wsrep::gtid& gtid) + { + return scan_from_c_str(buf, buf_len, gtid); + } + + /** + * Print a GTID into character buffer. + * + * @param gtid GTID to be printed. + * @param buf Pointer to the beginning of the buffer + * @param buf_len Buffer length + * + * @return Number of characters printed or negative value for error + */ + ssize_t print_to_c_str(const wsrep::gtid& gtid, char* buf, size_t buf_len); + + /* + * Deprecated version of the above for backwards compatibility. + * Will be removed when all the superprojects have been updated. + */ + static inline ssize_t gtid_print_to_c_str(const wsrep::gtid& gtid, + char* buf, size_t buf_len) + { + return print_to_c_str(gtid, buf, buf_len); + } + + /** + * Return minimum number of chars required to store any GTID. + */ + static inline size_t gtid_c_str_len() { return WSREP_LIB_GTID_C_STR_LEN; } + + /** + * Overload for ostream operator<<. + */ + std::ostream& operator<<(std::ostream&, const wsrep::gtid&); + + /** + * Overload for istream operator>>. + */ + std::istream& operator>>(std::istream&, wsrep::gtid&); +} + +#endif // WSREP_GTID_HPP diff --git a/wsrep-lib/include/wsrep/high_priority_service.hpp b/wsrep-lib/include/wsrep/high_priority_service.hpp new file mode 100644 index 00000000..f1d011ec --- /dev/null +++ b/wsrep-lib/include/wsrep/high_priority_service.hpp @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file high_priority_service.hpp + * + * Interface for services for applying high priority transactions. + */ +#ifndef WSREP_HIGH_PRIORITY_SERVICE_HPP +#define WSREP_HIGH_PRIORITY_SERVICE_HPP + +#include "xid.hpp" +#include "server_state.hpp" + +namespace wsrep +{ + class ws_handle; + class ws_meta; + class const_buffer; + class transaction; + class high_priority_service + { + public: + high_priority_service(wsrep::server_state& server_state) + : server_state_(server_state) + , must_exit_() { } + virtual ~high_priority_service() { } + + int apply(const ws_handle& ws_handle, const ws_meta& ws_meta, + const const_buffer& data) + { + return server_state_.on_apply(*this, ws_handle, ws_meta, data); + } + /** + * Start a new transaction + */ + virtual int start_transaction(const wsrep::ws_handle&, + const wsrep::ws_meta&) = 0; + + /** + * Start the next fragment of current transaction + */ + virtual int next_fragment(const wsrep::ws_meta&) = 0; + + /** + * Return transaction object associated to high priority + * service state. + */ + virtual const wsrep::transaction& transaction() const = 0; + + /** + * Adopt a transaction. + */ + virtual int adopt_transaction(const wsrep::transaction&) = 0; + + /** + * Apply a write set. + * + * A write set applying happens always + * as a part of the transaction. The caller must start a + * new transaction before applying a write set and must + * either commit to make changes persistent or roll back. + * + * @params ws_meta Write set meta data + * @params ws Write set buffer + * @params err Buffer to store error data + */ + virtual int apply_write_set(const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& ws, + wsrep::mutable_buffer& err) = 0; + + /** + * Append a fragment into fragment storage. This will be + * called after a non-committing fragment belonging to + * streaming transaction has been applied. The call will + * not be done within an open transaction, the implementation + * must start a new transaction and commit. + * + * Note that the call is not done from streaming transaction + * context, but from applier context. + */ + virtual int append_fragment_and_commit( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data, + const wsrep::xid& xid) = 0; + + /** + * Remove fragments belonging to streaming transaction. + * This method will be called within the streaming transaction + * before commit. The implementation must not commit the + * whole transaction. The call will be done from streaming + * transaction context. + * + * @param ws_meta Write set meta data for commit fragment. + * + * @return Zero on success, non-zero on failure. + */ + virtual int remove_fragments(const wsrep::ws_meta& ws_meta) = 0; + + /** + * Commit a transaction. + * An implementation must call + * wsrep::client_state::prepare_for_ordering() to set + * the ws_handle and ws_meta before the commit if the + * commit process will go through client state commit + * processing. Otherwise the implementation must release + * commit order explicitly via provider. + * + * @param ws_handle Write set handle + * @param ws_meta Write set meta + * + * @return Zero in case of success, non-zero in case of failure + */ + virtual int commit(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) = 0; + /** + * Roll back a transaction + * + * An implementation must call + * wsrep::client_state::prepare_for_ordering() to set + * the ws_handle and ws_meta before the rollback if + * the rollback process will go through client state + * rollback processing. Otherwise the implementation + * must release commit order explicitly via provider. + * + * @param ws_handle Write set handle + * @param ws_meta Write set meta + * + * @return Zero in case of success, non-zero in case of failure + */ + virtual int rollback(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) = 0; + + /** + * Apply a TOI operation. + * + * TOI operation is a standalone operation and should not + * be executed as a part of a transaction. + * + * @params ws_meta Write set meta data + * @params ws Write set buffer + * @params err Buffer to store error data + */ + virtual int apply_toi(const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& ws, + wsrep::mutable_buffer& err) = 0; + + /** + * Apply NBO begin event. + * + * The responsibility of the implementation is to start + * an asynchronous process which will complete the operation. + * The call is done under total order isolation, and the + * isolation is released by the caller after the method + * returns. It is a responsibility of the asynchronous process + * to complete the second phase of NBO. + * + * @param ws_meta Write set meta data. + * @param data Buffer containing the command to execute. + * @params err Buffer to store error data + * + * @return Zero in case of success, non-zero if the asynchronous + * process could not be started. + */ + virtual int apply_nbo_begin(const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data, + wsrep::mutable_buffer& err) = 0; + + /** + * Actions to take after applying a write set was completed. + */ + virtual void after_apply() = 0; + + /** + * Store global execution context for high priority service. + */ + virtual void store_globals() = 0; + + /** + * Reset global execution context for high priority service. + */ + virtual void reset_globals() = 0; + + /** + * Switch exection context to context of orig_hps. + * + * @param orig_hps Original high priority service. + */ + virtual void switch_execution_context( + wsrep::high_priority_service& orig_hps) = 0; + + /** + * Log a dummy write set which is either SR transaction fragment + * or roll back fragment. The implementation must release + * commit order inside the call. + * + * @params ws_handle Write set handle + * @params ws_meta Write set meta data + * @params err Optional applying error data buffer, may be modified + * + * @return Zero in case of success, non-zero on failure + */ + virtual int log_dummy_write_set(const ws_handle& ws_handle, + const ws_meta& ws_meta, + wsrep::mutable_buffer& err) = 0; + + /** + * Adopt (store) apply error description for further reporting + * to provider, source buffer may be modified. + */ + virtual void adopt_apply_error(wsrep::mutable_buffer& err) = 0; + + virtual bool is_replaying() const = 0; + + bool must_exit() const { return must_exit_; } + + /** + * Debug facility to crash the server at given point. + */ + virtual void debug_crash(const char* crash_point) = 0; + + protected: + wsrep::server_state& server_state_; + bool must_exit_; + }; + + class high_priority_switch + { + public: + high_priority_switch(high_priority_service& orig_service, + high_priority_service& current_service) + : orig_service_(orig_service) + , current_service_(current_service) + { + orig_service_.reset_globals(); + current_service_.switch_execution_context(orig_service_); + current_service_.store_globals(); + } + ~high_priority_switch() + { + current_service_.reset_globals(); + orig_service_.store_globals(); + } + private: + high_priority_service& orig_service_; + high_priority_service& current_service_; + }; +} + +#endif // WSREP_HIGH_PRIORITY_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/id.hpp b/wsrep-lib/include/wsrep/id.hpp new file mode 100644 index 00000000..fc1e82b2 --- /dev/null +++ b/wsrep-lib/include/wsrep/id.hpp @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file id.hpp + * + * A generic identifier utility class. + */ +#ifndef WSREP_ID_HPP +#define WSREP_ID_HPP + +#include "exception.hpp" + +#include <iosfwd> +#include <cstring> // std::memset() + +namespace wsrep +{ + /** + * The idientifier class stores identifiers either in UUID + * format or in string format. The storage format is decided + * upon construction. If the given string contains a valid + * UUID, the storage format will be binary. Otherwise the + * string will be copied verbatim. If the string format is used, + * the maximum length of the identifier is limited to 16 bytes. + */ + class id + { + public: + typedef struct native_type { unsigned char buf[16]; } native_type; + /** + * Default constructor. Constructs an empty identifier. + */ + id() : data_() { std::memset(data_.buf, 0, sizeof(data_.buf)); } + + /** + * Construct from string. The input string may contain either + * valid UUID or a string with maximum 16 bytes length. + */ + id(const std::string&); + + /** + * Construct from void pointer. + */ + id (const void* data, size_t size) : data_() + { + if (size > 16) + { + throw wsrep::runtime_error("Too long identifier"); + } + std::memset(data_.buf, 0, sizeof(data_.buf)); + std::memcpy(data_.buf, data, size); + } + + bool operator<(const id& other) const + { + return (std::memcmp(data_.buf, other.data_.buf, sizeof(data_.buf)) < 0); + } + + bool operator==(const id& other) const + { + return (std::memcmp(data_.buf, other.data_.buf, sizeof(data_.buf)) == 0); + } + bool operator!=(const id& other) const + { + return !(*this == other); + } + const void* data() const { return data_.buf; } + + size_t size() const { return sizeof(data_); } + + bool is_undefined() const + { + return (*this == undefined()); + } + + static const wsrep::id& undefined() + { + return undefined_; + } + private: + static const wsrep::id undefined_; + native_type data_; + }; + + std::ostream& operator<<(std::ostream&, const wsrep::id& id); + std::istream& operator>>(std::istream&, wsrep::id& id); +} + +#endif // WSREP_ID_HPP diff --git a/wsrep-lib/include/wsrep/key.hpp b/wsrep-lib/include/wsrep/key.hpp new file mode 100644 index 00000000..85c266c6 --- /dev/null +++ b/wsrep-lib/include/wsrep/key.hpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_KEY_HPP +#define WSREP_KEY_HPP + +#include "exception.hpp" +#include "buffer.hpp" + +#include <iosfwd> + +namespace wsrep +{ + /** @class key + * + * Certification key type. + */ + class key + { + public: + enum type + { + shared, + reference, + update, + exclusive + }; + + key(enum type type) + : type_(type) + , key_parts_() + , key_parts_len_() + { } + + /** + * Append key part to key. + * + * @param ptr Pointer to key part data. The caller is supposed to take + * care that the pointer remains valid over the lifetime + * if the key object. + * @param len Length of the key part data. + */ + void append_key_part(const void* ptr, size_t len) + { + if (key_parts_len_ == 3) + { + throw wsrep::runtime_error("key parts exceed maximum of 3"); + } + key_parts_[key_parts_len_] = wsrep::const_buffer(ptr, len); + ++key_parts_len_; + } + + enum type type() const + { + return type_; + } + + size_t size() const + { + return key_parts_len_; + } + + const wsrep::const_buffer* key_parts() const + { + return key_parts_; + } + private: + + enum type type_; + wsrep::const_buffer key_parts_[3]; + size_t key_parts_len_; + }; + + typedef std::vector<wsrep::key> key_array; + + std::ostream& operator<<(std::ostream&, enum wsrep::key::type); + std::ostream& operator<<(std::ostream&, const wsrep::key&); +} + +#endif // WSREP_KEY_HPP diff --git a/wsrep-lib/include/wsrep/lock.hpp b/wsrep-lib/include/wsrep/lock.hpp new file mode 100644 index 00000000..6537901a --- /dev/null +++ b/wsrep-lib/include/wsrep/lock.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_LOCK_HPP +#define WSREP_LOCK_HPP + +#include <mutex> + +namespace wsrep +{ + template <class C> + using unique_lock = std::unique_lock<C>; +} + +#endif // WSREP_LOCK_HPP diff --git a/wsrep-lib/include/wsrep/logger.hpp b/wsrep-lib/include/wsrep/logger.hpp new file mode 100644 index 00000000..a15873c2 --- /dev/null +++ b/wsrep-lib/include/wsrep/logger.hpp @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_LOGGER_HPP +#define WSREP_LOGGER_HPP + +#include "mutex.hpp" +#include "lock.hpp" +#include "atomic.hpp" + +#include <iosfwd> +#include <sstream> + +#define WSREP_LOG_DEBUG(debug_level_fn, debug_level, msg) \ + do { \ + if (debug_level_fn >= debug_level) wsrep::log_debug() << msg; \ + } while (0) + +namespace wsrep +{ + class log + { + public: + enum level + { + debug, + info, + warning, + error, + unknown + }; + + enum debug_level + { + debug_level_server_state = 1, + debug_level_transaction, + debug_level_streaming, + debug_level_client_state + }; + + /** + * Signature for user defined logger callback function. + * + * @param pfx optional internally defined prefix for the message + * @param msg message to log + */ + typedef void (*logger_fn_type)(level l, + const char* pfx, const char* msg); + + static const char* to_c_string(enum level level) + { + switch (level) + { + case debug: return "debug"; + case info: return "info"; + case warning: return "warning"; + case error: return "error"; + case unknown: break; + }; + return "unknown"; + } + + log(enum wsrep::log::level level, const char* prefix = "L:") + : level_(level) + , prefix_(prefix) + , oss_() + { } + + ~log() + { + if (logger_fn_) + { + logger_fn_(level_, prefix_, oss_.str().c_str()); + } + else + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + os_ << prefix_ << oss_.str() << std::endl; + } + } + + template <typename T> + std::ostream& operator<<(const T& val) + { + return (oss_ << val); + } + + /** + * Set user defined logger callback function. + */ + static void logger_fn(logger_fn_type); + + /** + * Set debug log level from client + */ + static void debug_log_level(int debug_level); + + /** + * Get current debug log level + */ + static int debug_log_level(); + + private: + log(const log&); + log& operator=(const log&); + enum level level_; + const char* prefix_; + std::ostringstream oss_; + static wsrep::mutex& mutex_; + static std::ostream& os_; + static logger_fn_type logger_fn_; + static std::atomic_int debug_log_level_; + }; + + class log_error : public log + { + public: + log_error() + : log(error) { } + }; + + class log_warning : public log + { + public: + log_warning() + : log(warning) { } + }; + + class log_info : public log + { + public: + log_info() + : log(info) { } + }; + + class log_debug : public log + { + public: + log_debug() + : log(debug) { } + }; +} + +#endif // WSREP_LOGGER_HPP diff --git a/wsrep-lib/include/wsrep/mutex.hpp b/wsrep-lib/include/wsrep/mutex.hpp new file mode 100644 index 00000000..9b0173bf --- /dev/null +++ b/wsrep-lib/include/wsrep/mutex.hpp @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_MUTEX_HPP +#define WSREP_MUTEX_HPP + +#include "compiler.hpp" +#include "exception.hpp" + + +#include <pthread.h> + +namespace wsrep +{ + /** + * Mutex interface. + */ + class mutex + { + public: + mutex() { } + virtual ~mutex() { } + virtual void lock() = 0; + virtual void unlock() = 0; + /* Return native handle */ + virtual void* native() = 0; + private: + mutex(const mutex& other); + mutex& operator=(const mutex& other); + }; + + // Default pthread implementation + class default_mutex : public wsrep::mutex + { + public: + default_mutex() + : wsrep::mutex(), + mutex_() + { + if (pthread_mutex_init(&mutex_, 0)) + { + throw wsrep::runtime_error("mutex init failed"); + } + } + ~default_mutex() + { + if (pthread_mutex_destroy(&mutex_)) ::abort(); + } + + void lock() WSREP_OVERRIDE + { + if (pthread_mutex_lock(&mutex_)) + { + throw wsrep::runtime_error("mutex lock failed"); + } + } + + void unlock() WSREP_OVERRIDE + { + if (pthread_mutex_unlock(&mutex_)) + { + throw wsrep::runtime_error("mutex unlock failed"); + } + } + + void* native() WSREP_OVERRIDE + { + return &mutex_; + } + + private: + pthread_mutex_t mutex_; + }; +} + +#endif // WSREP_MUTEX_HPP diff --git a/wsrep-lib/include/wsrep/provider.hpp b/wsrep-lib/include/wsrep/provider.hpp new file mode 100644 index 00000000..ea7e97fe --- /dev/null +++ b/wsrep-lib/include/wsrep/provider.hpp @@ -0,0 +1,507 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_PROVIDER_HPP +#define WSREP_PROVIDER_HPP + +#include "gtid.hpp" +#include "key.hpp" +#include "buffer.hpp" +#include "client_id.hpp" +#include "transaction_id.hpp" +#include "compiler.hpp" + +#include <cassert> +#include <cstring> + +#include <string> +#include <vector> +#include <ostream> + +/** + * Empty provider magic. If none provider is passed to make_provider(), + * a dummy provider is loaded. + */ +#define WSREP_LIB_PROVIDER_NONE "none" + +namespace wsrep +{ + class server_state; + class high_priority_service; + class thread_service; + class tls_service; + class allowlist_service; + class event_service; + + class stid + { + public: + stid() + : server_id_() + , transaction_id_() + , client_id_() + { } + stid(const wsrep::id& server_id, + wsrep::transaction_id transaction_id, + wsrep::client_id client_id) + : server_id_(server_id) + , transaction_id_(transaction_id) + , client_id_(client_id) + { } + const wsrep::id& server_id() const { return server_id_; } + wsrep::transaction_id transaction_id() const + { return transaction_id_; } + wsrep::client_id client_id() const { return client_id_; } + bool operator==(const stid& other) const + { + return ( + server_id_ == other.server_id_ && + transaction_id_ == other.transaction_id_ && + client_id_ == other.client_id_ + ); + } + private: + wsrep::id server_id_; + wsrep::transaction_id transaction_id_; + wsrep::client_id client_id_; + }; + + class ws_handle + { + public: + ws_handle() + : transaction_id_() + , opaque_() + { } + ws_handle(wsrep::transaction_id id) + : transaction_id_(id) + , opaque_() + { } + ws_handle(wsrep::transaction_id id, + void* opaque) + : transaction_id_(id) + , opaque_(opaque) + { } + + wsrep::transaction_id transaction_id() const + { return transaction_id_; } + + void* opaque() const { return opaque_; } + + bool operator==(const ws_handle& other) const + { + return ( + transaction_id_ == other.transaction_id_ && + opaque_ == other.opaque_ + ); + } + private: + wsrep::transaction_id transaction_id_; + void* opaque_; + }; + + class ws_meta + { + public: + ws_meta() + : gtid_() + , stid_() + , depends_on_() + , flags_() + { } + ws_meta(const wsrep::gtid& gtid, + const wsrep::stid& stid, + wsrep::seqno depends_on, + int flags) + : gtid_(gtid) + , stid_(stid) + , depends_on_(depends_on) + , flags_(flags) + { } + ws_meta(const wsrep::stid& stid) + : gtid_() + , stid_(stid) + , depends_on_() + , flags_() + { } + const wsrep::gtid& gtid() const { return gtid_; } + const wsrep::id& group_id() const + { + return gtid_.id(); + } + + wsrep::seqno seqno() const + { + return gtid_.seqno(); + } + + const wsrep::id& server_id() const + { + return stid_.server_id(); + } + + wsrep::client_id client_id() const + { + return stid_.client_id(); + } + + wsrep::transaction_id transaction_id() const + { + return stid_.transaction_id(); + } + + bool ordered() const { return !gtid_.is_undefined(); } + + wsrep::seqno depends_on() const { return depends_on_; } + + int flags() const { return flags_; } + + bool operator==(const ws_meta& other) const + { + return ( + gtid_ == other.gtid_ && + stid_ == other.stid_ && + depends_on_ == other.depends_on_ && + flags_ == other.flags_ + ); + } + private: + wsrep::gtid gtid_; + wsrep::stid stid_; + wsrep::seqno depends_on_; + int flags_; + }; + + std::string flags_to_string(int flags); + + std::ostream& operator<<(std::ostream& os, const wsrep::ws_meta& ws_meta); + + // Abstract interface for provider implementations + class provider + { + public: + class status_variable + { + public: + status_variable(const std::string& name, + const std::string& value) + : name_(name) + , value_(value) + { } + const std::string& name() const { return name_; } + const std::string& value() const { return value_; } + private: + std::string name_; + std::string value_; + }; + + /** + * Return value enumeration + * + * @todo Convert this to struct ec, get rid of prefixes. + */ + enum status + { + /** Success */ + success, + /** Warning*/ + error_warning, + /** Transaction was not found from provider */ + error_transaction_missing, + /** Certification failed */ + error_certification_failed, + /** Transaction was BF aborted */ + error_bf_abort, + /** Transaction size exceeded */ + error_size_exceeded, + /** Connectivity to cluster lost */ + error_connection_failed, + /** Internal provider failure or provider was closed, + provider must be reinitialized */ + error_provider_failed, + /** Fatal error, server must abort */ + error_fatal, + /** Requested functionality is not implemented by the provider */ + error_not_implemented, + /** Operation is not allowed */ + error_not_allowed, + /** Unknown error code from the provider */ + error_unknown + }; + + static std::string to_string(enum status); + + struct flag + { + static const int start_transaction = (1 << 0); + static const int commit = (1 << 1); + static const int rollback = (1 << 2); + static const int isolation = (1 << 3); + static const int pa_unsafe = (1 << 4); + static const int commutative = (1 << 5); + static const int native = (1 << 6); + static const int prepare = (1 << 7); + static const int snapshot = (1 << 8); + static const int implicit_deps = (1 << 9); + }; + + /** + * Provider capabilities. + */ + struct capability + { + static const int multi_master = (1 << 0); + static const int certification = (1 << 1); + static const int parallel_applying = (1 << 2); + static const int transaction_replay = (1 << 3); + static const int isolation = (1 << 4); + static const int pause = (1 << 5); + static const int causal_reads = (1 << 6); + static const int causal_transaction = (1 << 7); + static const int incremental_writeset = (1 << 8); + static const int session_locks = (1 << 9); + static const int distributed_locks = (1 << 10); + static const int consistency_check = (1 << 11); + static const int unordered = (1 << 12); + static const int annotation = (1 << 13); + static const int preordered = (1 << 14); + static const int streaming = (1 << 15); + static const int snapshot = (1 << 16); + static const int nbo = (1 << 17); + + /** decipher capability bitmask */ + static std::string str(int); + }; + + provider(wsrep::server_state& server_state) + : server_state_(server_state) + { } + virtual ~provider() { } + // Provider state management + virtual enum status connect(const std::string& cluster_name, + const std::string& cluster_url, + const std::string& state_donor, + bool bootstrap) = 0; + virtual int disconnect() = 0; + + virtual int capabilities() const = 0; + virtual int desync() = 0; + virtual int resync() = 0; + + virtual wsrep::seqno pause() = 0; + virtual int resume() = 0; + + // Applier interface + virtual enum status run_applier(wsrep::high_priority_service* + applier_ctx) = 0; + // Write set replication + // TODO: Rename to assing_read_view() + virtual int start_transaction(wsrep::ws_handle&) = 0; + virtual enum status assign_read_view( + wsrep::ws_handle&, const wsrep::gtid*) = 0; + virtual int append_key(wsrep::ws_handle&, const wsrep::key&) = 0; + virtual enum status append_data( + wsrep::ws_handle&, const wsrep::const_buffer&) = 0; + virtual enum status + certify(wsrep::client_id, wsrep::ws_handle&, + int, + wsrep::ws_meta&) = 0; + /** + * BF abort a transaction inside provider. + * + * @param[in] bf_seqno Seqno of the aborter transaction + * @param[in] victim_txt Transaction identifier of the victim + * @param[out] victim_seqno Sequence number of the victim transaction + * or WSREP_SEQNO_UNDEFINED if the victim was not ordered + * + * @return wsrep_status_t + */ + virtual enum status bf_abort(wsrep::seqno bf_seqno, + wsrep::transaction_id victim_trx, + wsrep::seqno& victim_seqno) = 0; + virtual enum status rollback(wsrep::transaction_id) = 0; + virtual enum status commit_order_enter(const wsrep::ws_handle&, + const wsrep::ws_meta&) = 0; + virtual int commit_order_leave(const wsrep::ws_handle&, + const wsrep::ws_meta&, + const wsrep::mutable_buffer& err) = 0; + virtual int release(wsrep::ws_handle&) = 0; + + /** + * Replay a transaction. + * + * @todo Inspect if the ws_handle could be made const + * + * @return Zero in case of success, non-zero on failure. + */ + virtual enum status replay( + const wsrep::ws_handle& ws_handle, + wsrep::high_priority_service* applier_ctx) = 0; + + /** + * Enter total order isolation critical section + */ + virtual enum status enter_toi(wsrep::client_id, + const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + wsrep::ws_meta& ws_meta, + int flags) = 0; + /** + * Leave total order isolation critical section + */ + virtual enum status leave_toi(wsrep::client_id, + const wsrep::mutable_buffer& err) = 0; + + /** + * Perform a causal read on cluster. + * + * @param timeout Timeout in seconds + * + * @return Provider status indicating the result of the call. + */ + virtual std::pair<wsrep::gtid, enum status> + causal_read(int timeout) const = 0; + virtual enum status wait_for_gtid(const wsrep::gtid&, int timeout) const = 0; + /** + * Return last committed GTID. + */ + virtual wsrep::gtid last_committed_gtid() const = 0; + virtual enum status sst_sent(const wsrep::gtid&, int) = 0; + virtual enum status sst_received(const wsrep::gtid&, int) = 0; + virtual enum status enc_set_key(const wsrep::const_buffer& key) = 0; + virtual std::vector<status_variable> status() const = 0; + virtual void reset_status() = 0; + + virtual std::string options() const = 0; + virtual enum status options(const std::string&) = 0; + /** + * Get provider name. + * + * @return Provider name string. + */ + virtual std::string name() const = 0; + + /** + * Get provider version. + * + * @return Provider version string. + */ + virtual std::string version() const = 0; + + /** + * Get provider vendor. + * + * @return Provider vendor string. + */ + virtual std::string vendor() const = 0; + + /** + * Return pointer to native provider handle. + */ + virtual void* native() const = 0; + + /** + * Services argument passed to make_provider. This struct contains + * optional services which are passed to the provider. + */ + struct services + { + wsrep::thread_service* thread_service; + wsrep::tls_service* tls_service; + wsrep::allowlist_service* allowlist_service; + wsrep::event_service* event_service; + + // some GCC and clang versions don't support C++11 default + // initializers fully, so we need to use explicit constructors + // instead: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88165 + // https://bugs.llvm.org/show_bug.cgi?id=36684 + services() + : thread_service() + , tls_service() + , allowlist_service() + , event_service() + { + } + + services(wsrep::thread_service* thr, + wsrep::tls_service* tls, + wsrep::allowlist_service* all, + wsrep::event_service* event) + : thread_service(thr) + , tls_service(tls) + , allowlist_service(all) + , event_service(event) + { + } + }; + /** + * Create a new provider. + * + * @param provider_spec Provider specification + * @param provider_options Initial options to provider + * @param thread_service Optional thread service implementation. + */ + static provider* make_provider(wsrep::server_state&, + const std::string& provider_spec, + const std::string& provider_options, + const wsrep::provider::services& services + = wsrep::provider::services()); + + protected: + wsrep::server_state& server_state_; + }; + + static inline bool starts_transaction(int flags) + { + return (flags & wsrep::provider::flag::start_transaction); + } + + static inline bool commits_transaction(int flags) + { + return (flags & wsrep::provider::flag::commit); + } + + static inline bool rolls_back_transaction(int flags) + { + return (flags & wsrep::provider::flag::rollback); + } + + static inline bool prepares_transaction(int flags) + { + return (flags & wsrep::provider::flag::prepare); + } + + static inline bool is_toi(int flags) + { + return (flags & wsrep::provider::flag::isolation); + } + + static inline bool is_commutative(int flags) + { + return (flags & wsrep::provider::flag::commutative); + } + + static inline bool is_native(int flags) + { + return (flags & wsrep::provider::flag::native); + } +} + +#endif // WSREP_PROVIDER_HPP diff --git a/wsrep-lib/include/wsrep/reporter.hpp b/wsrep-lib/include/wsrep/reporter.hpp new file mode 100644 index 00000000..75ed0389 --- /dev/null +++ b/wsrep-lib/include/wsrep/reporter.hpp @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file info.hpp + * + * Interface to report application status to external programs + * via JSON file. + */ + +#ifndef WSREP_REPORTER_HPP +#define WSREP_REPORTER_HPP + +#include "mutex.hpp" +#include "server_state.hpp" + +#include <string> +#include <deque> + +namespace wsrep +{ + class reporter + { + public: + reporter(mutex& mutex, + const std::string& file_name, + size_t max_msg); + + virtual ~reporter(); + + void report_state(enum server_state::state state); + + /** + * Report progres in the form of a JSON string (all values integers): + * { + * "from": FROM, // from wsrep API state number + * "to": TO, // to wsrep API state number + * "total": TOTAL, // total work to do + * "done": DONE, // work already done + * "indefinite": INDEFINITE // indefinite value of work constant + * } + */ + void report_progress(const std::string& json); + + enum log_level + { + error, + warning + }; + + // undefined timestamp value + static double constexpr undefined = 0.0; + + void report_log_msg(log_level, const std::string& msg, + double timestamp = undefined); + + private: + enum substates { + s_disconnected_disconnected, + s_disconnected_initializing, + s_disconnected_initialized, + s_connected_waiting, // to become joiner + s_joining_initialized, + s_joining_sst, + s_joining_initializing, + s_joining_ist, + s_joined_syncing, + s_synced_running, + s_donor_sending, + s_disconnecting_disconnecting, + substates_max + }; + + wsrep::mutex& mutex_; + std::string const file_name_; + std::string progress_; + char* template_; + substates state_; + bool initialized_; + + typedef struct { + double tstamp; + std::string msg; + } log_msg; + + std::deque<log_msg> err_msg_; + std::deque<log_msg> warn_msg_; + size_t const max_msg_; + + static void write_log_msgs(std::ostream& os, + const std::string& label, + const std::deque<log_msg>& msgs); + static void write_log_msg(std::ostream& os, + const log_msg& msg); + + substates substate_map(enum server_state::state state); + float progress_map(float progress) const; + void write_file(double timestamp); + + // make uncopyable + reporter(const wsrep::reporter&); + void operator=(const wsrep::reporter&); + }; /* reporter */ + +} /* wsrep */ + +#endif /* WSREP_REPORTER_HPP */ diff --git a/wsrep-lib/include/wsrep/seqno.hpp b/wsrep-lib/include/wsrep/seqno.hpp new file mode 100644 index 00000000..9d8cedbc --- /dev/null +++ b/wsrep-lib/include/wsrep/seqno.hpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_SEQNO_HPP +#define WSREP_SEQNO_HPP + +#include <iosfwd> + +namespace wsrep +{ + /** @class seqno + * + * Sequence number type. + */ + class seqno + { + public: + typedef long long native_type; + + seqno() + : seqno_(-1) + { } + + explicit seqno(long long seqno) + : seqno_(seqno) + { } + + long long get() const + { + return seqno_; + } + + bool is_undefined() const + { + return (seqno_ == -1); + } + + bool operator<(seqno other) const + { + return (seqno_ < other.seqno_); + } + + bool operator>(seqno other) const + { + return (seqno_ > other.seqno_); + } + + bool operator<=(seqno other) const + { + return !(seqno_ > other.seqno_); + } + + bool operator>=(seqno other) const + { + return !(seqno_ < other.seqno_); + } + + bool operator==(seqno other) const + { + return (seqno_ == other.seqno_); + } + bool operator!=(seqno other) const + { + return !(seqno_ == other.seqno_); + } + seqno operator+(seqno other) const + { + return (seqno(seqno_ + other.seqno_)); + } + seqno operator+(long long other) const + { + return (*this + seqno(other)); + } + static seqno undefined() { return seqno(-1); } + + private: + native_type seqno_; + }; + + std::ostream& operator<<(std::ostream& os, wsrep::seqno seqno); +} + +#endif // WSREP_SEQNO_HPP diff --git a/wsrep-lib/include/wsrep/server_service.hpp b/wsrep-lib/include/wsrep/server_service.hpp new file mode 100644 index 00000000..2021e66d --- /dev/null +++ b/wsrep-lib/include/wsrep/server_service.hpp @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + + +/** @file server_service.hpp + * + * An abstract interface for a DBMS server services. + * The interface will define methods which will be called from + * the wsrep-lib. + */ + +#ifndef WSREP_SERVER_SERVICE_HPP +#define WSREP_SERVER_SERVICE_HPP + +#include "logger.hpp" +#include "server_state.hpp" + +#include <string> + +namespace wsrep +{ + class client_service; + class storage_service; + class high_priority_service; + class ws_meta; + class gtid; + class view; + class server_service + { + public: + + virtual ~server_service() { } + virtual wsrep::storage_service* storage_service( + wsrep::client_service&) = 0; + virtual wsrep::storage_service* storage_service( + wsrep::high_priority_service&) = 0; + virtual void release_storage_service(wsrep::storage_service*) = 0; + + /** + * Create an applier state for streaming transaction applying. + * + * @param orig_cs Reference to client service which is + * requesting a new streaming applier service + * instance. + * + * @return Pointer to streaming applier client state. + */ + virtual wsrep::high_priority_service* + streaming_applier_service(wsrep::client_service& orig_cs) = 0; + + /** + * Create an applier state for streaming transaction applying. + * + * @param orig_hps Reference to high priority service which is + * requesting a new streaming applier service + * instance. + * + * @return Pointer to streaming applier client state. + */ + virtual wsrep::high_priority_service* + streaming_applier_service(wsrep::high_priority_service& orig_hps) = 0; + + /** + * Release a client state allocated by either local_client_state() + * or streaming_applier_client_state(). + */ + virtual void release_high_priority_service( + wsrep::high_priority_service*) = 0; + + /** + * Perform a background rollback for a transaction. + */ + virtual void background_rollback(wsrep::client_state&) = 0; + + /** + * Bootstrap a DBMS state for a new cluster. + * + * This method is called by the wsrep lib after the + * new cluster is bootstrapped and the server has reached + * initialized state. From this call the DBMS should initialize + * environment for the new cluster. + */ + virtual void bootstrap() = 0; + + /** + * Log message + * + * @param level Requested logging level + * @param message Message + */ + virtual void log_message(enum wsrep::log::level level, + const char* message) = 0; + /** + * Log a dummy write set. A dummy write set is usually either + * a remotely generated write set which failed certification in + * provider and had a GTID assigned or a streaming replication + * rollback write set. If the DBMS implements logging for + * applied transactions, logging dummy write sets which do not + * commit any transaction is neeeded to keep the GTID sequence + * continuous in the server. + */ + virtual void log_dummy_write_set(wsrep::client_state& client_state, + const wsrep::ws_meta& ws_meta) = 0; + + /** + * Log a cluster view change event. The method takes + * as an argument a pointer to high priority service associated + * to an applier thread if one is available. + * + * @param high_priority_service Pointer to high priority service + * @param view Reference to view object + */ + virtual void log_view( + wsrep::high_priority_service* high_priority_service, + const wsrep::view& view) = 0; + + /** + * Recover streaming appliers from the streaming log. + * The implementation must scan through log of stored streaming + * fragments and reconstruct the streaming applier service + * objects. + * + * This is overload for calls which are done from client context, + * e.g. after SST has been received. + * + * @param client_service Reference to client service object + */ + virtual void recover_streaming_appliers( + wsrep::client_service& client_service) = 0; + + /** + * Recover streaming appliers from the streaming log. + * The implementation must scan through log of stored streaming + * fragments and reconstruct the streaming applier service + * objects. + * + * This is overload for calls which are done from high priority + * context, e.g. when handling cluster view change events. + * + * @param high_priority_service Reference to high priority service + * object. + */ + virtual void recover_streaming_appliers( + wsrep::high_priority_service& high_priority_service) = 0; + + /** + * Recover a cluster view change event. + * The method takes own node ID. + * + * @param client_service Reference to client_service + * @param own_id this node ID obtained on connection to cluster + */ + virtual wsrep::view get_view( + wsrep::client_service& client_service, + const wsrep::id& own_id) = 0; + + /** + * Get the current replication position from the server + * storage. + * + * @param client_service Reference to client_service + * + * @return Current position GTID. + */ + virtual wsrep::gtid get_position( + wsrep::client_service& client_service) = 0; + + /** + * Set the current replication position of the server + * storage. + * + * @param client_service Reference to client_service + * @param gtid Reference to position to be set + */ + virtual void set_position( + wsrep::client_service& client_service, + const wsrep::gtid& gtid) = 0; + + /** + * Log a state change event. + * + * Note that this method may be called with server_state + * mutex locked, so calling server_state public methods + * should be avoided from within this call. + * + * @param prev_state Previous state server was in + * @param current_state Current state + */ + virtual void log_state_change( + enum wsrep::server_state::state prev_state, + enum wsrep::server_state::state current_state) = 0; + + /** + * Determine if the configured SST method requires SST to be + * performed before DBMS storage engine initialization. + * + * @return True if the SST must happen before storage engine init, + * otherwise false. + */ + virtual bool sst_before_init() const = 0; + + /** + * Return SST request which provides the donor server enough + * information how to donate the snapshot. + * + * @return A string containing a SST request. + */ + virtual std::string sst_request() = 0; + + /** + * Start a SST process. + * + * @param sst_request A string containing the SST request from + * the joiner + * @param gtid A GTID denoting the current replication position + * @param bypass Boolean bypass flag. + * + * @return Zero if the SST transfer was successfully started, + * non-zero otherwise. + */ + virtual int start_sst(const std::string& sst_request, + const wsrep::gtid& gtid, + bool bypass) = 0; + + + /** + * Wait until committing transactions have completed. + * Prior calling this method the server should have been + * desynced from the group to disallow further transactions + * to start committing. + */ + virtual int wait_committing_transactions(int timeout) = 0; + + /** + * Provide a server level debug sync point for a caller. + */ + virtual void debug_sync(const char* sync_point) = 0; + + }; +} + +#endif // WSREP_SERVER_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/server_state.hpp b/wsrep-lib/include/wsrep/server_state.hpp new file mode 100644 index 00000000..0fa6420b --- /dev/null +++ b/wsrep-lib/include/wsrep/server_state.hpp @@ -0,0 +1,745 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file server_state.hpp + * + * Server State Abstraction + * ========================== + * + * This file defines an interface for WSREP Server State. + * The Server State will encapsulate server identification, + * server state and server capabilities. The class also + * defines an interface for manipulating server state, applying + * of remote transaction write sets, processing SST requests, + * creating local client connections for local storage access + * operations. + * + * Concepts + * ======== + * + * State Snapshot Transfer + * ----------------------- + * + * Depending on SST type (physical or logical), the server storage + * engine initialization must be done before or after SST happens. + * In case of physical SST method (typically rsync, filesystem snapshot) + * the SST happens before the storage engine is initialized, in case + * of logical backup typically after the storage engine initialization. + * + * Rollback Mode + * ------------- + * + * When High Priority Transaction (HPT) write set is applied, it + * may be required that the HPT Brute Force Aborts (BFA) locally + * executing transaction. As HPT must be able to apply all its + * write sets without interruption, the locally executing transaction + * must yield immediately, otherwise a transaction processing + * may stop or even deadlock. Depending on DBMS implementation, + * the local transaction may need to be rolled back immediately + * (synchronous mode) or the rollback may happen later on + * (asynchronous mode). The Server Context implementation + * which derives from Server Context base class must provide + * the base class for the rollback mode which server operates on. + * + * ### Synchronous + * + * If the DBMS server implementation does not allow asynchronous rollback, + * the victim transaction must be rolled back immediately in order to + * allow transaction processing to proceed. Depending on DBMS process model, + * there may be either background thread which processes the rollback + * or the rollback can be done by the HTP applier. + * + * ### Asynchronous + * + * In asynchronous mode the BFA victim transaction is just marked + * to be aborted or in case of fully optimistic concurrency control, + * the conflict is detected at commit. + * + * + * # Return value conventions + * + * The calls which are proxies to corresponding provider functionality + * will return wsrep::provider::status enum as a result. Otherwise + * the return value is generally zero on success, non zero on failure. + */ + +#ifndef WSREP_SERVER_STATE_HPP +#define WSREP_SERVER_STATE_HPP + +#include "mutex.hpp" +#include "condition_variable.hpp" +#include "id.hpp" +#include "view.hpp" +#include "transaction_id.hpp" +#include "logger.hpp" +#include "provider.hpp" +#include "compiler.hpp" +#include "xid.hpp" + +#include <deque> +#include <vector> +#include <string> +#include <map> + +/** + * Magic string to tell provider to engage into trivial (empty) + * state transfer. No data will be passed, but the node shall be + * considered joined. + */ +#define WSREP_LIB_SST_TRIVIAL "trivial" + +namespace wsrep +{ + // Forward declarations + class ws_handle; + class ws_meta; + class client_state; + class transaction; + class const_buffer; + class server_service; + class client_service; + class encryption_service; + + /** @class Server Context + * + * + */ + class server_state + { + public: + /** + * Server state enumeration. + * + * @todo Fix UML generation + * + * Server state diagram if initialization happens before SST. + * + * [*] --> disconnected + * disconnected --> initializing + * initializing --> initialized + * initialized --> connected + * connected --> joiner + * joiner --> joined + * joined --> synced + * synced --> donor + * donor --> joined + * + * Server state diagram if SST happens before initialization. + * + * [*] --> disconnected + * disconnected --> connected + * connected --> joiner + * joiner --> initializing + * initializing --> initialized + * initialized --> joined + * joined --> synced + * synced --> donor + * donor --> joined + */ + enum state + { + /** Server is in disconnected state. */ + s_disconnected, + /** Server is initializing */ + s_initializing, + /** Server has been initialized */ + s_initialized, + /** Server is connected to the cluster */ + s_connected, + /** Server is receiving SST */ + s_joiner, + /** Server has received SST successfully but has not synced + with rest of the cluster yet. */ + s_joined, + /** Server is donating state snapshot transfer */ + s_donor, + /** Server has synced with the cluster */ + s_synced, + /** Server is disconnecting from group */ + s_disconnecting + }; + + static const int n_states_ = s_disconnecting + 1; + + /** + * Rollback Mode enumeration + */ + enum rollback_mode + { + /** Asynchronous rollback mode */ + rm_async, + /** Synchronous rollback mode */ + rm_sync + }; + + virtual ~server_state(); + + wsrep::encryption_service* encryption_service() + { return encryption_service_; } + + wsrep::server_service& server_service() { return server_service_; } + + /** + * Return human readable server name. + * + * @return Human readable server name string. + */ + const std::string& name() const { return name_; } + + /** + * Return Server identifier. + * + * @return Server identifier. + */ + const wsrep::id& id() const { return id_; } + + const std::string& incoming_address() const + { return incoming_address_; } + /** + * Return server group communication address. + * + * @return Return server group communication address. + */ + const std::string& address() const { return address_; } + + /** + * Return working directory + * + * @return String containing path to working directory. + */ + const std::string& working_dir() const { return working_dir_; } + + /** + * Return initial position for server. + */ + const wsrep::gtid& initial_position() const + { return initial_position_; } + /** + * Return maximum protocol version. + */ + int max_protocol_version() const { return max_protocol_version_;} + + /** + * Get the rollback mode which server is operating in. + * + * @return Rollback mode. + */ + enum rollback_mode rollback_mode() const { return rollback_mode_; } + + /** + * Registers a streaming client. + */ + void start_streaming_client(wsrep::client_state* client_state); + + void convert_streaming_client_to_applier( + wsrep::client_state* client_state); + void stop_streaming_client(wsrep::client_state* client_state); + + void start_streaming_applier( + const wsrep::id&, + const wsrep::transaction_id&, + wsrep::high_priority_service*); + + void stop_streaming_applier( + const wsrep::id&, const wsrep::transaction_id&); + + /** + * Find a streaming applier matching server and transaction ids + */ + wsrep::high_priority_service* find_streaming_applier( + const wsrep::id&, + const wsrep::transaction_id&) const; + + /** + * Find a streaming applier matching xid + */ + wsrep::high_priority_service* find_streaming_applier( + const wsrep::xid& xid) const; + + /** + * Queue a rollback fragment the transaction with given id + */ + void queue_rollback_event(const wsrep::transaction_id& id); + + /** + * Send rollback fragments for previously queued events via + * queue_rollback_event() + */ + enum wsrep::provider::status send_pending_rollback_events(); + + /** + * Load WSRep provider. + * + * @param provider WSRep provider library to be loaded. + * @param provider_options Provider specific options string + * to be passed for provider during initialization. + * @param services Application defined services passed to + * the provider. + * + * @return Zero on success, non-zero on error. + */ + int load_provider(const std::string& provider, + const std::string& provider_options, + const wsrep::provider::services& services + = wsrep::provider::services()); + + void unload_provider(); + + bool is_provider_loaded() const { return provider_ != 0; } + + /** + * Return reference to provider. + * + * @return Reference to provider + * + * @throw wsrep::runtime_error if provider has not been loaded + * + * @todo This should not be virtual. However, currently there + * is no mechanism for tests and integrations to provide + * their own provider implementations, so this is kept virtual + * for time being. + */ + virtual wsrep::provider& provider() const + { + if (provider_ == 0) + { + throw wsrep::runtime_error("provider not loaded"); + } + return *provider_; + } + + /** + * Initialize connection to cluster. + * + * @param cluster_name A string containing the name of the cluster + * @param cluster_address Cluster address string + * @param state_donor String containing a list of desired donors + * @param bootstrap Bootstrap option + * + * @return Zero in case of success, non-zero on error. + */ + int connect(const std::string& cluster_name, + const std::string& cluster_address, + const std::string& state_donor, + bool bootstrap); + + int disconnect(); + + /** + * A method which will be called when the server + * has been joined to the cluster + */ + void on_connect(const wsrep::view& view); + + /** + * A method which will be called when a view + * notification event has been delivered by the + * provider. + * + * @params view wsrep::view object which holds the new view + * information. + */ + void on_view(const wsrep::view& view, + wsrep::high_priority_service*); + + /** + * A method which will be called when the server + * has been synchronized with the cluster. + * + * This will have a side effect of changing the Server Context + * state to s_synced. + */ + void on_sync(); + + /** + * Wait until server reaches given state. + */ + void wait_until_state(enum state state) const + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + wait_until_state(lock, state); + } + + /** + * Return GTID at the position when server connected to + * the cluster. + */ + wsrep::gtid connected_gtid() const { return connected_gtid_; } + + /** + * Return current view + */ + const wsrep::view& current_view() const { return current_view_; } + + /** + * Wait until all the write sets up to given GTID have been + * committed. + * + * @return Zero on success, non-zero on failure. + */ + enum wsrep::provider::status + wait_for_gtid(const wsrep::gtid&, int timeout) const; + + /** + * Set encryption key + * + * @param key Encryption key + * + * @return Zero on success, non-zero on failure. + */ + int set_encryption_key(std::vector<unsigned char>& key); + + /** + * Return encryption key. + */ + const std::vector<unsigned char>& get_encryption_key() const + { return encryption_key_; } + + /** + * Perform a causal read in the cluster. After the call returns, + * all the causally preceding write sets have been committed + * or the error is returned. + * + * This operation may require communication with other processes + * in the DBMS cluster, so it may be relatively heavy operation. + * Method wait_for_gtid() should be used whenever possible. + * + * @param timeout Timeout in seconds + * + * @return Pair of GTID and result status from provider. + */ + + std::pair<wsrep::gtid, enum wsrep::provider::status> + causal_read(int timeout) const; + + /** + * Desynchronize the server. + * + * If the server state is synced, this call will desynchronize + * the server from the cluster. + */ + int desync() + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + return desync(lock); + } + + /** + * Resynchronize the server. + */ + void resync() + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + resync(lock); + } + + wsrep::seqno pause(); + + wsrep::seqno pause_seqno() const { return pause_seqno_; } + + void resume(); + + /** + * Desync and pause the provider on one go. Will return + * pause seqno if successful. In case of failure, + * undefined seqno will be returned. + */ + wsrep::seqno desync_and_pause(); + + /** + * Resume and resync the provider on one go. Prior this + * call the provider must have been both desynced and paused, + * by either desync_and_pause() or separate calls to desync() + * and pause(). + */ + void resume_and_resync(); + + /** + * True if server has issued and active desync and pause in one go, + * false otherwise. + */ + bool desynced_on_pause() const { return desynced_on_pause_; } + + /** + * Prepares server state for SST. + * + * @return String containing a SST request + */ + std::string prepare_for_sst(); + + /** + * Start a state snapshot transfer. + * + * @param sst_requets SST request string + * @param gtid Current GTID + * @param bypass Bypass flag + * + * @return Zero in case of success, non-zero otherwise + */ + int start_sst(const std::string& sst_request, + const wsrep::gtid& gtid, + bool bypass); + + /** + * + */ + void sst_sent(const wsrep::gtid& gtid, int error); + + /** + * This method must be called by the joiner after the SST + * transfer has been received. If the DBMS state has not been + * initialized, the call will shift the state to initializing + * and will wait until the initialization is complete. + * + * @param client_service + * @param error code of the SST operation + */ + void sst_received(wsrep::client_service& cs, int error); + + /** + * This method must be called after the server initialization + * has been completed. The call has a side effect of changing + * the Server Context state to s_initialized. + */ + void initialized(); + + /** + * Return true if the server has been initialized. + */ + bool is_initialized() const + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + return init_initialized_; + } + + /** + * This method will be called by the provider when + * a remote write set is being applied. It is the responsibility + * of the caller to set up transaction context and data properly. + * + * @todo Make this private, allow calls for provider implementations + * only. + * @param high_priority_service High priority applier service. + * @param transaction Transaction context. + * @param data Write set data + * + * @return Zero on success, non-zero on failure. + */ + int on_apply(wsrep::high_priority_service& high_priority_service, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data); + + enum state state() const + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + return state(lock); + } + + enum state state(wsrep::unique_lock<wsrep::mutex>& + lock WSREP_UNUSED) const + { + assert(lock.owns_lock()); + return state_; + } + + /** + * Get provider status variables. + */ + std::vector<wsrep::provider::status_variable> status() const; + + /** + * Set server wide wsrep debug logging level. + * + * Log levels are + * - 0 - No debug logging. + * - 1..n - Debug logging with increasing verbosity. + */ + void debug_log_level(int level) + { + wsrep::log::debug_log_level(level); + } + + wsrep::mutex& mutex() { return mutex_; } + + protected: + /** Server state constructor + * + * @param mutex Mutex provided by the DBMS implementation. + * @param name Human Readable Server Name. + * @param id Server Identifier String, UUID or some unique + * identifier. + * @param address Server address in form of IPv4 address, IPv6 address + * or hostname. + * @param working_dir Working directory for replication specific + * data files. + * @param rollback_mode Rollback mode which server operates on. + */ + server_state(wsrep::mutex& mutex, + wsrep::condition_variable& cond, + wsrep::server_service& server_service, + wsrep::encryption_service* encryption_service, + const std::string& name, + const std::string& incoming_address, + const std::string& address, + const std::string& working_dir, + const wsrep::gtid& initial_position, + int max_protocol_version, + enum rollback_mode rollback_mode) + : mutex_(mutex) + , cond_(cond) + , server_service_(server_service) + , encryption_service_(encryption_service) + , state_(s_disconnected) + , state_hist_() + , state_waiters_(n_states_) + , bootstrap_() + , initial_position_(initial_position) + , init_initialized_() + , init_synced_() + , sst_gtid_() + , desync_count_() + , desynced_on_pause_() + , pause_count_() + , pause_seqno_() + , streaming_clients_() + , streaming_appliers_() + , streaming_appliers_recovered_() + , provider_() + , name_(name) + , id_(wsrep::id::undefined()) + , incoming_address_(incoming_address) + , address_(address) + , working_dir_(working_dir) + , encryption_key_() + , max_protocol_version_(max_protocol_version) + , rollback_mode_(rollback_mode) + , connected_gtid_() + , previous_primary_view_() + , current_view_() + , rollback_event_queue_() + { } + + private: + + server_state(const server_state&); + server_state& operator=(const server_state&); + + int desync(wsrep::unique_lock<wsrep::mutex>&); + void resync(wsrep::unique_lock<wsrep::mutex>&); + void state(wsrep::unique_lock<wsrep::mutex>&, enum state); + void wait_until_state(wsrep::unique_lock<wsrep::mutex>&, enum state) const; + // Interrupt all threads which are waiting for state + void interrupt_state_waiters(wsrep::unique_lock<wsrep::mutex>&); + + // Recover streaming appliers if not already recoverd + template <class C> + void recover_streaming_appliers_if_not_recovered( + wsrep::unique_lock<wsrep::mutex>&, C&); + + // Close SR transcations whose origin is outside of current + // cluster view. + void close_orphaned_sr_transactions( + wsrep::unique_lock<wsrep::mutex>&, + wsrep::high_priority_service&); + + // Close transactions when handling disconnect from the group. + void close_transactions_at_disconnect(wsrep::high_priority_service&); + + // Handle primary view + void on_primary_view(const wsrep::view&, + wsrep::high_priority_service*); + // Handle non-primary view + void on_non_primary_view(const wsrep::view&, + wsrep::high_priority_service*); + // Common actions on final view + void go_final(wsrep::unique_lock<wsrep::mutex>&, + const wsrep::view&, wsrep::high_priority_service*); + + // Send rollback fragments for all transactions in + // rollback_event_queue_ + enum wsrep::provider::status send_pending_rollback_events( + wsrep::unique_lock<wsrep::mutex>& lock); + + wsrep::mutex& mutex_; + wsrep::condition_variable& cond_; + wsrep::server_service& server_service_; + wsrep::encryption_service* encryption_service_; + enum state state_; + std::vector<enum state> state_hist_; + mutable std::vector<int> state_waiters_; + bool bootstrap_; + const wsrep::gtid initial_position_; + bool init_initialized_; + bool init_synced_; + wsrep::gtid sst_gtid_; + size_t desync_count_; + // Boolean to denote if desync was succesfull when desyncing + // and pausing the provider on one go. + bool desynced_on_pause_; + size_t pause_count_; + wsrep::seqno pause_seqno_; + typedef std::map<wsrep::client_id, wsrep::client_state*> + streaming_clients_map; + streaming_clients_map streaming_clients_; + typedef std::map<std::pair<wsrep::id, wsrep::transaction_id>, + wsrep::high_priority_service*> streaming_appliers_map; + streaming_appliers_map streaming_appliers_; + bool streaming_appliers_recovered_; + wsrep::provider* provider_; + std::string name_; + wsrep::id id_; + std::string incoming_address_; + std::string address_; + std::string working_dir_; + std::vector<unsigned char> encryption_key_; + int max_protocol_version_; + enum rollback_mode rollback_mode_; + wsrep::gtid connected_gtid_; + wsrep::view previous_primary_view_; + wsrep::view current_view_; + std::deque<wsrep::transaction_id> rollback_event_queue_; + }; + + static inline const char* to_c_string( + enum wsrep::server_state::state state) + { + switch (state) + { + case wsrep::server_state::s_disconnected: return "disconnected"; + case wsrep::server_state::s_initializing: return "initializing"; + case wsrep::server_state::s_initialized: return "initialized"; + case wsrep::server_state::s_connected: return "connected"; + case wsrep::server_state::s_joiner: return "joiner"; + case wsrep::server_state::s_joined: return "joined"; + case wsrep::server_state::s_donor: return "donor"; + case wsrep::server_state::s_synced: return "synced"; + case wsrep::server_state::s_disconnecting: return "disconnecting"; + } + return "unknown"; + } + + static inline std::string to_string(enum wsrep::server_state::state state) + { + return (to_c_string(state)); + } + +} + +#endif // WSREP_SERVER_STATE_HPP diff --git a/wsrep-lib/include/wsrep/sr_key_set.hpp b/wsrep-lib/include/wsrep/sr_key_set.hpp new file mode 100644 index 00000000..abdb1e04 --- /dev/null +++ b/wsrep-lib/include/wsrep/sr_key_set.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_SR_KEY_SET_HPP +#define WSREP_SR_KEY_SET_HPP + +#include <set> +#include <map> + +namespace wsrep +{ + class sr_key_set + { + public: + typedef std::set<std::string> leaf_type; + typedef std::map<std::string, leaf_type > branch_type; + sr_key_set() + : root_() + { } + + void insert(const wsrep::key& key) + { + assert(key.size() >= 2); + if (key.size() < 2) + { + throw wsrep::runtime_error("Invalid key size"); + } + + root_[std::string( + static_cast<const char*>(key.key_parts()[0].data()), + key.key_parts()[0].size())].insert( + std::string(static_cast<const char*>(key.key_parts()[1].data()), + key.key_parts()[1].size())); + } + + const branch_type& root() const { return root_; } + void clear() { root_.clear(); } + bool empty() const { return root_.empty(); } + private: + branch_type root_; + }; +} + +#endif // WSREP_KEY_SET_HPP diff --git a/wsrep-lib/include/wsrep/storage_service.hpp b/wsrep-lib/include/wsrep/storage_service.hpp new file mode 100644 index 00000000..e68548b7 --- /dev/null +++ b/wsrep-lib/include/wsrep/storage_service.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file storage_service.hpp + * + * Abstract interface which defines required access to DBMS storage + * service. The service is used for storing streaming replication + * write set fragments into stable storage. The interface is used + * from locally processing transaction context only. Corresponding + * operations for high priority processing can be found from + * wsrep::high_priority_service interface. + */ +#ifndef WSREP_STORAGE_SERVICE_HPP +#define WSREP_STORAGE_SERVICE_HPP + +#include "transaction_id.hpp" +#include "id.hpp" +#include "buffer.hpp" +#include "xid.hpp" + +namespace wsrep +{ + // Forward declarations + class ws_handle; + class ws_meta; + class transaction; + + /** + * Storage service abstract interface. + */ + class storage_service + { + public: + virtual ~storage_service() { } + /** + * Start a new transaction for storage access. + * + * @param[out] ws_handle Write set handle for a new transaction + * + * @return Zero in case of success, non-zero on error. + */ + virtual int start_transaction(const wsrep::ws_handle&) = 0; + + virtual void adopt_transaction(const wsrep::transaction&) = 0; + /** + * Append fragment into stable storage. + */ + virtual int append_fragment(const wsrep::id& server_id, + wsrep::transaction_id client_id, + int flags, + const wsrep::const_buffer& data, + const wsrep::xid& xid) = 0; + + /** + * Update fragment meta data after certification process. + */ + virtual int update_fragment_meta(const wsrep::ws_meta&) = 0; + + /** + * Remove fragments from storage. The storage service must have + * adopted a transaction prior this call. + */ + virtual int remove_fragments() = 0; + + /** + * Commit the transaction. + */ + virtual int commit(const wsrep::ws_handle&, const wsrep::ws_meta&) = 0; + + /** + * Roll back the transaction. + */ + virtual int rollback(const wsrep::ws_handle&, + const wsrep::ws_meta&) = 0; + + + virtual void store_globals() = 0; + virtual void reset_globals() = 0; + }; +} + +#endif // WSREP_STORAGE_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/streaming_context.hpp b/wsrep-lib/include/wsrep/streaming_context.hpp new file mode 100644 index 00000000..9b205c5b --- /dev/null +++ b/wsrep-lib/include/wsrep/streaming_context.hpp @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_STREAMING_CONTEXT_HPP +#define WSREP_STREAMING_CONTEXT_HPP + +#include "compiler.hpp" +#include "logger.hpp" +#include "seqno.hpp" +#include "transaction_id.hpp" + +#include <vector> + +namespace wsrep +{ + class streaming_context + { + public: + enum fragment_unit + { + bytes, + row, + statement + }; + + streaming_context() + : fragments_certified_() + , fragments_() + , rollback_replicated_for_() + , fragment_unit_() + , fragment_size_() + , unit_counter_() + , log_position_() + { } + + /** + * Set streaming parameters. + * + * Calling this method has a side effect of resetting unit + * counter. + * + * @param fragment_unit Desired fragment unit. + * @param fragment_size Desired fragment size. + */ + void params(enum fragment_unit fragment_unit, size_t fragment_size) + { + if (fragment_size) + { + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_streaming, + "Enabling streaming: " + << fragment_unit << " " << fragment_size); + } + else + { + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_streaming, + "Disabling streaming"); + } + fragment_unit_ = fragment_unit; + fragment_size_ = fragment_size; + reset_unit_counter(); + } + + void enable(enum fragment_unit fragment_unit, size_t fragment_size) + { + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_streaming, + "Enabling streaming: " + << fragment_unit << " " << fragment_size); + assert(fragment_size > 0); + fragment_unit_ = fragment_unit; + fragment_size_ = fragment_size; + } + + enum fragment_unit fragment_unit() const { return fragment_unit_; } + + size_t fragment_size() const { return fragment_size_; } + + void disable() + { + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_streaming, + "Disabling streaming"); + fragment_size_ = 0; + } + + void certified() + { + ++fragments_certified_; + } + + size_t fragments_certified() const + { + return fragments_certified_; + } + + void stored(wsrep::seqno seqno) + { + check_fragment_seqno(seqno); + fragments_.push_back(seqno); + } + + size_t fragments_stored() const + { + return fragments_.size(); + } + + void applied(wsrep::seqno seqno) + { + check_fragment_seqno(seqno); + ++fragments_certified_; + fragments_.push_back(seqno); + } + + void rolled_back(wsrep::transaction_id id) + { + assert(rollback_replicated_for_ == wsrep::transaction_id::undefined()); + rollback_replicated_for_ = id; + } + + bool rolled_back() const + { + return (rollback_replicated_for_ != + wsrep::transaction_id::undefined()); + } + + size_t unit_counter() const + { + return unit_counter_; + } + + void set_unit_counter(size_t count) + { + unit_counter_ = count; + } + + void increment_unit_counter(size_t inc) + { + unit_counter_ += inc; + } + + void reset_unit_counter() + { + unit_counter_ = 0; + } + + size_t log_position() const + { + return log_position_; + } + + void set_log_position(size_t position) + { + log_position_ = position; + } + + const std::vector<wsrep::seqno>& fragments() const + { + return fragments_; + } + + bool fragment_size_exceeded() const + { + return unit_counter_ >= fragment_size_; + } + + void cleanup() + { + fragments_certified_ = 0; + fragments_.clear(); + rollback_replicated_for_ = wsrep::transaction_id::undefined(); + unit_counter_ = 0; + log_position_ = 0; + } + private: + + void check_fragment_seqno(wsrep::seqno seqno WSREP_UNUSED) + { + assert(seqno.is_undefined() == false); + assert(fragments_.empty() || fragments_.back() < seqno); + } + + size_t fragments_certified_; + std::vector<wsrep::seqno> fragments_; + wsrep::transaction_id rollback_replicated_for_; + enum fragment_unit fragment_unit_; + size_t fragment_size_; + size_t unit_counter_; + size_t log_position_; + }; +} + +#endif // WSREP_STREAMING_CONTEXT_HPP diff --git a/wsrep-lib/include/wsrep/thread.hpp b/wsrep-lib/include/wsrep/thread.hpp new file mode 100644 index 00000000..99b8cff5 --- /dev/null +++ b/wsrep-lib/include/wsrep/thread.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include <pthread.h> +#include <iosfwd> + +namespace wsrep +{ + class thread + { + public: + class id + { + public: + id() : thread_() { } + explicit id(pthread_t thread) : thread_(thread) { } + private: + friend bool operator==(thread::id left, thread::id right) + { + return (pthread_equal(left.thread_, right.thread_)); + } + friend std::ostream& operator<<(std::ostream&, const id&); + pthread_t thread_; + }; + + thread() + : id_(pthread_self()) + { } + private: + id id_; + }; + + namespace this_thread + { + static inline thread::id get_id() { return thread::id(pthread_self()); } + } + + std::ostream& operator<<(std::ostream&, const thread::id&); +}; diff --git a/wsrep-lib/include/wsrep/thread_service.hpp b/wsrep-lib/include/wsrep/thread_service.hpp new file mode 100644 index 00000000..702d71c1 --- /dev/null +++ b/wsrep-lib/include/wsrep/thread_service.hpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + + +/** @file thread_service.hpp + * + * Service interface for threads and synchronization primitives. + * The purpose of this interface is to provider provider implementations + * means to integrate with application thread implementation. + * + * Interface is designed to resemble POSIX threads, mutexes and + * condition variables. + */ + +#ifndef WSREP_THREAD_SERVICE_HPP +#define WSREP_THREAD_SERVICE_HPP + +#include <cstddef> // size_t +#include "compiler.hpp" + +struct timespec; +struct sched_param; + +namespace wsrep +{ + + class thread_service + { + public: + + thread_service() : exit() { } + virtual ~thread_service() { } + struct thread_key { }; + struct thread { }; + struct mutex_key { }; + struct mutex { }; + struct cond_key { }; + struct cond { }; + + /** + * Method will be called before library side thread + * service initialization. + */ + virtual int before_init() = 0; + + /** + * Method will be called after library side thread service + * has been initialized. + */ + virtual int after_init() = 0; + + /* Thread */ + virtual const thread_key* create_thread_key(const char* name) WSREP_NOEXCEPT + = 0; + virtual int create_thread(const thread_key*, thread**, + void* (*fn)(void*), void*) WSREP_NOEXCEPT + = 0; + virtual int detach(thread*) WSREP_NOEXCEPT = 0; + virtual int equal(thread*, thread*) WSREP_NOEXCEPT = 0; + + /* + * This unlike others is a function pointer to + * avoid having C++ methods on thread exit codepath. + */ + WSREP_NORETURN void (*exit)(thread*, void* retval); + virtual int join(thread*, void** retval) WSREP_NOEXCEPT = 0; + virtual thread* self() WSREP_NOEXCEPT = 0; + virtual int setschedparam(thread*, int, + const struct sched_param*) WSREP_NOEXCEPT + = 0; + virtual int getschedparam(thread*, int*, struct sched_param*) WSREP_NOEXCEPT + = 0; + + /* Mutex */ + virtual const mutex_key* create_mutex_key(const char* name) WSREP_NOEXCEPT + = 0; + virtual mutex* init_mutex(const mutex_key*, void*, size_t) WSREP_NOEXCEPT = 0; + virtual int destroy(mutex*) WSREP_NOEXCEPT = 0; + virtual int lock(mutex*) WSREP_NOEXCEPT = 0; + virtual int trylock(mutex*) WSREP_NOEXCEPT = 0; + virtual int unlock(mutex*) WSREP_NOEXCEPT = 0; + + /* Condition variable */ + virtual const cond_key* create_cond_key(const char* name) WSREP_NOEXCEPT = 0; + virtual cond* init_cond(const cond_key*, void*, size_t) WSREP_NOEXCEPT = 0; + virtual int destroy(cond*) WSREP_NOEXCEPT = 0; + virtual int wait(cond*, mutex*) WSREP_NOEXCEPT = 0; + virtual int timedwait(cond*, mutex*, const struct timespec*) WSREP_NOEXCEPT + = 0; + virtual int signal(cond*) WSREP_NOEXCEPT = 0; + virtual int broadcast(cond*) WSREP_NOEXCEPT = 0; + }; +} // namespace wsrep + +#endif // WSREP_THREAD_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/tls_service.hpp b/wsrep-lib/include/wsrep/tls_service.hpp new file mode 100644 index 00000000..07d20642 --- /dev/null +++ b/wsrep-lib/include/wsrep/tls_service.hpp @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + + +/** @file tls_service.hpp + * + * Service interface for interacting with DBMS provided + * TLS and encryption facilities. + */ + +#ifndef WSREP_TLS_SERVICE_HPP +#define WSREP_TLS_SERVICE_HPP + +#include "compiler.hpp" + +#include <sys/types.h> // ssize_t + +namespace wsrep +{ + + /* Type tags for TLS context and TLS stream. */ + struct tls_context { }; + struct tls_stream { }; + + /** @class tls_service + * + * TLS service interface. This provides an interface corresponding + * to wsrep-API TLS service. For details see wsrep-API/wsrep_tls_service.h + */ + class tls_service + { + public: + enum status + { + success = 0, + want_read, + want_write, + eof, + error + }; + + struct op_result + { + /** Status code of the operation of negative system error number. */ + ssize_t status; + /** Bytes transferred from/to given buffer during the operation. */ + size_t bytes_transferred; + }; + + virtual ~tls_service() { } + /** + * @return Zero on success, system error code on failure. + */ + virtual tls_stream* create_tls_stream(int fd) WSREP_NOEXCEPT = 0; + virtual void destroy(tls_stream*) WSREP_NOEXCEPT = 0; + + virtual int get_error_number(const tls_stream*) const WSREP_NOEXCEPT = 0; + virtual const void* get_error_category(const tls_stream*) const WSREP_NOEXCEPT = 0; + virtual const char* get_error_message(const tls_stream*, + int value, const void* category) + const WSREP_NOEXCEPT = 0; + /** + * @return Status enum. + */ + virtual status client_handshake(tls_stream*) WSREP_NOEXCEPT = 0; + + /** + * @return Status enum or negative error code. + */ + virtual status server_handshake(tls_stream*) WSREP_NOEXCEPT = 0; + + /** + * Read at most max_count bytes into buf. + */ + virtual op_result read(tls_stream*, + void* buf, size_t max_count) WSREP_NOEXCEPT = 0; + + /** + * Write at most count bytes from buf. + */ + virtual op_result write(tls_stream*, + const void* buf, size_t count) WSREP_NOEXCEPT = 0; + + /** + * Shutdown TLS stream. + */ + virtual status shutdown(tls_stream*) WSREP_NOEXCEPT = 0; + }; +} + +#endif // WSREP_TLS_SERVICE_HPP diff --git a/wsrep-lib/include/wsrep/transaction.hpp b/wsrep-lib/include/wsrep/transaction.hpp new file mode 100644 index 00000000..76835fd9 --- /dev/null +++ b/wsrep-lib/include/wsrep/transaction.hpp @@ -0,0 +1,313 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file transaction.hpp */ +#ifndef WSREP_TRANSACTION_HPP +#define WSREP_TRANSACTION_HPP + +#include "provider.hpp" +#include "server_state.hpp" +#include "transaction_id.hpp" +#include "streaming_context.hpp" +#include "lock.hpp" +#include "sr_key_set.hpp" +#include "buffer.hpp" +#include "client_service.hpp" +#include "xid.hpp" + +#include <cassert> +#include <vector> + +namespace wsrep +{ + class client_service; + class client_state; + class key; + class const_buffer; + + class transaction + { + public: + enum state + { + s_executing, + s_preparing, + s_prepared, + s_certifying, + s_committing, + s_ordered_commit, + s_committed, + s_cert_failed, + s_must_abort, + s_aborting, + s_aborted, + s_must_replay, + s_replaying + }; + static const int n_states = s_replaying + 1; + enum state state() const + { return state_; } + + transaction(wsrep::client_state& client_state); + ~transaction(); + // Accessors + wsrep::transaction_id id() const + { return id_; } + + const wsrep::id& server_id() const + { return server_id_; } + + bool active() const + { return (id_ != wsrep::transaction_id::undefined()); } + + + void state(wsrep::unique_lock<wsrep::mutex>&, enum state); + + // Return true if the certification of the last + // fragment succeeded + bool certified() const { return certified_; } + + wsrep::seqno seqno() const + { + return ws_meta_.seqno(); + } + // Return true if the last fragment was ordered by the + // provider + bool ordered() const + { return (ws_meta_.seqno().is_undefined() == false); } + + /** + * Return true if any fragments have been successfully certified + * for the transaction. + */ + bool is_streaming() const + { + return (streaming_context_.fragments_certified() > 0); + } + + /** + * Return number of fragments certified for current statement. + * + * This counts fragments which have been successfully certified + * since the construction of object or last after_statement() + * call. + * + * @return Number of fragments certified for current statement. + */ + size_t fragments_certified_for_statement() const + { + return fragments_certified_for_statement_; + } + + /** + * Return true if transaction has not generated any changes. + */ + bool is_empty() const + { + return sr_keys_.empty(); + } + + bool is_xa() const + { + return !xid_.is_null(); + } + + void assign_xid(const wsrep::xid& xid); + + const wsrep::xid& xid() const + { + return xid_; + } + + int restore_to_prepared_state(const wsrep::xid& xid); + + int commit_or_rollback_by_xid(const wsrep::xid& xid, bool commit); + + void xa_detach(); + + int xa_replay(wsrep::unique_lock<wsrep::mutex>&); + + bool pa_unsafe() const { return (flags() & wsrep::provider::flag::pa_unsafe); } + void pa_unsafe(bool pa_unsafe) { + if (pa_unsafe) { + flags(flags() | wsrep::provider::flag::pa_unsafe); + } else { + flags(flags() & ~wsrep::provider::flag::pa_unsafe); + } + } + bool implicit_deps() const { return implicit_deps_; } + void implicit_deps(bool implicit) { implicit_deps_ = implicit; } + + int start_transaction(const wsrep::transaction_id& id); + + int start_transaction(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta); + + int next_fragment(const wsrep::ws_meta& ws_meta); + + void adopt(const transaction& transaction); + void fragment_applied(wsrep::seqno seqno); + + int prepare_for_ordering(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + bool is_commit); + + int assign_read_view(const wsrep::gtid* gtid); + + int append_key(const wsrep::key&); + + int append_data(const wsrep::const_buffer&); + + int after_row(); + + int before_prepare(wsrep::unique_lock<wsrep::mutex>&); + + int after_prepare(wsrep::unique_lock<wsrep::mutex>&); + + int before_commit(); + + int ordered_commit(); + + int after_commit(); + + int before_rollback(); + + int after_rollback(); + + int before_statement(); + + int after_statement(); + + void after_command_must_abort(wsrep::unique_lock<wsrep::mutex>&); + + void after_applying(); + + bool bf_abort(wsrep::unique_lock<wsrep::mutex>& lock, + wsrep::seqno bf_seqno); + bool total_order_bf_abort(wsrep::unique_lock<wsrep::mutex>&, + wsrep::seqno bf_seqno); + + void clone_for_replay(const wsrep::transaction& other); + + bool bf_aborted() const + { + return (bf_abort_client_state_ != 0); + } + + bool bf_aborted_in_total_order() const + { + return bf_aborted_in_total_order_; + } + + int flags() const + { + return flags_; + } + + // wsrep::mutex& mutex(); + wsrep::ws_handle& ws_handle() { return ws_handle_; } + const wsrep::ws_handle& ws_handle() const { return ws_handle_; } + const wsrep::ws_meta& ws_meta() const { return ws_meta_; } + const wsrep::streaming_context& streaming_context() const + { return streaming_context_; } + wsrep::streaming_context& streaming_context() + { return streaming_context_; } + void adopt_apply_error(wsrep::mutable_buffer& buf) + { + apply_error_buf_ = std::move(buf); + } + const wsrep::mutable_buffer& apply_error() const + { return apply_error_buf_; } + private: + transaction(const transaction&); + transaction operator=(const transaction&); + + wsrep::provider& provider(); + void flags(int flags) { flags_ = flags; } + // Return true if the transaction must abort, is aborting, + // or has been aborted, or has been interrupted by DBMS + // as indicated by client_service::interrupted() call. + // The call will adjust transaction state and set client_state + // error status accordingly. + bool abort_or_interrupt(wsrep::unique_lock<wsrep::mutex>&); + int streaming_step(wsrep::unique_lock<wsrep::mutex>&, bool force = false); + int certify_fragment(wsrep::unique_lock<wsrep::mutex>&); + int certify_commit(wsrep::unique_lock<wsrep::mutex>&); + int append_sr_keys_for_commit(); + int release_commit_order(wsrep::unique_lock<wsrep::mutex>&); + void streaming_rollback(wsrep::unique_lock<wsrep::mutex>&); + int replay(wsrep::unique_lock<wsrep::mutex>&); + void xa_replay_common(wsrep::unique_lock<wsrep::mutex>&); + int xa_replay_commit(wsrep::unique_lock<wsrep::mutex>&); + void cleanup(); + void debug_log_state(const char*) const; + void debug_log_key_append(const wsrep::key& key) const; + + wsrep::server_service& server_service_; + wsrep::client_service& client_service_; + wsrep::client_state& client_state_; + wsrep::id server_id_; + wsrep::transaction_id id_; + enum state state_; + std::vector<enum state> state_hist_; + enum state bf_abort_state_; + enum wsrep::provider::status bf_abort_provider_status_; + int bf_abort_client_state_; + bool bf_aborted_in_total_order_; + wsrep::ws_handle ws_handle_; + wsrep::ws_meta ws_meta_; + int flags_; + bool implicit_deps_; + bool certified_; + size_t fragments_certified_for_statement_; + wsrep::streaming_context streaming_context_; + wsrep::sr_key_set sr_keys_; + wsrep::mutable_buffer apply_error_buf_; + wsrep::xid xid_; + bool streaming_rollback_in_progress_; + }; + + static inline const char* to_c_string(enum wsrep::transaction::state state) + { + switch (state) + { + case wsrep::transaction::s_executing: return "executing"; + case wsrep::transaction::s_preparing: return "preparing"; + case wsrep::transaction::s_prepared: return "prepared"; + case wsrep::transaction::s_certifying: return "certifying"; + case wsrep::transaction::s_committing: return "committing"; + case wsrep::transaction::s_ordered_commit: return "ordered_commit"; + case wsrep::transaction::s_committed: return "committed"; + case wsrep::transaction::s_cert_failed: return "cert_failed"; + case wsrep::transaction::s_must_abort: return "must_abort"; + case wsrep::transaction::s_aborting: return "aborting"; + case wsrep::transaction::s_aborted: return "aborted"; + case wsrep::transaction::s_must_replay: return "must_replay"; + case wsrep::transaction::s_replaying: return "replaying"; + } + return "unknown"; + } + static inline std::string to_string(enum wsrep::transaction::state state) + { + return to_c_string(state); + } + +} + +#endif // WSREP_TRANSACTION_HPP diff --git a/wsrep-lib/include/wsrep/transaction_id.hpp b/wsrep-lib/include/wsrep/transaction_id.hpp new file mode 100644 index 00000000..f5fb5dbc --- /dev/null +++ b/wsrep-lib/include/wsrep/transaction_id.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_TRANSACTION_ID_HPP +#define WSREP_TRANSACTION_ID_HPP + +#include <iostream> +#include <limits> + +namespace wsrep +{ + class transaction_id + { + public: + typedef unsigned long long type; + + + transaction_id() + : id_(std::numeric_limits<type>::max()) + { } + + template <typename I> + explicit transaction_id(I id) + : id_(static_cast<type>(id)) + { } + type get() const { return id_; } + static transaction_id undefined() { return transaction_id(-1); } + bool is_undefined() const { return (id_ == type(-1)); } + bool operator<(const transaction_id& other) const + { + return (id_ < other.id_); + } + bool operator==(const transaction_id& other) const + { return (id_ == other.id_); } + bool operator!=(const transaction_id& other) const + { return (id_ != other.id_); } + private: + type id_; + }; + + static inline std::ostream& operator<<(std::ostream& os, transaction_id id) + { + return (os << id.get()); + } +} + +#endif // WSREP_TRANSACTION_ID_HPP diff --git a/wsrep-lib/include/wsrep/version.hpp b/wsrep-lib/include/wsrep/version.hpp new file mode 100644 index 00000000..94312cf0 --- /dev/null +++ b/wsrep-lib/include/wsrep/version.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_VERSION_HPP +#define WSREP_VERSION_HPP + +/** @file version.hpp + * + * Wsrep library version numbers. The versioning follows Semantic + * Versioning 2.0.0 (https://semver.org/). + */ + +/** + * Major version number. + */ +#define WSREP_LIB_VERSION_MAJOR 1 +/** + * Minor version number. + */ +#define WSREP_LIB_VERSION_MINOR 0 +/** + * Patch version number. + */ +#define WSREP_LIB_VERSION_PATCH 0 + +// Range of supported wsrep-API versions. + +/** + * Lowest supported wsrep-API version. + */ +#define WSREP_LIB_MIN_API_VERSION 26 +/** + * Highest supported wsrep-API version. + */ +#define WSREP_LIB_MAX_API_VERSION 26 + +#endif // WSREP_VERSION_HPP diff --git a/wsrep-lib/include/wsrep/view.hpp b/wsrep-lib/include/wsrep/view.hpp new file mode 100644 index 00000000..d17c27f1 --- /dev/null +++ b/wsrep-lib/include/wsrep/view.hpp @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file view.hpp + * + * + */ + + +#ifndef WSREP_VIEW_HPP +#define WSREP_VIEW_HPP + +#include "id.hpp" +#include "seqno.hpp" +#include "gtid.hpp" +#include <vector> +#include <iostream> + +namespace wsrep +{ + class view + { + public: + enum status + { + primary, + non_primary, + disconnected + }; + class member + { + public: + member(const wsrep::id& id, + const std::string& name, + const std::string& incoming) + : id_(id) + , name_(name) + , incoming_(incoming) + { + } + const wsrep::id& id() const { return id_; } + const std::string& name() const { return name_; } + const std::string& incoming() const { return incoming_; } + private: + wsrep::id id_; + std::string name_; + std::string incoming_; + }; + + view() + : state_id_() + , view_seqno_() + , status_(disconnected) + , capabilities_() + , own_index_(-1) + , protocol_version_(0) + , members_() + { } + view(const wsrep::gtid& state_id, + wsrep::seqno view_seqno, + enum wsrep::view::status status, + int capabilities, + ssize_t own_index, + int protocol_version, + const std::vector<wsrep::view::member>& members) + : state_id_(state_id) + , view_seqno_(view_seqno) + , status_(status) + , capabilities_(capabilities) + , own_index_(own_index) + , protocol_version_(protocol_version) + , members_(members) + { } + + wsrep::gtid state_id() const + { return state_id_; } + + wsrep::seqno view_seqno() const + { return view_seqno_; } + + wsrep::view::status status() const + { return status_; } + + int capabilities() const + { return capabilities_; } + + ssize_t own_index() const + { return own_index_; } + + /** + * Return true if the two views have the same membership + */ + bool equal_membership(const wsrep::view& other) const; + + int protocol_version() const + { return protocol_version_; } + + const std::vector<member>& members() const + { return members_; } + + /** + * Return true if the view is final + */ + bool final() const + { + return (members_.empty() && own_index_ == -1); + } + + /** + * Return member index in the view. + * + * @return Member index if found, -1 if member is not present + * in the view. + */ + int member_index(const wsrep::id& member_id) const; + + /** + * Return true if id is member of this view + */ + bool is_member(const wsrep::id& id) const + { + return member_index(id) != -1; + } + + void print(std::ostream& os) const; + + private: + wsrep::gtid state_id_; + wsrep::seqno view_seqno_; + enum wsrep::view::status status_; + int capabilities_; + ssize_t own_index_; + int protocol_version_; + std::vector<wsrep::view::member> members_; + }; + + static inline + std::ostream& operator<<(std::ostream& os, const wsrep::view& v) + { + v.print(os); return os; + } + + static inline const char* to_c_string(enum wsrep::view::status status) + { + switch(status) + { + case wsrep::view::primary: return "primary"; + case wsrep::view::non_primary: return "non-primary"; + case wsrep::view::disconnected: return "disconnected"; + } + return "invalid status"; + } +} + +#endif // WSREP_VIEW diff --git a/wsrep-lib/include/wsrep/xid.hpp b/wsrep-lib/include/wsrep/xid.hpp new file mode 100644 index 00000000..c500d63b --- /dev/null +++ b/wsrep-lib/include/wsrep/xid.hpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_XID_HPP +#define WSREP_XID_HPP + +#include <iosfwd> +#include "buffer.hpp" +#include "exception.hpp" + +namespace wsrep +{ + class xid + { + public: + xid() + : format_id_(-1) + , gtrid_len_(0) + , bqual_len_(0) + , data_() + { } + + xid(long format_id, long gtrid_len, + long bqual_len, const char* data) + : format_id_(format_id) + , gtrid_len_(gtrid_len) + , bqual_len_(bqual_len) + , data_() + { + if (gtrid_len_ > 64 || bqual_len_ > 64) + { + throw wsrep::runtime_error("maximum wsrep::xid size exceeded"); + } + const long len = gtrid_len_ + bqual_len_; + if (len > 0) + { + data_.push_back(data, data + len); + } + } + + xid(const xid& xid) + : format_id_(xid.format_id_) + , gtrid_len_(xid.gtrid_len_) + , bqual_len_(xid.bqual_len_) + , data_(xid.data_) + { } + + bool is_null() const + { + return format_id_ == -1; + } + + void clear() + { + format_id_ = -1; + gtrid_len_ = 0; + bqual_len_ = 0; + data_.clear(); + } + + xid& operator= (const xid& other) + { + format_id_ = other.format_id_; + gtrid_len_ = other.gtrid_len_; + bqual_len_ = other.bqual_len_; + data_ = other.data_; + return *this; + } + + bool operator==(const xid& other) const + { + if (format_id_ != other.format_id_ || + gtrid_len_ != other.gtrid_len_ || + bqual_len_ != other.bqual_len_ || + data_.size() != other.data_.size()) + { + return false; + } + return data_ == other.data_; + } + + friend std::string to_string(const wsrep::xid& xid); + friend std::ostream& operator<<(std::ostream& os, const wsrep::xid& xid); + protected: + long format_id_; + long gtrid_len_; + long bqual_len_; + mutable_buffer data_; + }; + + std::string to_string(const wsrep::xid& xid); + std::ostream& operator<<(std::ostream& os, const wsrep::xid& xid); +} + +#endif // WSREP_XID_HPP diff --git a/wsrep-lib/scripts/benchmark-provider.sh b/wsrep-lib/scripts/benchmark-provider.sh new file mode 100755 index 00000000..87eb541e --- /dev/null +++ b/wsrep-lib/scripts/benchmark-provider.sh @@ -0,0 +1,46 @@ +#!/bin/bash -eu +# +# Copyright (C) 2018 Codership Oy <info@codership.com> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# + +set -x +PROVIDER=${PROVIDER:?"Provider library is required"} + +total_transactions=$((1 << 17)) + +function run_benchmark() +{ + servers=$1 + clients=$2 + transactions=$(($total_transactions/($servers*$clients))) + result_file=dbms-bench-$(basename $PROVIDER)-$servers-$clients + echo "Running benchmark for servers: $servers clients $clients" + rm -r *_data/ || : + command time -v ./src/dbms_simulator --servers=$servers --clients=$clients --transactions=$transactions --wsrep-provider="$PROVIDER" --fast-exit=1 >& $result_file +} + + +for servers in 1 2 4 8 16 +# for servers in 16 +do + for clients in 1 2 4 8 16 32 +# for clients in 4 8 16 32 + do + run_benchmark $servers $clients + done +done diff --git a/wsrep-lib/scripts/benchmark-report.sh b/wsrep-lib/scripts/benchmark-report.sh new file mode 100755 index 00000000..254e82b0 --- /dev/null +++ b/wsrep-lib/scripts/benchmark-report.sh @@ -0,0 +1,43 @@ +#!/bin/bash -eu +# +# Copyright (C) 2018 Codership Oy <info@codership.com> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# + +PROVIDER=${PROVIDER:?"Provider library is required"} + +function report() +{ + servers=$1 + clients=$2 + result_file=dbms-bench-$(basename $PROVIDER)-$servers-$clients + trx_per_sec=$(grep -a "Transactions per second" "$result_file" | cut -d ':' -f 2) + cpu=$(grep -a "Percent of CPU this job got" "$result_file" | cut -d ':' -f 2) + vol_ctx_switches=$(grep -a "Voluntary context switches" "$result_file" | cut -d ':' -f 2) + invol_ctx_switches=$(grep -a "Involuntary context switches" "$result_file" | cut -d ':' -f 2) + echo "$clients $cpu $trx_per_sec $vol_ctx_switches $invol_ctx_switches" +} + +for servers in 1 2 4 8 16 +do + echo "Servers: $servers" + for clients in 1 2 4 8 16 32 + do + report $servers $clients + done +done + diff --git a/wsrep-lib/scripts/stress-provider.sh b/wsrep-lib/scripts/stress-provider.sh new file mode 100755 index 00000000..b9a91533 --- /dev/null +++ b/wsrep-lib/scripts/stress-provider.sh @@ -0,0 +1,38 @@ +#!/bin/bash -eu +# +# Copyright (C) 2019 Codership Oy <info@codership.com> +# +# This file is part of wsrep-lib. +# +# Wsrep-lib is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# Wsrep-lib is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. +# + +set -x +WSREP_PROVIDER=${WSREP_PROVIDER:?"Provider library is required"} + +transactions=$((1 << 20)) +clients=16 +servers=3 +rows=100 +alg_freq=100 + +./dbsim/dbsim \ + --servers=$servers \ + --clients=$clients \ + --transactions=$(($transactions/($servers*$clients))) \ + --rows=$rows \ + --alg-freq=$alg_freq \ + --wsrep-provider=$WSREP_PROVIDER + +rm -r dbsim_1_data dbsim_2_data dbsim_3_data diff --git a/wsrep-lib/src/CMakeLists.txt b/wsrep-lib/src/CMakeLists.txt new file mode 100644 index 00000000..d5049e11 --- /dev/null +++ b/wsrep-lib/src/CMakeLists.txt @@ -0,0 +1,26 @@ +# +# Copyright (C) 2018 Codership Oy <info@codership.com> +# + +add_library(wsrep-lib + client_state.cpp + exception.cpp + gtid.cpp + id.cpp + xid.cpp + key.cpp + logger.cpp + provider.cpp + seqno.cpp + view.cpp + server_state.cpp + thread.cpp + thread_service_v1.cpp + tls_service_v1.cpp + event_service_v1.cpp + transaction.cpp + uuid.cpp + reporter.cpp + allowlist_service_v1.cpp + wsrep_provider_v26.cpp) +target_link_libraries(wsrep-lib wsrep_api_v26 pthread ${WSREP_LIB_LIBDL}) diff --git a/wsrep-lib/src/allowlist_service_v1.cpp b/wsrep-lib/src/allowlist_service_v1.cpp new file mode 100644 index 00000000..9e7bf2ae --- /dev/null +++ b/wsrep-lib/src/allowlist_service_v1.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "allowlist_service_v1.hpp" +#include "service_helpers.hpp" + +#include "wsrep/allowlist_service.hpp" +#include "wsrep/buffer.hpp" +#include "v26/wsrep_allowlist_service.h" + +#include <cassert> +#include <dlfcn.h> +#include <cerrno> + +namespace wsrep_allowlist_service_v1 +{ + // Pointer to allowlist service implementation provided by + // the application. + static wsrep::allowlist_service* allowlist_service_impl{ 0 }; + static std::atomic<size_t> use_count; + + enum wsrep::allowlist_service::allowlist_key allowlist_key_from_native(wsrep_allowlist_key_t key) + { + switch (key) + { + case WSREP_ALLOWLIST_KEY_IP: return wsrep::allowlist_service::allowlist_key::allowlist_ip; + case WSREP_ALLOWLIST_KEY_SSL: return wsrep::allowlist_service::allowlist_key::allowlist_ssl; + default: throw wsrep::runtime_error("Unknown allowlist key"); + } + } + + // + // allowlist service callbacks + // + + wsrep_status_t allowlist_cb( + wsrep_allowlist_context_t*, + wsrep_allowlist_key_t key, + const wsrep_buf_t* value + ) + { + assert(allowlist_service_impl); + wsrep::const_buffer allowlist_value(value->ptr, value->len); + if (allowlist_service_impl->allowlist_cb(allowlist_key_from_native(key), + allowlist_value)) + { + return WSREP_OK; + } + return WSREP_NOT_ALLOWED; + } + + static wsrep_allowlist_service_v1_t allowlist_service_callbacks + = { allowlist_cb, + 0 }; +} + +int wsrep::allowlist_service_v1_probe(void* dlh) +{ + typedef int (*init_fn)(wsrep_allowlist_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe<init_fn>( + dlh, WSREP_ALLOWLIST_SERVICE_INIT_FUNC_V1, "allowlist service v1") || + wsrep_impl::service_probe<deinit_fn>( + dlh, WSREP_ALLOWLIST_SERVICE_DEINIT_FUNC_V1, "allowlist service v1")) + { + wsrep::log_warning() << "Provider does not support allowlist service v1"; + return 1; + } + return 0; +} + +int wsrep::allowlist_service_v1_init(void* dlh, + wsrep::allowlist_service* allowlist_service) +{ + if (not (dlh && allowlist_service)) return EINVAL; + typedef int (*init_fn)(wsrep_allowlist_service_v1_t*); + wsrep_allowlist_service_v1::allowlist_service_impl = allowlist_service; + int ret(0); + if ((ret = wsrep_impl::service_init<init_fn>( + dlh, WSREP_ALLOWLIST_SERVICE_INIT_FUNC_V1, + &wsrep_allowlist_service_v1::allowlist_service_callbacks, + "allowlist service v1"))) + { + wsrep_allowlist_service_v1::allowlist_service_impl = 0; + } + else + { + ++wsrep_allowlist_service_v1::use_count; + } + return ret; +} + +void wsrep::allowlist_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit<deinit_fn>( + dlh, WSREP_ALLOWLIST_SERVICE_DEINIT_FUNC_V1, "allowlist service v1"); + --wsrep_allowlist_service_v1::use_count; + if (wsrep_allowlist_service_v1::use_count == 0) + { + wsrep_allowlist_service_v1::allowlist_service_impl = 0; + } +} diff --git a/wsrep-lib/src/allowlist_service_v1.hpp b/wsrep-lib/src/allowlist_service_v1.hpp new file mode 100644 index 00000000..eb73bd9c --- /dev/null +++ b/wsrep-lib/src/allowlist_service_v1.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_ALLOWLIST_SERVICE_V1_HPP +#define WSREP_ALLOWLIST_SERVICE_V1_HPP + +namespace wsrep +{ + class allowlist_service; + /** + * Probe allowlist_service_v1 support in loaded library. + * + * @param dlh Handle returned by dlopen(). + * + * @return Zero on success, non-zero system error code on failure. + */ + int allowlist_service_v1_probe(void *dlh); + + /** + * Initialize the allowlist service. + * + * @param dlh Handle returned by dlopen(). + * @param allowlist_service Pointer to wsrep::allowlist_service implementation. + * + * @return Zero on success, non-zero system error code on failure. + */ + int allowlist_service_v1_init(void* dlh, + wsrep::allowlist_service* allowlist_service); + + /** + * Deinitialize the allowlist service. + * + * @params dlh Handler returned by dlopen(). + */ + void allowlist_service_v1_deinit(void* dlh); + +} + +#endif // WSREP_allowlist_SERVICE_V1_HPP diff --git a/wsrep-lib/src/client_state.cpp b/wsrep-lib/src/client_state.cpp new file mode 100644 index 00000000..9d0a8167 --- /dev/null +++ b/wsrep-lib/src/client_state.cpp @@ -0,0 +1,883 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/client_state.hpp" +#include "wsrep/compiler.hpp" +#include "wsrep/logger.hpp" + +#include <unistd.h> // usleep() +#include <sstream> +#include <iostream> + +wsrep::provider& wsrep::client_state::provider() const +{ + return server_state_.provider(); +} + +void wsrep::client_state::open(wsrep::client_id id) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(state_ == s_none); + assert(keep_command_error_ == false); + debug_log_state("open: enter"); + owning_thread_id_ = wsrep::this_thread::get_id(); + rollbacker_active_ = false; + sync_wait_gtid_ = wsrep::gtid::undefined(); + last_written_gtid_ = wsrep::gtid::undefined(); + state(lock, s_idle); + id_ = id; + debug_log_state("open: leave"); +} + +void wsrep::client_state::close() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("close: enter"); + + while (mode_ == m_local && is_rollbacker_active()) { + cond_.wait(lock); + } + do_acquire_ownership(lock); + + state(lock, s_quitting); + keep_command_error_ = false; + lock.unlock(); + if (transaction_.active() && + (mode_ != m_local || + transaction_.state() != wsrep::transaction::s_prepared)) + { + client_service_.bf_rollback(); + transaction_.after_statement(); + } + if (mode_ == m_local) + { + disable_streaming(); + } + debug_log_state("close: leave"); +} + +void wsrep::client_state::cleanup() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + cleanup(lock); +} + +void wsrep::client_state::cleanup(wsrep::unique_lock<wsrep::mutex>& lock) +{ + debug_log_state("cleanup: enter"); + state(lock, s_none); + debug_log_state("cleanup: leave"); +} + +void wsrep::client_state::override_error(enum wsrep::client_error error, + enum wsrep::provider::status status) +{ + assert(wsrep::this_thread::get_id() == owning_thread_id_); + // Error state should not be cleared with success code without + // explicit reset_error() call. + assert(current_error_ == wsrep::e_success || + error != wsrep::e_success); + current_error_ = error; + current_error_status_ = status; +} + +int wsrep::client_state::before_command(bool keep_command_error) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("before_command: enter"); + // If the state is s_exec, the processing thread has already grabbed + // control with wait_rollback_complete_and_acquire_ownership() + if (state_ != s_exec) + { + assert(state_ == s_idle); + do_wait_rollback_complete_and_acquire_ownership(lock); + assert(state_ == s_exec); + client_service_.store_globals(); + } + else + { + // This thread must have acquired control by other means, + // for example via wait_rollback_complete_and_acquire_ownership(). + assert(wsrep::this_thread::get_id() == owning_thread_id_); + } + + keep_command_error_ = keep_command_error; + + // If the transaction is active, it must be either executing, + // aborted as rolled back by rollbacker, or must_abort if the + // client thread gained control via + // wait_rollback_complete_and_acquire_ownership() + // just before BF abort happened. + assert(transaction_.active() == false || + (transaction_.state() == wsrep::transaction::s_executing || + transaction_.state() == wsrep::transaction::s_prepared || + transaction_.state() == wsrep::transaction::s_aborted || + transaction_.state() == wsrep::transaction::s_must_abort)); + + if (transaction_.active()) + { + if (transaction_.state() == wsrep::transaction::s_must_abort || + transaction_.state() == wsrep::transaction::s_aborted) + { + if (transaction_.is_xa()) + { + // Client will rollback explicitly, return error. + debug_log_state("before_command: error"); + return 1; + } + + override_error(wsrep::e_deadlock_error); + if (transaction_.state() == wsrep::transaction::s_must_abort) + { + lock.unlock(); + client_service_.bf_rollback(); + lock.lock(); + + } + + if (keep_command_error_) + { + // Keep the error for the next command + debug_log_state("before_command: keep error"); + return 0; + } + + // Clean up the transaction and return error. + lock.unlock(); + (void)transaction_.after_statement(); + lock.lock(); + + assert(transaction_.active() == false); + assert(transaction_.state() == wsrep::transaction::s_aborted); + assert(current_error() != wsrep::e_success); + + debug_log_state("before_command: error"); + return 1; + } + } + debug_log_state("before_command: success"); + return 0; +} + +void wsrep::client_state::after_command_before_result() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("after_command_before_result: enter"); + assert(state() == s_exec); + if (transaction_.active() && + transaction_.state() == wsrep::transaction::s_must_abort) + { + transaction_.after_command_must_abort(lock); + // If keep current error is set, the result will be propagated + // back to client with some future command, so keep the transaction + // open here so that error handling can happen in before_command() + // hook. + if (not keep_command_error_) + { + lock.unlock(); + (void)transaction_.after_statement(); + lock.lock(); + } + + assert(transaction_.state() == wsrep::transaction::s_aborted); + } + state(lock, s_result); + debug_log_state("after_command_before_result: leave"); +} + +void wsrep::client_state::after_command_after_result() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("after_command_after_result_enter"); + assert(state() == s_result); + assert(transaction_.state() != wsrep::transaction::s_aborting); + if (transaction_.active() && + transaction_.state() == wsrep::transaction::s_must_abort) + { + transaction_.after_command_must_abort(lock); + assert(transaction_.state() == wsrep::transaction::s_aborted); + } + else if (transaction_.active() == false && not keep_command_error_) + { + current_error_ = wsrep::e_success; + current_error_status_ = wsrep::provider::success; + } + keep_command_error_ = false; + sync_wait_gtid_ = wsrep::gtid::undefined(); + state(lock, s_idle); + debug_log_state("after_command_after_result: leave"); +} + +int wsrep::client_state::before_statement() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("before_statement: enter"); +#if 0 + /** + * @todo It might be beneficial to implement timed wait for + * server synced state. + */ + if (allow_dirty_reads_ == false && + server_state_.state() != wsrep::server_state::s_synced) + { + return 1; + } +#endif // 0 + + if (transaction_.active() && + transaction_.state() == wsrep::transaction::s_must_abort) + { + // Rollback and cleanup will happen in after_command_before_result() + debug_log_state("before_statement_error"); + return 1; + } + debug_log_state("before_statement: success"); + return 0; +} + +int wsrep::client_state::after_statement() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("after_statement: enter"); + assert(state() == s_exec); + assert(mode() == m_local); + + if (transaction_.active() && + transaction_.state() == wsrep::transaction::s_must_abort) + { + lock.unlock(); + client_service_.bf_rollback(); + lock.lock(); + assert(transaction_.state() == wsrep::transaction::s_aborted); + // Error may be set already. For example, if fragment size + // exceeded the maximum size in certify_fragment(), then + // we already have wsrep::e_error_during_commit + if (current_error() == wsrep::e_success) + { + override_error(wsrep::e_deadlock_error); + } + } + lock.unlock(); + + (void)transaction_.after_statement(); + if (current_error() == wsrep::e_deadlock_error) + { + if (mode_ == m_local) + { + debug_log_state("after_statement: may_retry"); + return 1; + } + else + { + debug_log_state("after_statement: error"); + return 1; + } + } + debug_log_state("after_statement: success"); + return 0; +} + +////////////////////////////////////////////////////////////////////////////// +// Rollbacker synchronization // +////////////////////////////////////////////////////////////////////////////// + +void wsrep::client_state::sync_rollback_complete() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("sync_rollback_complete: enter"); + assert(state_ == s_idle && mode_ == m_local && + transaction_.state() == wsrep::transaction::s_aborted); + set_rollbacker_active(false); + cond_.notify_all(); + debug_log_state("sync_rollback_complete: leave"); +} + +void wsrep::client_state::wait_rollback_complete_and_acquire_ownership() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + debug_log_state("wait_rollback_complete_and_acquire_ownership: enter"); + if (state_ == s_idle) + { + do_wait_rollback_complete_and_acquire_ownership(lock); + } + assert(state_ == s_exec); + debug_log_state("wait_rollback_complete_and_acquire_ownership: leave"); +} + +////////////////////////////////////////////////////////////////////////////// +// Streaming // +////////////////////////////////////////////////////////////////////////////// + +void wsrep::client_state::streaming_params( + enum wsrep::streaming_context::fragment_unit fragment_unit, + size_t fragment_size) +{ + assert(mode_ == m_local); + transaction_.streaming_context().params(fragment_unit, fragment_size); +} + +int wsrep::client_state::enable_streaming( + enum wsrep::streaming_context::fragment_unit + fragment_unit, + size_t fragment_size) +{ + assert(mode_ == m_local); + if (transaction_.is_streaming() && + transaction_.streaming_context().fragment_unit() != + fragment_unit) + { + wsrep::log_error() + << "Changing fragment unit for active streaming transaction " + << "not allowed"; + return 1; + } + transaction_.streaming_context().enable(fragment_unit, fragment_size); + return 0; +} + +void wsrep::client_state::disable_streaming() +{ + assert(mode_ == m_local); + assert(state_ == s_exec || state_ == s_quitting); + transaction_.streaming_context().disable(); +} + +////////////////////////////////////////////////////////////////////////////// +// TOI // +////////////////////////////////////////////////////////////////////////////// + +enum wsrep::provider::status +wsrep::client_state::poll_enter_toi( + wsrep::unique_lock<wsrep::mutex>& lock, + const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + wsrep::ws_meta& meta, + int flags, + std::chrono::time_point<wsrep::clock> wait_until, + bool& timed_out) +{ + WSREP_LOG_DEBUG(debug_log_level(), + wsrep::log::debug_level_client_state, + "poll_enter_toi: " + << flags + << "," + << wait_until.time_since_epoch().count()); + enum wsrep::provider::status status; + timed_out = false; + wsrep::ws_meta poll_meta; // tmp var for polling, as enter_toi may clear meta arg on errors + do + { + lock.unlock(); + poll_meta = meta; + status = provider().enter_toi(id_, keys, buffer, poll_meta, flags); + if (status != wsrep::provider::success && + not poll_meta.gtid().is_undefined()) + { + // Successfully entered TOI, but the provider reported failure. + // This may happen for example if certification fails. + // Leave TOI before proceeding. + if (provider().leave_toi(id_, wsrep::mutable_buffer())) + { + wsrep::log_warning() + << "Failed to leave TOI after failure in " + << "poll_enter_toi()"; + } + poll_meta = wsrep::ws_meta(); + } + if (status == wsrep::provider::error_certification_failed || + status == wsrep::provider::error_connection_failed) + { + ::usleep(300000); + } + lock.lock(); + timed_out = !(wait_until.time_since_epoch().count() && + wsrep::clock::now() < wait_until); + } + while ((status == wsrep::provider::error_certification_failed || + status == wsrep::provider::error_connection_failed) && + not timed_out && + not client_service_.interrupted(lock)); + meta = poll_meta; + return status; +} + +void wsrep::client_state::enter_toi_common( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + toi_mode_ = mode_; + mode(lock, m_toi); +} + +int wsrep::client_state::enter_toi_local(const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + std::chrono::time_point<wsrep::clock> wait_until) +{ + debug_log_state("enter_toi_local: enter"); + assert(state_ == s_exec); + assert(mode_ == m_local); + int ret; + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + + bool timed_out; + auto const status(poll_enter_toi( + lock, keys, buffer, + toi_meta_, + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::commit, + wait_until, + timed_out)); + switch (status) + { + case wsrep::provider::success: + { + enter_toi_common(lock); + ret = 0; + break; + } + case wsrep::provider::error_certification_failed: + override_error(e_deadlock_error, status); + ret = 1; + break; + default: + if (timed_out) { + override_error(e_timeout_error); + } else { + override_error(e_error_during_commit, status); + } + ret = 1; + break; + } + + debug_log_state("enter_toi_local: leave"); + return ret; +} + +void wsrep::client_state::enter_toi_mode(const wsrep::ws_meta& ws_meta) +{ + debug_log_state("enter_toi_mode: enter"); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(mode_ == m_high_priority); + enter_toi_common(lock); + toi_meta_ = ws_meta; + debug_log_state("enter_toi_mode: leave"); +} + +void wsrep::client_state::leave_toi_common() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + mode(lock, toi_mode_); + toi_mode_ = m_undefined; + if (toi_meta_.gtid().is_undefined() == false) + { + update_last_written_gtid(toi_meta_.gtid()); + } + toi_meta_ = wsrep::ws_meta(); +} + +int wsrep::client_state::leave_toi_local(const wsrep::mutable_buffer& err) +{ + debug_log_state("leave_toi_local: enter"); + assert(toi_mode_ == m_local); + leave_toi_common(); + + debug_log_state("leave_toi_local: leave"); + return (provider().leave_toi(id_, err) == provider::success ? 0 : 1); +} + +void wsrep::client_state::leave_toi_mode() +{ + debug_log_state("leave_toi_mode: enter"); + assert(toi_mode_ == m_high_priority); + leave_toi_common(); + debug_log_state("leave_toi_mode: leave"); +} + +/////////////////////////////////////////////////////////////////////////////// +// RSU // +/////////////////////////////////////////////////////////////////////////////// + +int wsrep::client_state::begin_rsu(int timeout) +{ + if (server_state_.desync()) + { + wsrep::log_warning() << "Failed to desync server"; + return 1; + } + if (server_state_.server_service().wait_committing_transactions(timeout)) + { + wsrep::log_warning() << "RSU failed due to pending transactions"; + server_state_.resync(); + return 1; + } + wsrep::seqno pause_seqno(server_state_.pause()); + if (pause_seqno.is_undefined()) + { + wsrep::log_warning() << "Failed to pause provider"; + server_state_.resync(); + return 1; + } + wsrep::log_info() << "Provider paused at: " << pause_seqno; + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + toi_mode_ = mode_; + mode(lock, m_rsu); + return 0; +} + +int wsrep::client_state::end_rsu() +{ + int ret(0); + try + { + server_state_.resume(); + server_state_.resync(); + } + catch (const wsrep::runtime_error& e) + { + wsrep::log_warning() << "End RSU failed: " << e.what(); + ret = 1; + } + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + mode(lock, toi_mode_); + toi_mode_ = m_undefined; + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// NBO // +/////////////////////////////////////////////////////////////////////////////// + +int wsrep::client_state::begin_nbo_phase_one( + const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + std::chrono::time_point<wsrep::clock> wait_until) +{ + debug_log_state("begin_nbo_phase_one: enter"); + debug_log_keys(keys); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(state_ == s_exec); + assert(mode_ == m_local); + assert(toi_mode_ == m_undefined); + + int ret; + bool timed_out; + auto const status(poll_enter_toi( + lock, keys, buffer, + toi_meta_, + wsrep::provider::flag::start_transaction, + wait_until, + timed_out)); + switch (status) + { + case wsrep::provider::success: + toi_mode_ = mode_; + mode(lock, m_nbo); + ret= 0; + break; + case wsrep::provider::error_certification_failed: + override_error(e_deadlock_error, status); + ret = 1; + break; + default: + if (timed_out) { + override_error(e_timeout_error); + } else { + override_error(e_error_during_commit, status); + } + ret = 1; + break; + } + + debug_log_state("begin_nbo_phase_one: leave"); + return ret; +} + +int wsrep::client_state::end_nbo_phase_one(const wsrep::mutable_buffer& err) +{ + debug_log_state("end_nbo_phase_one: enter"); + assert(state_ == s_exec); + assert(mode_ == m_nbo); + assert(in_toi()); + + enum wsrep::provider::status status(provider().leave_toi(id_, err)); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + int ret; + switch (status) + { + case wsrep::provider::success: + ret = 0; + break; + default: + override_error(e_error_during_commit, status); + ret = 1; + break; + } + nbo_meta_ = toi_meta_; + toi_meta_ = wsrep::ws_meta(); + toi_mode_ = m_undefined; + debug_log_state("end_nbo_phase_one: leave"); + return ret; +} + +int wsrep::client_state::enter_nbo_mode(const wsrep::ws_meta& ws_meta) +{ + assert(state_ == s_exec); + assert(mode_ == m_local); + assert(toi_mode_ == m_undefined); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + nbo_meta_ = ws_meta; + mode(lock, m_nbo); + return 0; +} + +int wsrep::client_state::begin_nbo_phase_two( + const wsrep::key_array& keys, + std::chrono::time_point<wsrep::clock> wait_until) +{ + debug_log_state("begin_nbo_phase_two: enter"); + debug_log_keys(keys); + assert(state_ == s_exec); + assert(mode_ == m_nbo); + assert(toi_mode_ == m_undefined); + assert(!in_toi()); + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + // Note: nbo_meta_ is passed to enter_toi() as it is + // an input param containing gtid of NBO begin. + // Output stored in nbo_meta_ is copied to toi_meta_ for + // phase two end. + bool timed_out; + enum wsrep::provider::status status( + poll_enter_toi(lock, keys, + wsrep::const_buffer(), + nbo_meta_, + wsrep::provider::flag::commit, + wait_until, + timed_out)); + int ret; + switch (status) + { + case wsrep::provider::success: + ret= 0; + toi_meta_ = nbo_meta_; + toi_mode_ = m_local; + break; + case wsrep::provider::error_provider_failed: + override_error(e_interrupted_error, status); + ret= 1; + break; + default: + if (timed_out) + { + override_error(e_timeout_error, status); + } + else + { + override_error(e_error_during_commit, status); + } + ret= 1; + break; + } + + // Failed to grab TOI for completing NBO in order. This means that + // the operation cannot be ended in total order, so we end the + // NBO mode and let the DBMS to deal with the error. + if (ret) + { + mode(lock, m_local); + nbo_meta_ = wsrep::ws_meta(); + } + + debug_log_state("begin_nbo_phase_two: leave"); + return ret; +} + +int wsrep::client_state::end_nbo_phase_two(const wsrep::mutable_buffer& err) +{ + debug_log_state("end_nbo_phase_two: enter"); + assert(state_ == s_exec); + assert(mode_ == m_nbo); + assert(toi_mode_ == m_local); + assert(in_toi()); + enum wsrep::provider::status status( + provider().leave_toi(id_, err)); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + int ret; + switch (status) + { + case wsrep::provider::success: + ret = 0; + break; + default: + override_error(e_error_during_commit, status); + ret = 1; + break; + } + toi_meta_ = wsrep::ws_meta(); + toi_mode_ = m_undefined; + nbo_meta_ = wsrep::ws_meta(); + mode(lock, m_local); + debug_log_state("end_nbo_phase_two: leave"); + return ret; +} +/////////////////////////////////////////////////////////////////////////////// +// Misc // +/////////////////////////////////////////////////////////////////////////////// + +int wsrep::client_state::sync_wait(int timeout) +{ + std::pair<wsrep::gtid, enum wsrep::provider::status> result( + server_state_.causal_read(timeout)); + int ret(1); + switch (result.second) + { + case wsrep::provider::success: + sync_wait_gtid_ = result.first; + ret = 0; + break; + case wsrep::provider::error_not_implemented: + override_error(wsrep::e_not_supported_error); + break; + default: + override_error(wsrep::e_timeout_error); + break; + } + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// Private // +/////////////////////////////////////////////////////////////////////////////// + +void wsrep::client_state::do_acquire_ownership( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED) +{ + assert(lock.owns_lock()); + // Be strict about client state for clients in local mode. The + // owning_thread_id_ is used to detect bugs which are caused by + // more than one thread operating the client state at the time, + // for example thread handling the client session and background + // rollbacker. + assert(state_ == s_idle || mode_ != m_local); + owning_thread_id_ = wsrep::this_thread::get_id(); +} + +void wsrep::client_state::do_wait_rollback_complete_and_acquire_ownership( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + assert(state_ == s_idle); + while (is_rollbacker_active()) + { + cond_.wait(lock); + } + do_acquire_ownership(lock); + state(lock, s_exec); +} + +void wsrep::client_state::update_last_written_gtid(const wsrep::gtid& gtid) +{ + assert(last_written_gtid_.is_undefined() || + (last_written_gtid_.id() == gtid.id() && + !(last_written_gtid_.seqno() > gtid.seqno()))); + last_written_gtid_ = gtid; +} + +void wsrep::client_state::debug_log_state(const char* context) const +{ + WSREP_LOG_DEBUG(debug_log_level(), + wsrep::log::debug_level_client_state, + context + << "(" << id_.get() + << "," << to_c_string(state_) + << "," << to_c_string(mode_) + << "," << wsrep::to_string(current_error_) + << "," << current_error_status_ + << ",toi: " << toi_meta_.seqno() + << ",nbo: " << nbo_meta_.seqno() << ")"); +} + +void wsrep::client_state::debug_log_keys(const wsrep::key_array& keys) const +{ + for (size_t i(0); i < keys.size(); ++i) + { + WSREP_LOG_DEBUG(debug_log_level(), + wsrep::log::debug_level_client_state, + "TOI keys: " + << " id: " << id_ + << "key: " << keys[i]); + } +} + +void wsrep::client_state::state( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED, + enum wsrep::client_state::state state) +{ + // Verify that the current thread has gained control to the + // connection by calling before_command() + assert(wsrep::this_thread::get_id() == owning_thread_id_); + assert(lock.owns_lock()); + static const char allowed[state_max_][state_max_] = + { + /* none idle exec result quit */ + { 0, 1, 0, 0, 0}, /* none */ + { 0, 0, 1, 0, 1}, /* idle */ + { 0, 0, 0, 1, 0}, /* exec */ + { 0, 1, 0, 0, 0}, /* result */ + { 1, 0, 0, 0, 0} /* quit */ + }; + if (!allowed[state_][state]) + { + wsrep::log_warning() << "client_state: Unallowed state transition: " + << state_ << " -> " << state; + assert(0); + } + state_hist_.push_back(state_); + state_ = state; + if (state_hist_.size() > 10) + { + state_hist_.erase(state_hist_.begin()); + } + +} + +void wsrep::client_state::mode( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED, + enum mode mode) +{ + assert(lock.owns_lock()); + + static const char allowed[n_modes_][n_modes_] = + { /* u l h t r n */ + { 0, 0, 0, 0, 0, 0 }, /* undefined */ + { 0, 0, 1, 1, 1, 1 }, /* local */ + { 0, 1, 0, 1, 0, 1 }, /* high prio */ + { 0, 1, 1, 0, 0, 0 }, /* toi */ + { 0, 1, 0, 0, 0, 0 }, /* rsu */ + { 0, 1, 1, 0, 0, 0 } /* nbo */ + }; + if (!allowed[mode_][mode]) + { + wsrep::log_warning() << "client_state: Unallowed mode transition: " + << mode_ << " -> " << mode; + assert(0); + } + mode_ = mode; +} diff --git a/wsrep-lib/src/event_service_v1.cpp b/wsrep-lib/src/event_service_v1.cpp new file mode 100644 index 00000000..ffaf4ec4 --- /dev/null +++ b/wsrep-lib/src/event_service_v1.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "event_service_v1.hpp" + +#include "wsrep/event_service.hpp" +#include "wsrep/reporter.hpp" +#include "wsrep/logger.hpp" +#include "v26/wsrep_event_service.h" +#include "service_helpers.hpp" + +#include <cassert> + +namespace wsrep_event_service_v1 +{ + static std::atomic_flag initialized = ATOMIC_FLAG_INIT; + + static void callback( + wsrep_event_context_t* ctx, + const char* name, + const char* value) + { + if (ctx) + { + wsrep::event_service* const impl + (reinterpret_cast<wsrep::event_service*>(ctx)); + impl->process_event(name, value); + } + } + + static const char* const log_string = "event service v1"; +} + +int wsrep::event_service_v1_probe(void* dlh) +{ + typedef int (*init_fn)(wsrep_event_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe<init_fn>( + dlh, WSREP_EVENT_SERVICE_INIT_FUNC_V1, + wsrep_event_service_v1::log_string) || + wsrep_impl::service_probe<deinit_fn>( + dlh, WSREP_EVENT_SERVICE_DEINIT_FUNC_V1, + wsrep_event_service_v1::log_string)) + { + // diagnostic message was logged by wsrep_impl::service_probe() + return 1; + } + return 0; +} + +int wsrep::event_service_v1_init(void* dlh, + wsrep::event_service* event_service) +{ + if (not (dlh && event_service)) return EINVAL; + + if (wsrep_event_service_v1::initialized.test_and_set()) return EALREADY; + + wsrep_event_service_v1_t service = + { + wsrep_event_service_v1::callback, + reinterpret_cast<wsrep_event_context_t*>(event_service) + }; + + typedef int (*init_fn)(wsrep_event_service_v1_t*); + int const ret(wsrep_impl::service_init<init_fn>( + dlh, WSREP_EVENT_SERVICE_INIT_FUNC_V1, &service, + wsrep_event_service_v1::log_string)); + if (ret) + { + wsrep_event_service_v1::initialized.clear(); + } + + return ret; +} + +void wsrep::event_service_v1_deinit(void* dlh) +{ + if (wsrep_event_service_v1::initialized.test_and_set()) + { + // service was initialized + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit<deinit_fn>( + dlh, WSREP_EVENT_SERVICE_DEINIT_FUNC_V1, + wsrep_event_service_v1::log_string); + } + + wsrep_event_service_v1::initialized.clear(); +} diff --git a/wsrep-lib/src/event_service_v1.hpp b/wsrep-lib/src/event_service_v1.hpp new file mode 100644 index 00000000..1c85b01e --- /dev/null +++ b/wsrep-lib/src/event_service_v1.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_EVENT_SERVICE_V1_HPP +#define WSREP_EVENT_SERVICE_V1_HPP + +namespace wsrep +{ + class event_service; + /** + * Probe event_service_v1 support in loaded library. + * + * @param dlh Handle returned by dlopen(). + * + * @return Zero on success, non-zero system error code on failure. + */ + int event_service_v1_probe(void *dlh); + + /** + * Initialize event service. + * + * @param dlh Handle returned by dlopen(). + * @params event_service Pointer to wsrep::event_service implementation. + * + * @return Zero on success, non-zero system error code on failure. + */ + int event_service_v1_init(void* dlh, + wsrep::event_service* event_service); + + /** + * Deinitialize event service. + * + * @param dlh Handler returned by dlopen(). + */ + void event_service_v1_deinit(void* dlh); +} + +#endif // WSREP_EVENT_SERVICE_V1_HPP diff --git a/wsrep-lib/src/exception.cpp b/wsrep-lib/src/exception.cpp new file mode 100644 index 00000000..c55ed641 --- /dev/null +++ b/wsrep-lib/src/exception.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/exception.hpp" + +bool wsrep::abort_on_exception(false); diff --git a/wsrep-lib/src/gtid.cpp b/wsrep-lib/src/gtid.cpp new file mode 100644 index 00000000..af32d524 --- /dev/null +++ b/wsrep-lib/src/gtid.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/gtid.hpp" + +#include <cerrno> +#include <iostream> +#include <sstream> + +const wsrep::gtid wsrep::gtid::undefined_ = wsrep::gtid(); + +std::ostream& wsrep::operator<<(std::ostream& os, const wsrep::gtid& gtid) +{ + return (os << gtid.id() << ":" << gtid.seqno()); +} + +std::istream& wsrep::operator>>(std::istream& is, wsrep::gtid& gtid) +{ + std::string id_str; + std::getline(is, id_str, ':'); + long long seq; + is >> seq; + if (!is) + { + is.clear(std::ios_base::failbit); + return is; + } + try + { + // wsrep::id constructor will throw if it cannot parse the + // id_str. + gtid = wsrep::gtid(wsrep::id(id_str), wsrep::seqno(seq)); + } + catch (const wsrep::runtime_error& e) + { + // Formatting or extraction error. Clear the istream state and + // set failibit. + is.clear(std::ios_base::failbit); + } + return is; +} + +ssize_t wsrep::scan_from_c_str( + const char* buf, size_t buf_len, wsrep::gtid& gtid) +{ + std::istringstream is(std::string(buf, buf_len)); + is >> gtid; + // Some failure occurred + if (!is) + { + return -EINVAL; + } + // Whole string was consumed without failures + if (is.eof()) + { + return static_cast<ssize_t>(buf_len); + } + // The string was not consumed completely, return current position + // of the istream. + return static_cast<ssize_t>(is.tellg()); +} + +ssize_t wsrep::print_to_c_str( + const wsrep::gtid& gtid, char* buf, size_t buf_len) +{ + std::ostringstream os; + os << gtid; + if (os.str().size() > buf_len) + { + return -ENOBUFS; + } + std::strncpy(buf, os.str().c_str(), os.str().size()); + return static_cast<ssize_t>(os.str().size()); +} diff --git a/wsrep-lib/src/id.cpp b/wsrep-lib/src/id.cpp new file mode 100644 index 00000000..2da188fc --- /dev/null +++ b/wsrep-lib/src/id.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/id.hpp" +#include "uuid.hpp" + +#include <cctype> +#include <sstream> +#include <algorithm> + +const wsrep::id wsrep::id::undefined_ = wsrep::id(); + +wsrep::id::id(const std::string& str) + : data_() +{ + wsrep::uuid_t wsrep_uuid; + + if (str.size() == WSREP_LIB_UUID_STR_LEN && + wsrep::uuid_scan(str.c_str(), str.size(), &wsrep_uuid) == + WSREP_LIB_UUID_STR_LEN) + { + std::memcpy(data_.buf, wsrep_uuid.data, sizeof(data_.buf)); + } + else if (str.size() <= 16) + { + std::memcpy(data_.buf, str.c_str(), str.size()); + } + else + { + std::ostringstream os; + os << "String '" << str + << "' does not contain UUID or is longer thatn 16 bytes"; + throw wsrep::runtime_error(os.str()); + } +} + +std::ostream& wsrep::operator<<(std::ostream& os, const wsrep::id& id) +{ + const char* ptr(static_cast<const char*>(id.data())); + size_t size(id.size()); + if (static_cast<size_t>(std::count_if(ptr, ptr + size, ::isalnum)) == size) + { + return (os << std::string(ptr, size)); + } + else + { + char uuid_str[WSREP_LIB_UUID_STR_LEN + 1]; + wsrep::uuid_t uuid; + std::memcpy(uuid.data, ptr, sizeof(uuid.data)); + if (wsrep::uuid_print(&uuid, uuid_str, sizeof(uuid_str)) < 0) + { + throw wsrep::runtime_error("Could not print uuid"); + } + uuid_str[WSREP_LIB_UUID_STR_LEN] = '\0'; + return (os << uuid_str); + } +} + +std::istream& wsrep::operator>>(std::istream& is, wsrep::id& id) +{ + std::string id_str; + std::getline(is, id_str); + id = wsrep::id(id_str); + return is; +} diff --git a/wsrep-lib/src/key.cpp b/wsrep-lib/src/key.cpp new file mode 100644 index 00000000..fb94fefc --- /dev/null +++ b/wsrep-lib/src/key.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/key.hpp" +#include <ostream> +#include <iomanip> + +namespace +{ + void print_key_part(std::ostream& os, const void* ptr, size_t len) + { + std::ios::fmtflags flags_save(os.flags()); + os << len << ": "; + for (size_t i(0); i < len; ++i) + { + os << std::hex + << std::setfill('0') + << std::setw(2) + << static_cast<int>( + *(reinterpret_cast<const unsigned char*>(ptr) + i)) << " "; + } + os.flags(flags_save); + } +} + +std::ostream& wsrep::operator<<(std::ostream& os, + enum wsrep::key::type key_type) +{ + switch (key_type) + { + case wsrep::key::shared: os << "shared"; break; + case wsrep::key::reference: os << "reference"; break; + case wsrep::key::update: os << "update"; break; + case wsrep::key::exclusive: os << "exclusive"; break; + default: os << "unknown"; break; + } + return os; +} + +std::ostream& wsrep::operator<<(std::ostream& os, const wsrep::key& key) +{ + os << "type: " << key.type(); + for (size_t i(0); i < key.size(); ++i) + { + os << "\n "; + print_key_part(os, key.key_parts()[i].data(), key.key_parts()[i].size()); + } + return os; +} diff --git a/wsrep-lib/src/logger.cpp b/wsrep-lib/src/logger.cpp new file mode 100644 index 00000000..058a42a0 --- /dev/null +++ b/wsrep-lib/src/logger.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/logger.hpp" + +#include <iostream> + +std::ostream& wsrep::log::os_ = std::cout; +static wsrep::default_mutex log_mutex_; +wsrep::mutex& wsrep::log::mutex_ = log_mutex_; +wsrep::log::logger_fn_type wsrep::log::logger_fn_ = 0; +std::atomic_int wsrep::log::debug_log_level_(0); + +void wsrep::log::logger_fn(wsrep::log::logger_fn_type logger_fn) +{ + logger_fn_ = logger_fn; +} + +void wsrep::log::debug_log_level(int debug_log_level) +{ + debug_log_level_.store(debug_log_level, std::memory_order_relaxed); +} + +int wsrep::log::debug_log_level() +{ + return debug_log_level_.load(std::memory_order_relaxed); +} diff --git a/wsrep-lib/src/provider.cpp b/wsrep-lib/src/provider.cpp new file mode 100644 index 00000000..43ded86d --- /dev/null +++ b/wsrep-lib/src/provider.cpp @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/provider.hpp" +#include "wsrep/logger.hpp" + +#include "wsrep_provider_v26.hpp" + +#include <dlfcn.h> +#include <memory> + +wsrep::provider* wsrep::provider::make_provider( + wsrep::server_state& server_state, + const std::string& provider_spec, + const std::string& provider_options, + const wsrep::provider::services& services) +{ + try + { + return new wsrep::wsrep_provider_v26( + server_state, provider_options, provider_spec, services); + } + catch (const wsrep::runtime_error& e) + { + wsrep::log_error() << "Failed to create a new provider '" + << provider_spec << "'" + << " with options '" << provider_options + << "': " << e.what(); + } + catch (...) + { + wsrep::log_error() << "Caught unknown exception when trying to " + << "create a new provider '" + << provider_spec << "'" + << " with options '" << provider_options; + } + return 0; +} + +std::string +wsrep::provider::to_string(enum wsrep::provider::status const val) +{ + switch(val) + { + case success: + return "Success"; + case error_warning: + return "Warning"; + case error_transaction_missing: + return "Transaction not registered with provider"; + case error_certification_failed: + return "Certification failed"; + case error_bf_abort: + return "Transaction was BF aborted"; + case error_size_exceeded: + return "Transaction size exceeded"; + case error_connection_failed: + return "Not connected to Primary Component"; + case error_provider_failed: + return "Provider in bad state, needs to be reinitialized."; + case error_fatal: + return "Fatal error, must abort."; + case error_not_implemented: + return "Function not implemented"; + case error_not_allowed: + return "Operation not allowed"; + case error_unknown: + return "Unknown error"; + } + + assert(0); + + std::ostringstream os; + os << "Invalid error code: " << val; + return os.str(); +} + +std::string wsrep::provider::capability::str(int caps) +{ + std::ostringstream os; + +#define WSREP_PRINT_CAPABILITY(cap_value, cap_string) \ + if (caps & cap_value) { \ + os << cap_string ", "; \ + caps &= ~cap_value; \ + } + + WSREP_PRINT_CAPABILITY(multi_master, "MULTI-MASTER"); + WSREP_PRINT_CAPABILITY(certification, "CERTIFICATION"); + WSREP_PRINT_CAPABILITY(parallel_applying, "PARALLEL_APPLYING"); + WSREP_PRINT_CAPABILITY(transaction_replay, "REPLAY"); + WSREP_PRINT_CAPABILITY(isolation, "ISOLATION"); + WSREP_PRINT_CAPABILITY(pause, "PAUSE"); + WSREP_PRINT_CAPABILITY(causal_reads, "CAUSAL_READ"); + WSREP_PRINT_CAPABILITY(causal_transaction, "CAUSAL_TRX"); + WSREP_PRINT_CAPABILITY(incremental_writeset, "INCREMENTAL_WS"); + WSREP_PRINT_CAPABILITY(session_locks, "SESSION_LOCK"); + WSREP_PRINT_CAPABILITY(distributed_locks, "DISTRIBUTED_LOCK"); + WSREP_PRINT_CAPABILITY(consistency_check, "CONSISTENCY_CHECK"); + WSREP_PRINT_CAPABILITY(unordered, "UNORDERED"); + WSREP_PRINT_CAPABILITY(annotation, "ANNOTATION"); + WSREP_PRINT_CAPABILITY(preordered, "PREORDERED"); + WSREP_PRINT_CAPABILITY(streaming, "STREAMING"); + WSREP_PRINT_CAPABILITY(snapshot, "READ_VIEW"); + WSREP_PRINT_CAPABILITY(nbo, "NBO"); + +#undef WSREP_PRINT_CAPABILITY + + if (caps) + { + assert(caps == 0); // to catch missed capabilities + os << "UNKNOWN(" << caps << ") "; + } + + std::string ret(os.str()); + if (ret.size() > 2) ret.erase(ret.size() - 2); + return ret; +} + +std::string wsrep::flags_to_string(int flags) +{ + std::ostringstream oss; + if (flags & provider::flag::start_transaction) + oss << "start_transaction | "; + if (flags & provider::flag::commit) + oss << "commit | "; + if (flags & provider::flag::rollback) + oss << "rollback | "; + if (flags & provider::flag::isolation) + oss << "isolation | "; + if (flags & provider::flag::pa_unsafe) + oss << "pa_unsafe | "; + if (flags & provider::flag::prepare) + oss << "prepare | "; + if (flags & provider::flag::snapshot) + oss << "read_view | "; + if (flags & provider::flag::implicit_deps) + oss << "implicit_deps | "; + + std::string ret(oss.str()); + if (ret.size() > 3) ret.erase(ret.size() - 3); + return ret; +} + +std::ostream& wsrep::operator<<(std::ostream& os, const wsrep::ws_meta& ws_meta) +{ + os << "gtid: " << ws_meta.gtid() + << " server_id: " << ws_meta.server_id() + << " client_id: " << ws_meta.client_id() + << " trx_id: " << ws_meta.transaction_id() + << " flags: " << ws_meta.flags() + << " (" << wsrep::flags_to_string(ws_meta.flags()) << ")"; + return os; +} diff --git a/wsrep-lib/src/reporter.cpp b/wsrep-lib/src/reporter.cpp new file mode 100644 index 00000000..2a5c6f56 --- /dev/null +++ b/wsrep-lib/src/reporter.cpp @@ -0,0 +1,304 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/reporter.hpp" +#include "wsrep/logger.hpp" + +#include <sstream> +#include <iomanip> + +#include <cstring> // strerror() +#include <cstdlib> // mkstemp() +#include <cerrno> // errno +#include <unistd.h> // write() +#include <cstdio> // rename(), snprintf() +#include <ctime> // clock_gettime() +#include <cmath> // floor() + +static std::string const TEMP_EXTENSION(".XXXXXX"); + +static std::string make_progress_string(int const from, int const to, + int const total,int const done, + int const indefinite) +{ + std::ostringstream os; + + os << "{ \"from\": " << from << ", " + << "\"to\": " << to << ", " + << "\"total\": " << total << ", " + << "\"done\": " << done << ", " + << "\"indefinite\": " << indefinite << " }"; + + return os.str(); +} + +static std::string const indefinite_progress + (make_progress_string(-1, -1, -1, -1, -1)); +static std::string const steady_state + (make_progress_string(-1, -1, 0, 0, -1)); + +static inline double +timestamp() +{ + struct timespec time; + clock_gettime(CLOCK_REALTIME, &time); + return (double(time.tv_sec) + double(time.tv_nsec)*1.0e-9); +} + +wsrep::reporter::reporter(wsrep::mutex& mutex, + const std::string& file_name, + size_t const max_msg) + : mutex_(mutex) + , file_name_(file_name) + , progress_(indefinite_progress) + , template_(new char [file_name_.length() + TEMP_EXTENSION.length() + 1]) + , state_(wsrep::reporter::s_disconnected_disconnected) + , initialized_(false) + , err_msg_() + , warn_msg_() + , max_msg_(max_msg) +{ + template_[file_name_.length() + TEMP_EXTENSION.length()] = '\0'; + write_file(timestamp()); +} + +wsrep::reporter::~reporter() +{ + delete [] template_; +} + +wsrep::reporter::substates +wsrep::reporter::substate_map(enum wsrep::server_state::state const state) +{ + switch (state) + { + case wsrep::server_state::s_disconnected: + initialized_ = false; + return s_disconnected_disconnected; + case wsrep::server_state::s_initializing: + if (s_disconnected_disconnected == state_) + return s_disconnected_initializing; + else if (s_joining_sst == state_) + return s_joining_initializing; + else if (s_joining_initializing == state_) + return s_joining_initializing; // continuation + else + { + assert(0); + return state_; + } + case wsrep::server_state::s_initialized: + initialized_ = true; + if (s_disconnected_initializing >= state_) + return s_disconnected_initialized; + else if (s_joining_initializing == state_) + return s_joining_ist; + else if (s_joining_ist == state_) + return s_joining_ist; // continuation + else + { + assert(0); + return state_; + } + case wsrep::server_state::s_connected: + return s_connected_waiting; + case wsrep::server_state::s_joiner: + if (initialized_) + return s_joining_initialized; + else + return s_joining_sst; + case wsrep::server_state::s_joined: + return s_joined_syncing; + case wsrep::server_state::s_donor: + return s_donor_sending; + case wsrep::server_state::s_synced: + return s_synced_running; + case wsrep::server_state::s_disconnecting: + return s_disconnecting_disconnecting; + default: + assert(0); + return state_; + } +} + +void +wsrep::reporter::write_log_msg(std::ostream& os, + const log_msg& msg) +{ + os << "\t\t{\n"; + os << "\t\t\t\"timestamp\": " << std::showpoint << std::setprecision(18) + << msg.tstamp << ",\n"; + os << "\t\t\t\"msg\": \"" << msg.msg << "\"\n"; + os << "\t\t}"; +} + +void +wsrep::reporter::write_log_msgs(std::ostream& os, + const std::string& label, + const std::deque<log_msg>& msgs) +{ + os << "\t\"" << label << "\": [\n"; + for (size_t i(0); i < msgs.size(); ++i) + { + write_log_msg(os, msgs[i]); + os << (i+1 < msgs.size() ? ",\n" : "\n"); + } + os << "\t],\n"; +} + +// write data to temporary file and then rename it to target file for atomicity +void +wsrep::reporter::write_file(double const tstamp) +{ + enum progress_type { + t_indefinite = -1, // indefinite wait + t_progressive, // measurable progress + t_final // final state + }; + + struct strings { + const char* state; + const char* comment; + progress_type type; + }; + + static const struct strings strings[substates_max] = + { + { "DISCONNECTED", "Disconnected", t_indefinite }, + { "DISCONNECTED", "Initializing", t_indefinite }, + { "DISCONNECTED", "Connecting", t_indefinite }, + { "CONNECTED", "Waiting", t_indefinite }, + { "JOINING", "Receiving state", t_progressive }, + { "JOINING", "Receiving SST", t_progressive }, + { "JOINING", "Initializing", t_progressive }, + { "JOINING", "Receiving IST", t_progressive }, + { "JOINED", "Syncing", t_progressive }, + { "SYNCED", "Operational", t_final }, + { "DONOR", "Donating SST", t_progressive }, + { "DISCONNECTING", "Disconnecting", t_indefinite } + }; + + // prepare template for mkstemp() + file_name_.copy(template_, file_name_.length()); + TEMP_EXTENSION.copy(template_ +file_name_.length(),TEMP_EXTENSION.length()); + + int const fd(mkstemp(template_)); + if (fd < 0) + { + std::cerr << "Reporter could not open temporary file `" << template_ + << "': " << strerror(errno) << " (" << errno << ")\n"; + return; + } + + double const seconds(floor(tstamp)); + time_t const tt = time_t(seconds); + struct tm date; + localtime_r(&tt, &date); + + char date_str[85] = { '\0', }; + snprintf(date_str, sizeof(date_str) - 1, + "%04d-%02d-%02d %02d:%02d:%02d.%03d", + date.tm_year + 1900, date.tm_mon + 1, date.tm_mday, + date.tm_hour, date.tm_min, date.tm_sec, + (int)((tstamp-seconds)*1000)); + + std::ostringstream os; + os << "{\n"; + os << "\t\"date\": \"" << date_str << "\",\n"; + os << "\t\"timestamp\": " << std::showpoint << std::setprecision(18) + << tstamp << ",\n"; + write_log_msgs(os, "errors", err_msg_); + write_log_msgs(os, "warnings", warn_msg_); + os << "\t\"status\": {\n"; + os << "\t\t\"state\": \"" << strings[state_].state << "\",\n"; + os << "\t\t\"comment\": \"" << strings[state_].comment << "\",\n"; + os << "\t\t\"progress\": " << progress_ << "\n"; + os << "\t}\n"; + os << "}\n"; + + std::string str(os.str()); + ssize_t err(write(fd, str.c_str(), str.length())); + if (err < 0) + { + std::cerr << "Could not write " << str.length() + << " bytes to temporary file '" + << template_ << "': " << strerror(errno) + << " (" << errno << ")\n"; + return; + } + + rename(template_, file_name_.c_str()); +} + +void +wsrep::reporter::report_state(enum server_state::state const s) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + + substates const state(substate_map(s)); + + if (state != state_) + { + state_ = state; + + if (state_ == s_synced_running) + progress_ = steady_state; + else + progress_ = indefinite_progress; + + write_file(timestamp()); + } +} + +void +wsrep::reporter::report_progress(const std::string& json) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + + if (json != progress_) + { + if (state_ != s_synced_running) + { + // ignore any progress in SYNCED state + progress_ = json; + write_file(timestamp()); + } + } +} + +void +wsrep::reporter::report_log_msg(log_level const lvl, + const std::string& msg, + double tstamp) +{ + std::deque<log_msg>& deque(lvl == error ? err_msg_ : warn_msg_); + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + + if (deque.empty() || msg != deque.back().msg) + { + if (deque.size() == max_msg_) deque.pop_front(); + + if (tstamp <= undefined) tstamp = timestamp(); + + log_msg entry({tstamp, msg}); + deque.push_back(entry); + write_file(tstamp); + } +} diff --git a/wsrep-lib/src/seqno.cpp b/wsrep-lib/src/seqno.cpp new file mode 100644 index 00000000..19ff567d --- /dev/null +++ b/wsrep-lib/src/seqno.cpp @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/seqno.hpp" +#include <ostream> + +std::ostream& wsrep::operator<<(std::ostream& os, wsrep::seqno seqno) +{ + return (os << seqno.get()); +} diff --git a/wsrep-lib/src/server_state.cpp b/wsrep-lib/src/server_state.cpp new file mode 100644 index 00000000..f07de3d9 --- /dev/null +++ b/wsrep-lib/src/server_state.cpp @@ -0,0 +1,1612 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/server_state.hpp" +#include "wsrep/client_state.hpp" +#include "wsrep/server_service.hpp" +#include "wsrep/high_priority_service.hpp" +#include "wsrep/transaction.hpp" +#include "wsrep/view.hpp" +#include "wsrep/logger.hpp" +#include "wsrep/compiler.hpp" +#include "wsrep/id.hpp" + +#include <cassert> +#include <sstream> +#include <algorithm> + + +////////////////////////////////////////////////////////////////////////////// +// Helpers // +////////////////////////////////////////////////////////////////////////////// + + +// +// This method is used to deal with historical burden of several +// ways to bootstrap the cluster. Bootstrap happens if +// +// * bootstrap option is given +// * cluster_address is "gcomm://" (Galera provider) +// +static bool is_bootstrap(const std::string& cluster_address, bool bootstrap) +{ + return (bootstrap || cluster_address == "gcomm://"); +} + +// Helper method to provide detailed error message if transaction +// adopt for fragment removal fails. +static void log_adopt_error(const wsrep::transaction& transaction) +{ + wsrep::log_warning() << "Adopting a transaction (" + << transaction.server_id() << "," << transaction.id() + << ") for rollback failed, " + << "this may leave stale entries to streaming log " + << "which may need to be removed manually."; +} + +// resolve which of the two errors return to caller +static inline int resolve_return_error(bool const vote, + int const vote_err, + int const apply_err) +{ + if (vote) return vote_err; + return vote_err != 0 ? vote_err : apply_err; +} + +static void +discard_streaming_applier(wsrep::server_state& server_state, + wsrep::high_priority_service& high_priority_service, + wsrep::high_priority_service* streaming_applier, + const wsrep::ws_meta& ws_meta) +{ + server_state.stop_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id()); + server_state.server_service().release_high_priority_service( + streaming_applier); + high_priority_service.store_globals(); +} + +static int apply_fragment(wsrep::server_state& server_state, + wsrep::high_priority_service& high_priority_service, + wsrep::high_priority_service* streaming_applier, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data) +{ + int ret(0); + int apply_err; + wsrep::mutable_buffer err; + { + wsrep::high_priority_switch sw(high_priority_service, + *streaming_applier); + apply_err = streaming_applier->apply_write_set(ws_meta, data, err); + if (!apply_err) + { + assert(err.size() == 0); + streaming_applier->after_apply(); + } + else + { + bool const remove_fragments(streaming_applier->transaction( + ).streaming_context().fragments().size() > 0); + ret = streaming_applier->rollback(ws_handle, ws_meta); + ret = ret || (streaming_applier->after_apply(), 0); + + if (remove_fragments) + { + ret = ret || streaming_applier->start_transaction(ws_handle, + ws_meta); + ret = ret || (streaming_applier->adopt_apply_error(err), 0); + ret = ret || streaming_applier->remove_fragments(ws_meta); + ret = ret || streaming_applier->commit(ws_handle, ws_meta); + ret = ret || (streaming_applier->after_apply(), 0); + } + else + { + ret = streaming_applier->log_dummy_write_set(ws_handle, + ws_meta, err); + } + } + } + + if (!ret) + { + if (!apply_err) + { + high_priority_service.debug_crash("crash_apply_cb_before_append_frag"); + const wsrep::xid xid(streaming_applier->transaction().xid()); + ret = high_priority_service.append_fragment_and_commit( + ws_handle, ws_meta, data, xid); + high_priority_service.debug_crash("crash_apply_cb_after_append_frag"); + ret = ret || (high_priority_service.after_apply(), 0); + } + else + { + discard_streaming_applier(server_state, + high_priority_service, + streaming_applier, + ws_meta); + ret = resolve_return_error(err.size() > 0, ret, apply_err); + } + } + + return ret; +} + +static int commit_fragment(wsrep::server_state& server_state, + wsrep::high_priority_service& high_priority_service, + wsrep::high_priority_service* streaming_applier, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data) +{ + int ret(0); + { + wsrep::high_priority_switch sw( + high_priority_service, *streaming_applier); + wsrep::mutable_buffer err; + int const apply_err( + streaming_applier->apply_write_set(ws_meta, data, err)); + if (apply_err) + { + assert(streaming_applier->transaction( + ).streaming_context().fragments().size() > 0); + ret = streaming_applier->rollback(ws_handle, ws_meta); + ret = ret || (streaming_applier->after_apply(), 0); + ret = ret || streaming_applier->start_transaction( + ws_handle, ws_meta); + ret = ret || (streaming_applier->adopt_apply_error(err),0); + } + else + { + assert(err.size() == 0); + } + + const wsrep::transaction& trx(streaming_applier->transaction()); + // Fragment removal for XA is going to happen in after_commit + if (trx.state() != wsrep::transaction::s_prepared) + { + streaming_applier->debug_crash( + "crash_apply_cb_before_fragment_removal"); + + ret = ret || streaming_applier->remove_fragments(ws_meta); + + streaming_applier->debug_crash( + "crash_apply_cb_after_fragment_removal"); + } + + streaming_applier->debug_crash( + "crash_commit_cb_before_last_fragment_commit"); + ret = ret || streaming_applier->commit(ws_handle, ws_meta); + streaming_applier->debug_crash( + "crash_commit_cb_last_fragment_commit_success"); + ret = ret || (streaming_applier->after_apply(), 0); + ret = resolve_return_error(err.size() > 0, ret, apply_err); + } + + if (!ret) + { + discard_streaming_applier(server_state, high_priority_service, + streaming_applier, ws_meta); + } + + return ret; +} + +static int rollback_fragment(wsrep::server_state& server_state, + wsrep::high_priority_service& high_priority_service, + wsrep::high_priority_service* streaming_applier, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + int ret(0); + int adopt_error(0); + bool const remove_fragments(streaming_applier->transaction(). + streaming_context().fragments().size() > 0); + // If fragment removal is needed, adopt transaction state + // and start a transaction for it. + if (remove_fragments && + (adopt_error = high_priority_service.adopt_transaction( + streaming_applier->transaction()))) + { + log_adopt_error(streaming_applier->transaction()); + } + // Even if the adopt above fails we roll back the streaming transaction. + // Adopt failure will leave stale entries in streaming log which can + // be removed manually. + wsrep::const_buffer no_error; + { + wsrep::high_priority_switch ws( + high_priority_service, *streaming_applier); + // Streaming applier rolls back out of order. Fragment + // removal grabs commit order below. + ret = streaming_applier->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + ret = ret || (streaming_applier->after_apply(), 0); + } + + if (!ret) + { + discard_streaming_applier(server_state, high_priority_service, + streaming_applier, ws_meta); + + if (adopt_error == 0) + { + if (remove_fragments) + { + ret = high_priority_service.remove_fragments(ws_meta); + ret = ret || high_priority_service.commit(ws_handle, ws_meta); + if (ret) + { + high_priority_service.rollback(ws_handle, ws_meta); + } + high_priority_service.after_apply(); + } + else + { + if (ws_meta.ordered()) + { + wsrep::mutable_buffer no_error; + ret = high_priority_service.log_dummy_write_set( + ws_handle, ws_meta, no_error); + } + } + } + } + return ret; +} + +static int apply_write_set(wsrep::server_state& server_state, + wsrep::high_priority_service& high_priority_service, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data) +{ + int ret(0); + if (wsrep::rolls_back_transaction(ws_meta.flags())) + { + wsrep::mutable_buffer no_error; + if (wsrep::starts_transaction(ws_meta.flags())) + { + // No transaction existed before, log a dummy write set + ret = high_priority_service.log_dummy_write_set( + ws_handle, ws_meta, no_error); + } + else + { + wsrep::high_priority_service* sa( + server_state.find_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id())); + if (sa == 0) + { + // It is a known limitation that galera provider + // cannot always determine if certification test + // for interrupted transaction will pass or fail + // (see comments in transaction::certify_fragment()). + // As a consequence, unnecessary rollback fragments + // may be delivered here. The message below has + // been intentionally turned into a debug message, + // rather than warning. + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "Could not find applier context for " + << ws_meta.server_id() + << ": " << ws_meta.transaction_id()); + ret = high_priority_service.log_dummy_write_set( + ws_handle, ws_meta, no_error); + } + else + { + // rollback_fragment() consumes sa + ret = rollback_fragment(server_state, + high_priority_service, + sa, + ws_handle, + ws_meta); + } + } + } + else if (wsrep::starts_transaction(ws_meta.flags()) && + wsrep::commits_transaction(ws_meta.flags())) + { + ret = high_priority_service.start_transaction(ws_handle, ws_meta); + if (!ret) + { + wsrep::mutable_buffer err; + int const apply_err(high_priority_service.apply_write_set( + ws_meta, data, err)); + if (!apply_err) + { + assert(err.size() == 0); + ret = high_priority_service.commit(ws_handle, ws_meta); + ret = ret || (high_priority_service.after_apply(), 0); + } + else + { + ret = high_priority_service.rollback(ws_handle, ws_meta); + ret = ret || (high_priority_service.after_apply(), 0); + ret = ret || high_priority_service.log_dummy_write_set( + ws_handle, ws_meta, err); + ret = resolve_return_error(err.size() > 0, ret, apply_err); + } + } + } + else if (wsrep::starts_transaction(ws_meta.flags())) + { + assert(server_state.find_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id()) == 0); + wsrep::high_priority_service* sa( + server_state.server_service().streaming_applier_service( + high_priority_service)); + server_state.start_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id(), sa); + sa->start_transaction(ws_handle, ws_meta); + ret = apply_fragment(server_state, + high_priority_service, + sa, + ws_handle, + ws_meta, + data); + } + else if (ws_meta.flags() == 0 || ws_meta.flags() == wsrep::provider::flag::pa_unsafe || + wsrep::prepares_transaction(ws_meta.flags())) + { + wsrep::high_priority_service* sa( + server_state.find_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id())); + if (sa == 0) + { + // It is possible that rapid group membership changes + // may cause streaming transaction be rolled back before + // commit fragment comes in. Although this is a valid + // situation, log a warning if a sac cannot be found as + // it may be an indication of a bug too. + wsrep::log_warning() << "Could not find applier context for " + << ws_meta.server_id() + << ": " << ws_meta.transaction_id(); + wsrep::mutable_buffer no_error; + ret = high_priority_service.log_dummy_write_set( + ws_handle, ws_meta, no_error); + } + else + { + sa->next_fragment(ws_meta); + ret = apply_fragment(server_state, + high_priority_service, + sa, + ws_handle, + ws_meta, + data); + } + } + else if (wsrep::commits_transaction(ws_meta.flags())) + { + if (high_priority_service.is_replaying()) + { + wsrep::mutable_buffer unused; + ret = high_priority_service.start_transaction( + ws_handle, ws_meta) || + high_priority_service.apply_write_set(ws_meta, data, unused) || + high_priority_service.commit(ws_handle, ws_meta); + } + else + { + wsrep::high_priority_service* sa( + server_state.find_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id())); + if (sa == 0) + { + // It is possible that rapid group membership changes + // may cause streaming transaction be rolled back before + // commit fragment comes in. Although this is a valid + // situation, log a warning if a sac cannot be found as + // it may be an indication of a bug too. + wsrep::log_warning() + << "Could not find applier context for " + << ws_meta.server_id() + << ": " << ws_meta.transaction_id(); + wsrep::mutable_buffer no_error; + ret = high_priority_service.log_dummy_write_set( + ws_handle, ws_meta, no_error); + } + else + { + // Commit fragment consumes sa + sa->next_fragment(ws_meta); + ret = commit_fragment(server_state, + high_priority_service, + sa, + ws_handle, + ws_meta, + data); + } + } + } + else + { + assert(0); + } + if (ret) + { + wsrep::log_error() << "Failed to apply write set: " << ws_meta; + } + return ret; +} + +static int apply_toi(wsrep::provider& provider, + wsrep::high_priority_service& high_priority_service, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data) +{ + if (wsrep::starts_transaction(ws_meta.flags()) && + wsrep::commits_transaction(ws_meta.flags())) + { + // + // Regular TOI. + // + provider.commit_order_enter(ws_handle, ws_meta); + wsrep::mutable_buffer err; + int const apply_err(high_priority_service.apply_toi(ws_meta,data,err)); + int const vote_err(provider.commit_order_leave(ws_handle, ws_meta,err)); + return resolve_return_error(err.size() > 0, vote_err, apply_err); + } + else if (wsrep::starts_transaction(ws_meta.flags())) + { + provider.commit_order_enter(ws_handle, ws_meta); + wsrep::mutable_buffer err; + int const apply_err(high_priority_service.apply_nbo_begin(ws_meta, data, err)); + int const vote_err(provider.commit_order_leave(ws_handle, ws_meta, err)); + return resolve_return_error(err.size() > 0, vote_err, apply_err); + } + else if (wsrep::commits_transaction(ws_meta.flags())) + { + // NBO end event is ignored here, both local and applied + // have NBO end handled via local TOI calls. + provider.commit_order_enter(ws_handle, ws_meta); + wsrep::mutable_buffer err; + provider.commit_order_leave(ws_handle, ws_meta, err); + return 0; + } + else + { + assert(0); + return 0; + } +} + +////////////////////////////////////////////////////////////////////////////// +// Server State // +////////////////////////////////////////////////////////////////////////////// + +int wsrep::server_state::load_provider( + const std::string& provider_spec, const std::string& provider_options, + const wsrep::provider::services& services) +{ + wsrep::log_info() << "Loading provider " << provider_spec + << " initial position: " << initial_position_; + + provider_ = wsrep::provider::make_provider(*this, + provider_spec, + provider_options, + services); + return (provider_ ? 0 : 1); +} + +void wsrep::server_state::unload_provider() +{ + delete provider_; + provider_ = 0; +} + +int wsrep::server_state::connect(const std::string& cluster_name, + const std::string& cluster_address, + const std::string& state_donor, + bool bootstrap) +{ + bootstrap_ = is_bootstrap(cluster_address, bootstrap); + wsrep::log_info() << "Connecting with bootstrap option: " << bootstrap_; + return provider().connect(cluster_name, cluster_address, state_donor, + bootstrap_); +} + +int wsrep::server_state::disconnect() +{ + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + // In case of failure situations which are caused by provider + // being shut down some failing operation may also try to shut + // down the replication. Check the state here and + // return success if the provider disconnect is already in progress + // or has completed. + if (state(lock) == s_disconnecting || state(lock) == s_disconnected) + { + return 0; + } + state(lock, s_disconnecting); + interrupt_state_waiters(lock); + } + return provider().disconnect(); +} + +wsrep::server_state::~server_state() +{ + delete provider_; +} + +std::vector<wsrep::provider::status_variable> +wsrep::server_state::status() const +{ + return provider().status(); +} + + +wsrep::seqno wsrep::server_state::pause() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + // Disallow concurrent calls to pause to in order to have non-concurrent + // access to desynced_on_pause_ which is checked in resume() call. + wsrep::log_info() << "pause"; + while (pause_count_ > 0) + { + cond_.wait(lock); + } + ++pause_count_; + assert(pause_seqno_.is_undefined()); + lock.unlock(); + pause_seqno_ = provider().pause(); + lock.lock(); + if (pause_seqno_.is_undefined()) + { + --pause_count_; + } + return pause_seqno_; +} + +void wsrep::server_state::resume() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + wsrep::log_info() << "resume"; + assert(pause_seqno_.is_undefined() == false); + assert(pause_count_ == 1); + if (provider().resume()) + { + throw wsrep::runtime_error("Failed to resume provider"); + } + pause_seqno_ = wsrep::seqno::undefined(); + --pause_count_; + cond_.notify_all(); +} + +wsrep::seqno wsrep::server_state::desync_and_pause() +{ + wsrep::log_info() << "Desyncing and pausing the provider"; + // Temporary variable to store desync() return status. This will be + // assigned to desynced_on_pause_ after pause() call to prevent + // concurrent access to member variable desynced_on_pause_. + bool desync_successful; + if (desync()) + { + // Desync may give transient error if the provider cannot + // communicate with the rest of the cluster. However, this + // error can be tolerated because if the provider can be + // paused successfully below. + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "Failed to desync server before pause"); + desync_successful = false; + } + else + { + desync_successful = true; + } + wsrep::seqno ret(pause()); + if (ret.is_undefined()) + { + wsrep::log_warning() << "Failed to pause provider"; + resync(); + return wsrep::seqno::undefined(); + } + else + { + desynced_on_pause_ = desync_successful; + } + wsrep::log_info() << "Provider paused at: " << ret; + return ret; +} + +void wsrep::server_state::resume_and_resync() +{ + wsrep::log_info() << "Resuming and resyncing the provider"; + try + { + // Assign desynced_on_pause_ to local variable before resuming + // in order to avoid concurrent access to desynced_on_pause_ member + // variable. + bool do_resync = desynced_on_pause_; + desynced_on_pause_ = false; + resume(); + if (do_resync) + { + resync(); + } + } + catch (const wsrep::runtime_error& e) + { + wsrep::log_warning() + << "Resume and resync failed, server may have to be restarted"; + } +} + +std::string wsrep::server_state::prepare_for_sst() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + state(lock, s_joiner); + lock.unlock(); + return server_service_.sst_request(); +} + +int wsrep::server_state::start_sst(const std::string& sst_request, + const wsrep::gtid& gtid, + bool bypass) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + state(lock, s_donor); + int ret(0); + lock.unlock(); + if (server_service_.start_sst(sst_request, gtid, bypass)) + { + lock.lock(); + wsrep::log_warning() << "SST preparation failed"; + // v26 API does not have JOINED event, so in anticipation of SYNCED + // we must do it here. + state(lock, s_joined); + ret = 1; + } + return ret; +} + +void wsrep::server_state::sst_sent(const wsrep::gtid& gtid, int error) +{ + if (0 == error) + wsrep::log_info() << "SST sent: " << gtid; + else + wsrep::log_info() << "SST sending failed: " << error; + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + // v26 API does not have JOINED event, so in anticipation of SYNCED + // we must do it here. + state(lock, s_joined); + lock.unlock(); + enum provider::status const retval(provider().sst_sent(gtid, error)); + if (retval != provider::success) + { + std::string msg("wsrep::sst_sent() returned an error: "); + msg += wsrep::provider::to_string(retval); + server_service_.log_message(wsrep::log::warning, msg.c_str()); + } +} + +void wsrep::server_state::sst_received(wsrep::client_service& cs, + int const error) +{ + wsrep::log_info() << "SST received"; + wsrep::gtid gtid(wsrep::gtid::undefined()); + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(state_ == s_joiner || state_ == s_initialized); + + // Run initialization only if the SST was successful. + // In case of SST failure the system is in undefined state + // may not be recoverable. + if (error == 0) + { + if (server_service_.sst_before_init()) + { + if (init_initialized_ == false) + { + state(lock, s_initializing); + lock.unlock(); + server_service_.debug_sync("on_view_wait_initialized"); + lock.lock(); + wait_until_state(lock, s_initialized); + assert(init_initialized_); + } + } + lock.unlock(); + + if (id_.is_undefined()) + { + assert(0); + throw wsrep::runtime_error( + "wsrep::sst_received() called before connection to cluster"); + } + + gtid = server_service_.get_position(cs); + wsrep::log_info() << "Recovered position from storage: " << gtid; + + lock.lock(); + if (gtid.seqno() >= connected_gtid().seqno()) + { + /* Now the node has all the data the cluster has: part in + * storage, part in replication event queue. */ + state(lock, s_joined); + } + lock.unlock(); + + wsrep::view const v(server_service_.get_view(cs, id_)); + wsrep::log_info() << "Recovered view from SST:\n" << v; + + /* + * If the state id from recovered view has undefined ID, we may + * be upgrading from earlier version which does not provide + * view stored in stable storage. In this case we skip + * sanity checks and assigning the current view and wait + * until the first view delivery. + */ + if (v.state_id().id().is_undefined() == false) + { + if (v.state_id().id() != gtid.id() || + v.state_id().seqno() > gtid.seqno()) + { + /* Since IN GENERAL we may not be able to recover SST GTID from + * the state data, we have to rely on SST script passing the + * GTID value explicitly. + * Here we check if the passed GTID makes any sense: it should + * have the same UUID and greater or equal seqno than the last + * logged view. */ + std::ostringstream msg; + msg << "SST script passed bogus GTID: " << gtid + << ". Preceding view GTID: " << v.state_id(); + throw wsrep::runtime_error(msg.str()); + } + + if (current_view_.status() == wsrep::view::primary) + { + previous_primary_view_ = current_view_; + } + current_view_ = v; + server_service_.log_view(NULL /* this view is stored already */, v); + } + else + { + wsrep::log_warning() + << "View recovered from stable storage was empty. If the " + << "server is doing rolling upgrade from previous version " + << "which does not support storing view info into stable " + << "storage, this is ok. Otherwise this may be a sign of " + << "malfunction."; + } + lock.lock(); + recover_streaming_appliers_if_not_recovered(lock, cs); + lock.unlock(); + } + + enum provider::status const retval(provider().sst_received(gtid, error)); + if (retval != provider::success) + { + std::string msg("wsrep::sst_received() failed: "); + msg += wsrep::provider::to_string(retval); + throw wsrep::runtime_error(msg); + } +} + +void wsrep::server_state::initialized() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + wsrep::log_info() << "Server initialized"; + init_initialized_ = true; + if (server_service_.sst_before_init()) + { + state(lock, s_initialized); + } + else + { + state(lock, s_initializing); + state(lock, s_initialized); + } +} + +enum wsrep::provider::status +wsrep::server_state::wait_for_gtid(const wsrep::gtid& gtid, int timeout) + const +{ + return provider().wait_for_gtid(gtid, timeout); +} + +int +wsrep::server_state::set_encryption_key(std::vector<unsigned char>& key) +{ + encryption_key_ = key; + if (state_ != s_disconnected) + { + wsrep::const_buffer const key(encryption_key_.data(), + encryption_key_.size()); + enum provider::status const retval(provider_->enc_set_key(key)); + if (retval != provider::success) + { + wsrep::log_error() << "Failed to set encryption key: " + << provider::to_string(retval); + return 1; + } + } + return 0; +} + +std::pair<wsrep::gtid, enum wsrep::provider::status> +wsrep::server_state::causal_read(int timeout) const +{ + return provider().causal_read(timeout); +} + +void wsrep::server_state::on_connect(const wsrep::view& view) +{ + // Sanity checks + if (view.own_index() < 0 || + size_t(view.own_index()) >= view.members().size()) + { + std::ostringstream os; + os << "Invalid view on connect: own index out of range: " << view; +#ifndef NDEBUG + wsrep::log_error() << os.str(); + assert(0); +#endif + throw wsrep::runtime_error(os.str()); + } + + const size_t own_index(static_cast<size_t>(view.own_index())); + if (id_.is_undefined() == false && id_ != view.members()[own_index].id()) + { + std::ostringstream os; + os << "Connection in connected state.\n" + << "Connected view:\n" << view + << "Previous view:\n" << current_view_ + << "Current own ID: " << id_; +#ifndef NDEBUG + wsrep::log_error() << os.str(); + assert(0); +#endif + throw wsrep::runtime_error(os.str()); + } + else + { + id_ = view.members()[own_index].id(); + } + + wsrep::log_info() << "Server " + << name_ + << " connected to cluster at position " + << view.state_id() + << " with ID " + << id_; + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + connected_gtid_ = view.state_id(); + state(lock, s_connected); +} + +void wsrep::server_state::on_primary_view( + const wsrep::view& view, + wsrep::high_priority_service* high_priority_service) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + assert(view.final() == false); + // + // Reached primary from connected state. This may mean the following + // + // 1) Server was joined to the cluster and got SST transfer + // 2) Server was partitioned from the cluster and got back + // 3) A new cluster was bootstrapped from non-prim cluster + // + // There is no enough information here what was the cause + // of the primary component, so we need to walk through + // all states leading to joined to notify possible state + // waiters in other threads. + // + if (server_service_.sst_before_init()) + { + if (state_ == s_connected) + { + state(lock, s_joiner); + // We need to assign init_initialized_ here to local + // variable. If the value here was false, we need to skip + // the initializing -> initialized -> joined state cycle + // below. However, if we don't assign the value to + // local, it is possible that the main thread gets control + // between changing the state to initializing and checking + // initialized flag, which may cause the initialzing -> initialized + // state change to be executed even if it should not be. + const bool was_initialized(init_initialized_); + state(lock, s_initializing); + if (was_initialized) + { + // If server side has already been initialized, + // skip directly to s_joined. + state(lock, s_initialized); + } + } + } + else + { + if (state_ == s_connected) + { + state(lock, s_joiner); + } + } + if (init_initialized_ == false) + { + lock.unlock(); + server_service_.debug_sync("on_view_wait_initialized"); + lock.lock(); + wait_until_state(lock, s_initialized); + } + assert(init_initialized_); + + if (bootstrap_) + { + server_service_.bootstrap(); + bootstrap_ = false; + } + + assert(high_priority_service); + + if (high_priority_service) + { + recover_streaming_appliers_if_not_recovered(lock, + *high_priority_service); + close_orphaned_sr_transactions(lock, *high_priority_service); + } + + if (state(lock) < s_joined && + view.state_id().seqno() >= connected_gtid().seqno()) + { + // If we progressed beyond connected seqno, it means we have full state + state(lock, s_joined); + } +} + +void wsrep::server_state::on_non_primary_view( + const wsrep::view& view, + wsrep::high_priority_service* high_priority_service) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + wsrep::log_info() << "Non-primary view"; + if (view.final()) + { + go_final(lock, view, high_priority_service); + } + else if (state_ != s_disconnecting) + { + state(lock, s_connected); + } +} + +void wsrep::server_state::go_final(wsrep::unique_lock<wsrep::mutex>& lock, + const wsrep::view& view, + wsrep::high_priority_service* hps) +{ + (void)view; // avoid compiler warning "unused parameter 'view'" + assert(view.final()); + assert(hps); + if (hps) + { + close_transactions_at_disconnect(*hps); + } + state(lock, s_disconnected); + id_ = wsrep::id::undefined(); +} + +void wsrep::server_state::on_view(const wsrep::view& view, + wsrep::high_priority_service* high_priority_service) +{ + wsrep::log_info() + << "================================================\nView:\n" + << view + << "================================================="; + if (current_view_.status() == wsrep::view::primary) + { + previous_primary_view_ = current_view_; + } + current_view_ = view; + switch (view.status()) + { + case wsrep::view::primary: + on_primary_view(view, high_priority_service); + break; + case wsrep::view::non_primary: + on_non_primary_view(view, high_priority_service); + break; + case wsrep::view::disconnected: + { + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + go_final(lock, view, high_priority_service); + break; + } + default: + wsrep::log_warning() << "Unrecognized view status: " << view.status(); + assert(0); + } + + server_service_.log_view(high_priority_service, view); +} + +void wsrep::server_state::on_sync() +{ + wsrep::log_info() << "Server " << name_ << " synced with group"; + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + + // Initial sync + if (server_service_.sst_before_init() && init_synced_ == false) + { + switch (state_) + { + case s_synced: + break; + case s_connected: // Seed node path: provider becomes + state(lock, s_joiner); // synced with itself before anything + WSREP_FALLTHROUGH; // else. Then goes DB initialization. + case s_joiner: // | + state(lock, s_initializing); // V + break; + case s_donor: + assert(false); // this should never happen + state(lock, s_joined); + state(lock, s_synced); + break; + case s_initialized: + state(lock, s_joined); + WSREP_FALLTHROUGH; + default: + /* State */ + state(lock, s_synced); + }; + } + else + { + // Calls to on_sync() in synced state are possible if + // server desyncs itself from the group. Provider does not + // inform about this through callbacks. + if (state_ != s_synced) + { + state(lock, s_synced); + } + } + init_synced_ = true; + + enum wsrep::provider::status status(send_pending_rollback_events(lock)); + if (status) + { + // TODO should be retried? + wsrep::log_warning() + << "Failed to flush rollback event cache: " << status; + } +} + +int wsrep::server_state::on_apply( + wsrep::high_priority_service& high_priority_service, + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer& data) +{ + if (is_toi(ws_meta.flags())) + { + return apply_toi(provider(), high_priority_service, + ws_handle, ws_meta, data); + } + else if (is_commutative(ws_meta.flags()) || is_native(ws_meta.flags())) + { + // Not implemented yet. + assert(0); + return 0; + } + else + { + return apply_write_set(*this, high_priority_service, + ws_handle, ws_meta, data); + } +} + +void wsrep::server_state::start_streaming_client( + wsrep::client_state* client_state) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "Start streaming client: " << client_state->id()); + if (streaming_clients_.insert( + std::make_pair(client_state->id(), client_state)).second == false) + { + wsrep::log_warning() << "Failed to insert streaming client " + << client_state->id(); + assert(0); + } +} + +void wsrep::server_state::convert_streaming_client_to_applier( + wsrep::client_state* client_state) +{ + // create streaming_applier beforehand as server_state lock should + // not be held when calling server_service methods + wsrep::high_priority_service* streaming_applier( + server_service_.streaming_applier_service( + client_state->client_service())); + + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "Convert streaming client to applier " + << client_state->id()); + streaming_clients_map::iterator i( + streaming_clients_.find(client_state->id())); + assert(i != streaming_clients_.end()); + if (i == streaming_clients_.end()) + { + wsrep::log_warning() << "Unable to find streaming client " + << client_state->id(); + assert(0); + } + else + { + streaming_clients_.erase(i); + } + + // Convert to applier only if the state is not disconnected. In + // disconnected state the applier map is supposed to be empty + // and it will be reconstructed from fragment storage when + // joining back to cluster. + if (state(lock) != s_disconnected) + { + if (streaming_applier->adopt_transaction(client_state->transaction())) + { + log_adopt_error(client_state->transaction()); + streaming_applier->after_apply(); + server_service_.release_high_priority_service(streaming_applier); + return; + } + if (streaming_appliers_.insert( + std::make_pair( + std::make_pair(client_state->transaction().server_id(), + client_state->transaction().id()), + streaming_applier)).second == false) + { + wsrep::log_warning() << "Could not insert streaming applier " + << id_ + << ", " + << client_state->transaction().id(); + assert(0); + } + } + else + { + server_service_.release_high_priority_service(streaming_applier); + client_state->client_service().store_globals(); + } +} + + +void wsrep::server_state::stop_streaming_client( + wsrep::client_state* client_state) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "Stop streaming client: " << client_state->id()); + streaming_clients_map::iterator i( + streaming_clients_.find(client_state->id())); + assert(i != streaming_clients_.end()); + if (i == streaming_clients_.end()) + { + wsrep::log_warning() << "Unable to find streaming client " + << client_state->id(); + assert(0); + return; + } + else + { + streaming_clients_.erase(i); + cond_.notify_all(); + } +} + +void wsrep::server_state::start_streaming_applier( + const wsrep::id& server_id, + const wsrep::transaction_id& transaction_id, + wsrep::high_priority_service* sa) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + if (streaming_appliers_.insert( + std::make_pair(std::make_pair(server_id, transaction_id), + sa)).second == false) + { + wsrep::log_error() << "Could not insert streaming applier"; + throw wsrep::fatal_error(); + } +} + +void wsrep::server_state::stop_streaming_applier( + const wsrep::id& server_id, + const wsrep::transaction_id& transaction_id) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + streaming_appliers_map::iterator i( + streaming_appliers_.find(std::make_pair(server_id, transaction_id))); + assert(i != streaming_appliers_.end()); + if (i == streaming_appliers_.end()) + { + wsrep::log_warning() << "Could not find streaming applier for " + << server_id << ":" << transaction_id; + } + else + { + streaming_appliers_.erase(i); + cond_.notify_all(); + } +} + +wsrep::high_priority_service* wsrep::server_state::find_streaming_applier( + const wsrep::id& server_id, + const wsrep::transaction_id& transaction_id) const +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + streaming_appliers_map::const_iterator i( + streaming_appliers_.find(std::make_pair(server_id, transaction_id))); + return (i == streaming_appliers_.end() ? 0 : i->second); +} + +wsrep::high_priority_service* wsrep::server_state::find_streaming_applier( + const wsrep::xid& xid) const +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + streaming_appliers_map::const_iterator i(streaming_appliers_.begin()); + while (i != streaming_appliers_.end()) + { + wsrep::high_priority_service* sa(i->second); + if (sa->transaction().xid() == xid) + { + return sa; + } + i++; + } + return NULL; +} + +////////////////////////////////////////////////////////////////////////////// +// Private // +////////////////////////////////////////////////////////////////////////////// + +int wsrep::server_state::desync(wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + ++desync_count_; + lock.unlock(); + int ret(provider().desync()); + lock.lock(); + if (ret) + { + --desync_count_; + } + return ret; +} + +void wsrep::server_state::resync(wsrep::unique_lock<wsrep::mutex>& + lock WSREP_UNUSED) +{ + assert(lock.owns_lock()); + assert(desync_count_ > 0); + if (desync_count_ > 0) + { + --desync_count_; + if (provider().resync()) + { + throw wsrep::runtime_error("Failed to resync"); + } + } + else + { + wsrep::log_warning() << "desync_count " << desync_count_ + << " on resync"; + } +} + + +void wsrep::server_state::state( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED, + enum wsrep::server_state::state state) +{ + assert(lock.owns_lock()); + static const char allowed[n_states_][n_states_] = + { + /* dis, ing, ized, cted, jer, jed, dor, sed, ding to/from */ + { 0, 1, 0, 1, 0, 0, 0, 0, 0}, /* dis */ + { 1, 0, 1, 0, 0, 0, 0, 0, 1}, /* ing */ + { 1, 0, 0, 1, 0, 1, 0, 0, 1}, /* ized */ + { 1, 0, 0, 1, 1, 0, 0, 1, 1}, /* cted */ + { 1, 1, 0, 0, 0, 1, 0, 0, 1}, /* jer */ + { 1, 0, 0, 1, 0, 0, 1, 1, 1}, /* jed */ + { 1, 0, 0, 1, 0, 1, 0, 0, 1}, /* dor */ + { 1, 0, 0, 1, 0, 1, 1, 0, 1}, /* sed */ + { 1, 0, 0, 0, 0, 0, 0, 0, 0} /* ding */ + }; + + if (allowed[state_][state] == false) + { + std::ostringstream os; + os << "server: " << name_ << " unallowed state transition: " + << wsrep::to_string(state_) << " -> " << wsrep::to_string(state); + wsrep::log_warning() << os.str() << "\n"; + assert(0); + } + + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "server " << name_ << " state change: " + << to_c_string(state_) << " -> " + << to_c_string(state)); + state_hist_.push_back(state_); + server_service_.log_state_change(state_, state); + state_ = state; + cond_.notify_all(); + while (state_waiters_[state_]) + { + cond_.wait(lock); + } +} + +void wsrep::server_state::wait_until_state( + wsrep::unique_lock<wsrep::mutex>& lock, + enum wsrep::server_state::state state) const +{ + ++state_waiters_[state]; + while (state_ != state) + { + cond_.wait(lock); + // If the waiter waits for any other state than disconnecting + // or disconnected and the state has been changed to disconnecting, + // this usually means that some error was encountered + if (state != s_disconnecting && state != s_disconnected + && state_ == s_disconnecting) + { + throw wsrep::runtime_error("State wait was interrupted"); + } + } + --state_waiters_[state]; + cond_.notify_all(); +} + +void wsrep::server_state::interrupt_state_waiters( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED) +{ + assert(lock.owns_lock()); + cond_.notify_all(); +} + +template <class C> +void wsrep::server_state::recover_streaming_appliers_if_not_recovered( + wsrep::unique_lock<wsrep::mutex>& lock, C& c) +{ + assert(lock.owns_lock()); + if (streaming_appliers_recovered_ == false) + { + lock.unlock(); + server_service_.recover_streaming_appliers(c); + lock.lock(); + } + streaming_appliers_recovered_ = true; +} + +class transaction_state_cmp +{ +public: + transaction_state_cmp(const enum wsrep::transaction::state s) + : state_(s) { } + bool operator()(const std::pair<wsrep::client_id, + wsrep::client_state*>& vt) const + { + return vt.second->transaction().state() == state_; + } +private: + enum wsrep::transaction::state state_; +}; + +void wsrep::server_state::close_orphaned_sr_transactions( + wsrep::unique_lock<wsrep::mutex>& lock, + wsrep::high_priority_service& high_priority_service) +{ + assert(lock.owns_lock()); + + // When the originator of an SR transaction leaves the primary + // component of the cluster, that SR must be rolled back. When two + // consecutive primary views have the same membership, the system + // may have been in a state with no primary components. + // Example with 2 node cluster: + // - (1,2 primary) + // - (1 non-primary) and (2 non-primary) + // - (1,2 primary) + // We need to rollback SRs owned by both 1 and 2. + // Notice that since the introduction of rollback_event_queue_, + // checking for equal consecutive views is no longer needed. + // However, we must keep it here for the time being, for backwards + // compatibility. + const bool equal_consecutive_views = + current_view_.equal_membership(previous_primary_view_); + + if (current_view_.own_index() == -1 || equal_consecutive_views) + { + streaming_clients_map::iterator i; + transaction_state_cmp prepared_state_cmp(wsrep::transaction::s_prepared); + while ((i = std::find_if_not(streaming_clients_.begin(), + streaming_clients_.end(), + prepared_state_cmp)) + != streaming_clients_.end()) + { + wsrep::client_id client_id(i->first); + wsrep::transaction_id transaction_id(i->second->transaction().id()); + // It is safe to unlock the server state temporarily here. + // The processing happens inside view handler which is + // protected by the provider commit ordering critical + // section. The lock must be unlocked temporarily to + // allow converting the current client to streaming + // applier in transaction::streaming_rollback(). + // The iterator i may be invalidated when the server state + // remains unlocked, so it should not be accessed after + // the bf abort call. + lock.unlock(); + i->second->total_order_bf_abort(current_view_.view_seqno()); + lock.lock(); + streaming_clients_map::const_iterator found_i; + while ((found_i = streaming_clients_.find(client_id)) != + streaming_clients_.end() && + found_i->second->transaction().id() == transaction_id) + { + cond_.wait(lock); + } + } + } + + streaming_appliers_map::iterator i(streaming_appliers_.begin()); + while (i != streaming_appliers_.end()) + { + wsrep::high_priority_service* streaming_applier(i->second); + + // Rollback SR on equal consecutive primary views or if its + // originator is not in the current view. + // Transactions in prepared state must be committed or + // rolled back explicitly, those are never rolled back here. + if ((streaming_applier->transaction().state() != + wsrep::transaction::s_prepared) && + (equal_consecutive_views || + not current_view_.is_member( + streaming_applier->transaction().server_id()))) + { + WSREP_LOG_DEBUG(wsrep::log::debug_log_level(), + wsrep::log::debug_level_server_state, + "Removing SR fragments for " + << i->first.first + << ", " << i->first.second); + wsrep::id server_id(i->first.first); + wsrep::transaction_id transaction_id(i->first.second); + int adopt_error; + if ((adopt_error = high_priority_service.adopt_transaction( + streaming_applier->transaction()))) + { + log_adopt_error(streaming_applier->transaction()); + } + // Even if the transaction adopt above fails, we roll back + // the transaction. Adopt error will leave stale entries + // in the streaming log which can be removed manually. + { + wsrep::high_priority_switch sw(high_priority_service, + *streaming_applier); + streaming_applier->rollback( + wsrep::ws_handle(), wsrep::ws_meta()); + streaming_applier->after_apply(); + } + + streaming_appliers_.erase(i++); + server_service_.release_high_priority_service(streaming_applier); + high_priority_service.store_globals(); + wsrep::ws_meta ws_meta( + wsrep::gtid(), + wsrep::stid(server_id, transaction_id, wsrep::client_id()), + wsrep::seqno::undefined(), 0); + lock.unlock(); + if (adopt_error == 0) + { + high_priority_service.remove_fragments(ws_meta); + high_priority_service.commit(wsrep::ws_handle(transaction_id, 0), + ws_meta); + } + high_priority_service.after_apply(); + lock.lock(); + } + else + { + ++i; + } + } +} + +void wsrep::server_state::close_transactions_at_disconnect( + wsrep::high_priority_service& high_priority_service) +{ + // Close streaming applier without removing fragments + // from fragment storage. When the server is started again, + // it must be able to recover ongoing streaming transactions. + streaming_appliers_map::iterator i(streaming_appliers_.begin()); + while (i != streaming_appliers_.end()) + { + wsrep::high_priority_service* streaming_applier(i->second); + { + wsrep::high_priority_switch sw(high_priority_service, + *streaming_applier); + streaming_applier->rollback( + wsrep::ws_handle(), wsrep::ws_meta()); + streaming_applier->after_apply(); + } + streaming_appliers_.erase(i++); + server_service_.release_high_priority_service(streaming_applier); + high_priority_service.store_globals(); + } + streaming_appliers_recovered_ = false; +} + +// +// Rollback event queue +// + +void wsrep::server_state::queue_rollback_event( + const wsrep::transaction_id& id) +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); +#ifndef NDEBUG + // Make sure we don't have duplicate + // transaction ids in rollback event queue. + // There is no need to do this in release + // build given that caller (streaming_rollback()) + // should avoid duplicates. + for (auto i : rollback_event_queue_) + { + assert(id != i); + } +#endif + rollback_event_queue_.push_back(id); +} + +enum wsrep::provider::status +wsrep::server_state::send_pending_rollback_events( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED) +{ + assert(lock.owns_lock()); + while (not rollback_event_queue_.empty()) + { + const wsrep::transaction_id& id(rollback_event_queue_.front()); + const enum wsrep::provider::status status(provider().rollback(id)); + if (status) + { + return status; + } + rollback_event_queue_.pop_front(); + } + return wsrep::provider::success; +} + +enum wsrep::provider::status +wsrep::server_state::send_pending_rollback_events() +{ + wsrep::unique_lock<wsrep::mutex> lock(mutex_); + return send_pending_rollback_events(lock); +} diff --git a/wsrep-lib/src/service_helpers.hpp b/wsrep-lib/src/service_helpers.hpp new file mode 100644 index 00000000..6e9f9ca3 --- /dev/null +++ b/wsrep-lib/src/service_helpers.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_SERVICE_HELPERS_HPP +#define WSREP_SERVICE_HELPERS_HPP + +#include "wsrep/logger.hpp" + +#include <dlfcn.h> +#include <cerrno> + +namespace wsrep_impl +{ + template <typename InitFn> + int service_probe(void* dlh, const char* symbol, const char* service_name) + { + union { + InitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + return 0; + } + else + { + wsrep::log_warning() << "Support for " << service_name + << " " << symbol + << " not found from provider: " + << dlerror(); + return ENOTSUP; + } + } + + + template <typename InitFn, typename ServiceCallbacks> + int service_init(void* dlh, + const char* symbol, + ServiceCallbacks service_callbacks, + const char* service_name) + { + union { + InitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + wsrep::log_info() << "Initializing " << service_name; + return (*alias.dlfun)(service_callbacks); + } + else + { + wsrep::log_info() + << "Provider does not support " << service_name; + return ENOTSUP; + } + } + + template <typename DeinitFn> + void service_deinit(void* dlh, const char* symbol, const char* service_name) + { + union { + DeinitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + wsrep::log_info() << "Deinitializing " << service_name; + (*alias.dlfun)(); + } + else + { + wsrep::log_info() + << "Provider does not support deinitializing of " + << service_name; + } + } +} + +#endif // WSREP_SERVICE_HELPERS_HPP + diff --git a/wsrep-lib/src/thread.cpp b/wsrep-lib/src/thread.cpp new file mode 100644 index 00000000..5fec0ff5 --- /dev/null +++ b/wsrep-lib/src/thread.cpp @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/thread.hpp" + +#include <ostream> + +std::ostream& wsrep::operator<<(std::ostream& os, const wsrep::thread::id& id) +{ + std::ios_base::fmtflags orig_flags(os.flags()); + os << std::hex << id.thread_; + os.flags(orig_flags); + return os; +} diff --git a/wsrep-lib/src/thread_service_v1.cpp b/wsrep-lib/src/thread_service_v1.cpp new file mode 100644 index 00000000..ee81ba31 --- /dev/null +++ b/wsrep-lib/src/thread_service_v1.cpp @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2019-2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "thread_service_v1.hpp" +#include "service_helpers.hpp" + +#include "wsrep/thread_service.hpp" +#include "wsrep/logger.hpp" +#include "v26/wsrep_thread_service.h" + +#include <cassert> +#include <dlfcn.h> +#include <cerrno> + +namespace wsrep_thread_service_v1 +{ + // + // Thread service callbacks + // + + // Pointer to thread service implementation provided by + // the application. + static wsrep::thread_service* thread_service_impl{ 0 }; + static std::atomic<size_t> use_count; + + static const wsrep_thread_key_t* thread_key_create_cb(const char* name) + { + assert(thread_service_impl); + return reinterpret_cast<const wsrep_thread_key_t*>( + thread_service_impl->create_thread_key(name)); + ; + } + + static int thread_create_cb(const wsrep_thread_key_t* key, + wsrep_thread_t** thread, + void* (*fn)(void*), void* args) + { + assert(thread_service_impl); + return thread_service_impl->create_thread( + reinterpret_cast<const wsrep::thread_service::thread_key*>(key), + reinterpret_cast<wsrep::thread_service::thread**>(thread), fn, + args); + } + + int thread_detach_cb(wsrep_thread_t* thread) + { + assert(thread_service_impl); + return thread_service_impl->detach( + reinterpret_cast<wsrep::thread_service::thread*>(thread)); + } + + int thread_equal_cb(wsrep_thread_t* thread_1, wsrep_thread_t* thread_2) + { + assert(thread_service_impl); + return thread_service_impl->equal( + reinterpret_cast<wsrep::thread_service::thread*>(thread_1), + reinterpret_cast<wsrep::thread_service::thread*>(thread_2)); + } + + __attribute__((noreturn)) + void thread_exit_cb(wsrep_thread_t* thread, void* retval) + { + assert(thread_service_impl); + thread_service_impl->exit( + reinterpret_cast<wsrep::thread_service::thread*>(thread), retval); + throw; // Implementation broke the contract and returned. + } + + int thread_join_cb(wsrep_thread_t* thread, void** retval) + { + assert(thread_service_impl); + return thread_service_impl->join( + reinterpret_cast<wsrep::thread_service::thread*>(thread), retval); + } + + wsrep_thread_t* thread_self_cb(void) + { + assert(thread_service_impl); + return reinterpret_cast<wsrep_thread_t*>(thread_service_impl->self()); + } + + int thread_setschedparam_cb(wsrep_thread_t* thread, int policy, + const struct sched_param* sp) + { + assert(thread_service_impl); + return thread_service_impl->setschedparam( + reinterpret_cast<wsrep::thread_service::thread*>(thread), + policy, sp); + } + + int thread_getschedparam_cb(wsrep_thread_t* thread, int* policy, + struct sched_param* sp) + { + assert(thread_service_impl); + return thread_service_impl->getschedparam( + reinterpret_cast<wsrep::thread_service::thread*>(thread), policy, + sp); + } + + const wsrep_mutex_key_t* mutex_key_create_cb(const char* name) + { + assert(thread_service_impl); + return reinterpret_cast<const wsrep_mutex_key_t*>( + thread_service_impl->create_mutex_key(name)); + } + + wsrep_mutex_t* mutex_init_cb(const wsrep_mutex_key_t* key, void* memblock, + size_t memblock_size) + { + assert(thread_service_impl); + return reinterpret_cast<wsrep_mutex_t*>( + thread_service_impl->init_mutex( + reinterpret_cast<const wsrep::thread_service::mutex_key*>(key), + memblock, memblock_size)); + } + + int mutex_destroy_cb(wsrep_mutex_t* mutex) + { + assert(thread_service_impl); + return thread_service_impl->destroy( + reinterpret_cast<wsrep::thread_service::mutex*>(mutex)); + } + int mutex_lock_cb(wsrep_mutex_t* mutex) + { + assert(thread_service_impl); + return thread_service_impl->lock( + reinterpret_cast<wsrep::thread_service::mutex*>(mutex)); + } + + int mutex_trylock_cb(wsrep_mutex_t* mutex) + { + assert(thread_service_impl); + return thread_service_impl->trylock( + reinterpret_cast<wsrep::thread_service::mutex*>(mutex)); + } + + int mutex_unlock_cb(wsrep_mutex_t* mutex) + { + assert(thread_service_impl); + return thread_service_impl->unlock( + reinterpret_cast<wsrep::thread_service::mutex*>(mutex)); + } + + const wsrep_cond_key_t* cond_key_create_cb(const char* name) + { + assert(thread_service_impl); + return reinterpret_cast<const wsrep_cond_key_t*>( + thread_service_impl->create_cond_key(name)); + } + + wsrep_cond_t* cond_init_cb(const wsrep_cond_key_t* key, void* memblock, + size_t memblock_size) + { + assert(thread_service_impl); + return reinterpret_cast<wsrep_cond_t*>(thread_service_impl->init_cond( + reinterpret_cast<const wsrep::thread_service::cond_key*>(key), + memblock, memblock_size)); + } + + int cond_destroy_cb(wsrep_cond_t* cond) + { + assert(thread_service_impl); + return thread_service_impl->destroy( + reinterpret_cast<wsrep::thread_service::cond*>(cond)); + } + + int cond_wait_cb(wsrep_cond_t* cond, wsrep_mutex_t* mutex) + { + assert(thread_service_impl); + return thread_service_impl->wait( + reinterpret_cast<wsrep::thread_service::cond*>(cond), + reinterpret_cast<wsrep::thread_service::mutex*>(mutex)); + } + + int cond_timedwait_cb(wsrep_cond_t* cond, wsrep_mutex_t* mutex, + const struct timespec* ts) + { + assert(thread_service_impl); + return thread_service_impl->timedwait( + reinterpret_cast<wsrep::thread_service::cond*>(cond), + reinterpret_cast<wsrep::thread_service::mutex*>(mutex), ts); + } + + int cond_signal_cb(wsrep_cond_t* cond) + { + assert(thread_service_impl); + return thread_service_impl->signal( + reinterpret_cast<wsrep::thread_service::cond*>(cond)); + } + + int cond_broadcast_cb(wsrep_cond_t* cond) + { + assert(thread_service_impl); + return thread_service_impl->broadcast( + reinterpret_cast<wsrep::thread_service::cond*>(cond)); + } + + static wsrep_thread_service_v1_t thread_service_callbacks + = { thread_key_create_cb, + thread_create_cb, + thread_detach_cb, + thread_equal_cb, + thread_exit_cb, + thread_join_cb, + thread_self_cb, + thread_setschedparam_cb, + thread_getschedparam_cb, + mutex_key_create_cb, + mutex_init_cb, + mutex_destroy_cb, + mutex_lock_cb, + mutex_trylock_cb, + mutex_unlock_cb, + cond_key_create_cb, + cond_init_cb, + cond_destroy_cb, + cond_wait_cb, + cond_timedwait_cb, + cond_signal_cb, + cond_broadcast_cb }; +} + +int wsrep::thread_service_v1_probe(void* dlh) +{ + typedef int (*init_fn)(wsrep_thread_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe<init_fn>( + dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, "thread service v1") || + wsrep_impl::service_probe<deinit_fn>( + dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1")) + { + wsrep::log_warning() << "Provider does not support thread service v1"; + return 1; + } + return 0; +} + +int wsrep::thread_service_v1_init(void* dlh, + wsrep::thread_service* thread_service) +{ + if (not (dlh && thread_service)) return EINVAL; + typedef int (*init_fn)(wsrep_thread_service_v1_t*); + wsrep_thread_service_v1::thread_service_impl = thread_service; + int ret(0); + if ((ret = wsrep_impl::service_init<init_fn>( + dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, + &wsrep_thread_service_v1::thread_service_callbacks, + "thread service v1"))) + { + wsrep_thread_service_v1::thread_service_impl = 0; + } + else + { + ++wsrep_thread_service_v1::use_count; + } + return ret; +} + +void wsrep::thread_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit<deinit_fn>( + dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1"); + --wsrep_thread_service_v1::use_count; + if (wsrep_thread_service_v1::use_count == 0) + { + wsrep_thread_service_v1::thread_service_impl = 0; + } +} diff --git a/wsrep-lib/src/thread_service_v1.hpp b/wsrep-lib/src/thread_service_v1.hpp new file mode 100644 index 00000000..b8300041 --- /dev/null +++ b/wsrep-lib/src/thread_service_v1.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_THREAD_SERVICE_V1_HPP +#define WSREP_THREAD_SERVICE_V1_HPP + +namespace wsrep +{ + class thread_service; + /** + * Probe thread_service_v1 support in loaded library. + * + * @param dlh Handle returned by dlopen(). + * + * @return Zero on success, non-zero system error code on failure. + */ + int thread_service_v1_probe(void *dlh); + + /** + * Initialize the thread service. + * + * @param dlh Handle returned by dlopen(). + * @params thread_service Pointer to wsrep::thread_service implementation. + * + * @return Zero on success, non-zero system error code on failure. + */ + int thread_service_v1_init(void* dlh, + wsrep::thread_service* thread_service); + + /** + * Deinitialize the thread service. + * + * @params dlh Handler returned by dlopen(). + */ + void thread_service_v1_deinit(void* dlh); + +} + +#endif // WSREP_THREAD_SERVICE_V1_HPP diff --git a/wsrep-lib/src/tls_service_v1.cpp b/wsrep-lib/src/tls_service_v1.cpp new file mode 100644 index 00000000..8746a767 --- /dev/null +++ b/wsrep-lib/src/tls_service_v1.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "tls_service_v1.hpp" + +#include "wsrep/tls_service.hpp" +#include "wsrep/logger.hpp" +#include "v26/wsrep_tls_service.h" +#include "service_helpers.hpp" + +#include <cassert> + +namespace wsrep_tls_service_v1 +{ + static wsrep::tls_service* tls_service_impl{0}; + static std::atomic<size_t> use_count; + + static int tls_stream_init_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + stream->opaque = + tls_service_impl->create_tls_stream(stream->fd); + if (not stream->opaque) + { + return ENOMEM; + } + return 0; + } + + static void tls_stream_deinit_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + tls_service_impl->destroy( + reinterpret_cast<wsrep::tls_stream*>(stream->opaque)); + } + + static int tls_stream_get_error_number_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return tls_service_impl->get_error_number( + reinterpret_cast<const wsrep::tls_stream*>(stream->opaque)); + } + + static const void* tls_stream_get_error_category_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return tls_service_impl->get_error_category( + reinterpret_cast<const wsrep::tls_stream*>(stream->opaque)); + } + + static const char* tls_error_message_get_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream, + int value, const void* category) + { + assert(tls_service_impl); + return tls_service_impl->get_error_message( + reinterpret_cast<const wsrep::tls_stream*>(stream->opaque), value, category); + } + + static enum wsrep_tls_result map_return_value(ssize_t status) + { + switch (status) + { + case wsrep::tls_service::success: + return wsrep_tls_result_success; + case wsrep::tls_service::want_read: + return wsrep_tls_result_want_read; + case wsrep::tls_service::want_write: + return wsrep_tls_result_want_write; + case wsrep::tls_service::eof: + return wsrep_tls_result_eof; + case wsrep::tls_service::error: + return wsrep_tls_result_error; + default: + assert(status < 0); + return wsrep_tls_result_error; + } + } + + + static enum wsrep_tls_result + tls_stream_client_handshake_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return map_return_value( + tls_service_impl->client_handshake( + reinterpret_cast<wsrep::tls_stream*>(stream->opaque))); + } + + static enum wsrep_tls_result + tls_stream_server_handshake_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return map_return_value( + tls_service_impl->server_handshake( + reinterpret_cast<wsrep::tls_stream*>(stream->opaque))); + } + + static enum wsrep_tls_result tls_stream_read_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + void* buf, + size_t max_count, + size_t* bytes_transferred) + { + assert(tls_service_impl); + auto result(tls_service_impl->read( + reinterpret_cast<wsrep::tls_stream*>(stream->opaque), + buf, max_count)); + *bytes_transferred = result.bytes_transferred; + return map_return_value(result.status); + } + + static enum wsrep_tls_result tls_stream_write_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + const void* buf, + size_t count, + size_t* bytes_transferred) + { + assert(tls_service_impl); + auto result(tls_service_impl->write( + reinterpret_cast<wsrep::tls_stream*>(stream->opaque), + buf, count)); + *bytes_transferred = result.bytes_transferred; + return map_return_value(result.status); + } + + static enum wsrep_tls_result + tls_stream_shutdown_cb(wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + // @todo Handle other values than success. + return map_return_value( + tls_service_impl->shutdown( + reinterpret_cast<wsrep::tls_stream*>(stream->opaque))); + } + + static wsrep_tls_service_v1_t tls_service_callbacks = + { + tls_stream_init_cb, + tls_stream_deinit_cb, + tls_stream_get_error_number_cb, + tls_stream_get_error_category_cb, + tls_stream_client_handshake_cb, + tls_stream_server_handshake_cb, + tls_stream_read_cb, + tls_stream_write_cb, + tls_stream_shutdown_cb, + tls_error_message_get_cb, + 0 // we pass NULL context for now. + }; +} + +int wsrep::tls_service_v1_probe(void* dlh) +{ + typedef int (*init_fn)(wsrep_tls_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe<init_fn>( + dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, "tls service v1") || + wsrep_impl::service_probe<deinit_fn>( + dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1")) + { + wsrep::log_warning() << "Provider does not support tls service v1"; + return 1; + } + return 0; +} + +int wsrep::tls_service_v1_init(void* dlh, + wsrep::tls_service* tls_service) +{ + if (not (dlh && tls_service)) return EINVAL; + + typedef int (*init_fn)(wsrep_tls_service_v1_t*); + wsrep_tls_service_v1::tls_service_impl = tls_service; + int ret(0); + if ((ret = wsrep_impl::service_init<init_fn>( + dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, + &wsrep_tls_service_v1::tls_service_callbacks, + "tls service v1"))) + { + wsrep_tls_service_v1::tls_service_impl = 0; + } + else + { + ++wsrep_tls_service_v1::use_count; + } + return ret; +} + +void wsrep::tls_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit<deinit_fn>( + dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1"); + --wsrep_tls_service_v1::use_count; + if (wsrep_tls_service_v1::use_count == 0) + { + wsrep_tls_service_v1::tls_service_impl = 0; + } +} diff --git a/wsrep-lib/src/tls_service_v1.hpp b/wsrep-lib/src/tls_service_v1.hpp new file mode 100644 index 00000000..fe17e206 --- /dev/null +++ b/wsrep-lib/src/tls_service_v1.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_TLS_SERVICE_V1_HPP +#define WSREP_TLS_SERVICE_V1_HPP + +namespace wsrep +{ + class tls_service; + /** + * Probe tls_service_v1 support in loaded library. + * + * @param dlh Handle returned by dlopen(). + * + * @return Zero on success, non-zero system error code on failure. + */ + int tls_service_v1_probe(void *dlh); + + /** + * Initialize TLS service. + * + * @param dlh Handle returned by dlopen(). + * @params tls_service Pointer to wsrep::thread_service implementation. + * + * @return Zero on success, non-zero system error code on failure. + */ + int tls_service_v1_init(void* dlh, + wsrep::tls_service* tls_service); + + /** + * Deinitialize TLS service. + * + * @param dlh Handler returned by dlopen(). + */ + void tls_service_v1_deinit(void* dlh); +} + +#endif // WSREP_TLS_SERVICE_V1_HPP diff --git a/wsrep-lib/src/transaction.cpp b/wsrep-lib/src/transaction.cpp new file mode 100644 index 00000000..764b1838 --- /dev/null +++ b/wsrep-lib/src/transaction.cpp @@ -0,0 +1,2114 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/transaction.hpp" +#include "wsrep/client_state.hpp" +#include "wsrep/server_state.hpp" +#include "wsrep/storage_service.hpp" +#include "wsrep/high_priority_service.hpp" +#include "wsrep/key.hpp" +#include "wsrep/logger.hpp" +#include "wsrep/compiler.hpp" + +#include <sstream> +#include <memory> + +namespace +{ + class storage_service_deleter + { + public: + storage_service_deleter(wsrep::server_service& server_service) + : server_service_(server_service) + { } + void operator()(wsrep::storage_service* storage_service) + { + server_service_.release_storage_service(storage_service); + } + private: + wsrep::server_service& server_service_; + }; + + template <class D> + class scoped_storage_service + { + public: + scoped_storage_service(wsrep::client_service& client_service, + wsrep::storage_service* storage_service, + D deleter) + : client_service_(client_service) + , storage_service_(storage_service) + , deleter_(deleter) + { + if (storage_service_ == 0) + { + throw wsrep::runtime_error("Null client_state provided"); + } + client_service_.reset_globals(); + storage_service_->store_globals(); + } + + wsrep::storage_service& storage_service() + { + return *storage_service_; + } + + ~scoped_storage_service() + { + deleter_(storage_service_); + client_service_.store_globals(); + } + private: + scoped_storage_service(const scoped_storage_service&); + scoped_storage_service& operator=(const scoped_storage_service&); + wsrep::client_service& client_service_; + wsrep::storage_service* storage_service_; + D deleter_; + }; +} + +// Public + +wsrep::transaction::transaction( + wsrep::client_state& client_state) + : server_service_(client_state.server_state().server_service()) + , client_service_(client_state.client_service()) + , client_state_(client_state) + , server_id_() + , id_(transaction_id::undefined()) + , state_(s_executing) + , state_hist_() + , bf_abort_state_(s_executing) + , bf_abort_provider_status_() + , bf_abort_client_state_() + , bf_aborted_in_total_order_() + , ws_handle_() + , ws_meta_() + , flags_() + , implicit_deps_(false) + , certified_(false) + , fragments_certified_for_statement_() + , streaming_context_() + , sr_keys_() + , apply_error_buf_() + , xid_() + , streaming_rollback_in_progress_(false) +{ } + + +wsrep::transaction::~transaction() +{ +} + +int wsrep::transaction::start_transaction( + const wsrep::transaction_id& id) +{ + debug_log_state("start_transaction enter"); + assert(active() == false); + assert(is_xa() == false); + assert(flags() == 0); + server_id_ = client_state_.server_state().id(); + id_ = id; + state_ = s_executing; + state_hist_.clear(); + ws_handle_ = wsrep::ws_handle(id); + flags(wsrep::provider::flag::start_transaction); + switch (client_state_.mode()) + { + case wsrep::client_state::m_high_priority: + debug_log_state("start_transaction success"); + return 0; + case wsrep::client_state::m_local: + debug_log_state("start_transaction success"); + return provider().start_transaction(ws_handle_); + default: + debug_log_state("start_transaction error"); + assert(0); + return 1; + } +} + +int wsrep::transaction::start_transaction( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + debug_log_state("start_transaction enter"); + if (state() != s_replaying) + { + // assert(ws_meta.flags()); + assert(active() == false); + assert(flags() == 0); + server_id_ = ws_meta.server_id(); + id_ = ws_meta.transaction_id(); + assert(client_state_.mode() == wsrep::client_state::m_high_priority); + state_ = s_executing; + state_hist_.clear(); + ws_handle_ = ws_handle; + ws_meta_ = ws_meta; + flags(wsrep::provider::flag::start_transaction); + certified_ = true; + } + else + { + ws_meta_ = ws_meta; + assert(ws_meta_.flags() & wsrep::provider::flag::commit); + assert(active()); + assert(client_state_.mode() == wsrep::client_state::m_high_priority); + assert(state() == s_replaying); + assert(ws_meta_.seqno().is_undefined() == false); + certified_ = true; + } + debug_log_state("start_transaction leave"); + return 0; +} + +int wsrep::transaction::next_fragment( + const wsrep::ws_meta& ws_meta) +{ + debug_log_state("next_fragment enter"); + ws_meta_ = ws_meta; + debug_log_state("next_fragment leave"); + return 0; +} + +void wsrep::transaction::adopt(const wsrep::transaction& transaction) +{ + debug_log_state("adopt enter"); + assert(transaction.is_streaming()); + start_transaction(transaction.id()); + server_id_ = transaction.server_id_; + flags_ = transaction.flags(); + streaming_context_ = transaction.streaming_context(); + debug_log_state("adopt leave"); +} + +void wsrep::transaction::fragment_applied(wsrep::seqno seqno) +{ + assert(active()); + streaming_context_.applied(seqno); +} + +int wsrep::transaction::prepare_for_ordering( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + bool is_commit) +{ + assert(active()); + + if (state_ != s_replaying) + { + ws_handle_ = ws_handle; + ws_meta_ = ws_meta; + certified_ = is_commit; + } + return 0; +} + +int wsrep::transaction::assign_read_view(const wsrep::gtid* const gtid) +{ + try + { + return provider().assign_read_view(ws_handle_, gtid); + } + catch (...) + { + wsrep::log_error() << "Failed to assign read view"; + return 1; + } +} + +int wsrep::transaction::append_key(const wsrep::key& key) +{ + assert(active()); + try + { + debug_log_key_append(key); + sr_keys_.insert(key); + return provider().append_key(ws_handle_, key); + } + catch (...) + { + wsrep::log_error() << "Failed to append key"; + return 1; + } +} + +int wsrep::transaction::append_data(const wsrep::const_buffer& data) +{ + assert(active()); + return provider().append_data(ws_handle_, data); +} + +int wsrep::transaction::after_row() +{ + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("after_row_enter"); + int ret(0); + if (streaming_context_.fragment_size() && + streaming_context_.fragment_unit() != streaming_context::statement) + { + ret = streaming_step(lock); + } + debug_log_state("after_row_leave"); + return ret; +} + +int wsrep::transaction::before_prepare( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + int ret(0); + debug_log_state("before_prepare_enter"); + assert(state() == s_executing || state() == s_must_abort || + state() == s_replaying); + + if (state() == s_must_abort) + { + assert(client_state_.mode() == wsrep::client_state::m_local); + client_state_.override_error(wsrep::e_deadlock_error); + return 1; + } + + switch (client_state_.mode()) + { + case wsrep::client_state::m_local: + if (is_streaming()) + { + client_service_.debug_crash( + "crash_last_fragment_commit_before_fragment_removal"); + lock.unlock(); + if (client_service_.statement_allowed_for_streaming() == false) + { + client_state_.override_error( + wsrep::e_error_during_commit, + wsrep::provider::error_not_allowed); + ret = 1; + } + else if (!is_xa()) + { + // Note: we can't remove fragments here for XA, + // the transaction has already issued XA END and + // is in IDLE state, no more changes allowed! + ret = client_service_.remove_fragments(); + if (ret) + { + client_state_.override_error(wsrep::e_deadlock_error); + } + } + lock.lock(); + client_service_.debug_crash( + "crash_last_fragment_commit_after_fragment_removal"); + if (state() == s_must_abort) + { + client_state_.override_error(wsrep::e_deadlock_error); + ret = 1; + } + } + + if (ret == 0) + { + if (is_xa()) + { + // Force fragment replication on XA prepare + flags(flags() | wsrep::provider::flag::prepare); + pa_unsafe(true); + append_sr_keys_for_commit(); + const bool force_streaming_step = true; + ret = streaming_step(lock, force_streaming_step); + if (ret == 0) + { + assert(state() == s_executing); + state(lock, s_preparing); + } + } + else + { + ret = certify_commit(lock); + } + + assert((ret == 0 && state() == s_preparing) || + (state() == s_must_abort || + state() == s_must_replay || + state() == s_cert_failed)); + + if (ret) + { + assert(state() == s_must_replay || + client_state_.current_error()); + ret = 1; + } + } + + break; + case wsrep::client_state::m_high_priority: + // Note: fragment removal is done from applying + // context for high priority mode. + if (is_xa()) + { + assert(state() == s_executing || + state() == s_replaying); + if (state() == s_replaying) + { + break; + } + } + state(lock, s_preparing); + break; + default: + assert(0); + break; + } + + assert(state() == s_preparing || + (is_xa() && state() == s_replaying) || + (ret && (state() == s_must_abort || + state() == s_must_replay || + state() == s_cert_failed || + state() == s_aborted))); + debug_log_state("before_prepare_leave"); + return ret; +} + +int wsrep::transaction::after_prepare( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + + int ret = 0; + debug_log_state("after_prepare_enter"); + if (is_xa()) + { + switch (state()) + { + case s_preparing: + assert(client_state_.mode() == wsrep::client_state::m_local || + (certified() && ordered())); + state(lock, s_prepared); + break; + case s_must_abort: + // prepared locally, but client has not received + // a result yet. We can still abort. + assert(client_state_.mode() == wsrep::client_state::m_local); + client_state_.override_error(wsrep::e_deadlock_error); + ret = 1; + break; + case s_replaying: + assert(client_state_.mode() == wsrep::client_state::m_high_priority); + break; + default: + assert(0); + } + } + else + { + assert(certified() && ordered()); + assert(state() == s_preparing || state() == s_must_abort); + + if (state() == s_must_abort) + { + assert(client_state_.mode() == wsrep::client_state::m_local); + state(lock, s_must_replay); + ret = 1; + } + else + { + state(lock, s_committing); + } + } + debug_log_state("after_prepare_leave"); + return ret; +} + +int wsrep::transaction::before_commit() +{ + int ret(1); + + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("before_commit_enter"); + assert(client_state_.mode() != wsrep::client_state::m_toi); + assert(state() == s_executing || + state() == s_prepared || + state() == s_committing || + state() == s_must_abort || + state() == s_replaying); + assert((state() != s_committing && state() != s_replaying) || + certified()); + + switch (client_state_.mode()) + { + case wsrep::client_state::m_local: + if (state() == s_executing) + { + ret = before_prepare(lock) || after_prepare(lock); + assert((ret == 0 && + (state() == s_committing || state() == s_prepared)) + || + (state() == s_must_abort || + state() == s_must_replay || + state() == s_cert_failed || + state() == s_aborted)); + } + else if (state() != s_committing && state() != s_prepared) + { + assert(state() == s_must_abort); + if (certified() || + (is_xa() && is_streaming())) + { + state(lock, s_must_replay); + } + else + { + client_state_.override_error(wsrep::e_deadlock_error); + } + } + else + { + // 2PC commit, prepare was done before + ret = 0; + } + + if (ret == 0 && state() == s_prepared) + { + ret = certify_commit(lock); + assert((ret == 0 && state() == s_committing) || + (state() == s_must_abort || + state() == s_must_replay || + state() == s_cert_failed || + state() == s_prepared)); + } + + if (ret == 0) + { + assert(certified()); + assert(ordered()); + lock.unlock(); + client_service_.debug_sync("wsrep_before_commit_order_enter"); + enum wsrep::provider::status + status(provider().commit_order_enter(ws_handle_, ws_meta_)); + lock.lock(); + switch (status) + { + case wsrep::provider::success: + break; + case wsrep::provider::error_bf_abort: + if (state() != s_must_abort) + { + state(lock, s_must_abort); + } + state(lock, s_must_replay); + ret = 1; + break; + default: + ret = 1; + assert(0); + break; + } + } + break; + case wsrep::client_state::m_high_priority: + assert(certified()); + assert(ordered()); + if (is_xa()) + { + assert(state() == s_prepared || + state() == s_replaying); + state(lock, s_committing); + ret = 0; + } + else if (state() == s_executing || state() == s_replaying) + { + ret = before_prepare(lock) || after_prepare(lock); + } + else + { + ret = 0; + } + lock.unlock(); + ret = ret || provider().commit_order_enter(ws_handle_, ws_meta_); + lock.lock(); + if (ret) + { + state(lock, s_must_abort); + state(lock, s_aborting); + } + break; + default: + assert(0); + break; + } + debug_log_state("before_commit_leave"); + return ret; +} + +int wsrep::transaction::ordered_commit() +{ + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("ordered_commit_enter"); + assert(state() == s_committing); + assert(ordered()); + client_service_.debug_sync("wsrep_before_commit_order_leave"); + int ret(provider().commit_order_leave(ws_handle_, ws_meta_, + apply_error_buf_)); + client_service_.debug_sync("wsrep_after_commit_order_leave"); + // Should always succeed: + // 1) If before commit before succeeds, the transaction handle + // in the provider is guaranteed to exist and the commit + // has been ordered + // 2) The transaction which has been ordered for commit cannot be BF + // aborted anymore + // 3) The provider should always guarantee that the transactions which + // have been ordered for commit can finish committing. + // + // The exception here is a storage service transaction which is running + // in high priority mode. The fragment storage commit may get BF + // aborted in the provider after commit ordering has been + // established since the transaction is operating in streaming + // mode. + if (ret) + { + assert(client_state_.mode() == wsrep::client_state::m_high_priority); + state(lock, s_must_abort); + state(lock, s_aborting); + } + else + { + state(lock, s_ordered_commit); + } + debug_log_state("ordered_commit_leave"); + return ret; +} + +int wsrep::transaction::after_commit() +{ + int ret(0); + + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("after_commit_enter"); + assert(state() == s_ordered_commit); + + if (is_streaming()) + { + assert(client_state_.mode() == wsrep::client_state::m_local || + client_state_.mode() == wsrep::client_state::m_high_priority); + + if (is_xa()) + { + // XA fragment removal happens here, + // see comment in before_prepare + lock.unlock(); + scoped_storage_service<storage_service_deleter> + sr_scope( + client_service_, + server_service_.storage_service(client_service_), + storage_service_deleter(server_service_)); + wsrep::storage_service& storage_service( + sr_scope.storage_service()); + storage_service.adopt_transaction(*this); + storage_service.remove_fragments(); + storage_service.commit(wsrep::ws_handle(), wsrep::ws_meta()); + lock.lock(); + } + + if (client_state_.mode() == wsrep::client_state::m_local) + { + lock.unlock(); + client_state_.server_state_.stop_streaming_client(&client_state_); + lock.lock(); + } + streaming_context_.cleanup(); + } + + switch (client_state_.mode()) + { + case wsrep::client_state::m_local: + ret = provider().release(ws_handle_); + break; + case wsrep::client_state::m_high_priority: + break; + default: + assert(0); + break; + } + assert(ret == 0); + state(lock, s_committed); + + debug_log_state("after_commit_leave"); + return ret; +} + +int wsrep::transaction::before_rollback() +{ + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("before_rollback_enter"); + assert(state() == s_executing || + state() == s_preparing || + state() == s_prepared || + state() == s_must_abort || + // Background rollbacker or rollback initiated from SE + state() == s_aborting || + state() == s_cert_failed || + state() == s_must_replay); + + switch (client_state_.mode()) + { + case wsrep::client_state::m_local: + if (is_streaming()) + { + client_service_.debug_sync("wsrep_before_SR_rollback"); + } + switch (state()) + { + case s_preparing: + // Error detected during prepare phase + state(lock, s_must_abort); + WSREP_FALLTHROUGH; + case s_prepared: + WSREP_FALLTHROUGH; + case s_executing: + // Voluntary rollback + if (is_streaming()) + { + streaming_rollback(lock); + } + state(lock, s_aborting); + break; + case s_must_abort: + if (certified()) + { + state(lock, s_must_replay); + } + else + { + if (is_streaming()) + { + streaming_rollback(lock); + } + state(lock, s_aborting); + } + break; + case s_cert_failed: + if (is_streaming()) + { + streaming_rollback(lock); + } + state(lock, s_aborting); + break; + case s_aborting: + if (is_streaming()) + { + streaming_rollback(lock); + } + break; + case s_must_replay: + break; + default: + assert(0); + break; + } + break; + case wsrep::client_state::m_high_priority: + // Rollback by rollback write set or BF abort + assert(state_ == s_executing || state_ == s_prepared || state_ == s_aborting); + if (state_ != s_aborting) + { + state(lock, s_aborting); + } + break; + default: + assert(0); + break; + } + + debug_log_state("before_rollback_leave"); + return 0; +} + +int wsrep::transaction::after_rollback() +{ + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("after_rollback_enter"); + assert(state() == s_aborting || + state() == s_must_replay); + + if (is_streaming() && bf_aborted_in_total_order_) + { + lock.unlock(); + // Storage service scope + { + scoped_storage_service<storage_service_deleter> + sr_scope( + client_service_, + server_service_.storage_service(client_service_), + storage_service_deleter(server_service_)); + wsrep::storage_service& storage_service( + sr_scope.storage_service()); + storage_service.adopt_transaction(*this); + storage_service.remove_fragments(); + storage_service.commit(wsrep::ws_handle(), wsrep::ws_meta()); + } + lock.lock(); + streaming_context_.cleanup(); + } + + if (is_streaming() && state() != s_must_replay) + { + streaming_context_.cleanup(); + } + + if (state() == s_aborting) + { + state(lock, s_aborted); + } + + // Releasing the transaction from provider is postponed into + // after_statement() hook. Depending on DBMS system all the + // resources acquired by transaction may or may not be released + // during actual rollback. If the transaction has been ordered, + // releasing the commit ordering critical section should be + // also postponed until all resources have been released. + debug_log_state("after_rollback_leave"); + return 0; +} + +int wsrep::transaction::release_commit_order( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + lock.unlock(); + int ret(provider().commit_order_enter(ws_handle_, ws_meta_)); + lock.lock(); + if (!ret) + { + server_service_.set_position(client_service_, ws_meta_.gtid()); + ret = provider().commit_order_leave(ws_handle_, ws_meta_, + apply_error_buf_); + } + return ret; +} + +int wsrep::transaction::after_statement() +{ + int ret(0); + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex()); + debug_log_state("after_statement_enter"); + assert(client_state_.mode() == wsrep::client_state::m_local); + assert(state() == s_executing || + state() == s_prepared || + state() == s_committed || + state() == s_aborted || + state() == s_must_abort || + state() == s_cert_failed || + state() == s_must_replay); + + if (state() == s_executing && + streaming_context_.fragment_size() && + streaming_context_.fragment_unit() == streaming_context::statement) + { + ret = streaming_step(lock); + } + + switch (state()) + { + case s_executing: + // ? + break; + case s_prepared: + assert(is_xa()); + break; + case s_committed: + assert(is_streaming() == false); + break; + case s_must_abort: + case s_cert_failed: + // Error may be set already. For example, if fragment size + // exceeded the maximum size in certify_fragment(), then + // we already have wsrep::e_error_during_commit + if (client_state_.current_error() == wsrep::e_success) + { + client_state_.override_error(wsrep::e_deadlock_error); + } + lock.unlock(); + ret = client_service_.bf_rollback(); + lock.lock(); + if (state() != s_must_replay) + { + break; + } + // Continue to replay if rollback() changed the state to s_must_replay + WSREP_FALLTHROUGH; + case s_must_replay: + { + if (is_xa() && !ordered()) + { + ret = xa_replay_commit(lock); + } + else + { + ret = replay(lock); + } + break; + } + case s_aborted: + // Raise a deadlock error if the transaction was BF aborted and + // rolled back by client outside of transaction hooks. + if (bf_aborted() && client_state_.current_error() == wsrep::e_success && + !client_service_.is_xa_rollback()) + { + client_state_.override_error(wsrep::e_deadlock_error); + } + break; + default: + assert(0); + break; + } + + assert(state() == s_executing || + state() == s_prepared || + state() == s_committed || + state() == s_aborted || + state() == s_must_replay); + + if (state() == s_aborted) + { + if (ordered()) + { + ret = release_commit_order(lock); + } + lock.unlock(); + provider().release(ws_handle_); + lock.lock(); + } + + if (state() != s_executing && + (!client_service_.is_explicit_xa() || + client_state_.state() == wsrep::client_state::s_quitting)) + { + cleanup(); + } + fragments_certified_for_statement_ = 0; + debug_log_state("after_statement_leave"); + assert(ret == 0 || state() == s_aborted); + return ret; +} + +void wsrep::transaction::after_command_must_abort( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + debug_log_state("after_command_must_abort enter"); + assert(active()); + assert(state_ == s_must_abort); + + if (is_xa() && is_streaming()) + { + state(lock, s_must_replay); + } + + lock.unlock(); + client_service_.bf_rollback(); + lock.lock(); + + if (is_xa() && is_streaming()) + { + xa_replay(lock); + } + else + { + client_state_.override_error(wsrep::e_deadlock_error); + } + + debug_log_state("after_command_must_abort leave"); +} + +void wsrep::transaction::after_applying() +{ + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex_); + debug_log_state("after_applying enter"); + assert(state_ == s_executing || + state_ == s_prepared || + state_ == s_committed || + state_ == s_aborted || + state_ == s_replaying); + + if (state_ != s_executing && state_ != s_prepared) + { + if (state_ == s_replaying) state(lock, s_aborted); + cleanup(); + } + else + { + // State remains executing or prepared, so this is a streaming applier. + // Reset the meta data to avoid releasing commit order + // critical section above if the next fragment is rollback + // fragment. Rollback fragment ordering will be handled by + // another instance while removing the fragments from + // storage. + ws_meta_ = wsrep::ws_meta(); + } + debug_log_state("after_applying leave"); +} + +bool wsrep::transaction::bf_abort( + wsrep::unique_lock<wsrep::mutex>& lock, + wsrep::seqno bf_seqno) +{ + bool ret(false); + const enum wsrep::transaction::state state_at_enter(state()); + assert(lock.owns_lock()); + + if (active() == false) + { + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "Transaction not active, skipping bf abort"); + } + else + { + switch (state_at_enter) + { + case s_executing: + case s_preparing: + case s_prepared: + case s_certifying: + case s_committing: + { + wsrep::seqno victim_seqno; + enum wsrep::provider::status + status(client_state_.provider().bf_abort( + bf_seqno, id_, victim_seqno)); + switch (status) + { + case wsrep::provider::success: + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "Seqno " << bf_seqno + << " successfully BF aborted " << id_ + << " victim_seqno " << victim_seqno); + bf_abort_state_ = state_at_enter; + state(lock, s_must_abort); + ret = true; + break; + default: + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "Seqno " << bf_seqno + << " failed to BF abort " << id_ + << " with status " << status + << " victim_seqno " << victim_seqno); + break; + } + break; + } + default: + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "BF abort not allowed in state " + << wsrep::to_string(state_at_enter)); + break; + } + } + + if (ret) + { + bf_abort_client_state_ = client_state_.state(); + // If the transaction is in executing state, we must initiate + // streaming rollback to ensure that the rollback fragment gets + // replicated before the victim starts to roll back and release locks. + // In other states the BF abort will be detected outside of + // storage engine operations and streaming rollback will be + // handled from before_rollback() call. + if (client_state_.mode() == wsrep::client_state::m_local && + is_streaming() && state_at_enter == s_executing) + { + streaming_rollback(lock); + } + + if ((client_state_.state() == wsrep::client_state::s_idle && + client_state_.server_state().rollback_mode() == + wsrep::server_state::rm_sync) // locally processing idle + || + // high priority streaming + (client_state_.mode() == wsrep::client_state::m_high_priority && + is_streaming())) + { + // We need to change the state to aborting under the + // lock protection to avoid a race between client thread, + // otherwise it could happen that the client gains control + // between releasing the lock and before background + // rollbacker gets control. + if (is_xa() && state_at_enter == s_prepared) + { + state(lock, s_must_replay); + client_state_.set_rollbacker_active(true); + } + else + { + state(lock, s_aborting); + client_state_.set_rollbacker_active(true); + if (client_state_.mode() == wsrep::client_state::m_high_priority) + { + lock.unlock(); + client_state_.server_state().stop_streaming_applier( + server_id_, id_); + lock.lock(); + } + } + + lock.unlock(); + server_service_.background_rollback(client_state_); + } + } + return ret; +} + +bool wsrep::transaction::total_order_bf_abort( + wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED, + wsrep::seqno bf_seqno) +{ + bool ret(bf_abort(lock, bf_seqno)); + if (ret) + { + bf_aborted_in_total_order_ = true; + } + return ret; +} + +void wsrep::transaction::clone_for_replay(const wsrep::transaction& other) +{ + assert(other.state() == s_replaying); + id_ = other.id_; + xid_ = other.xid_; + server_id_ = other.server_id_; + ws_handle_ = other.ws_handle_; + ws_meta_ = other.ws_meta_; + streaming_context_ = other.streaming_context_; + state_ = s_replaying; +} + +void wsrep::transaction::assign_xid(const wsrep::xid& xid) +{ + assert(active()); + assert(!is_xa()); + xid_ = xid; +} + +int wsrep::transaction::restore_to_prepared_state(const wsrep::xid& xid) +{ + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex_); + assert(active()); + assert(is_empty()); + assert(state() == s_executing || state() == s_replaying); + flags(flags() & ~wsrep::provider::flag::start_transaction); + if (state() == s_executing) + { + state(lock, s_certifying); + } + else + { + state(lock, s_preparing); + } + state(lock, s_prepared); + xid_ = xid; + return 0; +} + +int wsrep::transaction::commit_or_rollback_by_xid(const wsrep::xid& xid, + bool commit) +{ + debug_log_state("commit_or_rollback_by_xid enter"); + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex_); + wsrep::server_state& server_state(client_state_.server_state()); + wsrep::high_priority_service* sa(server_state.find_streaming_applier(xid)); + + if (!sa) + { + assert(sa); + client_state_.override_error(wsrep::e_error_during_commit); + return 1; + } + + if (commit) + { + flags(wsrep::provider::flag::commit); + } + else + { + flags(wsrep::provider::flag::rollback); + } + pa_unsafe(true); + wsrep::stid stid(sa->transaction().server_id(), + sa->transaction().id(), + client_state_.id()); + wsrep::ws_meta meta(stid); + + const enum wsrep::provider::status cert_ret( + provider().certify(client_state_.id(), + ws_handle_, + flags(), + meta)); + + int ret; + if (cert_ret == wsrep::provider::success) + { + if (commit) + { + state(lock, s_certifying); + state(lock, s_committing); + state(lock, s_committed); + } + else + { + state(lock, s_aborting); + state(lock, s_aborted); + } + ret = 0; + } + else + { + client_state_.override_error(wsrep::e_error_during_commit, + cert_ret); + wsrep::log_error() << "Failed to commit_or_rollback_by_xid," + << " xid: " << xid + << " error: " << cert_ret; + ret = 1; + } + debug_log_state("commit_or_rollback_by_xid leave"); + return ret; +} + +void wsrep::transaction::xa_detach() +{ + debug_log_state("xa_detach enter"); + assert(state() == s_prepared || + client_state_.state() == wsrep::client_state::s_quitting); + if (state() == s_prepared) + { + wsrep::server_state& server_state(client_state_.server_state()); + server_state.convert_streaming_client_to_applier(&client_state_); + client_service_.store_globals(); + client_service_.cleanup_transaction(); + } + wsrep::unique_lock<wsrep::mutex> lock(client_state_.mutex_); + streaming_context_.cleanup(); + state(lock, s_aborting); + state(lock, s_aborted); + provider().release(ws_handle_); + cleanup(); + debug_log_state("xa_detach leave"); +} + +void wsrep::transaction::xa_replay_common(wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + assert(is_xa()); + assert(is_streaming()); + assert(state() == s_must_replay); + assert(bf_aborted()); + + state(lock, s_replaying); + + enum wsrep::provider::status status; + wsrep::server_state& server_state(client_state_.server_state()); + + lock.unlock(); + server_state.convert_streaming_client_to_applier(&client_state_); + status = client_service_.replay_unordered(); + client_service_.store_globals(); + lock.lock(); + + if (status != wsrep::provider::success) + { + client_service_.emergency_shutdown(); + } +} + +int wsrep::transaction::xa_replay(wsrep::unique_lock<wsrep::mutex>& lock) +{ + debug_log_state("xa_replay enter"); + xa_replay_common(lock); + state(lock, s_aborted); + streaming_context_.cleanup(); + provider().release(ws_handle_); + cleanup(); + client_service_.signal_replayed(); + debug_log_state("xa_replay leave"); + return 0; +} + +int wsrep::transaction::xa_replay_commit(wsrep::unique_lock<wsrep::mutex>& lock) +{ + debug_log_state("xa_replay_commit enter"); + xa_replay_common(lock); + lock.unlock(); + enum wsrep::provider::status status(client_service_.commit_by_xid()); + lock.lock(); + int ret(1); + switch (status) + { + case wsrep::provider::success: + state(lock, s_committed); + streaming_context_.cleanup(); + provider().release(ws_handle_); + cleanup(); + ret = 0; + break; + default: + log_warning() << "Failed to commit by xid during replay"; + // Commit by xid failed, return a commit + // error and let the client retry + state(lock, s_preparing); + state(lock, s_prepared); + client_state_.override_error(wsrep::e_error_during_commit, status); + } + + client_service_.signal_replayed(); + debug_log_state("xa_replay_commit leave"); + return ret; +} + +//////////////////////////////////////////////////////////////////////////////// +// Private // +//////////////////////////////////////////////////////////////////////////////// + +inline wsrep::provider& wsrep::transaction::provider() +{ + return client_state_.server_state().provider(); +} + +void wsrep::transaction::state( + wsrep::unique_lock<wsrep::mutex>& lock __attribute__((unused)), + enum wsrep::transaction::state next_state) +{ + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "client: " << client_state_.id().get() + << " txc: " << id().get() + << " state: " << to_string(state_) + << " -> " << to_string(next_state)); + + assert(lock.owns_lock()); + // BF aborter is allowed to change the state without gaining control + // to the state if the next state is s_must_abort, s_aborting or + // s_must_replay (for xa idle replay). + assert(client_state_.owning_thread_id_ == wsrep::this_thread::get_id() || + next_state == s_must_abort || + next_state == s_must_replay || + next_state == s_aborting); + + static const char allowed[n_states][n_states] = + { /* ex pg pd ce co oc ct cf ma ab ad mr re */ + { 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0}, /* ex */ + { 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0}, /* pg */ + { 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0}, /* pd */ + { 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0}, /* ce */ + { 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0}, /* co */ + { 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0}, /* oc */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, /* ct */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0}, /* cf */ + { 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0}, /* ma */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}, /* ab */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, /* ad */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, /* mr */ + { 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0} /* re */ + }; + + if (!allowed[state_][next_state]) + { + wsrep::log_debug() << "unallowed state transition for transaction " + << id_ << ": " << wsrep::to_string(state_) + << " -> " << wsrep::to_string(next_state); + assert(0); + } + + state_hist_.push_back(state_); + if (state_hist_.size() == 12) + { + state_hist_.erase(state_hist_.begin()); + } + state_ = next_state; + + if (state_ == s_must_replay) + { + client_service_.will_replay(); + } +} + +bool wsrep::transaction::abort_or_interrupt( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + if (state() == s_must_abort) + { + client_state_.override_error(wsrep::e_deadlock_error); + return true; + } + else if (state() == s_aborting || state() == s_aborted) + { + // Somehow we got there after BF abort without setting error + // status. This may happen if the DBMS side aborts the transaction + // but still tries to continue processing rows. Raise the error here + // and assert so that the error will be caught in debug builds. + if (bf_abort_client_state_ && + client_state_.current_error() == wsrep::e_success) + { + client_state_.override_error(wsrep::e_deadlock_error); + assert(0); + } + return true; + } + else if (client_service_.interrupted(lock)) + { + client_state_.override_error(wsrep::e_interrupted_error); + if (state() != s_must_abort) + { + state(lock, s_must_abort); + } + return true; + } + return false; +} + +int wsrep::transaction::streaming_step(wsrep::unique_lock<wsrep::mutex>& lock, + bool force) +{ + assert(lock.owns_lock()); + assert(streaming_context_.fragment_size() || is_xa()); + + if (client_service_.bytes_generated() < + streaming_context_.log_position()) + { + /* Something went wrong on DBMS side in keeping track of + generated bytes. Return an error to abort the transaction. */ + wsrep::log_warning() << "Bytes generated " + << client_service_.bytes_generated() + << " less than bytes certified " + << streaming_context_.log_position() + << ", aborting streaming transaction"; + return 1; + } + int ret(0); + + const size_t bytes_to_replicate(client_service_.bytes_generated() - + streaming_context_.log_position()); + + switch (streaming_context_.fragment_unit()) + { + case streaming_context::row: + WSREP_FALLTHROUGH; + case streaming_context::statement: + streaming_context_.increment_unit_counter(1); + break; + case streaming_context::bytes: + streaming_context_.set_unit_counter(bytes_to_replicate); + break; + } + + // Some statements have no effect. Do not atttempt to + // replicate a fragment if no data has been generated since + // last fragment replication. + // We use `force = true` on XA prepare: a fragment will be + // generated even if no data is pending replication. + if (bytes_to_replicate <= 0 && !force) + { + assert(bytes_to_replicate == 0); + return ret; + } + + if (streaming_context_.fragment_size_exceeded() || force) + { + streaming_context_.reset_unit_counter(); + ret = certify_fragment(lock); + } + + return ret; +} + +int wsrep::transaction::certify_fragment( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + + assert(client_state_.mode() == wsrep::client_state::m_local); + assert(streaming_context_.rolled_back() == false || + state() == s_must_abort); + + client_service_.wait_for_replayers(lock); + if (abort_or_interrupt(lock)) + { + return 1; + } + + state(lock, s_certifying); + lock.unlock(); + client_service_.debug_sync("wsrep_before_fragment_certification"); + + enum wsrep::provider::status status( + client_state_.server_state_.send_pending_rollback_events()); + if (status) + { + wsrep::log_warning() + << "Failed to replicate pending rollback events: " + << status << " (" + << wsrep::provider::to_string(status) << ")"; + lock.lock(); + state(lock, s_must_abort); + return 1; + } + + wsrep::mutable_buffer data; + size_t log_position(0); + if (client_service_.prepare_fragment_for_replication(data, log_position)) + { + lock.lock(); + state(lock, s_must_abort); + client_state_.override_error(wsrep::e_error_during_commit); + return 1; + } + streaming_context_.set_log_position(log_position); + + if (data.size() == 0) + { + wsrep::log_warning() << "Attempt to replicate empty data buffer"; + lock.lock(); + state(lock, s_executing); + return 0; + } + + if (provider().append_data(ws_handle_, + wsrep::const_buffer(data.data(), data.size()))) + { + lock.lock(); + state(lock, s_must_abort); + client_state_.override_error(wsrep::e_error_during_commit); + return 1; + } + + if (is_xa()) + { + // One more check to see if the transaction + // has been aborted. This is necessary because + // until the append_data above will make sure + // that the transaction exists in provider. + lock.lock(); + if (abort_or_interrupt(lock)) + { + return 1; + } + lock.unlock(); + } + + if (is_streaming() == false) + { + client_state_.server_state_.start_streaming_client(&client_state_); + } + + if (implicit_deps()) + { + flags(flags() | wsrep::provider::flag::implicit_deps); + } + + int ret(0); + enum wsrep::client_error error(wsrep::e_success); + enum wsrep::provider::status cert_ret(wsrep::provider::success); + // Storage service scope + { + scoped_storage_service<storage_service_deleter> + sr_scope( + client_service_, + server_service_.storage_service(client_service_), + storage_service_deleter(server_service_)); + wsrep::storage_service& storage_service( + sr_scope.storage_service()); + + // First the fragment is appended to the stable storage. + // This is done to ensure that there is enough capacity + // available to store the fragment. The fragment meta data + // is updated after certification. + wsrep::id server_id(client_state_.server_state().id()); + + if (server_id.is_undefined()) { + // Server disconnected from cluster, do not + // append a fragment with undefined server_id. + ret = 1; + error = wsrep::e_append_fragment_error; + } + + if (ret == 0 && + (storage_service.start_transaction(ws_handle_) || + storage_service.append_fragment( + server_id, + id(), + flags(), + wsrep::const_buffer(data.data(), data.size()), + xid()))) + { + ret = 1; + error = wsrep::e_append_fragment_error; + } + + if (ret == 0) + { + client_service_.debug_crash( + "crash_replicate_fragment_before_certify"); + + wsrep::ws_meta sr_ws_meta; + cert_ret = provider().certify(client_state_.id(), + ws_handle_, + flags(), + sr_ws_meta); + client_service_.debug_crash( + "crash_replicate_fragment_after_certify"); + + switch (cert_ret) + { + case wsrep::provider::success: + ++fragments_certified_for_statement_; + assert(sr_ws_meta.seqno().is_undefined() == false); + streaming_context_.certified(); + if (storage_service.update_fragment_meta(sr_ws_meta)) + { + storage_service.rollback(wsrep::ws_handle(), + wsrep::ws_meta()); + ret = 1; + error = wsrep::e_deadlock_error; + break; + } + if (storage_service.commit(ws_handle_, sr_ws_meta)) + { + ret = 1; + error = wsrep::e_deadlock_error; + } + else + { + streaming_context_.stored(sr_ws_meta.seqno()); + } + client_service_.debug_crash( + "crash_replicate_fragment_success"); + break; + case wsrep::provider::error_bf_abort: + case wsrep::provider::error_certification_failed: + // Streaming transcation got BF aborted, so it must roll + // back. Roll back the fragment storage operation out of + // order as the commit order will be grabbed later on + // during rollback process. Mark the fragment as certified + // though in streaming context in order to enter streaming + // rollback codepath. + // + // Note that despite we handle error_certification_failed + // here, we mark the transaction as streaming. Apparently + // the provider may return status corresponding to certification + // failure even if the fragment has passed certification. + // This may be a bug in provider implementation or a limitation + // of error codes defined in wsrep-API. In order to make + // sure that the transaction will be cleaned on other servers, + // we take a risk of sending one rollback fragment for nothing. + storage_service.rollback(wsrep::ws_handle(), + wsrep::ws_meta()); + streaming_context_.certified(); + ret = 1; + error = wsrep::e_deadlock_error; + break; + default: + // Storage service rollback must be done out of order, + // otherwise there may be a deadlock between BF aborter + // and the rollback process. + storage_service.rollback(wsrep::ws_handle(), wsrep::ws_meta()); + ret = 1; + error = wsrep::e_deadlock_error; + break; + } + } + } + + // Note: This does not release the handle in the provider + // since streaming is still on. However it is needed to + // make provider internal state to transition for the + // next fragment. If any of the operations above failed, + // the handle needs to be left unreleased for the following + // rollback process. + if (ret == 0) + { + assert(error == wsrep::e_success); + ret = provider().release(ws_handle_); + if (ret) + { + error = wsrep::e_deadlock_error; + } + } + lock.lock(); + if (ret) + { + assert(error != wsrep::e_success); + if (is_streaming() == false) + { + lock.unlock(); + client_state_.server_state_.stop_streaming_client(&client_state_); + lock.lock(); + } + else + { + streaming_rollback(lock); + } + if (state_ != s_must_abort) + { + state(lock, s_must_abort); + } + client_state_.override_error(error, cert_ret); + } + else if (state_ == s_must_abort) + { + if (is_streaming()) + { + streaming_rollback(lock); + } + client_state_.override_error(wsrep::e_deadlock_error, cert_ret); + ret = 1; + } + else + { + assert(state_ == s_certifying); + state(lock, s_executing); + flags(flags() & ~wsrep::provider::flag::start_transaction); + flags(flags() & ~wsrep::provider::flag::pa_unsafe); + } + return ret; +} + +int wsrep::transaction::certify_commit( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + assert(lock.owns_lock()); + assert(active()); + client_service_.wait_for_replayers(lock); + + assert(lock.owns_lock()); + + if (abort_or_interrupt(lock)) + { + if (is_xa() && state() == s_must_abort) + { + state(lock, s_must_replay); + } + return 1; + } + + state(lock, s_certifying); + lock.unlock(); + + enum wsrep::provider::status status( + client_state_.server_state_.send_pending_rollback_events()); + if (status) + { + wsrep::log_warning() + << "Failed to replicate pending rollback events: " + << status << " (" + << wsrep::provider::to_string(status) << ")"; + + // We failed to replicate some pending rollback fragment. + // Meaning that some transaction that was rolled back + // locally might still be active out there in the cluster. + // To avoid a potential BF-BF conflict, we need to abort + // and give up on this one. + // Notice that we can't abort a prepared XA that wants to + // commit. Fortunately, there is no need to in this case: + // the commit fragment for XA does not cause any changes and + // can't possibly conflict with other transactions out there. + if (!is_xa()) + { + lock.lock(); + state(lock, s_must_abort); + return 1; + } + } + + if (is_streaming()) + { + if (!is_xa()) + { + append_sr_keys_for_commit(); + } + pa_unsafe(true); + } + + if (implicit_deps()) + { + flags(flags() | wsrep::provider::flag::implicit_deps); + } + + flags(flags() | wsrep::provider::flag::commit); + flags(flags() & ~wsrep::provider::flag::prepare); + + if (client_service_.prepare_data_for_replication()) + { + lock.lock(); + // Here we fake that the size exceeded error came from provider, + // even though it came from the client service. This requires + // some consideration how to get meaningful error codes from + // the client service. + client_state_.override_error(wsrep::e_size_exceeded_error, + wsrep::provider::error_size_exceeded); + if (state_ != s_must_abort) + { + state(lock, s_must_abort); + } + return 1; + } + + client_service_.debug_sync("wsrep_before_certification"); + enum wsrep::provider::status + cert_ret(provider().certify(client_state_.id(), + ws_handle_, + flags(), + ws_meta_)); + client_service_.debug_sync("wsrep_after_certification"); + + lock.lock(); + + assert(state() == s_certifying || state() == s_must_abort); + + int ret(1); + switch (cert_ret) + { + case wsrep::provider::success: + assert(ordered()); + certified_ = true; + ++fragments_certified_for_statement_; + switch (state()) + { + case s_certifying: + if (is_xa()) + { + state(lock, s_committing); + } + else + { + state(lock, s_preparing); + } + ret = 0; + break; + case s_must_abort: + // We got BF aborted after succesful certification + // and before acquiring client state lock. The trasaction + // must be replayed. + state(lock, s_must_replay); + break; + default: + assert(0); + break; + } + break; + case wsrep::provider::error_warning: + assert(ordered() == false); + state(lock, s_must_abort); + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + break; + case wsrep::provider::error_transaction_missing: + state(lock, s_must_abort); + // The execution should never reach this point if the + // transaction has not generated any keys or data. + wsrep::log_warning() << "Transaction was missing in provider"; + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + break; + case wsrep::provider::error_bf_abort: + // Transaction was replicated successfully and it was either + // certified successfully or the result of certifying is not + // yet known. Therefore the transaction must roll back + // and go through replay either to replay and commit the whole + // transaction or to determine failed certification status. + if (state() != s_must_abort) + { + state(lock, s_must_abort); + } + state(lock, s_must_replay); + break; + case wsrep::provider::error_certification_failed: + state(lock, s_cert_failed); + client_state_.override_error(wsrep::e_deadlock_error); + break; + case wsrep::provider::error_size_exceeded: + state(lock, s_must_abort); + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + break; + case wsrep::provider::error_connection_failed: + // Galera provider may return CONN_FAIL if the trx is + // BF aborted O_o. If we see here that the trx was BF aborted, + // return deadlock error instead of error during commit + // to reduce number of error state combinations elsewhere. + if (state() == s_must_abort) + { + if (is_xa()) + { + state(lock, s_must_replay); + } + client_state_.override_error(wsrep::e_deadlock_error); + } + else + { + client_state_.override_error(wsrep::e_error_during_commit, + cert_ret); + if (is_xa()) + { + state(lock, s_prepared); + } + else + { + state(lock, s_must_abort); + } + } + break; + case wsrep::provider::error_provider_failed: + if (state() != s_must_abort) + { + state(lock, s_must_abort); + } + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + break; + case wsrep::provider::error_fatal: + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + state(lock, s_must_abort); + client_service_.emergency_shutdown(); + break; + case wsrep::provider::error_not_implemented: + case wsrep::provider::error_not_allowed: + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + state(lock, s_must_abort); + wsrep::log_warning() << "Certification operation was not allowed: " + << "id: " << id().get() + << " flags: " << std::hex << flags() << std::dec; + break; + default: + state(lock, s_must_abort); + client_state_.override_error(wsrep::e_error_during_commit, cert_ret); + break; + } + + return ret; +} + +int wsrep::transaction::append_sr_keys_for_commit() +{ + int ret(0); + assert(client_state_.mode() == wsrep::client_state::m_local); + for (wsrep::sr_key_set::branch_type::const_iterator + i(sr_keys_.root().begin()); + ret == 0 && i != sr_keys_.root().end(); ++i) + { + for (wsrep::sr_key_set::leaf_type::const_iterator + j(i->second.begin()); + ret == 0 && j != i->second.end(); ++j) + { + wsrep::key key(wsrep::key::shared); + key.append_key_part(i->first.data(), i->first.size()); + key.append_key_part(j->data(), j->size()); + ret = provider().append_key(ws_handle_, key); + } + } + return ret; +} + +void wsrep::transaction::streaming_rollback( + wsrep::unique_lock<wsrep::mutex>& lock) +{ + debug_log_state("streaming_rollback enter"); + assert(state_ != s_must_replay); + assert(is_streaming()); + assert(lock.owns_lock()); + + // Prevent streaming_rollback() to be executed simultaneously. + // Notice that lock is unlocked when calling into server_state + // methods, to avoid violating lock order. + // The condition variable below prevents a thread to go + // through streaming_rollback() while another thread is busy + // stopping or converting the streaming_client(). + // This would be problematic if a thread is performing BF abort, + // while the original client manages to complete its rollback + // and therefore change the state of the transaction, causing + // assertions to fire. + while (streaming_rollback_in_progress_) + client_state_.cond_.wait(lock); + streaming_rollback_in_progress_ = true; + + if (streaming_context_.rolled_back() == false) + { + if (bf_aborted_in_total_order_) + { + lock.unlock(); + client_state_.server_state_.stop_streaming_client(&client_state_); + lock.lock(); + } + else + { + // Create a high priority applier which will handle the + // rollback fragment or clean up on configuration change. + // Adopt transaction will copy fragment set and appropriate + // meta data. + lock.unlock(); + server_service_.debug_sync("wsrep_streaming_rollback"); + client_state_.server_state_.convert_streaming_client_to_applier( + &client_state_); + lock.lock(); + streaming_context_.cleanup(); + + enum wsrep::provider::status status(provider().rollback(id_)); + if (status) + { + lock.unlock(); + client_state_.server_state_.queue_rollback_event(id_); + lock.lock(); + wsrep::log_debug() + << "Failed to replicate rollback fragment for " << id_ + << ": " << status << " ( " + << wsrep::provider::to_string(status) << ")"; + } + } + + // Mark the streaming context as rolled back, + // so that this block is executed once. + streaming_context_.rolled_back(id_); + } + + debug_log_state("streaming_rollback leave"); + streaming_rollback_in_progress_ = false; + client_state_.cond_.notify_all(); +} + +int wsrep::transaction::replay(wsrep::unique_lock<wsrep::mutex>& lock) +{ + int ret(0); + state(lock, s_replaying); + // Need to remember streaming state before replay, entering + // after_commit() after succesful replay will clear + // fragments. + const bool was_streaming(is_streaming()); + lock.unlock(); + client_service_.debug_sync("wsrep_before_replay"); + enum wsrep::provider::status replay_ret(client_service_.replay()); + client_service_.signal_replayed(); + if (was_streaming) + { + client_state_.server_state_.stop_streaming_client(&client_state_); + } + lock.lock(); + switch (replay_ret) + { + case wsrep::provider::success: + if (state() == s_replaying) + { + // Replay was done by using different client state, adjust state + // to committed. + state(lock, s_committed); + } + if (is_streaming()) + { + streaming_context_.cleanup(); + } + provider().release(ws_handle_); + break; + case wsrep::provider::error_certification_failed: + client_state_.override_error( + wsrep::e_deadlock_error); + if (is_streaming()) + { + client_service_.remove_fragments(); + streaming_context_.cleanup(); + } + state(lock, s_aborted); + ret = 1; + break; + default: + client_service_.emergency_shutdown(); + break; + } + + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "replay returned" << replay_ret); + return ret; +} + +void wsrep::transaction::cleanup() +{ + debug_log_state("cleanup_enter"); + assert(is_streaming() == false); + assert(state() == s_committed || state() == s_aborted); + id_ = wsrep::transaction_id::undefined(); + ws_handle_ = wsrep::ws_handle(); + // Keep the state history for troubleshooting. Reset + // at start_transaction(). + // state_hist_.clear(); + if (ordered()) + { + client_state_.update_last_written_gtid(ws_meta_.gtid()); + } + bf_abort_state_ = s_executing; + bf_abort_provider_status_ = wsrep::provider::success; + bf_abort_client_state_ = 0; + bf_aborted_in_total_order_ = false; + ws_meta_ = wsrep::ws_meta(); + flags_ = 0; + certified_ = false; + implicit_deps_ = false; + sr_keys_.clear(); + streaming_context_.cleanup(); + client_service_.cleanup_transaction(); + apply_error_buf_.clear(); + xid_.clear(); + debug_log_state("cleanup_leave"); +} + +void wsrep::transaction::debug_log_state( + const char* context) const +{ + WSREP_LOG_DEBUG( + client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + context + << "\n server: " << server_id_ + << ", client: " << int64_t(client_state_.id().get()) + << ", state: " << wsrep::to_c_string(client_state_.state()) + << ", mode: " << wsrep::to_c_string(client_state_.mode()) + << "\n trx_id: " << int64_t(id_.get()) + << ", seqno: " << ws_meta_.seqno().get() + << ", flags: " << flags() + << "\n" + << " state: " << wsrep::to_c_string(state_) + << ", bfa_state: " << wsrep::to_c_string(bf_abort_state_) + << ", error: " << wsrep::to_c_string(client_state_.current_error()) + << ", status: " << client_state_.current_error_status() + << "\n" + << " is_sr: " << is_streaming() + << ", frags: " << streaming_context_.fragments_certified() + << ", frags size: " << streaming_context_.fragments().size() + << ", unit: " << streaming_context_.fragment_unit() + << ", size: " << streaming_context_.fragment_size() + << ", counter: " << streaming_context_.unit_counter() + << ", log_pos: " << streaming_context_.log_position() + << ", sr_rb: " << streaming_context_.rolled_back() + << "\n own: " << (client_state_.owning_thread_id_ == wsrep::this_thread::get_id()) + << " thread_id: " << client_state_.owning_thread_id_ + << ""); +} + +void wsrep::transaction::debug_log_key_append(const wsrep::key& key) const +{ + WSREP_LOG_DEBUG(client_state_.debug_log_level(), + wsrep::log::debug_level_transaction, + "key_append: " + << "trx_id: " + << int64_t(id().get()) + << " append key:\n" << key); +} diff --git a/wsrep-lib/src/uuid.cpp b/wsrep-lib/src/uuid.cpp new file mode 100644 index 00000000..9fe41779 --- /dev/null +++ b/wsrep-lib/src/uuid.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "uuid.hpp" + +#include <cstring> +#include <cerrno> +#include <cstdio> +#include <cctype> + +int wsrep::uuid_scan (const char* str, size_t str_len, wsrep::uuid_t* uuid) +{ + unsigned int uuid_len = 0; + unsigned int uuid_offt = 0; + + while (uuid_len + 1 < str_len) { + /* We are skipping potential '-' after uuid_offt == 4, 6, 8, 10 + * which means + * (uuid_offt >> 1) == 2, 3, 4, 5, + * which in turn means + * (uuid_offt >> 1) - 2 <= 3 + * since it is always >= 0, because uuid_offt is unsigned */ + if (((uuid_offt >> 1) - 2) <= 3 && str[uuid_len] == '-') { + // skip dashes after 4th, 6th, 8th and 10th positions + uuid_len += 1; + continue; + } + + if (isxdigit(str[uuid_len]) && isxdigit(str[uuid_len + 1])) { + // got hex digit, scan another byte to uuid, increment uuid_offt + sscanf (str + uuid_len, "%2hhx", uuid->data + uuid_offt); + uuid_len += 2; + uuid_offt += 1; + if (sizeof (uuid->data) == uuid_offt) + return static_cast<int>(uuid_len); + } + else { + break; + } + } + + *uuid = wsrep::uuid_initializer; + return -EINVAL; +} + +int wsrep::uuid_print (const wsrep::uuid_t* uuid, char* str, size_t str_len) +{ + if (str_len > WSREP_LIB_UUID_STR_LEN) { + const unsigned char* u = uuid->data; + return snprintf(str, str_len, "%02x%02x%02x%02x-%02x%02x-%02x%02x-" + "%02x%02x-%02x%02x%02x%02x%02x%02x", + u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6], u[ 7], + u[ 8], u[ 9], u[10], u[11], u[12], u[13], u[14], u[15]); + } + else { + return -EMSGSIZE; + } +} diff --git a/wsrep-lib/src/uuid.hpp b/wsrep-lib/src/uuid.hpp new file mode 100644 index 00000000..72812a76 --- /dev/null +++ b/wsrep-lib/src/uuid.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file uuid.hpp + * + * Helper methods to parse and print UUIDs, intended to use + * internally in wsrep-lib. + * + * The implementation is copied from wsrep-API v26. + */ + +#ifndef WSREP_UUID_HPP +#define WSREP_UUID_HPP + +#include "wsrep/compiler.hpp" + +#include <cstddef> + +/** + * Length of UUID string representation, not including terminating '\0'. + */ +#define WSREP_LIB_UUID_STR_LEN 36 + +namespace wsrep +{ + /** + * UUID type. + */ + typedef union uuid_ + { + unsigned char data[16]; + size_t alignment; + } uuid_t; + + static const wsrep::uuid_t uuid_initializer = {{0, }}; + + /** + * Read UUID from string. + * + * @param str String to read from + * @param str_len Length of string + * @param[out] UUID to read to + * + * @return Number of bytes read or negative error code in case + * of error. + */ + int uuid_scan(const char* str, size_t str_len, wsrep::uuid_t* uuid); + + /** + * Write UUID to string. The caller must allocate at least + * WSREP_LIB_UUID_STR_LEN + 1 space for the output str parameter. + * + * @param uuid UUID to print + * @param str[out] Output buffer + * @param str_len Size of output buffer + * + * @return Number of chars printerd, negative error code in case of + * error. + */ + int uuid_print(const wsrep::uuid_t* uuid, char* str, size_t str_len); +} + +#endif // WSREP_UUID_HPP diff --git a/wsrep-lib/src/view.cpp b/wsrep-lib/src/view.cpp new file mode 100644 index 00000000..d5bff099 --- /dev/null +++ b/wsrep-lib/src/view.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/view.hpp" +#include "wsrep/provider.hpp" + +int wsrep::view::member_index(const wsrep::id& member_id) const +{ + for (std::vector<member>::const_iterator i(members_.begin()); + i != members_.end(); ++i) + { + if (i->id() == member_id) + { + return static_cast<int>(i - members_.begin()); + } + } + return -1; +} + +bool wsrep::view::equal_membership(const wsrep::view& other) const +{ + if (members_.size() != other.members_.size()) + { + return false; + } + // we can't assume members ordering + for (std::vector<member>::const_iterator i(members_.begin()); + i != members_.end(); ++i) + { + if (other.member_index(i->id()) == -1) + { + return false; + } + } + return true; +} + +void wsrep::view::print(std::ostream& os) const +{ + os << " id: " << state_id() << "\n" + << " status: " << to_c_string(status()) << "\n" + << " protocol_version: " << protocol_version() << "\n" + << " capabilities: " << provider::capability::str(capabilities())<<"\n" + << " final: " << (final() ? "yes" : "no") << "\n" + << " own_index: " << own_index() << "\n" + << " members(" << members().size() << "):\n"; + + for (std::vector<wsrep::view::member>::const_iterator i(members().begin()); + i != members().end(); ++i) + { + os << "\t" << (i - members().begin()) /* ordinal index */ + << ": " << i->id() + << ", " << i->name() << "\n"; + } +} diff --git a/wsrep-lib/src/wsrep_provider_v26.cpp b/wsrep-lib/src/wsrep_provider_v26.cpp new file mode 100644 index 00000000..eb8679c7 --- /dev/null +++ b/wsrep-lib/src/wsrep_provider_v26.cpp @@ -0,0 +1,1183 @@ +/* + * Copyright (C) 2018-2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep_provider_v26.hpp" + +#include "wsrep/encryption_service.hpp" +#include "wsrep/server_state.hpp" +#include "wsrep/high_priority_service.hpp" +#include "wsrep/view.hpp" +#include "wsrep/exception.hpp" +#include "wsrep/logger.hpp" +#include "wsrep/thread_service.hpp" +#include "wsrep/tls_service.hpp" +#include "wsrep/allowlist_service.hpp" + +#include "thread_service_v1.hpp" +#include "tls_service_v1.hpp" +#include "allowlist_service_v1.hpp" +#include "event_service_v1.hpp" +#include "v26/wsrep_api.h" + + +#include <dlfcn.h> +#include <cassert> +#include <climits> + +#include <iostream> +#include <sstream> +#include <cstring> // strerror() + +namespace +{ + ///////////////////////////////////////////////////////////////////// + // Helpers // + ///////////////////////////////////////////////////////////////////// + + enum wsrep::provider::status map_return_value(wsrep_status_t status) + { + switch (status) + { + case WSREP_OK: + return wsrep::provider::success; + case WSREP_WARNING: + return wsrep::provider::error_warning; + case WSREP_TRX_MISSING: + return wsrep::provider::error_transaction_missing; + case WSREP_TRX_FAIL: + return wsrep::provider::error_certification_failed; + case WSREP_BF_ABORT: + return wsrep::provider::error_bf_abort; + case WSREP_SIZE_EXCEEDED: + return wsrep::provider::error_size_exceeded; + case WSREP_CONN_FAIL: + return wsrep::provider::error_connection_failed; + case WSREP_NODE_FAIL: + return wsrep::provider::error_provider_failed; + case WSREP_FATAL: + return wsrep::provider::error_fatal; + case WSREP_NOT_IMPLEMENTED: + return wsrep::provider::error_not_implemented; + case WSREP_NOT_ALLOWED: + return wsrep::provider::error_not_allowed; + } + + wsrep::log_warning() << "Unexpected value for wsrep_status_t: " + << status << " (" + << (status < 0 ? strerror(-status) : "") << ')'; + + return wsrep::provider::error_unknown; + } + + wsrep_key_type_t map_key_type(enum wsrep::key::type type) + { + switch (type) + { + case wsrep::key::shared: return WSREP_KEY_SHARED; + case wsrep::key::reference: return WSREP_KEY_REFERENCE; + case wsrep::key::update: return WSREP_KEY_UPDATE; + case wsrep::key::exclusive: return WSREP_KEY_EXCLUSIVE; + } + assert(0); + throw wsrep::runtime_error("Invalid key type"); + } + + static inline wsrep_seqno_t seqno_to_native(wsrep::seqno seqno) + { + return seqno.get(); + } + + static inline wsrep::seqno seqno_from_native(wsrep_seqno_t seqno) + { + return wsrep::seqno(seqno); + } + + template <typename F, typename T> + inline uint32_t map_one(const int flags, const F from, + const T to) + { + // INT_MAX because GCC 4.4 does not eat numeric_limits<int>::max() + // in static_assert + static_assert(WSREP_FLAGS_LAST < INT_MAX, + "WSREP_FLAGS_LAST < INT_MAX"); + return static_cast<uint32_t>((flags & static_cast<int>(from)) ? + static_cast<int>(to) : 0); + } + + uint32_t map_flags_to_native(int flags) + { + using wsrep::provider; + return static_cast<uint32_t>( + map_one(flags, provider::flag::start_transaction, + WSREP_FLAG_TRX_START) | + map_one(flags, provider::flag::commit, WSREP_FLAG_TRX_END) | + map_one(flags, provider::flag::rollback, WSREP_FLAG_ROLLBACK) | + map_one(flags, provider::flag::isolation, WSREP_FLAG_ISOLATION) | + map_one(flags, provider::flag::pa_unsafe, WSREP_FLAG_PA_UNSAFE) | + // map_one(flags, provider::flag::commutative, WSREP_FLAG_COMMUTATIVE) + // | + // map_one(flags, provider::flag::native, WSREP_FLAG_NATIVE) | + map_one(flags, provider::flag::prepare, WSREP_FLAG_TRX_PREPARE) | + map_one(flags, provider::flag::snapshot, WSREP_FLAG_SNAPSHOT) | + map_one(flags, provider::flag::implicit_deps, + WSREP_FLAG_IMPLICIT_DEPS)); + } + + int map_flags_from_native(uint32_t flags_u32) + { + using wsrep::provider; + const int flags(static_cast<int>(flags_u32)); + return static_cast<int>( + map_one(flags, WSREP_FLAG_TRX_START, + provider::flag::start_transaction) | + map_one(flags, WSREP_FLAG_TRX_END, provider::flag::commit) | + map_one(flags, WSREP_FLAG_ROLLBACK, provider::flag::rollback) | + map_one(flags, WSREP_FLAG_ISOLATION, provider::flag::isolation) | + map_one(flags, WSREP_FLAG_PA_UNSAFE, provider::flag::pa_unsafe) | + // map_one(flags, provider::flag::commutative, WSREP_FLAG_COMMUTATIVE) + // | + // map_one(flags, provider::flag::native, WSREP_FLAG_NATIVE) | + map_one(flags, WSREP_FLAG_TRX_PREPARE, provider::flag::prepare) | + map_one(flags, WSREP_FLAG_SNAPSHOT, provider::flag::snapshot) | + map_one(flags, WSREP_FLAG_IMPLICIT_DEPS, + provider::flag::implicit_deps)); + } + + class mutable_ws_handle + { + public: + mutable_ws_handle(wsrep::ws_handle& ws_handle) + : ws_handle_(ws_handle) + , native_((wsrep_ws_handle_t) + { + ws_handle_.transaction_id().get(), + ws_handle_.opaque() + }) + { } + + ~mutable_ws_handle() + { + ws_handle_ = wsrep::ws_handle( + wsrep::transaction_id(native_.trx_id), native_.opaque); + } + + wsrep_ws_handle_t* native() + { + return &native_; + } + private: + wsrep::ws_handle& ws_handle_; + wsrep_ws_handle_t native_; + }; + + class const_ws_handle + { + public: + const_ws_handle(const wsrep::ws_handle& ws_handle) + : ws_handle_(ws_handle) + , native_((wsrep_ws_handle_t) + { + ws_handle_.transaction_id().get(), + ws_handle_.opaque() + }) + { } + + ~const_ws_handle() + { + assert(ws_handle_.transaction_id().get() == native_.trx_id); + assert(ws_handle_.opaque() == native_.opaque); + } + + const wsrep_ws_handle_t* native() const + { + return &native_; + } + private: + const wsrep::ws_handle& ws_handle_; + const wsrep_ws_handle_t native_; + }; + + class mutable_ws_meta + { + public: + mutable_ws_meta(wsrep::ws_meta& ws_meta, int flags) + : ws_meta_(ws_meta) + , trx_meta_() + , flags_(flags) + { + std::memcpy(trx_meta_.gtid.uuid.data, ws_meta.group_id().data(), + sizeof(trx_meta_.gtid.uuid.data)); + trx_meta_.gtid.seqno = seqno_to_native(ws_meta.seqno()); + std::memcpy(trx_meta_.stid.node.data, ws_meta.server_id().data(), + sizeof(trx_meta_.stid.node.data)); + trx_meta_.stid.conn = ws_meta.client_id().get(); + trx_meta_.stid.trx = ws_meta.transaction_id().get(); + trx_meta_.depends_on = seqno_to_native(ws_meta.depends_on()); + } + + ~mutable_ws_meta() + { + ws_meta_ = wsrep::ws_meta( + wsrep::gtid( + wsrep::id(trx_meta_.gtid.uuid.data, + sizeof(trx_meta_.gtid.uuid.data)), + seqno_from_native(trx_meta_.gtid.seqno)), + wsrep::stid(wsrep::id(trx_meta_.stid.node.data, + sizeof(trx_meta_.stid.node.data)), + wsrep::transaction_id(trx_meta_.stid.trx), + wsrep::client_id(trx_meta_.stid.conn)), + seqno_from_native(trx_meta_.depends_on), flags_); + } + + wsrep_trx_meta* native() { return &trx_meta_; } + uint32_t native_flags() const { return map_flags_to_native(flags_); } + private: + wsrep::ws_meta& ws_meta_; + wsrep_trx_meta_t trx_meta_; + int flags_; + }; + + class const_ws_meta + { + public: + const_ws_meta(const wsrep::ws_meta& ws_meta) + : trx_meta_() + { + std::memcpy(trx_meta_.gtid.uuid.data, ws_meta.group_id().data(), + sizeof(trx_meta_.gtid.uuid.data)); + trx_meta_.gtid.seqno = seqno_to_native(ws_meta.seqno()); + std::memcpy(trx_meta_.stid.node.data, ws_meta.server_id().data(), + sizeof(trx_meta_.stid.node.data)); + trx_meta_.stid.conn = ws_meta.client_id().get(); + trx_meta_.stid.trx = ws_meta.transaction_id().get(); + trx_meta_.depends_on = seqno_to_native(ws_meta.depends_on()); + } + + ~const_ws_meta() + { + } + + const wsrep_trx_meta* native() const { return &trx_meta_; } + private: + wsrep_trx_meta_t trx_meta_; + }; + + enum wsrep::view::status map_view_status_from_native( + wsrep_view_status_t status) + { + switch (status) + { + case WSREP_VIEW_PRIMARY: return wsrep::view::primary; + case WSREP_VIEW_NON_PRIMARY: return wsrep::view::non_primary; + case WSREP_VIEW_DISCONNECTED: return wsrep::view::disconnected; + default: throw wsrep::runtime_error("Unknown view status"); + } + } + + /** @todo Currently capabilities defined in provider.hpp + * are one to one with wsrep_api.h. However, the mapping should + * be made explicit. */ + int map_capabilities_from_native(wsrep_cap_t capabilities) + { + return static_cast<int>(capabilities); + } + wsrep::view view_from_native(const wsrep_view_info& view_info, + const wsrep::id& own_id) + { + std::vector<wsrep::view::member> members; + for (int i(0); i < view_info.memb_num; ++i) + { + wsrep::id id(view_info.members[i].id.data, sizeof(view_info.members[i].id.data)); + std::string name( + view_info.members[i].name, + strnlen(view_info.members[i].name, + sizeof(view_info.members[i].name))); + std::string incoming( + view_info.members[i].incoming, + strnlen(view_info.members[i].incoming, + sizeof(view_info.members[i].incoming))); + members.push_back(wsrep::view::member(id, name, incoming)); + } + + int own_idx(-1); + if (own_id.is_undefined()) + { + // If own ID is undefined, obtain it from the view. This is + // the case on the initial connect to cluster. + own_idx = view_info.my_idx; + } + else + { + // If the node has already obtained its ID from cluster, + // its position in the view (or lack thereof) must be determined + // by the ID. + for (size_t i(0); i < members.size(); ++i) + { + if (own_id == members[i].id()) + { + own_idx = static_cast<int>(i); + break; + } + } + } + + return wsrep::view( + wsrep::gtid( + wsrep::id(view_info.state_id.uuid.data, + sizeof(view_info.state_id.uuid.data)), + wsrep::seqno(view_info.state_id.seqno)), + wsrep::seqno(view_info.view), + map_view_status_from_native(view_info.status), + map_capabilities_from_native(view_info.capabilities), + own_idx, + view_info.proto_ver, + members); + } + + ///////////////////////////////////////////////////////////////////// + // Callbacks // + ///////////////////////////////////////////////////////////////////// + + wsrep_cb_status_t connected_cb( + void* app_ctx, + const wsrep_view_info_t* view_info) + { + assert(app_ctx); + wsrep::server_state& server_state( + *reinterpret_cast<wsrep::server_state*>(app_ctx)); + wsrep::view view(view_from_native(*view_info, server_state.id())); + const ssize_t own_index(view.own_index()); + assert(own_index >= 0); + if (own_index < 0) + { + wsrep::log_error() << "Connected without being in reported view"; + return WSREP_CB_FAILURE; + } + assert(// first connect + server_state.id().is_undefined() || + // reconnect to primary component + server_state.id() == + view.members()[static_cast<size_t>(own_index)].id()); + try + { + server_state.on_connect(view); + return WSREP_CB_SUCCESS; + } + catch (const wsrep::runtime_error& e) + { + wsrep::log_error() << "Exception: " << e.what(); + return WSREP_CB_FAILURE; + } + } + + wsrep_cb_status_t view_cb(void* app_ctx, + void* recv_ctx, + const wsrep_view_info_t* view_info, + const char*, + size_t) + { + assert(app_ctx); + assert(view_info); + wsrep::server_state& server_state( + *reinterpret_cast<wsrep::server_state*>(app_ctx)); + wsrep::high_priority_service* high_priority_service( + reinterpret_cast<wsrep::high_priority_service*>(recv_ctx)); + try + { + wsrep::view view(view_from_native(*view_info, server_state.id())); + server_state.on_view(view, high_priority_service); + return WSREP_CB_SUCCESS; + } + catch (const wsrep::runtime_error& e) + { + wsrep::log_error() << "Exception: " << e.what(); + return WSREP_CB_FAILURE; + } + } + + wsrep_cb_status_t sst_request_cb(void* app_ctx, + void **sst_req, size_t* sst_req_len) + { + assert(app_ctx); + wsrep::server_state& server_state( + *reinterpret_cast<wsrep::server_state*>(app_ctx)); + + try + { + std::string req(server_state.prepare_for_sst()); + if (req.size() > 0) + { + *sst_req = ::malloc(req.size() + 1); + memcpy(*sst_req, req.data(), req.size() + 1); + *sst_req_len = req.size() + 1; + } + else + { + *sst_req = 0; + *sst_req_len = 0; + } + return WSREP_CB_SUCCESS; + } + catch (const wsrep::runtime_error& e) + { + return WSREP_CB_FAILURE; + } + } + + int encrypt_cb(void* app_ctx, + wsrep_enc_ctx_t* enc_ctx, + const wsrep_buf_t* input, + void* output, + wsrep_enc_direction_t direction, + bool last) + { + assert(app_ctx); + wsrep::server_state& server_state( + *static_cast<wsrep::server_state*>(app_ctx)); + + assert(server_state.encryption_service()); + if (server_state.encryption_service() == 0) + { + wsrep::log_error() << "Encryption service not defined in encrypt_cb()"; + return -1; + } + + wsrep::const_buffer key(enc_ctx->key->ptr, enc_ctx->key->len); + wsrep::const_buffer in(input->ptr, input->len); + try + { + return server_state.encryption_service()->do_crypt(&enc_ctx->ctx, + key, + enc_ctx->iv, + in, + output, + direction == WSREP_ENC, + last); + } + catch (const wsrep::runtime_error& e) + { + free(enc_ctx->ctx); + // Return negative value in case of callback error + return -1; + } + } + + wsrep_cb_status_t apply_cb(void* ctx, + const wsrep_ws_handle_t* wsh, + uint32_t flags, + const wsrep_buf_t* buf, + const wsrep_trx_meta_t* meta, + wsrep_bool_t* exit_loop) + { + wsrep::high_priority_service* high_priority_service( + reinterpret_cast<wsrep::high_priority_service*>(ctx)); + assert(high_priority_service); + + wsrep::const_buffer data(buf->ptr, buf->len); + wsrep::ws_handle ws_handle(wsrep::transaction_id(wsh->trx_id), + wsh->opaque); + wsrep::ws_meta ws_meta( + wsrep::gtid(wsrep::id(meta->gtid.uuid.data, + sizeof(meta->gtid.uuid.data)), + wsrep::seqno(meta->gtid.seqno)), + wsrep::stid(wsrep::id(meta->stid.node.data, + sizeof(meta->stid.node.data)), + wsrep::transaction_id(meta->stid.trx), + wsrep::client_id(meta->stid.conn)), + wsrep::seqno(seqno_from_native(meta->depends_on)), + map_flags_from_native(flags)); + try + { + if (high_priority_service->apply(ws_handle, ws_meta, data)) + { + return WSREP_CB_FAILURE; + } + *exit_loop = high_priority_service->must_exit(); + return WSREP_CB_SUCCESS; + } + catch (const wsrep::runtime_error& e) + { + wsrep::log_error() << "Caught runtime error while applying " + << ws_meta.flags() << ": " + << e.what(); + return WSREP_CB_FAILURE; + } + } + + wsrep_cb_status_t synced_cb(void* app_ctx) + { + assert(app_ctx); + wsrep::server_state& server_state( + *reinterpret_cast<wsrep::server_state*>(app_ctx)); + try + { + server_state.on_sync(); + return WSREP_CB_SUCCESS; + } + catch (const wsrep::runtime_error& e) + { + std::cerr << "On sync failed: " << e.what() << "\n"; + return WSREP_CB_FAILURE; + } + } + + + wsrep_cb_status_t sst_donate_cb(void* app_ctx, + void* , + const wsrep_buf_t* req_buf, + const wsrep_gtid_t* req_gtid, + const wsrep_buf_t*, + bool bypass) + { + assert(app_ctx); + wsrep::server_state& server_state( + *reinterpret_cast<wsrep::server_state*>(app_ctx)); + try + { + std::string req(reinterpret_cast<const char*>(req_buf->ptr), + req_buf->len); + wsrep::gtid gtid(wsrep::id(req_gtid->uuid.data, + sizeof(req_gtid->uuid.data)), + wsrep::seqno(req_gtid->seqno)); + if (server_state.start_sst(req, gtid, bypass)) + { + return WSREP_CB_FAILURE; + } + return WSREP_CB_SUCCESS; + } + catch (const wsrep::runtime_error& e) + { + return WSREP_CB_FAILURE; + } + } + + void logger_cb(wsrep_log_level_t level, const char* msg) + { + static const char* const pfx("P:"); // "provider" + wsrep::log::level ll(wsrep::log::unknown); + switch (level) + { + case WSREP_LOG_FATAL: + case WSREP_LOG_ERROR: + ll = wsrep::log::error; + break; + case WSREP_LOG_WARN: + ll = wsrep::log::warning; + break; + case WSREP_LOG_INFO: + ll = wsrep::log::info; + break; + case WSREP_LOG_DEBUG: + ll = wsrep::log::debug; + break; + } + wsrep::log(ll, pfx) << msg; + } + + static int init_thread_service(void* dlh, + wsrep::thread_service* thread_service) + { + assert(thread_service); + if (wsrep::thread_service_v1_probe(dlh)) + { + // No support in library. + return 1; + } + else + { + if (thread_service->before_init()) + { + wsrep::log_error() << "Thread service before init failed"; + return 1; + } + wsrep::thread_service_v1_init(dlh, thread_service); + if (thread_service->after_init()) + { + wsrep::log_error() << "Thread service after init failed"; + return 1; + } + } + return 0; + } + + static void deinit_thread_service(void* dlh) + { + // assert(not wsrep::thread_service_v1_probe(dlh)); + wsrep::thread_service_v1_deinit(dlh); + } + + static int init_tls_service(void* dlh, + wsrep::tls_service* tls_service) + { + assert(tls_service); + if (not wsrep::tls_service_v1_probe(dlh)) + { + return wsrep::tls_service_v1_init(dlh, tls_service); + } + return 1; + } + + static void deinit_tls_service(void* dlh) + { + // assert(not wsrep::tls_service_v1_probe(dlh)); + wsrep::tls_service_v1_deinit(dlh); + } + + static int init_allowlist_service(void* dlh, + wsrep::allowlist_service* allowlist_service) + { + assert(allowlist_service); + if (not wsrep::allowlist_service_v1_probe(dlh)) + { + return wsrep::allowlist_service_v1_init(dlh, allowlist_service); + } + return 1; + } + + static void deinit_allowlist_service(void* dlh) + { + // assert(not wsrep::allowlist_service_v1_probe(dlh)); + wsrep::allowlist_service_v1_deinit(dlh); + } + + static int init_event_service(void* dlh, + wsrep::event_service* service) + { + assert(service); + if (not wsrep::event_service_v1_probe(dlh)) + { + return wsrep::event_service_v1_init(dlh, service); + } + return 1; + } + + static void deinit_event_service(void* dlh) + { + wsrep::event_service_v1_deinit(dlh); + } +} + + + +void wsrep::wsrep_provider_v26::init_services( + const wsrep::provider::services& services) +{ + if (services.thread_service) + { + if (init_thread_service(wsrep_->dlh, services.thread_service)) + { + throw wsrep::runtime_error("Failed to initialize thread service"); + } + services_enabled_.thread_service = services.thread_service; + } + if (services.tls_service) + { + if (init_tls_service(wsrep_->dlh, services.tls_service)) + { + throw wsrep::runtime_error("Failed to initialize TLS service"); + } + services_enabled_.tls_service = services.tls_service; + } + if (services.allowlist_service) + { + if (init_allowlist_service(wsrep_->dlh, services.allowlist_service)) + { + throw wsrep::runtime_error("Failed to initialize allowlist service"); + } + services_enabled_.allowlist_service = services.allowlist_service; + } + if (services.event_service) + { + if (init_event_service(wsrep_->dlh, services.event_service)) + { + wsrep::log_warning() << "Failed to initialize event service"; + // provider does not produce events, ignore + } + else + { + services_enabled_.event_service = services.event_service; + } + } +} + +void wsrep::wsrep_provider_v26::deinit_services() +{ + if (services_enabled_.event_service) + deinit_event_service(wsrep_->dlh); + if (services_enabled_.tls_service) + deinit_tls_service(wsrep_->dlh); + if (services_enabled_.thread_service) + deinit_thread_service(wsrep_->dlh); + if (services_enabled_.allowlist_service) + deinit_allowlist_service(wsrep_->dlh); +} + +wsrep::wsrep_provider_v26::wsrep_provider_v26( + wsrep::server_state& server_state, + const std::string& provider_options, + const std::string& provider_spec, + const wsrep::provider::services& services) + : provider(server_state) + , wsrep_() + , services_enabled_() +{ + wsrep_gtid_t state_id; + bool encryption_enabled = server_state.encryption_service() && + server_state.encryption_service()->encryption_enabled(); + std::memcpy(state_id.uuid.data, + server_state.initial_position().id().data(), + sizeof(state_id.uuid.data)); + state_id.seqno = server_state.initial_position().seqno().get(); + struct wsrep_init_args init_args; + memset(&init_args, 0, sizeof(init_args)); + init_args.app_ctx = &server_state; + init_args.node_name = server_state_.name().c_str(); + init_args.node_address = server_state_.address().c_str(); + init_args.node_incoming = server_state_.incoming_address().c_str(); + init_args.data_dir = server_state_.working_dir().c_str(); + init_args.options = provider_options.c_str(); + init_args.proto_ver = server_state.max_protocol_version(); + init_args.state_id = &state_id; + init_args.state = 0; + init_args.logger_cb = &logger_cb; + init_args.connected_cb = &connected_cb; + init_args.view_cb = &view_cb; + init_args.sst_request_cb = &sst_request_cb; + init_args.encrypt_cb = encryption_enabled ? encrypt_cb : NULL; + init_args.apply_cb = &apply_cb; + init_args.unordered_cb = 0; + init_args.sst_donate_cb = &sst_donate_cb; + init_args.synced_cb = &synced_cb; + + if (wsrep_load(provider_spec.c_str(), &wsrep_, logger_cb)) + { + throw wsrep::runtime_error("Failed to load wsrep library"); + } + + init_services(services); + + if (wsrep_->init(wsrep_, &init_args) != WSREP_OK) + { + throw wsrep::runtime_error("Failed to initialize wsrep provider"); + } + + if (encryption_enabled) + { + const std::vector<unsigned char>& key = server_state.get_encryption_key(); + if (key.size()) + { + wsrep::const_buffer const_key(key.data(), key.size()); + enum status const retval(enc_set_key(const_key)); + if (retval != success) + { + std::string msg("Failed to set encryption key: "); + msg += to_string(retval); + throw wsrep::runtime_error(msg); + } + } + } +} + +wsrep::wsrep_provider_v26::~wsrep_provider_v26() +{ + wsrep_->free(wsrep_); + deinit_services(); + wsrep_unload(wsrep_); +} + +enum wsrep::provider::status wsrep::wsrep_provider_v26::connect( + const std::string& cluster_name, + const std::string& cluster_url, + const std::string& state_donor, + bool bootstrap) +{ + return map_return_value(wsrep_->connect(wsrep_, + cluster_name.c_str(), + cluster_url.c_str(), + state_donor.c_str(), + bootstrap)); +} + +int wsrep::wsrep_provider_v26::disconnect() +{ + int ret(0); + wsrep_status_t wret; + if ((wret = wsrep_->disconnect(wsrep_)) != WSREP_OK) + { + std::cerr << "Failed to disconnect from cluster: " + << wret << "\n"; + ret = 1; + } + return ret; +} + +int wsrep::wsrep_provider_v26::capabilities() const +{ + return map_capabilities_from_native(wsrep_->capabilities(wsrep_)); +} +int wsrep::wsrep_provider_v26::desync() +{ + return (wsrep_->desync(wsrep_) != WSREP_OK); +} + +int wsrep::wsrep_provider_v26::resync() +{ + return (wsrep_->resync(wsrep_) != WSREP_OK); +} + +wsrep::seqno wsrep::wsrep_provider_v26::pause() +{ + return wsrep::seqno(wsrep_->pause(wsrep_)); +} + +int wsrep::wsrep_provider_v26::resume() +{ + return (wsrep_->resume(wsrep_) != WSREP_OK); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::run_applier( + wsrep::high_priority_service *applier_ctx) +{ + return map_return_value(wsrep_->recv(wsrep_, applier_ctx)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::assign_read_view(wsrep::ws_handle& ws_handle, + const wsrep::gtid* gtid) +{ + const wsrep_gtid_t* gtid_ptr(NULL); + wsrep_gtid_t tmp; + + if (gtid) + { + ::memcpy(&tmp.uuid, gtid->id().data(), sizeof(tmp.uuid)); + tmp.seqno = gtid->seqno().get(); + gtid_ptr = &tmp; + } + + mutable_ws_handle mwsh(ws_handle); + return map_return_value(wsrep_->assign_read_view(wsrep_, mwsh.native(), + gtid_ptr)); +} + +int wsrep::wsrep_provider_v26::append_key(wsrep::ws_handle& ws_handle, + const wsrep::key& key) +{ + if (key.size() > 3) + { + assert(0); + return 1; + } + wsrep_buf_t key_parts[3]; + for (size_t i(0); i < key.size(); ++i) + { + key_parts[i].ptr = key.key_parts()[i].ptr(); + key_parts[i].len = key.key_parts()[i].size(); + } + wsrep_key_t wsrep_key = {key_parts, key.size()}; + mutable_ws_handle mwsh(ws_handle); + return (wsrep_->append_key( + wsrep_, mwsh.native(), + &wsrep_key, 1, map_key_type(key.type()), true) + != WSREP_OK); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::append_data(wsrep::ws_handle& ws_handle, + const wsrep::const_buffer& data) +{ + const wsrep_buf_t wsrep_buf = {data.data(), data.size()}; + mutable_ws_handle mwsh(ws_handle); + return map_return_value( + wsrep_->append_data(wsrep_, mwsh.native(), &wsrep_buf, + 1, WSREP_DATA_ORDERED, true)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::certify(wsrep::client_id client_id, + wsrep::ws_handle& ws_handle, + int flags, + wsrep::ws_meta& ws_meta) +{ + mutable_ws_handle mwsh(ws_handle); + mutable_ws_meta mmeta(ws_meta, flags); + return map_return_value( + wsrep_->certify(wsrep_, client_id.get(), mwsh.native(), + mmeta.native_flags(), + mmeta.native())); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::bf_abort( + wsrep::seqno bf_seqno, + wsrep::transaction_id victim_id, + wsrep::seqno& victim_seqno) +{ + wsrep_seqno_t wsrep_victim_seqno; + wsrep_status_t ret( + wsrep_->abort_certification( + wsrep_, seqno_to_native(bf_seqno), + victim_id.get(), &wsrep_victim_seqno)); + victim_seqno = seqno_from_native(wsrep_victim_seqno); + return map_return_value(ret); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::rollback(wsrep::transaction_id id) +{ + return map_return_value(wsrep_->rollback(wsrep_, id.get(), 0)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::commit_order_enter( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + const_ws_handle cwsh(ws_handle); + const_ws_meta cwsm(ws_meta); + return map_return_value( + wsrep_->commit_order_enter(wsrep_, cwsh.native(), cwsm.native())); +} + +int +wsrep::wsrep_provider_v26::commit_order_leave( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::mutable_buffer& err) +{ + const_ws_handle cwsh(ws_handle); + const_ws_meta cwsm(ws_meta); + wsrep_buf_t const err_buf = { err.data(), err.size() }; + int ret(wsrep_->commit_order_leave( + wsrep_, cwsh.native(), cwsm.native(), &err_buf) != WSREP_OK); + return ret; +} + +int wsrep::wsrep_provider_v26::release(wsrep::ws_handle& ws_handle) +{ + mutable_ws_handle mwsh(ws_handle); + return (wsrep_->release(wsrep_, mwsh.native()) != WSREP_OK); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::replay(const wsrep::ws_handle& ws_handle, + wsrep::high_priority_service* reply_service) +{ + const_ws_handle mwsh(ws_handle); + return map_return_value( + wsrep_->replay_trx(wsrep_, mwsh.native(), reply_service)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::enter_toi( + wsrep::client_id client_id, + const wsrep::key_array& keys, + const wsrep::const_buffer& buffer, + wsrep::ws_meta& ws_meta, + int flags) +{ + mutable_ws_meta mmeta(ws_meta, flags); + std::vector<std::vector<wsrep_buf_t> > key_parts; + std::vector<wsrep_key_t> wsrep_keys; + wsrep_buf_t wsrep_buf = {buffer.data(), buffer.size()}; + for (size_t i(0); i < keys.size(); ++i) + { + key_parts.push_back(std::vector<wsrep_buf_t>()); + for (size_t kp(0); kp < keys[i].size(); ++kp) + { + wsrep_buf_t buf = {keys[i].key_parts()[kp].data(), + keys[i].key_parts()[kp].size()}; + key_parts[i].push_back(buf); + } + } + for (size_t i(0); i < key_parts.size(); ++i) + { + wsrep_key_t key = {key_parts[i].data(), key_parts[i].size()}; + wsrep_keys.push_back(key); + } + return map_return_value(wsrep_->to_execute_start( + wsrep_, + client_id.get(), + wsrep_keys.data(), + wsrep_keys.size(), + &wsrep_buf, + buffer.size() ? 1 : 0, + mmeta.native_flags(), + mmeta.native())); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::leave_toi(wsrep::client_id client_id, + const wsrep::mutable_buffer& err) +{ + const wsrep_buf_t err_buf = { err.data(), err.size() }; + return map_return_value(wsrep_->to_execute_end( + wsrep_, client_id.get(), &err_buf)); +} + +std::pair<wsrep::gtid, enum wsrep::provider::status> +wsrep::wsrep_provider_v26::causal_read(int timeout) const +{ + wsrep_gtid_t wsrep_gtid; + wsrep_status_t ret(wsrep_->sync_wait(wsrep_, 0, timeout, &wsrep_gtid)); + wsrep::gtid gtid(ret == WSREP_OK ? + wsrep::gtid(wsrep::id(wsrep_gtid.uuid.data, + sizeof(wsrep_gtid.uuid.data)), + wsrep::seqno(wsrep_gtid.seqno)) : + wsrep::gtid::undefined()); + return std::make_pair(gtid, map_return_value(ret)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::wait_for_gtid(const wsrep::gtid& gtid, int timeout) + const +{ + wsrep_gtid_t wsrep_gtid; + std::memcpy(wsrep_gtid.uuid.data, gtid.id().data(), + sizeof(wsrep_gtid.uuid.data)); + wsrep_gtid.seqno = gtid.seqno().get(); + return map_return_value(wsrep_->sync_wait(wsrep_, &wsrep_gtid, timeout, 0)); +} + +wsrep::gtid wsrep::wsrep_provider_v26::last_committed_gtid() const +{ + wsrep_gtid_t wsrep_gtid; + if (wsrep_->last_committed_id(wsrep_, &wsrep_gtid) != WSREP_OK) + { + throw wsrep::runtime_error("Failed to read last committed id"); + } + return wsrep::gtid( + wsrep::id(wsrep_gtid.uuid.data, sizeof(wsrep_gtid.uuid.data)), + wsrep::seqno(wsrep_gtid.seqno)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::sst_sent(const wsrep::gtid& gtid, int err) +{ + wsrep_gtid_t wsrep_gtid; + std::memcpy(wsrep_gtid.uuid.data, gtid.id().data(), + sizeof(wsrep_gtid.uuid.data)); + wsrep_gtid.seqno = gtid.seqno().get(); + return map_return_value(wsrep_->sst_sent(wsrep_, &wsrep_gtid, err)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::sst_received(const wsrep::gtid& gtid, int err) +{ + wsrep_gtid_t wsrep_gtid; + std::memcpy(wsrep_gtid.uuid.data, gtid.id().data(), + sizeof(wsrep_gtid.uuid.data)); + wsrep_gtid.seqno = gtid.seqno().get(); + return map_return_value(wsrep_->sst_received(wsrep_, &wsrep_gtid, 0, err)); +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::enc_set_key(const wsrep::const_buffer& key) +{ + wsrep_enc_key_t enc_key = {key.data(), key.size()}; + return map_return_value(wsrep_->enc_set_key(wsrep_, &enc_key)); +} + +std::vector<wsrep::provider::status_variable> +wsrep::wsrep_provider_v26::status() const +{ + std::vector<status_variable> ret; + wsrep_stats_var* const stats(wsrep_->stats_get(wsrep_)); + wsrep_stats_var* i(stats); + if (i) + { + while (i->name) + { + switch (i->type) + { + case WSREP_VAR_STRING: + ret.push_back(status_variable(i->name, i->value._string)); + break; + case WSREP_VAR_INT64: + { + std::ostringstream os; + os << i->value._int64; + ret.push_back(status_variable(i->name, os.str())); + break; + } + case WSREP_VAR_DOUBLE: + { + std::ostringstream os; + os << i->value._double; + ret.push_back(status_variable(i->name, os.str())); + break; + } + default: + assert(0); + break; + } + ++i; + } + wsrep_->stats_free(wsrep_, stats); + } + return ret; +} + +void wsrep::wsrep_provider_v26::reset_status() +{ + wsrep_->stats_reset(wsrep_); +} + +std::string wsrep::wsrep_provider_v26::options() const +{ + std::string ret; + char* opts; + if ((opts = wsrep_->options_get(wsrep_))) + { + ret = opts; + free(opts); + } + else + { + throw wsrep::runtime_error("Failed to get provider options"); + } + return ret; +} + +enum wsrep::provider::status +wsrep::wsrep_provider_v26::options(const std::string& opts) +{ + return map_return_value(wsrep_->options_set(wsrep_, opts.c_str())); +} + +std::string wsrep::wsrep_provider_v26::name() const +{ + return (wsrep_->provider_name ? wsrep_->provider_name : "unknown"); +} + +std::string wsrep::wsrep_provider_v26::version() const +{ + return (wsrep_->provider_version ? wsrep_->provider_version : "unknown"); +} + +std::string wsrep::wsrep_provider_v26::vendor() const +{ + return (wsrep_->provider_vendor ? wsrep_->provider_vendor : "unknown"); +} + +void* wsrep::wsrep_provider_v26::native() const +{ + return wsrep_; +} diff --git a/wsrep-lib/src/wsrep_provider_v26.hpp b/wsrep-lib/src/wsrep_provider_v26.hpp new file mode 100644 index 00000000..1859b679 --- /dev/null +++ b/wsrep-lib/src/wsrep_provider_v26.hpp @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_WSREP_PROVIDER_V26_HPP +#define WSREP_WSREP_PROVIDER_V26_HPP + +#include "wsrep/provider.hpp" + +struct wsrep_st; + +namespace wsrep +{ + class thread_service; + class wsrep_provider_v26 : public wsrep::provider + { + public: + void init_services(const wsrep::provider::services& services); + void deinit_services(); + wsrep_provider_v26(wsrep::server_state&, const std::string&, + const std::string&, + const wsrep::provider::services& services); + ~wsrep_provider_v26(); + enum wsrep::provider::status + connect(const std::string&, const std::string&, const std::string&, + bool) WSREP_OVERRIDE; + int disconnect() WSREP_OVERRIDE; + int capabilities() const WSREP_OVERRIDE; + + int desync() WSREP_OVERRIDE; + int resync() WSREP_OVERRIDE; + wsrep::seqno pause() WSREP_OVERRIDE; + int resume() WSREP_OVERRIDE; + + enum wsrep::provider::status + run_applier(wsrep::high_priority_service*) WSREP_OVERRIDE; + int start_transaction(wsrep::ws_handle&) WSREP_OVERRIDE { return 0; } + enum wsrep::provider::status + assign_read_view(wsrep::ws_handle&, const wsrep::gtid*) WSREP_OVERRIDE; + int append_key(wsrep::ws_handle&, const wsrep::key&) WSREP_OVERRIDE; + enum wsrep::provider::status + append_data(wsrep::ws_handle&, const wsrep::const_buffer&) + WSREP_OVERRIDE; + enum wsrep::provider::status + certify(wsrep::client_id, wsrep::ws_handle&, + int, + wsrep::ws_meta&) WSREP_OVERRIDE; + enum wsrep::provider::status + bf_abort(wsrep::seqno, + wsrep::transaction_id, + wsrep::seqno&) WSREP_OVERRIDE; + enum wsrep::provider::status + rollback(const wsrep::transaction_id) WSREP_OVERRIDE; + enum wsrep::provider::status + commit_order_enter(const wsrep::ws_handle&, + const wsrep::ws_meta&) WSREP_OVERRIDE; + int commit_order_leave(const wsrep::ws_handle&, + const wsrep::ws_meta&, + const wsrep::mutable_buffer&) WSREP_OVERRIDE; + int release(wsrep::ws_handle&) WSREP_OVERRIDE; + enum wsrep::provider::status replay(const wsrep::ws_handle&, + wsrep::high_priority_service*) + WSREP_OVERRIDE; + enum wsrep::provider::status enter_toi(wsrep::client_id, + const wsrep::key_array&, + const wsrep::const_buffer&, + wsrep::ws_meta&, + int) + WSREP_OVERRIDE; + enum wsrep::provider::status leave_toi(wsrep::client_id, + const wsrep::mutable_buffer&) + WSREP_OVERRIDE; + std::pair<wsrep::gtid, enum wsrep::provider::status> + causal_read(int) const WSREP_OVERRIDE; + enum wsrep::provider::status wait_for_gtid(const wsrep::gtid&, int) + const WSREP_OVERRIDE; + wsrep::gtid last_committed_gtid() const WSREP_OVERRIDE; + enum wsrep::provider::status sst_sent(const wsrep::gtid&, int) + WSREP_OVERRIDE; + enum wsrep::provider::status sst_received(const wsrep::gtid& gtid, int) + WSREP_OVERRIDE; + enum wsrep::provider::status enc_set_key(const wsrep::const_buffer& key) + WSREP_OVERRIDE; + std::vector<status_variable> status() const WSREP_OVERRIDE; + void reset_status() WSREP_OVERRIDE; + std::string options() const WSREP_OVERRIDE; + enum wsrep::provider::status options(const std::string&) WSREP_OVERRIDE; + std::string name() const WSREP_OVERRIDE; + std::string version() const WSREP_OVERRIDE; + std::string vendor() const WSREP_OVERRIDE; + void* native() const WSREP_OVERRIDE; + private: + wsrep_provider_v26(const wsrep_provider_v26&); + wsrep_provider_v26& operator=(const wsrep_provider_v26); + struct wsrep_st* wsrep_; + services services_enabled_; + }; +} + + +#endif // WSREP_WSREP_PROVIDER_V26_HPP diff --git a/wsrep-lib/src/xid.cpp b/wsrep-lib/src/xid.cpp new file mode 100644 index 00000000..7757e445 --- /dev/null +++ b/wsrep-lib/src/xid.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/xid.hpp" +#include <ostream> + +std::string wsrep::to_string(const wsrep::xid& xid) +{ + return std::string(xid.data_.data(), xid.data_.size()); +} + +std::ostream& wsrep::operator<<(std::ostream& os, const wsrep::xid& xid) +{ + return os << to_string(xid); +} diff --git a/wsrep-lib/test/CMakeLists.txt b/wsrep-lib/test/CMakeLists.txt new file mode 100644 index 00000000..366cc478 --- /dev/null +++ b/wsrep-lib/test/CMakeLists.txt @@ -0,0 +1,47 @@ +# +# Copyright (C) 2018 Codership Oy <info@codership.com> +# + + +set(TEST_SOURCES + mock_client_state.cpp + mock_high_priority_service.cpp + mock_storage_service.cpp + test_utils.cpp + buffer_test.cpp + gtid_test.cpp + id_test.cpp + nbo_test.cpp + rsu_test.cpp + server_context_test.cpp + toi_test.cpp + transaction_test.cpp + transaction_test_2pc.cpp + transaction_test_xa.cpp + view_test.cpp + xid_test.cpp + wsrep-lib_test.cpp + ) + +if (WSREP_LIB_WITH_UNIT_TESTS_EXTRA) + set(TEST_SOURCES ${TEST_SOURCES} + reporter_test.cpp + ) +endif() + +add_executable(wsrep-lib_test ${TEST_SOURCES}) + +target_link_libraries(wsrep-lib_test wsrep-lib) + +add_test(NAME wsrep-lib_test + COMMAND wsrep-lib_test) + +if (WSREP_LIB_WITH_AUTO_TEST) + set(UNIT_TEST wsrep-lib_test) + add_custom_command( + TARGET ${UNIT_TEST} + COMMENT "Run tests" + POST_BUILD + COMMAND ${UNIT_TEST} + ) +endif() diff --git a/wsrep-lib/test/buffer_test.cpp b/wsrep-lib/test/buffer_test.cpp new file mode 100644 index 00000000..84d4a483 --- /dev/null +++ b/wsrep-lib/test/buffer_test.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/buffer.hpp" +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_CASE(buffer_test_empty_access) +{ + wsrep::mutable_buffer buf; + BOOST_REQUIRE(buf.size() == 0); + (void)buf.data(); +} diff --git a/wsrep-lib/test/client_state_fixture.hpp b/wsrep-lib/test/client_state_fixture.hpp new file mode 100644 index 00000000..a26adeed --- /dev/null +++ b/wsrep-lib/test/client_state_fixture.hpp @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_TEST_CLIENT_CONTEXT_FIXTURE_HPP +#define WSREP_TEST_CLIENT_CONTEXT_FIXTURE_HPP + +#include "mock_server_state.hpp" +#include "mock_client_state.hpp" + + +#include <boost/test/unit_test.hpp> + +namespace +{ + struct replicating_client_fixture_sync_rm + { + replicating_client_fixture_sync_rm() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc(sc, wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct replicating_two_clients_fixture_sync_rm + { + replicating_two_clients_fixture_sync_rm() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc1(sc, wsrep::client_id(1), + wsrep::client_state::m_local) + , cc2(sc, wsrep::client_id(2), + wsrep::client_state::m_local) + , tc(cc1.transaction()) + { + sc.mock_connect(); + cc1.open(cc1.id()); + BOOST_REQUIRE(cc1.before_command() == 0); + BOOST_REQUIRE(cc1.before_statement() == 0); + cc2.open(cc2.id()); + BOOST_REQUIRE(cc2.before_command() == 0); + BOOST_REQUIRE(cc2.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc1; + wsrep::mock_client cc2; + const wsrep::transaction& tc; + }; + + struct replicating_client_fixture_async_rm + { + replicating_client_fixture_async_rm() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_async, server_service) + , cc(sc, wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct replicating_client_fixture_2pc + { + replicating_client_fixture_2pc() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc(sc, wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + cc.do_2pc_ = true; + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct replicating_client_fixture_autocommit + { + replicating_client_fixture_autocommit() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc(sc, wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + cc.is_autocommit_ = true; + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct applying_client_fixture + { + applying_client_fixture() + : server_service(sc) + , sc("s1", + wsrep::server_state::rm_async, server_service) + , cc(sc, + wsrep::client_id(1), + wsrep::client_state::m_high_priority) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + } + void start_transaction(wsrep::transaction_id id, + wsrep::seqno seqno) + { + wsrep::ws_handle ws_handle(id, (void*)1); + wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("1"), seqno), + wsrep::stid(sc.id(), + wsrep::transaction_id(1), + cc.id()), + wsrep::seqno(0), + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::commit); + BOOST_REQUIRE(cc.start_transaction(ws_handle, ws_meta) == 0); + BOOST_REQUIRE(tc.active() == true); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + } + + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct applying_client_fixture_2pc + { + applying_client_fixture_2pc() + : server_service(sc) + , sc("s1", + wsrep::server_state::rm_async, server_service) + , cc(sc, + wsrep::client_id(1), + wsrep::client_state::m_high_priority) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + cc.do_2pc_ = true; + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + wsrep::ws_handle ws_handle(wsrep::transaction_id(1), (void*)1); + wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(1)), + wsrep::stid(sc.id(), + wsrep::transaction_id(1), + cc.id()), + wsrep::seqno(0), + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::commit); + BOOST_REQUIRE(cc.start_transaction(ws_handle, ws_meta) == 0); + BOOST_REQUIRE(tc.active() == true); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct streaming_client_fixture_row + { + streaming_client_fixture_row() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc(sc, + wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + cc.enable_streaming(wsrep::streaming_context::row, 1); + } + + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct streaming_client_fixture_byte + { + streaming_client_fixture_byte() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc(sc, + wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + cc.enable_streaming(wsrep::streaming_context::bytes, 1); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; + + struct streaming_client_fixture_statement + { + streaming_client_fixture_statement() + : server_service(sc) + , sc("s1", wsrep::server_state::rm_sync, server_service) + , cc(sc, + wsrep::client_id(1), + wsrep::client_state::m_local) + , tc(cc.transaction()) + { + sc.mock_connect(); + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Verify initial state + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + cc.enable_streaming(wsrep::streaming_context::statement, 1); + } + + wsrep::mock_server_service server_service; + wsrep::mock_server_state sc; + wsrep::mock_client cc; + const wsrep::transaction& tc; + }; +} +#endif // WSREP_TEST_CLIENT_CONTEXT_FIXTURE_HPP diff --git a/wsrep-lib/test/gtid_test.cpp b/wsrep-lib/test/gtid_test.cpp new file mode 100644 index 00000000..4f4d644b --- /dev/null +++ b/wsrep-lib/test/gtid_test.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/gtid.hpp" +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_CASE(gtid_test_scan_from_string_uuid) +{ + std::string gtid_str("6a20d44a-6e17-11e8-b1e2-9061aec0cdad:123456"); + wsrep::gtid gtid; + ssize_t ret(wsrep::scan_from_c_str( + gtid_str.c_str(), + gtid_str.size(), gtid)); + BOOST_REQUIRE_MESSAGE(ret == ssize_t(gtid_str.size()), + "Expected " << gtid_str.size() << " got " << ret); + BOOST_REQUIRE(gtid.seqno().get() == 123456); +} + +BOOST_AUTO_TEST_CASE(gtid_test_scan_from_string_uuid_too_long) +{ + std::string gtid_str("6a20d44a-6e17-11e8-b1e2-9061aec0cdadx:123456"); + wsrep::gtid gtid; + ssize_t ret(wsrep::scan_from_c_str( + gtid_str.c_str(), + gtid_str.size(), gtid)); + BOOST_REQUIRE_MESSAGE(ret == -EINVAL, + "Expected " << -EINVAL << " got " << ret); +} + +BOOST_AUTO_TEST_CASE(gtid_test_scan_from_string_seqno_out_of_range) +{ + std::string gtid_str("6a20d44a-6e17-11e8-b1e2-9061aec0cdad:9223372036854775808"); + wsrep::gtid gtid; + ssize_t ret(wsrep::scan_from_c_str( + gtid_str.c_str(), + gtid_str.size(), gtid)); + BOOST_REQUIRE_MESSAGE(ret == -EINVAL, + "Expected " << -EINVAL << " got " << ret); + + gtid_str = "6a20d44a-6e17-11e8-b1e2-9061aec0cdad:-9223372036854775809"; + ret = wsrep::scan_from_c_str( + gtid_str.c_str(), + gtid_str.size(), gtid); + BOOST_REQUIRE_MESSAGE(ret == -EINVAL, + "Expected " << -EINVAL << " got " << ret); +} diff --git a/wsrep-lib/test/id_test.cpp b/wsrep-lib/test/id_test.cpp new file mode 100644 index 00000000..5a87ba16 --- /dev/null +++ b/wsrep-lib/test/id_test.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/id.hpp" +#include <boost/test/unit_test.hpp> + +#include <sstream> + +namespace +{ + bool exception_check(const wsrep::runtime_error&) { return true; } +} + +BOOST_AUTO_TEST_CASE(id_test_uuid) +{ + std::string uuid_str("6a20d44a-6e17-11e8-b1e2-9061aec0cdad"); + wsrep::id id(uuid_str); + std::ostringstream os; + os << id; + BOOST_REQUIRE(uuid_str == os.str()); +} + +BOOST_AUTO_TEST_CASE(id_test_string) +{ + std::string id_str("1234567890123456"); + wsrep::id id(id_str); + std::ostringstream os; + os << id; + BOOST_REQUIRE(id_str == os.str()); +} + +BOOST_AUTO_TEST_CASE(id_test_string_too_long) +{ + std::string id_str("12345678901234567"); + BOOST_REQUIRE_EXCEPTION(wsrep::id id(id_str), wsrep::runtime_error, + exception_check); +} + +BOOST_AUTO_TEST_CASE(id_test_binary) +{ + char data[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4 ,5 ,6}; + wsrep::id id(data, sizeof(data)); + std::ostringstream os; + os << id; + BOOST_REQUIRE(os.str() == "01020304-0506-0708-0900-010203040506"); +} + +BOOST_AUTO_TEST_CASE(id_test_binary_too_long) +{ + char data[17] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4 ,5 ,6, 7}; + BOOST_REQUIRE_EXCEPTION(wsrep::id id(data, sizeof(data)), + wsrep::runtime_error, exception_check);; +} diff --git a/wsrep-lib/test/mock_client_state.cpp b/wsrep-lib/test/mock_client_state.cpp new file mode 100644 index 00000000..03e90d04 --- /dev/null +++ b/wsrep-lib/test/mock_client_state.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/transaction.hpp" +#include "mock_client_state.hpp" +#include "mock_high_priority_service.hpp" + +int wsrep::mock_client_service::bf_rollback() +{ + int ret(0); + if (client_state_.before_rollback()) + { + ret = 1; + } + else if (client_state_.after_rollback()) + { + ret = 1; + } + return ret; +} + +enum wsrep::provider::status +wsrep::mock_client_service::replay() +{ + wsrep::mock_high_priority_service hps(client_state_.server_state(), + &client_state_, true); + enum wsrep::provider::status ret( + client_state_.provider().replay( + client_state_.transaction().ws_handle(), + &hps)); + ++replays_; + return ret; +} diff --git a/wsrep-lib/test/mock_client_state.hpp b/wsrep-lib/test/mock_client_state.hpp new file mode 100644 index 00000000..56434f73 --- /dev/null +++ b/wsrep-lib/test/mock_client_state.hpp @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_MOCK_CLIENT_CONTEXT_HPP +#define WSREP_MOCK_CLIENT_CONTEXT_HPP + +#include "wsrep/client_state.hpp" +#include "wsrep/mutex.hpp" +#include "wsrep/compiler.hpp" + +#include "test_utils.hpp" + +namespace wsrep +{ + class mock_client_state : public wsrep::client_state + { + public: + mock_client_state(wsrep::server_state& server_state, + wsrep::client_service& client_service, + const wsrep::client_id& id, + enum wsrep::client_state::mode mode) + : wsrep::client_state(mutex_, cond_, server_state, client_service, + id, mode) + , mutex_() + , cond_() + { } + ~mock_client_state() + { + if (transaction().active()) + { + (void)client_service().bf_rollback(); + } + } + private: + wsrep::default_mutex mutex_; + wsrep::default_condition_variable cond_; + public: + private: + }; + + + class mock_client_service : public wsrep::client_service + { + public: + mock_client_service(wsrep::mock_client_state& client_state) + : wsrep::client_service() + , is_autocommit_() + , do_2pc_() + // , fail_next_applying_() + // , fail_next_toi_() + , bf_abort_during_wait_() + , bf_abort_during_fragment_removal_() + , error_during_prepare_data_() + , killed_before_certify_() + , sync_point_enabled_() + , sync_point_action_() + , bytes_generated_() + , client_state_(client_state) + , will_replay_called_() + , replays_() + , unordered_replays_() + , aborts_() + { } + + int bf_rollback() WSREP_OVERRIDE; + + bool interrupted(wsrep::unique_lock<wsrep::mutex>&) + const WSREP_OVERRIDE + { return killed_before_certify_; } + + + void emergency_shutdown() WSREP_OVERRIDE { ++aborts_; } + + int remove_fragments() WSREP_OVERRIDE + { + if (bf_abort_during_fragment_removal_) + { + client_state_.before_rollback(); + client_state_.after_rollback(); + return 1; + } + else + { + return 0; + } + } + + void will_replay() WSREP_OVERRIDE { will_replay_called_ = true; } + + void signal_replayed() WSREP_OVERRIDE { } + + enum wsrep::provider::status replay() WSREP_OVERRIDE; + + enum wsrep::provider::status replay_unordered() WSREP_OVERRIDE + { + unordered_replays_++; + return wsrep::provider::success; + } + + void wait_for_replayers( + wsrep::unique_lock<wsrep::mutex>& lock) + WSREP_OVERRIDE + { + lock.unlock(); + if (bf_abort_during_wait_) + { + wsrep_test::bf_abort_unordered(client_state_); + } + lock.lock(); + } + + int prepare_data_for_replication() WSREP_OVERRIDE + { + if (error_during_prepare_data_) + { + return 1; + } + static const char buf[1] = { 1 }; + wsrep::const_buffer data = wsrep::const_buffer(buf, 1); + return client_state_.append_data(data); + } + + void cleanup_transaction() WSREP_OVERRIDE { } + + size_t bytes_generated() const WSREP_OVERRIDE + { + return bytes_generated_; + } + + bool statement_allowed_for_streaming() const WSREP_OVERRIDE + { return true; } + int prepare_fragment_for_replication(wsrep::mutable_buffer& buffer, size_t& position) + WSREP_OVERRIDE + { + if (error_during_prepare_data_) + { + return 1; + } + static const char buf[1] = { 1 }; + buffer.push_back(&buf[0], &buf[1]); + wsrep::const_buffer data(buffer.data(), buffer.size()); + position = buffer.size(); + return client_state_.append_data(data); + } + + void store_globals() WSREP_OVERRIDE { } + void reset_globals() WSREP_OVERRIDE { } + + enum wsrep::provider::status commit_by_xid() WSREP_OVERRIDE + { + return wsrep::provider::success; + } + + bool is_explicit_xa() WSREP_OVERRIDE + { + return false; + } + + bool is_xa_rollback() WSREP_OVERRIDE + { + return false; + } + + void debug_sync(const char* sync_point) WSREP_OVERRIDE + { + if (sync_point_enabled_ == sync_point) + { + switch (sync_point_action_) + { + case spa_bf_abort_unordered: + wsrep_test::bf_abort_unordered(client_state_); + break; + case spa_bf_abort_ordered: + wsrep_test::bf_abort_ordered(client_state_); + break; + } + } + } + + void debug_crash(const char*) WSREP_OVERRIDE + { + // Not going to do this while unit testing + } + + + // + // Knobs to tune the behavior + // + bool is_autocommit_; + bool do_2pc_; + // bool fail_next_applying_; + // bool fail_next_toi_; + bool bf_abort_during_wait_; + bool bf_abort_during_fragment_removal_; + bool error_during_prepare_data_; + bool killed_before_certify_; + std::string sync_point_enabled_; + enum sync_point_action + { + spa_bf_abort_unordered, + spa_bf_abort_ordered + } sync_point_action_; + size_t bytes_generated_; + + // + // Verifying the state + // + bool will_replay_called() const { return will_replay_called_; } + size_t replays() const { return replays_; } + size_t unordered_replays() const { return unordered_replays_; } + size_t aborts() const { return aborts_; } + private: + wsrep::mock_client_state& client_state_; + bool will_replay_called_; + size_t replays_; + size_t unordered_replays_; + size_t aborts_; + }; + + class mock_client + : public mock_client_state + , public mock_client_service + { + public: + mock_client(wsrep::server_state& server_state, + const wsrep::client_id& id, + enum wsrep::client_state::mode mode) + : mock_client_state(server_state, *this, id, mode) + , mock_client_service(static_cast<mock_client_state&>(*this)) + { } + + int after_row() + { + bytes_generated_++; + return wsrep::client_state::after_row(); + } + }; +} + +#endif // WSREP_MOCK_CLIENT_CONTEXT_HPP diff --git a/wsrep-lib/test/mock_high_priority_service.cpp b/wsrep-lib/test/mock_high_priority_service.cpp new file mode 100644 index 00000000..bb67c9a9 --- /dev/null +++ b/wsrep-lib/test/mock_high_priority_service.cpp @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "mock_high_priority_service.hpp" +#include "mock_server_state.hpp" +#include <sstream> + +int wsrep::mock_high_priority_service::start_transaction( + const wsrep::ws_handle& ws_handle, const wsrep::ws_meta& ws_meta) +{ + return client_state_->start_transaction(ws_handle, ws_meta); +} + +int wsrep::mock_high_priority_service::next_fragment( + const wsrep::ws_meta& ws_meta) +{ + return client_state_->next_fragment(ws_meta); +} + +int wsrep::mock_high_priority_service::adopt_transaction( + const wsrep::transaction& transaction) +{ + client_state_->adopt_transaction(transaction); + if (transaction.state() == wsrep::transaction::s_prepared) + { + client_state_->restore_xid(transaction.xid()); + } + return 0; +} + +int wsrep::mock_high_priority_service::apply_write_set( + const wsrep::ws_meta& meta, + const wsrep::const_buffer&, + wsrep::mutable_buffer& err) +{ + assert(client_state_->toi_meta().seqno().is_undefined()); + assert(client_state_->transaction().state() == wsrep::transaction::s_executing || + client_state_->transaction().state() == wsrep::transaction::s_prepared || + client_state_->transaction().state() == wsrep::transaction::s_replaying); + if (fail_next_applying_) + { + std::ostringstream os; + os << "failed " << meta; + err.push_back(os.str()); + assert(err.size() > 0); + return 1; + } + else + { + int ret(0); + if (!(meta.flags() & wsrep::provider::flag::commit)) + { + client_state_->fragment_applied(meta.seqno()); + } + if ((meta.flags() & wsrep::provider::flag::prepare)) + { + client_state_->assign_xid(wsrep::xid(1, 3, 1, "xid")); + ret = client_state_->before_prepare() || + client_state_->after_prepare(); + } + return ret; + }; +} + +int wsrep::mock_high_priority_service::commit( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + int ret(0); + client_state_->prepare_for_ordering(ws_handle, ws_meta, true); + if (do_2pc_) + { + ret = client_state_->before_prepare() || + client_state_->after_prepare(); + } + const bool is_ordered= !ws_meta.seqno().is_undefined(); + if (!is_ordered) + { + client_state_->before_rollback(); + client_state_->after_rollback(); + return 0; + } + else + { + return (ret || client_state_->before_commit() || + client_state_->ordered_commit() || + client_state_->after_commit()); + } +} + +int wsrep::mock_high_priority_service::rollback( + const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + client_state_->prepare_for_ordering(ws_handle, ws_meta, false); + return (client_state_->before_rollback() || + client_state_->after_rollback()); +} + +int wsrep::mock_high_priority_service::apply_toi(const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) +{ + assert(client_state_->transaction().active() == false); + assert(client_state_->toi_meta().seqno().is_undefined() == false); + return (fail_next_toi_ ? 1 : 0); +} + +int wsrep::mock_high_priority_service::apply_nbo_begin( + const wsrep::ws_meta& ws_meta, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) +{ + const int nbo_begin_flags __attribute__((unused)) + (wsrep::provider::flag::isolation | + wsrep::provider::flag::start_transaction); + assert(ws_meta.flags() & nbo_begin_flags); + assert((ws_meta.flags() & ~nbo_begin_flags) == 0); + + if (fail_next_toi_) + { + return 1; + } + else + { + nbo_cs_ = std::unique_ptr<wsrep::mock_client>( + new wsrep::mock_client(client_state_->server_state(), + wsrep::client_id(1), + wsrep::client_state::m_local)); + nbo_cs_->open(wsrep::client_id(1)); + nbo_cs_->before_command(); + nbo_cs_->before_statement(); + return nbo_cs_->enter_nbo_mode(ws_meta); + } +} + +void wsrep::mock_high_priority_service::adopt_apply_error( + wsrep::mutable_buffer& err) +{ + client_state_->adopt_apply_error(err); +} + +void wsrep::mock_high_priority_service::after_apply() +{ + client_state_->after_applying(); +} + +int wsrep::mock_high_priority_service::log_dummy_write_set( + const wsrep::ws_handle&, + const wsrep::ws_meta&, + wsrep::mutable_buffer& err) +{ + return err.size() > 0; +} diff --git a/wsrep-lib/test/mock_high_priority_service.hpp b/wsrep-lib/test/mock_high_priority_service.hpp new file mode 100644 index 00000000..caf40b2c --- /dev/null +++ b/wsrep-lib/test/mock_high_priority_service.hpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_MOCK_HIGH_PRIORITY_SERVICE_HPP +#define WSREP_MOCK_HIGH_PRIORITY_SERVICE_HPP + +#include "wsrep/high_priority_service.hpp" +#include "mock_client_state.hpp" + +#include <memory> + +namespace wsrep +{ + class mock_high_priority_service : public wsrep::high_priority_service + { + public: + mock_high_priority_service( + wsrep::server_state& server_state, + wsrep::mock_client_state* client_state, + bool replaying) + : wsrep::high_priority_service(server_state) + , do_2pc_() + , fail_next_applying_() + , fail_next_toi_() + , client_state_(client_state) + , replaying_(replaying) + , nbo_cs_() + { } + + ~mock_high_priority_service() + { } + int start_transaction(const wsrep::ws_handle&, const wsrep::ws_meta&) + WSREP_OVERRIDE; + + int next_fragment(const wsrep::ws_meta&) WSREP_OVERRIDE; + + const wsrep::transaction& transaction() const WSREP_OVERRIDE + { return client_state_->transaction(); } + int adopt_transaction(const wsrep::transaction&) WSREP_OVERRIDE; + int apply_write_set(const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) WSREP_OVERRIDE; + int append_fragment_and_commit( + const wsrep::ws_handle&, + const wsrep::ws_meta&, + const wsrep::const_buffer&, + const wsrep::xid&) WSREP_OVERRIDE + { return 0; } + int remove_fragments(const wsrep::ws_meta&) WSREP_OVERRIDE + { return 0; } + int commit(const wsrep::ws_handle&, const wsrep::ws_meta&) + WSREP_OVERRIDE; + int rollback(const wsrep::ws_handle&, const wsrep::ws_meta&) WSREP_OVERRIDE; + int apply_toi(const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) WSREP_OVERRIDE; + int apply_nbo_begin(const wsrep::ws_meta&, + const wsrep::const_buffer&, + wsrep::mutable_buffer&) WSREP_OVERRIDE; + void adopt_apply_error(wsrep::mutable_buffer& err) WSREP_OVERRIDE; + void after_apply() WSREP_OVERRIDE; + void store_globals() WSREP_OVERRIDE { } + void reset_globals() WSREP_OVERRIDE { } + void switch_execution_context(wsrep::high_priority_service&) + WSREP_OVERRIDE { } + int log_dummy_write_set(const wsrep::ws_handle&, + const wsrep::ws_meta&, + wsrep::mutable_buffer&) WSREP_OVERRIDE; + bool is_replaying() const WSREP_OVERRIDE { return replaying_; } + void debug_crash(const char*) WSREP_OVERRIDE { /* Not in unit tests*/} + + wsrep::client_state& client_state() + { + return *client_state_; + } + bool do_2pc_; + bool fail_next_applying_; + bool fail_next_toi_; + + wsrep::mock_client* nbo_cs() const { return nbo_cs_.get(); } + + private: + mock_high_priority_service(const mock_high_priority_service&); + mock_high_priority_service& operator=(const mock_high_priority_service&); + wsrep::mock_client_state* client_state_; + bool replaying_; + + /* Client state associated to NBO processing. This should be + * stored elsewhere (like mock_server_state), but is kept + * here for convenience. */ + std::unique_ptr<wsrep::mock_client> nbo_cs_; + }; +} + +#endif // WSREP_MOCK_HIGH_PRIORITY_SERVICE_HPP diff --git a/wsrep-lib/test/mock_provider.hpp b/wsrep-lib/test/mock_provider.hpp new file mode 100644 index 00000000..c2a89f05 --- /dev/null +++ b/wsrep-lib/test/mock_provider.hpp @@ -0,0 +1,357 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_MOCK_PROVIDER_HPP +#define WSREP_MOCK_PROVIDER_HPP + +#include "wsrep/provider.hpp" +#include "wsrep/logger.hpp" +#include "wsrep/buffer.hpp" +#include "wsrep/high_priority_service.hpp" + +#include <cstring> +#include <map> +#include <iostream> // todo: proper logging + +#include <boost/test/unit_test.hpp> + +namespace wsrep +{ + class mock_provider : public wsrep::provider + { + public: + typedef std::map<wsrep::transaction_id, wsrep::seqno> bf_abort_map; + + mock_provider(wsrep::server_state& server_state) + : provider(server_state) + , certify_result_() + , commit_order_enter_result_() + , commit_order_leave_result_() + , release_result_() + , replay_result_() + , group_id_("1") + , server_id_("1") + , group_seqno_(0) + , bf_abort_map_() + , start_fragments_() + , fragments_() + , commit_fragments_() + , rollback_fragments_() + , toi_write_sets_() + , toi_start_transaction_() + , toi_commit_() + { } + + enum wsrep::provider::status + connect(const std::string&, const std::string&, const std::string&, + bool) WSREP_OVERRIDE + { return wsrep::provider::success; } + int disconnect() WSREP_OVERRIDE { return 0; } + int capabilities() const WSREP_OVERRIDE { return 0; } + int desync() WSREP_OVERRIDE { return 0; } + int resync() WSREP_OVERRIDE { return 0; } + wsrep::seqno pause() WSREP_OVERRIDE { return wsrep::seqno(0); } + int resume() WSREP_OVERRIDE { return 0; } + enum wsrep::provider::status run_applier(wsrep::high_priority_service*) + WSREP_OVERRIDE + { + return wsrep::provider::success; + } + // Provider implemenatation interface + int start_transaction(wsrep::ws_handle&) WSREP_OVERRIDE { return 0; } + enum wsrep::provider::status + certify(wsrep::client_id client_id, + wsrep::ws_handle& ws_handle, + int flags, + wsrep::ws_meta& ws_meta) + WSREP_OVERRIDE + { + ws_handle = wsrep::ws_handle(ws_handle.transaction_id(), (void*)1); + wsrep::log_debug() << "provider certify: " + << "client: " << client_id.get() + << " flags: " << std::hex << flags + << std::dec + << " certify_status: " << certify_result_; + if (certify_result_) + { + return certify_result_; + } + + ++fragments_; + if (starts_transaction(flags)) + { + ++start_fragments_; + } + if (commits_transaction(flags)) + { + ++commit_fragments_; + } + if (rolls_back_transaction(flags)) + { + ++rollback_fragments_; + } + + wsrep::stid stid(server_id_, + ws_handle.transaction_id(), + client_id); + bf_abort_map::iterator it(bf_abort_map_.find( + ws_handle.transaction_id())); + if (it == bf_abort_map_.end()) + { + ++group_seqno_; + wsrep::gtid gtid(group_id_, wsrep::seqno(group_seqno_)); + ws_meta = wsrep::ws_meta(gtid, stid, + wsrep::seqno(group_seqno_ - 1), + flags); + return wsrep::provider::success; + } + else + { + enum wsrep::provider::status ret; + if (it->second.is_undefined()) + { + ws_meta = wsrep::ws_meta(wsrep::gtid(), wsrep::stid(), + wsrep::seqno::undefined(), 0); + ret = wsrep::provider::error_certification_failed; + } + else + { + ++group_seqno_; + wsrep::gtid gtid(group_id_, wsrep::seqno(group_seqno_)); + ws_meta = wsrep::ws_meta(gtid, stid, + wsrep::seqno(group_seqno_ - 1), + flags); + ret = wsrep::provider::error_bf_abort; + } + bf_abort_map_.erase(it); + return ret; + } + } + + enum wsrep::provider::status + assign_read_view(wsrep::ws_handle&, const wsrep::gtid*) + WSREP_OVERRIDE + { return wsrep::provider::success; } + int append_key(wsrep::ws_handle&, const wsrep::key&) + WSREP_OVERRIDE + { return 0; } + enum wsrep::provider::status + append_data(wsrep::ws_handle&, const wsrep::const_buffer&) + WSREP_OVERRIDE + { return wsrep::provider::success; } + enum wsrep::provider::status rollback(const wsrep::transaction_id) + WSREP_OVERRIDE + { + ++fragments_; + ++rollback_fragments_; + return wsrep::provider::success; + } + enum wsrep::provider::status + commit_order_enter(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) + WSREP_OVERRIDE + { + BOOST_REQUIRE(ws_handle.opaque()); + BOOST_REQUIRE(ws_meta.seqno().is_undefined() == false); + return commit_order_enter_result_; + } + + int commit_order_leave(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta, + const wsrep::mutable_buffer& err) + WSREP_OVERRIDE + { + BOOST_REQUIRE(ws_handle.opaque()); + BOOST_REQUIRE(ws_meta.seqno().is_undefined() == false); + return err.size() > 0 ? + wsrep::provider::error_fatal : + commit_order_leave_result_; + } + + int release(wsrep::ws_handle& ) + WSREP_OVERRIDE + { + // BOOST_REQUIRE(ws_handle.opaque()); + return release_result_; + } + + enum wsrep::provider::status replay( + const wsrep::ws_handle& ws_handle, + wsrep::high_priority_service* hps) + WSREP_OVERRIDE + { + wsrep::mock_high_priority_service& high_priority_service( + *static_cast<wsrep::mock_high_priority_service*>(hps)); + wsrep::mock_client_state& cc( + static_cast<wsrep::mock_client_state&>( + high_priority_service.client_state())); + wsrep::high_priority_context high_priority_context(cc); + const wsrep::transaction& tc(cc.transaction()); + wsrep::ws_meta ws_meta; + if (replay_result_ == wsrep::provider::success) + { + // If the ws_meta was not assigned yet, the certify + // returned early due to BF abort. + if (tc.ws_meta().seqno().is_undefined()) + { + ++group_seqno_; + ws_meta = wsrep::ws_meta( + wsrep::gtid(group_id_, wsrep::seqno(group_seqno_)), + wsrep::stid(server_id_, tc.id(), cc.id()), + wsrep::seqno(group_seqno_ - 1), + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::commit); + } + else + { + ws_meta = tc.ws_meta(); + } + } + else + { + return replay_result_; + } + + if (high_priority_service.apply(ws_handle, ws_meta, + wsrep::const_buffer())) + { + return wsrep::provider::error_fatal; + } + return wsrep::provider::success; + } + + enum wsrep::provider::status enter_toi(wsrep::client_id client_id, + const wsrep::key_array&, + const wsrep::const_buffer&, + wsrep::ws_meta& toi_meta, + int flags) + WSREP_OVERRIDE + { + ++group_seqno_; + wsrep::gtid gtid(group_id_, wsrep::seqno(group_seqno_)); + wsrep::stid stid(server_id_, + wsrep::transaction_id::undefined(), + client_id); + toi_meta = wsrep::ws_meta(gtid, stid, + wsrep::seqno(group_seqno_ - 1), + flags); + ++toi_write_sets_; + if (flags & wsrep::provider::flag::start_transaction) + ++toi_start_transaction_; + if (flags & wsrep::provider::flag::commit) + ++toi_commit_; + return certify_result_; + } + + enum wsrep::provider::status leave_toi(wsrep::client_id, + const wsrep::mutable_buffer&) + WSREP_OVERRIDE + { return wsrep::provider::success; } + + std::pair<wsrep::gtid, enum wsrep::provider::status> + causal_read(int) const WSREP_OVERRIDE + { + return std::make_pair(wsrep::gtid::undefined(), + wsrep::provider::error_not_implemented); + } + enum wsrep::provider::status wait_for_gtid(const wsrep::gtid&, + int) const WSREP_OVERRIDE + { return wsrep::provider::success; } + wsrep::gtid last_committed_gtid() const WSREP_OVERRIDE + { return wsrep::gtid(); } + enum wsrep::provider::status sst_sent(const wsrep::gtid&, int) + WSREP_OVERRIDE + { return wsrep::provider::success; } + enum wsrep::provider::status sst_received(const wsrep::gtid&, int) + WSREP_OVERRIDE + { return wsrep::provider::success; } + + enum wsrep::provider::status enc_set_key(const wsrep::const_buffer&) + WSREP_OVERRIDE + { return wsrep::provider::success; } + + std::vector<status_variable> status() const WSREP_OVERRIDE + { + return std::vector<status_variable>(); + } + void reset_status() WSREP_OVERRIDE { } + std::string options() const WSREP_OVERRIDE { return ""; } + enum wsrep::provider::status options(const std::string&) + WSREP_OVERRIDE + { return wsrep::provider::success; } + std::string name() const WSREP_OVERRIDE { return "mock"; } + std::string version() const WSREP_OVERRIDE { return "0.0"; } + std::string vendor() const WSREP_OVERRIDE { return "mock"; } + void* native() const WSREP_OVERRIDE { return 0; } + + // + // Methods to modify mock state + // + /** Inject BF abort event into the provider. + * + * @param bf_seqno Aborter sequence number + * @param trx_id Trx id to be aborted + * @param[out] victim_seqno + */ + enum wsrep::provider::status + bf_abort(wsrep::seqno bf_seqno, + wsrep::transaction_id trx_id, + wsrep::seqno& victim_seqno) + WSREP_OVERRIDE + { + bf_abort_map_.insert(std::make_pair(trx_id, bf_seqno)); + if (bf_seqno.is_undefined() == false) + { + group_seqno_ = bf_seqno.get(); + } + victim_seqno = wsrep::seqno::undefined(); + return wsrep::provider::success; + } + + // Parameters to control return value from the call + enum wsrep::provider::status certify_result_; + enum wsrep::provider::status commit_order_enter_result_; + enum wsrep::provider::status commit_order_leave_result_; + enum wsrep::provider::status release_result_; + enum wsrep::provider::status replay_result_; + + size_t start_fragments() const { return start_fragments_; } + size_t fragments() const { return fragments_; } + size_t commit_fragments() const { return commit_fragments_; } + size_t rollback_fragments() const { return rollback_fragments_; } + size_t toi_write_sets() const { return toi_write_sets_; } + size_t toi_start_transaction() const { return toi_start_transaction_; } + size_t toi_commit() const { return toi_commit_; } + private: + wsrep::id group_id_; + wsrep::id server_id_; + long long group_seqno_; + bf_abort_map bf_abort_map_; + size_t start_fragments_; + size_t fragments_; + size_t commit_fragments_; + size_t rollback_fragments_; + size_t toi_write_sets_; + size_t toi_start_transaction_; + size_t toi_commit_; + }; +} + + +#endif // WSREP_MOCK_PROVIDER_HPP diff --git a/wsrep-lib/test/mock_server_state.hpp b/wsrep-lib/test/mock_server_state.hpp new file mode 100644 index 00000000..730c9c04 --- /dev/null +++ b/wsrep-lib/test/mock_server_state.hpp @@ -0,0 +1,302 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_MOCK_SERVER_STATE_HPP +#define WSREP_MOCK_SERVER_STATE_HPP + +#include "wsrep/server_state.hpp" +#include "wsrep/server_service.hpp" +#include "mock_client_state.hpp" +#include "mock_high_priority_service.hpp" +#include "mock_storage_service.hpp" +#include "mock_provider.hpp" + +#include "wsrep/compiler.hpp" + +namespace wsrep +{ + class mock_server_service : public wsrep::server_service + { + public: + mock_server_service(wsrep::server_state& server_state) + : sync_point_enabled_() + , sync_point_action_() + , sst_before_init_() + , server_state_(server_state) + , last_client_id_(0) + , last_transaction_id_(0) + , logged_view_() + , position_() + { } + + wsrep::storage_service* storage_service(wsrep::client_service&) + WSREP_OVERRIDE + { + return new wsrep::mock_storage_service(server_state_, + wsrep::client_id(++last_client_id_)); + } + + wsrep::storage_service* storage_service(wsrep::high_priority_service&) + WSREP_OVERRIDE + { + return new wsrep::mock_storage_service(server_state_, + wsrep::client_id(++last_client_id_)); + } + + void release_storage_service(wsrep::storage_service* storage_service) + WSREP_OVERRIDE + { + delete storage_service; + } + + wsrep::high_priority_service* streaming_applier_service( + wsrep::client_service&) + WSREP_OVERRIDE + { + wsrep::mock_client* cs(new wsrep::mock_client( + server_state_, + wsrep::client_id(++last_client_id_), + wsrep::client_state::m_high_priority)); + wsrep::mock_high_priority_service* ret( + new wsrep::mock_high_priority_service(server_state_, + cs, false)); + cs->open(cs->id()); + cs->before_command(); + return ret; + } + + wsrep::high_priority_service* streaming_applier_service( + wsrep::high_priority_service&) WSREP_OVERRIDE + { + wsrep::mock_client* cs(new wsrep::mock_client( + server_state_, + wsrep::client_id(++last_client_id_), + wsrep::client_state::m_high_priority)); + wsrep::mock_high_priority_service* ret( + new wsrep::mock_high_priority_service(server_state_, + cs, false)); + cs->open(cs->id()); + cs->before_command(); + return ret; + } + + void release_high_priority_service( + wsrep::high_priority_service *high_priority_service) + WSREP_OVERRIDE + { + mock_high_priority_service* mhps( + static_cast<mock_high_priority_service*>(high_priority_service)); + wsrep::mock_client* cs(&static_cast<wsrep::mock_client&>( + mhps->client_state())); + cs->after_command_before_result(); + cs->after_command_after_result(); + cs->close(); + cs->cleanup(); + delete cs; + delete mhps; + } + void bootstrap() WSREP_OVERRIDE { } + void log_message(enum wsrep::log::level level, const char* message) + WSREP_OVERRIDE + { + wsrep::log(level, server_state_.name().c_str()) << message; + } + void log_dummy_write_set(wsrep::client_state&, + const wsrep::ws_meta&) + WSREP_OVERRIDE + { + } + void log_view(wsrep::high_priority_service*, const wsrep::view& view) + WSREP_OVERRIDE + { + logged_view_ = view; + } + + void recover_streaming_appliers(wsrep::client_service&) + WSREP_OVERRIDE + { } + + void recover_streaming_appliers(wsrep::high_priority_service&) + WSREP_OVERRIDE + { } + + wsrep::view get_view(wsrep::client_service&, const wsrep::id& own_id) + WSREP_OVERRIDE + { + int const my_idx(logged_view_.member_index(own_id)); + wsrep::view my_view( + logged_view_.state_id(), + logged_view_.view_seqno(), + logged_view_.status(), + logged_view_.capabilities(), + my_idx, + logged_view_.protocol_version(), + logged_view_.members() + ); + return my_view; + } + + wsrep::gtid get_position(wsrep::client_service&) WSREP_OVERRIDE + { + return position_; + } + + void set_position(wsrep::client_service&, + const wsrep::gtid& gtid) WSREP_OVERRIDE + { + position_ = gtid; + } + + void log_state_change(enum wsrep::server_state::state, + enum wsrep::server_state::state) + WSREP_OVERRIDE + { } + bool sst_before_init() const WSREP_OVERRIDE + { return sst_before_init_; } + std::string sst_request() WSREP_OVERRIDE { return ""; } + int start_sst(const std::string&, + const wsrep::gtid&, + bool) WSREP_OVERRIDE { return 0; } + void background_rollback(wsrep::client_state& client_state) + WSREP_OVERRIDE + { + client_state.before_rollback(); + client_state.after_rollback(); + } + + int wait_committing_transactions(int) WSREP_OVERRIDE { return 0; } + + wsrep::transaction_id next_transaction_id() + { + return wsrep::transaction_id(++last_transaction_id_); + } + + void debug_sync(const char* sync_point) WSREP_OVERRIDE + { + if (sync_point_enabled_ == sync_point) + { + switch (sync_point_action_) + { + case spa_none: + break; + case spa_initialize: + server_state_.initialized(); + break; + case spa_initialize_error: + throw wsrep::runtime_error("Inject initialization error"); + break; + } + } + } + + std::string sync_point_enabled_; + enum sync_point_action + { + spa_none, + spa_initialize, + spa_initialize_error + + } sync_point_action_; + bool sst_before_init_; + + void logged_view(const wsrep::view& view) + { + logged_view_ = view; + } + void position(const wsrep::gtid& position) + { + position_ = position; + } + private: + wsrep::server_state& server_state_; + unsigned long long last_client_id_; + unsigned long long last_transaction_id_; + wsrep::view logged_view_; + wsrep::gtid position_; + }; + + + class mock_server_state : public wsrep::server_state + { + public: + mock_server_state(const std::string& name, + enum wsrep::server_state::rollback_mode rollback_mode, + wsrep::server_service& server_service) + : wsrep::server_state(mutex_, cond_, server_service, NULL, + name, "", "", "./", + wsrep::gtid::undefined(), + 1, + rollback_mode) + , mutex_() + , cond_() + , provider_(*this) + { } + + wsrep::mock_provider& provider() const WSREP_OVERRIDE + { return provider_; } + + // mock connected state for tests without overriding the connect() + // method. + int mock_connect(const std::string& own_id, + const std::string& cluster_name, + const std::string& cluster_address, + const std::string& state_donor, + bool bootstrap) + { + int const ret(server_state::connect(cluster_name, + cluster_address, + state_donor, + bootstrap)); + if (0 == ret) + { + wsrep::id cluster_id("1"); + wsrep::gtid state_id(cluster_id, wsrep::seqno(0)); + std::vector<wsrep::view::member> members; + members.push_back(wsrep::view::member(wsrep::id(own_id), + "name", "")); + wsrep::view bootstrap_view(state_id, + wsrep::seqno(1), + wsrep::view::primary, + 0, + 0, + 1, + members); + server_state::on_connect(bootstrap_view); + } + else + { + assert(0); + } + + return ret; + } + + int mock_connect() + { + return mock_connect(name(), "cluster", "local", "0", false); + } + + private: + wsrep::default_mutex mutex_; + wsrep::default_condition_variable cond_; + mutable wsrep::mock_provider provider_; + }; +} + +#endif // WSREP_MOCK_SERVER_STATE_HPP diff --git a/wsrep-lib/test/mock_storage_service.cpp b/wsrep-lib/test/mock_storage_service.cpp new file mode 100644 index 00000000..a455b9bf --- /dev/null +++ b/wsrep-lib/test/mock_storage_service.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "mock_storage_service.hpp" +#include "mock_server_state.hpp" + +#include "wsrep/client_state.hpp" + +wsrep::mock_storage_service::mock_storage_service( + wsrep::server_state& server_state, + wsrep::client_id client_id) + : client_service_(client_state_) + , client_state_(server_state, client_service_, client_id, + wsrep::client_state::m_high_priority) +{ + client_state_.open(client_id); + client_state_.before_command(); +} + + +wsrep::mock_storage_service::~mock_storage_service() +{ + client_state_.after_command_before_result(); + client_state_.after_command_after_result(); + client_state_.close(); + client_state_.cleanup(); +} + +int wsrep::mock_storage_service::start_transaction( + const wsrep::ws_handle& ws_handle) +{ + return client_state_.start_transaction(ws_handle.transaction_id()); +} + +void wsrep::mock_storage_service::adopt_transaction( + const wsrep::transaction& transaction) +{ + client_state_.adopt_transaction(transaction); +} + +int wsrep::mock_storage_service::commit(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + // the logic here matches mariadb's wsrep_storage_service::commit + bool ret = 0; + const bool is_ordered = !ws_meta.seqno().is_undefined(); + if (is_ordered) + { + ret = ret || client_state_.prepare_for_ordering(ws_handle, ws_meta, true); + ret = ret || client_state_.before_commit(); + ret = ret || client_state_.ordered_commit(); + ret = ret || client_state_.after_commit(); + } + + if (!is_ordered) + { + client_state_.before_rollback(); + client_state_.after_rollback(); + } + else if (ret) + { + client_state_.prepare_for_ordering(wsrep::ws_handle(), wsrep::ws_meta(), false); + } + + client_state_.after_applying(); + return ret; +} + +int wsrep::mock_storage_service::rollback(const wsrep::ws_handle& ws_handle, + const wsrep::ws_meta& ws_meta) +{ + int ret(client_state_.prepare_for_ordering( + ws_handle, ws_meta, false) || + client_state_.before_rollback() || + client_state_.after_rollback()); + client_state_.after_applying(); + return ret; +} diff --git a/wsrep-lib/test/mock_storage_service.hpp b/wsrep-lib/test/mock_storage_service.hpp new file mode 100644 index 00000000..b0275b97 --- /dev/null +++ b/wsrep-lib/test/mock_storage_service.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef WSREP_MOCK_STORAGE_SERVICE_HPP +#define WSREP_MOCK_STORAGE_SERVICE_HPP + +#include "wsrep/storage_service.hpp" +#include "mock_client_state.hpp" + +namespace wsrep +{ +class mock_server_state; + class mock_storage_service : public wsrep::storage_service + { + public: + mock_storage_service(wsrep::server_state&, wsrep::client_id); + ~mock_storage_service(); + + int start_transaction(const wsrep::ws_handle&) WSREP_OVERRIDE; + + void adopt_transaction(const wsrep::transaction&) WSREP_OVERRIDE; + + int append_fragment(const wsrep::id&, + wsrep::transaction_id, + int, + const wsrep::const_buffer&, + const wsrep::xid&) WSREP_OVERRIDE + { return 0; } + + int update_fragment_meta(const wsrep::ws_meta&) WSREP_OVERRIDE + { return 0; } + int remove_fragments() WSREP_OVERRIDE { return 0; } + int commit(const wsrep::ws_handle&, const wsrep::ws_meta&) + WSREP_OVERRIDE; + + int rollback(const wsrep::ws_handle&, const wsrep::ws_meta&) + WSREP_OVERRIDE; + + void store_globals() WSREP_OVERRIDE { } + void reset_globals() WSREP_OVERRIDE { } + private: + wsrep::mock_client_service client_service_; + wsrep::mock_client_state client_state_; + }; +} + +#endif // WSREP_MOCK_STORAGE_SERVICE_HPP diff --git a/wsrep-lib/test/nbo_test.cpp b/wsrep-lib/test/nbo_test.cpp new file mode 100644 index 00000000..238a4da5 --- /dev/null +++ b/wsrep-lib/test/nbo_test.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/client_state.hpp" + +#include "client_state_fixture.hpp" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_CASE(test_local_nbo, + replicating_client_fixture_sync_rm) +{ + // NBO is executed in two consecutive TOI operations + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + // First phase certifies the write set and enters TOI + wsrep::key key(wsrep::key::exclusive); + key.append_key_part("k1", 2); + key.append_key_part("k2", 2); + wsrep::key_array keys{key}; + std::string data("data"); + BOOST_REQUIRE(cc.begin_nbo_phase_one( + keys, + wsrep::const_buffer(data.data(), + data.size())) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_nbo); + BOOST_REQUIRE(cc.in_toi()); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local); + // After required resoureces have been grabbed, NBO leave should + // end TOI but leave the client state in m_nbo. + const wsrep::mutable_buffer err; + BOOST_REQUIRE(cc.end_nbo_phase_one(err) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_nbo); + BOOST_REQUIRE(cc.in_toi() == false); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); + // Second phase replicates the NBO end event and grabs TOI + // again for finalizing the NBO. + BOOST_REQUIRE(cc.begin_nbo_phase_two(keys) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_nbo); + BOOST_REQUIRE(cc.in_toi()); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local); + // End of NBO phase will leave TOI and put the client state back to + // m_local + BOOST_REQUIRE(cc.end_nbo_phase_two(err) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + BOOST_REQUIRE(cc.in_toi() == false); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); + + // There must have been two toi write sets, one with + // start transaction flag, another with commit flag. + BOOST_REQUIRE(sc.provider().toi_write_sets() == 2); + BOOST_REQUIRE(sc.provider().toi_start_transaction() == 1); + BOOST_REQUIRE(sc.provider().toi_commit() == 1); +} + +BOOST_FIXTURE_TEST_CASE(test_local_nbo_cert_failure, + replicating_client_fixture_sync_rm) +{ + // NBO is executed in two consecutive TOI operations + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + // First phase certifies the write set and enters TOI + wsrep::key key(wsrep::key::exclusive); + key.append_key_part("k1", 2); + key.append_key_part("k2", 2); + wsrep::key_array keys{key}; + std::string data("data"); + sc.provider().certify_result_ = wsrep::provider::error_certification_failed; + BOOST_REQUIRE(cc.begin_nbo_phase_one( + keys, + wsrep::const_buffer(data.data(), + data.size())) == 1); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + BOOST_REQUIRE(cc.current_error_status() == + wsrep::provider::error_certification_failed); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + BOOST_REQUIRE(cc.in_toi() == false); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); +} + +// This test case operates through server_state object in order to +// verify that the high priority service is called with appropriate +// arguments. +BOOST_FIXTURE_TEST_CASE(test_applying_nbo, + applying_client_fixture) +{ + + wsrep::mock_high_priority_service hps(sc, &cc, false); + wsrep::ws_handle ws_handle(wsrep::transaction_id::undefined(), (void*)(1)); + const int nbo_begin_flags(wsrep::provider::flag::start_transaction | + wsrep::provider::flag::isolation); + wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("cluster1"), + wsrep::seqno(1)), + wsrep::stid(wsrep::id("s1"), + wsrep::transaction_id::undefined(), + wsrep::client_id(1)), + wsrep::seqno(0), + nbo_begin_flags); + std::string nbo_begin("nbo_begin"); + BOOST_REQUIRE(sc.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer(nbo_begin.data(), + nbo_begin.size())) == 0); + wsrep::mock_client* nbo_cs(hps.nbo_cs()); + BOOST_REQUIRE(nbo_cs); + BOOST_REQUIRE(nbo_cs->toi_mode() == wsrep::client_state::m_undefined); + BOOST_REQUIRE(nbo_cs->mode() == wsrep::client_state::m_nbo); + + // After this point the control is on local process and applier + // has released toi critical section. + wsrep::key key(wsrep::key::exclusive); + key.append_key_part("k1", 2); + key.append_key_part("k2", 2); + wsrep::key_array keys{key}; + // Starting phase two should put nbo_cs in toi mode. + BOOST_REQUIRE(nbo_cs->begin_nbo_phase_two(keys) == 0); + BOOST_REQUIRE(nbo_cs->mode() == wsrep::client_state::m_nbo); + BOOST_REQUIRE(nbo_cs->in_toi()); + BOOST_REQUIRE(nbo_cs->toi_mode() == wsrep::client_state::m_local); + // Ending phase two should make nbo_cs leave TOI mode and + // return to m_local mode. + const wsrep::mutable_buffer err; + BOOST_REQUIRE(nbo_cs->end_nbo_phase_two(err) == 0); + BOOST_REQUIRE(nbo_cs->mode() == wsrep::client_state::m_local); + BOOST_REQUIRE(nbo_cs->in_toi() == false); + BOOST_REQUIRE(nbo_cs->toi_mode() == wsrep::client_state::m_undefined); + + // There must have been one toi write set with commit flag. + BOOST_REQUIRE(sc.provider().toi_write_sets() == 1); + BOOST_REQUIRE(sc.provider().toi_start_transaction() == 0); + BOOST_REQUIRE(sc.provider().toi_commit() == 1); +} + +// This test case operates through server_state object in order to +// verify that the high priority service is called with appropriate +// arguments. The test checks that error is returned in the case if +// launching asynchronous process for NBO fails. +BOOST_FIXTURE_TEST_CASE(test_applying_nbo_fail, + applying_client_fixture) +{ + + wsrep::mock_high_priority_service hps(sc, &cc, false); + wsrep::ws_handle ws_handle(wsrep::transaction_id::undefined(), (void*)(1)); + const int nbo_begin_flags(wsrep::provider::flag::start_transaction | + wsrep::provider::flag::isolation); + wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("cluster1"), + wsrep::seqno(1)), + wsrep::stid(wsrep::id("s1"), + wsrep::transaction_id::undefined(), + wsrep::client_id(1)), + wsrep::seqno(0), + nbo_begin_flags); + std::string nbo_begin("nbo_begin"); + hps.fail_next_toi_ = true; + BOOST_REQUIRE(sc.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer(nbo_begin.data(), + nbo_begin.size())) == 1); +} diff --git a/wsrep-lib/test/reporter_test.cpp b/wsrep-lib/test/reporter_test.cpp new file mode 100644 index 00000000..bed7a2a6 --- /dev/null +++ b/wsrep-lib/test/reporter_test.cpp @@ -0,0 +1,636 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/reporter.hpp" + +#include <boost/test/unit_test.hpp> + +#include <boost/json/src.hpp> + +#include <fstream> +#include <deque> +#include <vector> +#include <unistd.h> // unlink() for cleanup + +namespace json = boost::json; + +//////// HELPERS /////// + +static inline double +timestamp() +{ + struct timespec time; + clock_gettime(CLOCK_REALTIME, &time); + return (double(time.tv_sec) + double(time.tv_nsec)*1.0e-9); +} + +static std::string make_progress_string(int const from, int const to, + int const total,int const done, + int const indefinite) +{ + std::ostringstream os; + + os << "{ \"from\": " << from << ", " + << "\"to\": " << to << ", " + << "\"total\": " << total << ", " + << "\"done\": " << done << ", " + << "\"indefinite\": " << indefinite << " }"; + + return os.str(); +} + + +static json::value +read_file(const char* const filename) +{ + std::ifstream input(filename, std::ios::binary); + std::vector<char> buffer(std::istreambuf_iterator<char>(input), {}); + json::stream_parser parser; + json::error_code err; + + parser.write(buffer.data(), buffer.size(), err); + if (err) + { + assert(0); + return nullptr; + } + + parser.finish(err); + if (err) + { + assert(0); + return nullptr; + } + + return parser.release(); +} + +struct logs +{ + std::deque<double> tstamp_; + std::deque<std::string> msg_; +}; + +struct progress +{ + int from_; + int to_; + int total_; + int done_; + int indefinite_; + + bool indefinite() const { return total_ == indefinite_; } + bool steady() const { return total_ == 0; } + progress(int const from, int const to, int const total, int const done, + int const indefinite) + : from_(from) + , to_(to) + , total_(total) + , done_(done) + , indefinite_(indefinite) + {} +}; + +static bool +operator==(const progress& left, const progress& right) +{ + if (left.indefinite() && right.indefinite()) return true; + if (left.steady() && right.steady()) return true; + + return (left.from_ == right.from_ && + left.to_ == right.to_ && + left.total_ == right.total_ && + left.done_ == right.done_ && + left.indefinite_ == right.indefinite_); +} + +static bool +operator!=(const progress& left, const progress& right) +{ + return !(left == right); +} + +static std::ostream& +operator<<(std::ostream& os, const progress& p) +{ + os << make_progress_string(p.from_, p.to_, p.total_, p.done_,p.indefinite_); + return os; +} + +struct result +{ + logs errors_; + logs warnings_; + struct + { + std::string state_; + std::string comment_; + progress progress_; + } status_; +}; + +static void +parse_result(const json::value& value, struct result& res, + const std::string& path = "") +{ + //std::cout << "Parsing " << path << ": " << value << ": " << value.kind() << std::endl; + switch (value.kind()) + { + case json::kind::object: + { + auto const obj(value.get_object()); + if (!obj.empty()) + { + for (auto it = obj.begin(); it != obj.end(); ++it) + { + std::string const key(it->key().data(), it->key().length()); + parse_result(it->value(), res, path + "." + key); + } + } + return; + } + case json::kind::array: + { + auto const arr(value.get_array()); + if (!arr.empty()) + { + for (auto it = arr.begin(); it != arr.end(); ++it) + { + parse_result(*it, res, path + ".[]"); + } + } + return; + } + case json::kind::string: + { + auto const val(value.get_string().c_str()); + if (path == ".errors.[].msg") + { + res.errors_.msg_.push_back(val); + } + else if (path == ".warnings.[].msg") + { + res.warnings_.msg_.push_back(val); + } + else if (path == ".status.state") + { + res.status_.state_ = val; + } + else if (path == ".status.comment") + { + res.status_.comment_ = val; + } + return; + } + case json::kind::uint64: + WSREP_FALLTHROUGH; + case json::kind::int64: + if (path == ".status.progress.from") + { + res.status_.progress_.from_ = int(value.get_int64()); + } + else if (path == ".status.progress.to") + { + res.status_.progress_.to_ = int(value.get_int64()); + } + else if (path == ".status.progress.total") + { + res.status_.progress_.total_ = int(value.get_int64()); + } + else if (path == ".status.progress.done") + { + res.status_.progress_.done_ = int(value.get_int64()); + } + else if (path == ".status.progress.indefinite") + { + res.status_.progress_.indefinite_ = int(value.get_int64()); + } + return; + case json::kind::double_: + if (path == ".errors.[].timestamp") + { + res.errors_.tstamp_.push_back(value.get_double()); + } + else if (path == ".warnings.[].timestamp") + { + res.warnings_.tstamp_.push_back(value.get_double()); + } + + return; + case json::kind::bool_: + return; + case json::kind::null: + return; + } + + assert(0); +} + +static bool +equal(const std::string& left, const std::string& right) +{ + return left == right; +} + +static bool +equal(double const left, double const right) +{ + return ::fabs(left - right) < 0.0001; // we are looking for ms precision +} + +template <typename T> +static bool +operator!=(const std::deque<T>& left, const std::deque<T>& right) +{ + if (left.size() != right.size()) return true; + + for (size_t i(0); i < left.size(); ++i) + if (!equal(left[i], right[i])) return true; + + return false; +} + +static bool +operator!=(const logs& left, const logs& right) +{ + if (left.tstamp_ != right.tstamp_) return true; + return (left.msg_ != right.msg_); +} + +static bool +operator==(const result& left, const result& right) +{ + if (left.errors_ != right.errors_) return false; + if (left.warnings_ != right.warnings_) return false; + + if (left.status_.state_ != right.status_.state_) return false; + if (left.status_.comment_ != right.status_.comment_) return false; + if (left.status_.progress_ != right.status_.progress_) return false; + + return true; +} + +template <typename T> +static void +print_deque(std::ostream& os,const std::deque<T> left,const std::deque<T> right) +{ + auto const max(std::max(left.size(), right.size())); + for (size_t i(0); i < max; ++i) + { + os << "|\t'"; + + if (i < left.size()) + os << left[i] << "'"; + else + os << "'\t"; + + if (i < right.size()) + os << "\t'" << right[i] << "'"; + else + os << "\t''"; + + if (!equal(left[i], right[i])) os << "\t!!!"; + + os << "\n"; + } +} + +static void +print_logs(std::ostream& os, const logs& left, const logs& right) +{ + os << "|\t" << left.msg_.size() << "\t" << right.msg_.size() << "\n"; + print_deque(os, left.tstamp_, right.tstamp_); + print_deque(os, left.msg_, right.msg_); +} + +// print two results against each other +static std::string +print(const result& left, const result& right, size_t it) +{ + std::ostringstream os; + + os << std::showpoint << std::setprecision(18); + + os << "Iteration " << it << "\nerrors:\n"; + print_logs(os, left.errors_, right.errors_); + os << "warnings:\n"; + print_logs(os, left.warnings_, right.warnings_); + os << "state:\n"; + os << "\t" << left.status_.state_ << "\t" << right.status_.state_ + << "\n"; + os << "\t" << left.status_.comment_ << "\t" << right.status_.comment_ + << "\n"; + os << "\t" << left.status_.progress_ << "\t" << right.status_.progress_ + << "\n"; + + return os.str(); +} + +#define VERIFY_RESULT(left, right, it) \ + BOOST_CHECK_MESSAGE(left == right, print(left, right, it)); + +static const char* +const REPORT = "report.json"; + +static progress +const indefinite(-1, -1, -1, -1, -1); + +static progress +const steady(-1, -1, 0, 0, -1); + +static struct logs +const LOGS_INIT = { std::deque<double>(), std::deque<std::string>() }; +static struct result +const RES_INIT = { LOGS_INIT, LOGS_INIT, + { "DISCONNECTED", "Disconnected", indefinite } }; + +static void +test_log(const char* const fname, + wsrep::reporter& rep, + result& check, + wsrep::reporter::log_level const lvl, + double const tstamp, + const std::string& msg, + size_t const iteration) +{ + // this is implementaiton detail, if it changes in the code, it needs + // to be changed here + size_t const MAX_ERROR(4); + + logs& log(lvl == wsrep::reporter::error ? check.errors_ : check.warnings_); + log.tstamp_.push_back(tstamp); + if (log.tstamp_.size() > MAX_ERROR) log.tstamp_.pop_front(); + log.msg_.push_back(msg); + if (log.msg_.size() > MAX_ERROR) log.msg_.pop_front(); + + rep.report_log_msg(lvl, msg, tstamp); + + auto value = read_file(fname); + auto res = RES_INIT; + parse_result(value, res); + VERIFY_RESULT(res, check, iteration); +} + +static size_t const MAX_MSG = 4; + +BOOST_AUTO_TEST_CASE(log_msg_test) +{ + wsrep::default_mutex m; + wsrep::reporter rep(m, REPORT, MAX_MSG); + + auto value = read_file(REPORT); + BOOST_REQUIRE(value != nullptr); + + struct result res(RES_INIT), check(RES_INIT); + parse_result(value, res); + VERIFY_RESULT(res, check, -1); + + struct entry + { + double tstamp_; + std::string msg_; + }; + std::vector<entry> msgs = + { + { 0.1, "a" }, + { 0.2, "bb" }, + { 0.3, "ccc" }, + { 0.4, "dddd" }, + { 0.5, "eeeee" }, + { 0.6, "ffffff" } + }; + for (size_t i(0); i < msgs.size(); ++i) + { + test_log(REPORT, rep, check, + wsrep::reporter::error, msgs[i].tstamp_, msgs[i].msg_, i); + test_log(REPORT, rep, check, + wsrep::reporter::warning, msgs[i].tstamp_, msgs[i].msg_, i); + } + + // test indefinite timestmap + std::string const msg("err"); + rep.report_log_msg(wsrep::reporter::error, msg); + value = read_file(REPORT); + res = RES_INIT; + parse_result(value, res); + BOOST_REQUIRE(res.errors_.tstamp_.back() > 0); + BOOST_REQUIRE(res.errors_.msg_.back() == msg); + + ::unlink(REPORT); +} + +BOOST_AUTO_TEST_CASE(state_test) +{ + using wsrep::server_state; + + wsrep::default_mutex m; + wsrep::reporter rep(m, REPORT, MAX_MSG); + double const err_tstamp(timestamp()); + std::string const err_msg("Error!"); + + struct test + { + struct + { + enum wsrep::server_state::state state; + float progress; + } input; + struct result output; + }; + + logs const ERRS_INIT = { {err_tstamp}, {err_msg} }; + + std::vector<test> tests = + { + {{ server_state::s_disconnected, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DISCONNECTED", "Disconnected", indefinite }}}, + {{ server_state::s_initializing, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DISCONNECTED", "Initializing", indefinite }}}, + {{ server_state::s_initialized, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DISCONNECTED", "Connecting", indefinite }}}, + {{ server_state::s_connected, 0 }, + { ERRS_INIT, LOGS_INIT, + { "CONNECTED", "Waiting", indefinite }}}, + {{ server_state::s_joiner, 0 }, + { ERRS_INIT, LOGS_INIT, + { "JOINING", "Receiving state", indefinite }}}, + {{ server_state::s_disconnecting, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DISCONNECTING", "Disconnecting", indefinite }}}, + {{ server_state::s_disconnected, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DISCONNECTED", "Disconnected", indefinite }}}, + {{ server_state::s_connected, 0 }, + { ERRS_INIT, LOGS_INIT, + { "CONNECTED", "Waiting", indefinite }}}, + {{ server_state::s_joiner, 0 }, + { ERRS_INIT, LOGS_INIT, + { "JOINING", "Receiving SST", indefinite }}}, + {{ server_state::s_initializing, 0 }, + { ERRS_INIT, LOGS_INIT, + { "JOINING", "Initializing", indefinite }}}, + {{ server_state::s_initialized, 0 }, + { ERRS_INIT, LOGS_INIT, + { "JOINING", "Receiving IST", indefinite }}}, + {{ server_state::s_joined, 0 }, + { ERRS_INIT, LOGS_INIT, + { "JOINED", "Syncing", indefinite }}}, + {{ server_state::s_synced, 0 }, + { ERRS_INIT, LOGS_INIT, + { "SYNCED", "Operational", steady }}}, + {{ server_state::s_donor, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DONOR", "Donating SST", indefinite }}}, + {{ server_state::s_joined, 0 }, + { ERRS_INIT, LOGS_INIT, + { "JOINED", "Syncing", indefinite }}}, + {{ server_state::s_synced, 0 }, + { ERRS_INIT, LOGS_INIT, + { "SYNCED", "Operational", steady }}}, + {{ server_state::s_disconnecting, 0 }, + { ERRS_INIT, LOGS_INIT, + { "DISCONNECTING", "Disconnecting", indefinite }}}, + }; + + rep.report_log_msg(wsrep::reporter::error, err_msg, err_tstamp); + + for (auto i(tests.begin()); i != tests.end(); ++i) + { + rep.report_state(i->input.state); + auto value = read_file(REPORT); + result res(RES_INIT); + parse_result(value, res); + auto check(i->output); + VERIFY_RESULT(res, check, i - tests.begin()); + } + + ::unlink(REPORT); +} + +BOOST_AUTO_TEST_CASE(progress_test) +{ + using wsrep::server_state; + + wsrep::default_mutex m; + wsrep::reporter rep(m, REPORT, MAX_MSG); + double const warn_tstamp(timestamp()); + std::string const warn_msg("Warn!"); + + static progress const progress5_0(-1, -1, 5, 0, -1); + static progress const progress5_2(-1, -1, 5, 2, -1); + static progress const progress5_5(-1, -1, 5, 5, -1); + + struct test + { + struct + { + enum wsrep::server_state::state state; + int total; + int done; + } input; + struct result output; + }; + + logs const WARN_INIT = { {warn_tstamp}, {warn_msg} }; + + std::vector<test> tests = + { + {{ server_state::s_initialized, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "DISCONNECTED", "Connecting", indefinite }}}, + {{ server_state::s_connected, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "CONNECTED", "Waiting", indefinite }}}, + {{ server_state::s_joiner, 5, 0 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving state", progress5_0 }}}, + {{ server_state::s_joiner, 5, 2 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving state", progress5_2 }}}, + {{ server_state::s_disconnected, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "DISCONNECTED", "Disconnected", indefinite }}}, + {{ server_state::s_connected, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "CONNECTED", "Waiting", indefinite }}}, + {{ server_state::s_joiner, 5, 0 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving SST", progress5_0 }}}, + {{ server_state::s_joiner, 5, 2 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving SST", progress5_2 }}}, + {{ server_state::s_joiner, 5, 5 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving SST", progress5_5 }}}, + {{ server_state::s_initializing, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Initializing", indefinite }}}, + {{ server_state::s_initializing, 5, 2 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Initializing", progress5_2 }}}, + {{ server_state::s_initialized, 5, 0 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving IST", progress5_0 }}}, + {{ server_state::s_initialized, 5, 5 }, + { LOGS_INIT, WARN_INIT, + { "JOINING", "Receiving IST", progress5_5 }}}, + {{ server_state::s_joined, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "JOINED", "Syncing", indefinite }}}, + {{ server_state::s_joined, 5, 2 }, + { LOGS_INIT, WARN_INIT, + { "JOINED", "Syncing", progress5_2 }}}, + {{ server_state::s_synced, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "SYNCED", "Operational", steady }}}, + {{ server_state::s_donor, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "DONOR", "Donating SST", indefinite }}}, + {{ server_state::s_donor, 5, 0 }, + { LOGS_INIT, WARN_INIT, + { "DONOR", "Donating SST", progress5_0 }}}, + {{ server_state::s_joined, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "JOINED", "Syncing", indefinite }}}, + {{ server_state::s_synced, -1, -1 }, + { LOGS_INIT, WARN_INIT, + { "SYNCED", "Operational", steady }}}, + }; + + rep.report_log_msg(wsrep::reporter::warning, warn_msg, warn_tstamp); + + for (auto i(tests.begin()); i != tests.end(); ++i) + { + rep.report_state(i->input.state); + if (i->input.total >= 0) + rep.report_progress(make_progress_string(-1, -1, + i->input.total, i->input.done, + -1)); + auto value = read_file(REPORT); + result res(RES_INIT); + parse_result(value, res); + auto check(i->output); + VERIFY_RESULT(res, check, i - tests.begin()); + } + + ::unlink(REPORT); +} diff --git a/wsrep-lib/test/rsu_test.cpp b/wsrep-lib/test/rsu_test.cpp new file mode 100644 index 00000000..b032ef8c --- /dev/null +++ b/wsrep-lib/test/rsu_test.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/client_state.hpp" + +#include "client_state_fixture.hpp" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_CASE(test_rsu, + replicating_client_fixture_sync_rm) +{ + BOOST_REQUIRE(cc.begin_rsu(1) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_rsu); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local); + + BOOST_REQUIRE(cc.end_rsu() == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); +} diff --git a/wsrep-lib/test/server_context_test.cpp b/wsrep-lib/test/server_context_test.cpp new file mode 100644 index 00000000..1c0b8bde --- /dev/null +++ b/wsrep-lib/test/server_context_test.cpp @@ -0,0 +1,914 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "mock_server_state.hpp" + +#include <boost/test/unit_test.hpp> + +namespace +{ + struct server_fixture_base + { + server_fixture_base() + : server_service(ss) + , ss("s1", + wsrep::server_state::rm_sync, server_service) + , cc(ss, + wsrep::client_id(1), + wsrep::client_state::m_high_priority) + , hps(ss, &cc, false) + , ws_handle(wsrep::transaction_id(1), (void*)1) + , ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(1)), + wsrep::stid(wsrep::id("1"), wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(0), + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::commit) + , cluster_id("1") + , bootstrap_view() + , second_view() + , third_view() + { + wsrep::gtid state_id(cluster_id, wsrep::seqno(0)); + std::vector<wsrep::view::member> members; + members.push_back(wsrep::view::member( + wsrep::id("s1"), "s1", "")); + bootstrap_view = wsrep::view(state_id, + wsrep::seqno(1), + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members); + + members.push_back(wsrep::view::member( + wsrep::id("s2"), "s2", "")); + second_view = wsrep::view(wsrep::gtid(cluster_id, wsrep::seqno(1)), + wsrep::seqno(2), + wsrep::view::primary, + 0, // capabilities + 1, // own index + 1, // protocol version + members); + + members.push_back(wsrep::view::member( + wsrep::id("s3"), "s3", "")); + + third_view = wsrep::view(wsrep::gtid(cluster_id, wsrep::seqno(2)), + wsrep::seqno(3), + wsrep::view::primary, + 0, // capabilities + 1, // own index + 1, // protocol version + members); + + cc.open(cc.id()); + BOOST_REQUIRE(cc.before_command() == 0); + } + wsrep::mock_server_service server_service; + wsrep::mock_server_state ss; + wsrep::mock_client cc; + wsrep::mock_high_priority_service hps; + wsrep::ws_handle ws_handle; + wsrep::ws_meta ws_meta; + wsrep::id cluster_id; + wsrep::view bootstrap_view; + wsrep::view second_view; + wsrep::view third_view; + + void connect_in_view(const wsrep::view& view) + { + BOOST_REQUIRE(ss.connect("cluster", "local", "0", false) == 0); + ss.on_connect(view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + } + + void prepare_for_sst() + { + ss.prepare_for_sst(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner); + } + + void final_view() + { + BOOST_REQUIRE(ss.state() != wsrep::server_state::s_disconnected); + wsrep::view view(wsrep::gtid(), // state_id + wsrep::seqno::undefined(), // view seqno + wsrep::view::disconnected, // status + 0, // capabilities + -1, // own_index + 0, // protocol ver + std::vector<wsrep::view::member>() // members + ); + ss.on_view(view, &hps); + } + + void disconnect() + { + BOOST_REQUIRE(ss.state() != wsrep::server_state::s_disconnecting); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting); + final_view(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); + } + + }; + + struct applying_server_fixture : server_fixture_base + { + applying_server_fixture() + : server_fixture_base() + { + ss.mock_connect(); + } + }; + + struct sst_first_server_fixture : server_fixture_base + { + sst_first_server_fixture() + : server_fixture_base() + { + server_service.sst_before_init_ = true; + } + + void sst_received_action() + { + server_service.sync_point_enabled_ = "on_view_wait_initialized"; + server_service.sync_point_action_ = server_service.spa_initialize; + } + + void initialization_failure_action() + { + server_service.sync_point_enabled_ = "on_view_wait_initialized"; + server_service.sync_point_action_ = + server_service.spa_initialize_error; + } + + void clear_sync_point_action() + { + server_service.sync_point_enabled_ = ""; + server_service.sync_point_action_ = server_service.spa_none; + } + + // Helper method to bootstrap the server with bootstrap view + void bootstrap() + { + connect_in_view(bootstrap_view); + + sst_received_action(); + ss.on_view(bootstrap_view, &hps); + clear_sync_point_action(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); + } + + }; + + struct init_first_server_fixture : server_fixture_base + { + init_first_server_fixture() + : server_fixture_base() + { + server_service.sst_before_init_ = false; + } + + // Helper method to bootstrap the server with bootstrap view + void bootstrap() + { + ss.initialized(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_initialized); + BOOST_REQUIRE(ss.connect("cluster", "local", "0", false) == 0); + ss.on_connect(bootstrap_view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + ss.on_view(bootstrap_view, &hps); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); + } + }; + + // Helper to pass to BOOST_REQUIRE_EXCEPTION. Always returns true. + bool exception_check(const wsrep::runtime_error&) { return true; } +} + +// Test on_apply() method for 1pc +BOOST_FIXTURE_TEST_CASE(server_state_applying_1pc, + applying_server_fixture) +{ + char buf[1] = { 1 }; + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer(buf, 1)) == 0); + const wsrep::transaction& txc(cc.transaction()); + // ::abort(); + BOOST_REQUIRE_MESSAGE( + txc.state() == wsrep::transaction::s_committed, + "Transaction state " << txc.state() << " not committed"); +} + +// Test on_apply() method for 2pc +BOOST_FIXTURE_TEST_CASE(server_state_applying_2pc, + applying_server_fixture) +{ + hps.do_2pc_ = true; + char buf[1] = { 1 }; + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer(buf, 1)) == 0); + const wsrep::transaction& txc(cc.transaction()); + BOOST_REQUIRE(txc.state() == wsrep::transaction::s_committed); +} + +// Test on_apply() method for 1pc transaction which +// fails applying and rolls back +BOOST_FIXTURE_TEST_CASE(server_state_applying_1pc_rollback, + applying_server_fixture) +{ + /* make sure default success result is flipped to error_fatal */ + ss.provider().commit_order_leave_result_ = wsrep::provider::success; + hps.fail_next_applying_ = true; + char buf[1] = { 1 }; + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer(buf, 1)) == 1); + const wsrep::transaction& txc(cc.transaction()); + BOOST_REQUIRE(txc.state() == wsrep::transaction::s_aborted); +} + +// Test on_apply() method for 2pc transaction which +// fails applying and rolls back +BOOST_FIXTURE_TEST_CASE(server_state_applying_2pc_rollback, + applying_server_fixture) +{ + /* make sure default success result is flipped to error_fatal */ + ss.provider().commit_order_leave_result_ = wsrep::provider::success; + hps.do_2pc_ = true; + hps.fail_next_applying_ = true; + char buf[1] = { 1 }; + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer(buf, 1)) == 1); + const wsrep::transaction& txc(cc.transaction()); + BOOST_REQUIRE(txc.state() == wsrep::transaction::s_aborted); +} + +BOOST_FIXTURE_TEST_CASE(server_state_streaming, applying_server_fixture) +{ + ws_meta = wsrep::ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(1)), + wsrep::stid(wsrep::id("1"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(0), + wsrep::provider::flag::start_transaction); + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id())); + ws_meta = wsrep::ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(2)), + wsrep::stid(wsrep::id("1"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(1), + 0); + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer("1", 1)) == 0); + ws_meta = wsrep::ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(2)), + wsrep::stid(wsrep::id("1"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(1), + wsrep::provider::flag::commit); + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, ws_meta, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + ws_meta.server_id(), ws_meta.transaction_id()) == 0); +} + + +BOOST_AUTO_TEST_CASE(server_state_state_strings) +{ + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_disconnected) == "disconnected"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_initializing) == "initializing"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_initialized) == "initialized"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_connected) == "connected"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_joiner) == "joiner"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_joined) == "joined"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_donor) == "donor"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_synced) == "synced"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::server_state::s_disconnecting) == "disconnecting"); +} + +/////////////////////////////////////////////////////////////////////////////// +// Test cases for SST first // +/////////////////////////////////////////////////////////////////////////////// + +BOOST_FIXTURE_TEST_CASE(server_state_sst_first_boostrap, + sst_first_server_fixture) +{ + bootstrap(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + + +BOOST_FIXTURE_TEST_CASE(server_state_sst_first_join_with_sst, + sst_first_server_fixture) +{ + connect_in_view(second_view); + prepare_for_sst(); + sst_received_action(); + // Mock server service get_view() gets view from logged_view_. + // Get_view() is called from sst_received(). This emulates the + // case where SST contains the view in which SST happens. + server_service.logged_view(second_view); + server_service.position(wsrep::gtid(cluster_id, wsrep::seqno(2))); + ss.sst_received(cc, 0); + clear_sync_point_action(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +BOOST_FIXTURE_TEST_CASE(server_state_sst_first_join_with_ist, + sst_first_server_fixture) +{ + connect_in_view(second_view); + // Mock server service get_view() gets view from logged_view_. + // Get_view() is called from sst_received(). This emulates the + // case where the view is stored in stable storage. + server_service.logged_view(second_view); + sst_received_action(); + ss.on_view(second_view, &hps); + clear_sync_point_action(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_view(third_view, &hps); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + + +// Cycle from synced state to disconnected and back to synced. Server +// storage engines remain initialized. +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_synced_disconnected_synced_no_sst, + sst_first_server_fixture) +{ + bootstrap(); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting); + final_view(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); + + // Connect back as a sole member in the cluster + BOOST_REQUIRE(ss.connect("cluster", "local", "0", false) == 0); + // @todo: s_connecting state would be good to have + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); + // Server state must keep the initialized state + BOOST_REQUIRE(ss.is_initialized() == true); + std::vector<wsrep::view::member> members; + members.push_back(wsrep::view::member(wsrep::id("s1"), "name", "")); + wsrep::view view(wsrep::gtid(cluster_id, wsrep::seqno(1)), + wsrep::seqno(2), + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members); + ss.on_connect(view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + // As storage engines have been initialized, there should not be + // any reason to wait for initialization. State should jump directly + // to s_joined after handling the view. + ss.on_view(view, &hps); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +// +// Error after connecting to cluster. This scenario may happen if SST +// request preparation fails. +// +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_error_on_connect, + sst_first_server_fixture) +{ + connect_in_view(second_view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + disconnect(); +} + +// Error during SST. +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_error_on_joiner, + sst_first_server_fixture) +{ + connect_in_view(second_view); + ss.prepare_for_sst(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner); + server_service.position(wsrep::gtid::undefined()); + ss.sst_received(cc, 1); + disconnect(); +} + +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_error_on_initializing, + sst_first_server_fixture) +{ + connect_in_view(second_view); + ss.prepare_for_sst(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner); + initialization_failure_action(); + server_service.position(wsrep::gtid(second_view.state_id())); + BOOST_REQUIRE_EXCEPTION(ss.sst_received(cc, 0), + wsrep::runtime_error, exception_check); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_initializing); + BOOST_REQUIRE_EXCEPTION(ss.on_view(second_view, &hps), + wsrep::runtime_error, exception_check); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_initializing); + disconnect(); +} + +// Error or shutdown happens during catchup phase after receiving +// SST successfully. +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_error_on_joined, + sst_first_server_fixture) +{ + connect_in_view(second_view); + ss.prepare_for_sst(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner); + sst_received_action(); + // Mock server service get_view() gets view from logged_view_. + // Get_view() is called from sst_received(). This emulates the + // case where SST contains the view in which SST happens. + server_service.logged_view(second_view); + server_service.position(wsrep::gtid(cluster_id, wsrep::seqno(2))); + ss.sst_received(cc, 0); + clear_sync_point_action(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); +} + +// Error or shutdown happens when donating a snapshot. +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_error_on_donor, + sst_first_server_fixture) +{ + bootstrap(); + ss.start_sst("", wsrep::gtid(cluster_id, wsrep::seqno(2)), false); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_donor); + disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); +} + +/////////////////////////////////////////////////////////////////////////////// +// Test cases for init first // +/////////////////////////////////////////////////////////////////////////////// + +BOOST_FIXTURE_TEST_CASE(server_state_init_first_boostrap, + init_first_server_fixture) +{ + bootstrap(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +BOOST_FIXTURE_TEST_CASE(server_state_init_first_join_with_sst, + init_first_server_fixture) +{ + ss.initialized(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_initialized); + connect_in_view(second_view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + prepare_for_sst(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner); + server_service.logged_view(second_view); + server_service.position(wsrep::gtid(cluster_id, wsrep::seqno(2))); + ss.sst_received(cc, 0); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +BOOST_FIXTURE_TEST_CASE(server_state_init_first_join_with_ist, + init_first_server_fixture) +{ + ss.initialized(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_initialized); + connect_in_view(second_view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + server_service.logged_view(second_view); + ss.on_view(second_view, &hps); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_view(third_view, &hps); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + + +// Cycle from synced state to disconnected and back to synced. Server +// storage engines remain initialized. +BOOST_FIXTURE_TEST_CASE( + server_state_init_first_synced_disconnected_synced_no_sst, + init_first_server_fixture) +{ + bootstrap(); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting); + final_view(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); + + // Connect back as a sole member in the cluster + BOOST_REQUIRE(ss.connect("cluster", "local", "0", false) == 0); + // @todo: s_connecting state would be good to have + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); + // Server state must keep the initialized state + BOOST_REQUIRE(ss.is_initialized() == true); + std::vector<wsrep::view::member> members; + members.push_back(wsrep::view::member(wsrep::id("s1"), "name", "")); + wsrep::view view(wsrep::gtid(cluster_id, wsrep::seqno(1)), + wsrep::seqno(2), + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members); + ss.on_connect(view); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected); + // As storage engines have been initialized, there should not be + // any reason to wait for initialization. State should jump directly + // to s_joined after handling the view. + ss.on_view(view, &hps); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +///////////////////////////////////////////////////////////////////////////// +// Donor state transitions // +///////////////////////////////////////////////////////////////////////////// + +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_donate_success, + sst_first_server_fixture) +{ + bootstrap(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); + ss.start_sst("", wsrep::gtid(cluster_id, wsrep::seqno(2)), false); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_donor); + ss.sst_sent(wsrep::gtid(cluster_id, wsrep::seqno(2)), 0); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_donate_error, + sst_first_server_fixture) +{ + bootstrap(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); + ss.start_sst("", wsrep::gtid(cluster_id, wsrep::seqno(2)), false); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_donor); + ss.sst_sent(wsrep::gtid(cluster_id, wsrep::seqno(2)), 1); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined); + ss.on_sync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +///////////////////////////////////////////////////////////////////////////// +// Pause/Resume and Desync/Resync // +///////////////////////////////////////////////////////////////////////////// + +BOOST_FIXTURE_TEST_CASE( + server_state_sst_first_desync_and_pause_resync_and_resume, + sst_first_server_fixture) +{ + bootstrap(); + ss.desync_and_pause(); + // @todo: Should we have here different state than synced + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); + ss.resume_and_resync(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced); +} + +///////////////////////////////////////////////////////////////////////////// +// Disconnect // +///////////////////////////////////////////////////////////////////////////// + +BOOST_FIXTURE_TEST_CASE( + server_state_disconnect, + sst_first_server_fixture) +{ + bootstrap(); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting); + final_view(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); +} + +// This test case verifies that the disconnect can be initiated +// concurrently by several callers. This may happen in failure situations +// where provider shutdown causes cascading failures and the failing operations +// try to disconnect the provider. +BOOST_FIXTURE_TEST_CASE( + server_state_disconnect_twice, + sst_first_server_fixture) +{ + bootstrap(); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting); + final_view(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); + ss.disconnect(); + BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected); +} + +///////////////////////////////////////////////////////////////////////////// +// Orphaned SR // +///////////////////////////////////////////////////////////////////////////// + +// Test the behavior of server_state::close_orphaned_sr_transactions(). +// In this test we check the scenario where we initially have 3 nodes in +// the cluster (s1, s2, s3), and this server_state delivers one streaming +// fragment from s2 and s3 each, followed by view changes: +// +// view 1: primary (s1, s2, s3) +// view 2: primary (s1, s2) +// view 3: non-primary (s1) +// view 4: non-primary (s1, s3) +// view 5: primary (s1, s2, s3) +// +// We expect that on view 2, transaction originated from s3 is considered +// orphaned, so it should be rolled back. +// Transaction from s2 should never be considered orphaned in this scenario, +// we expect it to survive until the end of the test. That's because +// transactions are rolled back in primary views only, and because s2 +// is member of all primary views in this scenario. +BOOST_FIXTURE_TEST_CASE(server_state_close_orphaned_transactions, + sst_first_server_fixture) +{ + connect_in_view(third_view); + server_service.logged_view(third_view); + sst_received_action(); + ss.on_view(third_view, &hps); + + // initially we have members (s1, s2, s3) + std::vector<wsrep::view::member> members(ss.current_view().members()); + + // apply a fragment coming from s2 + wsrep::ws_meta meta_s2(wsrep::gtid(wsrep::id("s2"), wsrep::seqno(1)), + wsrep::stid(wsrep::id("s2"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(1), + wsrep::provider::flag::start_transaction); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_s2, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + + // apply a fragment coming from s3 + wsrep::ws_meta meta_s3(wsrep::gtid(wsrep::id("s3"), wsrep::seqno(2)), + wsrep::stid(wsrep::id("s3"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(2), + wsrep::provider::flag::start_transaction); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_s3, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + // s3 drops out of the cluster, deliver primary view (s1, s2) + wsrep::view::member s3(members.back()); + members.pop_back(); + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno() + 1, + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members), &hps); + + // transaction from s2 is still present + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + // transaction from s3 is gone + BOOST_REQUIRE(not ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + // s2 drops out of the cluster, deliver non-primary view (s1) + wsrep::view::member s2(members.back()); + members.pop_back(); + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno(), + wsrep::view::non_primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members), &hps); + + // no streaming appliers are closed on non-primary view, + // so transaction from s2 is still present + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + + // s3 comes back, deliver non-primary view (s1, s3) + members.push_back(s3); + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno() + 1, + wsrep::view::non_primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members), &hps); + + // transaction s2 is still present after non-primary view + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + + // s2 comes back, deliver primary-view (s1, s2, s3) + members.push_back(s2); + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno() + 1, + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members), &hps); + + // finally, transaction from s2 is still present (part of primary view) + // and transaction from s3 is gone + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + BOOST_REQUIRE(not ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + // cleanup + wsrep::ws_meta meta_commit_s2(wsrep::gtid(wsrep::id("s2"), wsrep::seqno(3)), + wsrep::stid(wsrep::id("s2"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(3), + wsrep::provider::flag::commit); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_commit_s2, + wsrep::const_buffer("1", 1)) == 0); + + BOOST_REQUIRE(not ss.find_streaming_applier( + meta_commit_s2.server_id(), meta_commit_s2.transaction_id())); +} + + +// Test the case where two consecutive primary views with the +// same members are delivered (provider may do so). +// Expect SR transactions to be rolled back on equal consecutive views +BOOST_FIXTURE_TEST_CASE(server_state_equal_consecutive_views, + sst_first_server_fixture) +{ + connect_in_view(third_view); + server_service.logged_view(third_view); + sst_received_action(); + ss.on_view(third_view, &hps); + + // apply a fragment coming from s2 + wsrep::ws_meta meta_s2(wsrep::gtid(wsrep::id("s2"), wsrep::seqno(1)), + wsrep::stid(wsrep::id("s2"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(1), + wsrep::provider::flag::start_transaction); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_s2, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + + // apply a fragment coming from s3 + wsrep::ws_meta meta_s3(wsrep::gtid(wsrep::id("s3"), wsrep::seqno(2)), + wsrep::stid(wsrep::id("s3"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(2), + wsrep::provider::flag::start_transaction); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_s3, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + // deliver primary view with the same members (s1, s2, s3) + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno() + 1, + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + ss.current_view().members()), &hps); + + // transaction from s2 and s3 are gone + BOOST_REQUIRE(not ss.find_streaming_applier( + meta_s2.server_id(), meta_s2.transaction_id())); + BOOST_REQUIRE(not ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); +} + +// Verify that prepared XA transactions are not rolled back +// by close_orphaned_transactions() +BOOST_FIXTURE_TEST_CASE(server_state_xa_not_orphaned, + sst_first_server_fixture) +{ + connect_in_view(third_view); + server_service.logged_view(third_view); + sst_received_action(); + ss.on_view(third_view, &hps); + + // initially we have members (s1, s2, s3) + std::vector<wsrep::view::member> members(ss.current_view().members()); + + + wsrep::ws_meta meta_s3(wsrep::gtid(wsrep::id("s3"), wsrep::seqno(1)), + wsrep::stid(wsrep::id("s3"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(1), + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::prepare); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_s3, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + + // s3 drops out of the cluster, deliver primary view (s1, s2) + wsrep::view::member s3(members.back()); + members.pop_back(); + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno() + 1, + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members), &hps); + + // transaction from s3 is still present + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + // s3 comes back, deliver primary view (s1, s2, s3) + members.push_back(s3); + ss.on_view(wsrep::view(ss.current_view().state_id(), + ss.current_view().view_seqno() + 1, + wsrep::view::primary, + 0, // capabilities + 0, // own index + 1, // protocol version + members), &hps); + + // transaction from s3 is still present + BOOST_REQUIRE(ss.find_streaming_applier( + meta_s3.server_id(), meta_s3.transaction_id())); + + // cleanup + wsrep::ws_meta meta_commit_s3(wsrep::gtid(wsrep::id("s3"), wsrep::seqno(3)), + wsrep::stid(wsrep::id("s3"), + wsrep::transaction_id(1), + wsrep::client_id(1)), + wsrep::seqno(3), + wsrep::provider::flag::commit); + + BOOST_REQUIRE(ss.on_apply(hps, ws_handle, meta_commit_s3, + wsrep::const_buffer("1", 1)) == 0); + BOOST_REQUIRE(not ss.find_streaming_applier( + meta_commit_s3.server_id(), meta_commit_s3.transaction_id())); +} diff --git a/wsrep-lib/test/test_utils.cpp b/wsrep-lib/test/test_utils.cpp new file mode 100644 index 00000000..3646e96c --- /dev/null +++ b/wsrep-lib/test/test_utils.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "test_utils.hpp" +#include "wsrep/client_state.hpp" +#include "mock_server_state.hpp" + + +// Simple BF abort method to BF abort unordered transasctions +void wsrep_test::bf_abort_unordered(wsrep::client_state& cc) +{ + assert(cc.transaction().ordered() == false); + cc.bf_abort(wsrep::seqno(1)); +} + +void wsrep_test::bf_abort_ordered(wsrep::client_state& cc) +{ + assert(cc.transaction().ordered()); + cc.bf_abort(wsrep::seqno(0)); +} +// BF abort method to abort transactions via provider +void wsrep_test::bf_abort_provider(wsrep::mock_server_state& sc, + const wsrep::transaction& tc, + wsrep::seqno bf_seqno) +{ + wsrep::seqno victim_seqno; + sc.provider().bf_abort(bf_seqno, tc.id(), victim_seqno); + (void)victim_seqno; +} + +void wsrep_test::terminate_streaming_applier( + wsrep::mock_server_state& sc, + const wsrep::id& server_id, + wsrep::transaction_id transaction_id) +{ + // Note that all other arguments than server_id and + // transaction_id are chosen arbitrarily and it is hoped + // that the mock implementation does not freak out about it. + wsrep::mock_client mc(sc, wsrep::client_id(10), + wsrep::client_state::m_high_priority); + mc.open(wsrep::client_id(10)); + mc.before_command(); + wsrep::mock_high_priority_service hps(sc, &mc, false); + wsrep::ws_handle ws_handle(transaction_id, (void*)(1)); + wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("cluster1"), + wsrep::seqno(100)), + wsrep::stid(server_id, + transaction_id, + wsrep::client_id(1)), + wsrep::seqno(0), + wsrep::provider::flag::rollback); + wsrep::const_buffer data(0, 0); + sc.on_apply(hps, ws_handle, ws_meta, data); +} diff --git a/wsrep-lib/test/test_utils.hpp b/wsrep-lib/test/test_utils.hpp new file mode 100644 index 00000000..69ce7002 --- /dev/null +++ b/wsrep-lib/test/test_utils.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +// Forward declarations +namespace wsrep +{ + class client_state; + class mock_server_state; +} + +#include "wsrep/transaction.hpp" +#include "wsrep/provider.hpp" + +// +// Utility functions +// +namespace wsrep_test +{ + + // Simple BF abort method to BF abort unordered transasctions + void bf_abort_unordered(wsrep::client_state& cc); + + // Simple BF abort method to BF abort unordered transasctions + void bf_abort_ordered(wsrep::client_state& cc); + + // BF abort method to abort transactions via provider + void bf_abort_provider(wsrep::mock_server_state& sc, + const wsrep::transaction& tc, + wsrep::seqno bf_seqno); + + // Terminate streaming applier by applying rollback fragment. + void terminate_streaming_applier( + wsrep::mock_server_state& sc, + const wsrep::id& server_id, + wsrep::transaction_id transaction_id); +} diff --git a/wsrep-lib/test/toi_test.cpp b/wsrep-lib/test/toi_test.cpp new file mode 100644 index 00000000..e6370170 --- /dev/null +++ b/wsrep-lib/test/toi_test.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/client_state.hpp" + +#include "client_state_fixture.hpp" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_CASE(test_toi_mode, + replicating_client_fixture_sync_rm) +{ + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); + wsrep::key key(wsrep::key::exclusive); + key.append_key_part("k1", 2); + key.append_key_part("k2", 2); + wsrep::key_array keys{key}; + wsrep::const_buffer buf("toi", 3); + BOOST_REQUIRE(cc.enter_toi_local(keys, buf) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_toi); + BOOST_REQUIRE(cc.in_toi()); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local); + wsrep::mutable_buffer err; + BOOST_REQUIRE(cc.leave_toi_local(err) == 0); + BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); + BOOST_REQUIRE(sc.provider().toi_write_sets() == 1); + BOOST_REQUIRE(sc.provider().toi_start_transaction() == 1); + BOOST_REQUIRE(sc.provider().toi_commit() == 1); +} + +BOOST_FIXTURE_TEST_CASE(test_toi_applying, + applying_client_fixture) +{ + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); + wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(2)), + wsrep::stid(sc.id(), + wsrep::transaction_id::undefined(), + cc.id()), + wsrep::seqno(1), + wsrep::provider::flag::start_transaction | + wsrep::provider::flag::commit); + cc.enter_toi_mode(ws_meta); + BOOST_REQUIRE(cc.in_toi()); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_high_priority); + cc.leave_toi_mode(); + BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_undefined); + cc.after_applying(); +} diff --git a/wsrep-lib/test/transaction_test.cpp b/wsrep-lib/test/transaction_test.cpp new file mode 100644 index 00000000..493bda8d --- /dev/null +++ b/wsrep-lib/test/transaction_test.cpp @@ -0,0 +1,1667 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/transaction.hpp" +#include "wsrep/provider.hpp" + +#include "test_utils.hpp" +#include "client_state_fixture.hpp" + +#include <boost/mpl/vector.hpp> + +namespace +{ + typedef + boost::mpl::vector<replicating_client_fixture_sync_rm, + replicating_client_fixture_async_rm> + replicating_fixtures; +} + +BOOST_FIXTURE_TEST_CASE(transaction_append_key_data, + replicating_client_fixture_sync_rm) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.is_empty()); + int vals[3] = {1, 2, 3}; + wsrep::key key(wsrep::key::exclusive); + for (int i(0); i < 3; ++i) + { + key.append_key_part(&vals[i], sizeof(vals[i])); + } + BOOST_REQUIRE(cc.append_key(key) == 0); + BOOST_REQUIRE(tc.is_empty() == false); + wsrep::const_buffer data(&vals[2], sizeof(vals[2])); + BOOST_REQUIRE(cc.append_data(data) == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + cc.after_statement(); +} +// +// Test a succesful 1PC transaction lifecycle +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE(transaction_1pc, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + // Establish default read view + BOOST_REQUIRE(0 == cc.assign_read_view(NULL)); + + // Verify that the commit can be successfully executed in separate command + BOOST_REQUIRE(cc.after_statement() == 0); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + // Run before commit + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + + // Run ordered commit + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + + // Run after commit + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + + +// +// Test a voluntary rollback +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE(transaction_rollback, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + // Run before commit + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + + // Run after commit + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test a 1PC transaction which gets BF aborted before before_commit +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_before_before_commit, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + wsrep_test::bf_abort_unordered(cc); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + + + +// +// Test a 1PC transaction which gets BF aborted during before_commit via +// provider before the write set was ordered and certified. +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_during_before_commit_uncertified, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + wsrep_test::bf_abort_provider(sc, tc, wsrep::seqno::undefined()); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_cert_failed); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + +// +// Test a 1PC transaction which gets BF aborted during before_commit +// when waiting for replayers +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_during_commit_wait_for_replayers, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + cc.bf_abort_during_wait_ = true; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_during_commit_order_enter, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + auto& sc(T::sc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().commit_order_enter_result_ = wsrep::provider::error_bf_abort; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + + sc.provider().commit_order_enter_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + + // Replay from after_statement() + cc.after_statement(); + BOOST_REQUIRE(cc.replays() > 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(not cc.current_error()); +} + +// +// Test a 1PC transaction for which prepare data fails +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_error_during_prepare_data, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + cc.error_during_prepare_data_ = true; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.current_error() == wsrep::e_size_exceeded_error); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + +// +// Test a 1PC transaction which gets killed by DBMS before certification +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_killed_before_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + cc.killed_before_certify_ = true; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.current_error() == wsrep::e_interrupted_error); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + +// +// Test a transaction which gets BF aborted inside provider before +// certification result is known. Replaying will be successful +// +BOOST_FIXTURE_TEST_CASE( + transaction_bf_before_cert_result_replay_success, + replicating_client_fixture_sync_rm) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + sc.provider().certify_result_ = wsrep::provider::error_bf_abort; + sc.provider().replay_result_ = wsrep::provider::success; + + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test a transaction which gets BF aborted inside provider before +// certification result is known. Replaying will fail because of +// certification failure. +// +BOOST_FIXTURE_TEST_CASE( + transaction_bf_before_cert_result_replay_cert_fail, + replicating_client_fixture_sync_rm) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + sc.provider().certify_result_ = wsrep::provider::error_bf_abort; + sc.provider().replay_result_ = wsrep::provider::error_certification_failed; + + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(cc.after_statement() ); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + BOOST_REQUIRE(tc.active() == false); +} + +// +// Test a 1PC transaction which gets BF aborted during before_commit via +// provider after the write set was ordered and certified. This must +// result replaying of transaction. +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_during_before_commit_certified, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + wsrep_test::bf_abort_provider(sc, tc, wsrep::seqno(1)); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == true); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(cc.replays() == 1); +} + +// +// Test a 1PC transaction which gets BF aborted simultaneously with +// certification failure. BF abort should not succeed as the +// transaction is going to roll back anyway. Certification failure +// should not generate seqno for write set meta. +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_before_unordered_cert_failure, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + cc.sync_point_enabled_ = "wsrep_before_certification"; + cc.sync_point_action_ = wsrep::mock_client_service::spa_bf_abort_unordered; + sc.provider().certify_result_ = wsrep::provider::error_certification_failed; + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_cert_failed); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.after_statement() ); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); +} + +// +// Test a 1PC transaction which gets "warning error" from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_warning_error_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_warning; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets transaction missing from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_transaction_missing_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_transaction_missing; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets size exceeded error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_size_exceeded_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_size_exceeded; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets connection failed error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_connection_failed_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_connection_failed; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets not allowed error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_no_allowed_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_not_allowed; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets fatal error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_fatal_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_fatal; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); + BOOST_REQUIRE(cc.aborts() == 1); +} + +// +// Test a 1PC transaction which gets unknown from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_unknown_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_state& sc(T::sc); + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + sc.provider().certify_result_ = wsrep::provider::error_unknown; + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().certify_result_ = wsrep::provider::success; + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets BF aborted before grabbing lock +// after certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_abort_before_certify_regain_lock, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + cc.sync_point_enabled_ = "wsrep_after_certification"; + cc.sync_point_action_ = wsrep::mock_client_service::spa_bf_abort_ordered; + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(cc.replays() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test a transaction which gets BF aborted before before_statement. +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_before_before_statement, T, + replicating_fixtures, T) +{ + wsrep::client_state& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + cc.after_statement(); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.before_command() == 0); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.before_statement() == 1); + BOOST_REQUIRE(tc.active()); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_after_result(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); +} + +// +// Test a transaction which gets BF aborted before after_statement. +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_before_after_statement, T, + replicating_fixtures, T) +{ + wsrep::client_state& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + wsrep_test::bf_abort_unordered(cc); + + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_abort_after_after_statement, T, + replicating_fixtures, T) +{ + wsrep::client_state& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); +} + +BOOST_FIXTURE_TEST_CASE( + transaction_1pc_autocommit_retry_bf_aborted, + replicating_client_fixture_autocommit) +{ + + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(cc.after_statement()); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); +} + +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_1pc_bf_abort_after_after_command_before_result, T, + replicating_fixtures, T) +{ + wsrep::client_state& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_result); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + wsrep_test::bf_abort_unordered(cc); + // The result is being sent to client. We need to mark transaction + // as must_abort but not override error yet as this might cause + // a race condition resulting incorrect result returned to the DBMS client. + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + // After the result has been sent to the DBMS client, the after result + // processing should roll back the transaction and set the error. + cc.after_command_after_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + BOOST_REQUIRE(tc.active() == true); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + cc.sync_rollback_complete(); + BOOST_REQUIRE(cc.before_command() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +BOOST_FIXTURE_TEST_CASE( + transaction_1pc_bf_abort_after_after_command_after_result_sync_rm, + replicating_client_fixture_sync_rm) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_result); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(tc.active()); + cc.sync_rollback_complete(); + BOOST_REQUIRE(cc.before_command() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(tc.active() == false); +} + +// Check the case where client program calls wait_rollback_complete() to +// gain control before before_command(). +BOOST_FIXTURE_TEST_CASE( + transaction_1pc_bf_abort_after_after_command_after_result_sync_rm_wait_rollback, + replicating_client_fixture_sync_rm) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_result); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(tc.active()); + cc.sync_rollback_complete(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + cc.wait_rollback_complete_and_acquire_ownership(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + // Idempotent + cc.wait_rollback_complete_and_acquire_ownership(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + BOOST_REQUIRE(cc.before_command() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(tc.active() == false); +} + +// Check the case where BF abort happens between client calls to +// wait_rollback_complete_and_acquire_ownership() +// and before before_command(). +BOOST_FIXTURE_TEST_CASE( + transaction_1pc_bf_abort_after_acquire_before_before_command_sync_rm, + replicating_client_fixture_sync_rm) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_result); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + cc.wait_rollback_complete_and_acquire_ownership(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + // As the control is now on client, the BF abort must just change + // the state to s_must_abort. + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(cc.before_command() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(tc.active() == false); +} + +BOOST_FIXTURE_TEST_CASE( + transaction_1pc_bf_abort_after_after_command_after_result_async_rm, + replicating_client_fixture_async_rm) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_result); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(cc.before_command() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + BOOST_REQUIRE(tc.active() == false); +} + +// +// Test before_command() with keep_command_error param +// Failure free case is not affected by keep_command_error +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE(transaction_keep_error, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + cc.after_statement(); + cc.after_command_before_result(); + cc.after_command_after_result(); + + bool keep_command_error(true); + BOOST_REQUIRE(cc.before_command(keep_command_error) == 0); + BOOST_REQUIRE(tc.active()); + cc.after_command_before_result(); + cc.after_command_after_result(); + + keep_command_error = false; + BOOST_REQUIRE(cc.before_command(keep_command_error) == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + cc.after_statement(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test before_command() with keep_command_error param +// BF abort while idle +// +BOOST_FIXTURE_TEST_CASE(transaction_keep_error_bf_idle_sync_rm, + replicating_client_fixture_sync_rm) +{ + cc.start_transaction(wsrep::transaction_id(1)); + cc.after_statement(); + cc.after_command_before_result(); + cc.after_command_after_result(); + + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + wsrep_test::bf_abort_unordered(cc); + cc.sync_rollback_complete(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + + bool keep_command_error(true); + BOOST_REQUIRE(cc.before_command(keep_command_error) == 0); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + + keep_command_error = false; + BOOST_REQUIRE(cc.before_command(keep_command_error) == 1); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test before_command() with keep_command_error param +// BF abort after ownership is acquired and before before_command() +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE(transaction_keep_error_bf_after_ownership, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + cc.after_statement(); + cc.after_command_before_result(); + cc.after_command_after_result(); + + cc.wait_rollback_complete_and_acquire_ownership(); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + + bool keep_command_error(true); + BOOST_REQUIRE(cc.before_command(keep_command_error) == 0); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + + keep_command_error = false; + BOOST_REQUIRE(cc.before_command(keep_command_error) == 1); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test before_command() with keep_command_error param +// BF abort right after before_command() +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE(transaction_keep_error_bf_after_before_command, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + cc.after_statement(); + cc.after_command_before_result(); + cc.after_command_after_result(); + + bool keep_command_error(true); + BOOST_REQUIRE(cc.before_command(keep_command_error) == 0); + BOOST_REQUIRE(tc.active()); + + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + + keep_command_error = false; + BOOST_REQUIRE(cc.before_command(keep_command_error) == 1); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test before_command() with keep_command_error param +// BF abort right after after_command_before_result() +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE(transaction_keep_error_bf_after_after_command_before_result, T, + replicating_fixtures, T) +{ + wsrep::mock_client& cc(T::cc); + const wsrep::transaction& tc(T::tc); + + cc.start_transaction(wsrep::transaction_id(1)); + cc.after_statement(); + cc.after_command_before_result(); + cc.after_command_after_result(); + + bool keep_command_error(true); + BOOST_REQUIRE(cc.before_command(keep_command_error) == 0); + BOOST_REQUIRE(tc.active()); + + cc.after_command_before_result(); + + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + + cc.after_command_after_result(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + + keep_command_error = false; + BOOST_REQUIRE(cc.before_command(keep_command_error) == 1); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +BOOST_FIXTURE_TEST_CASE(transaction_1pc_applying, + applying_client_fixture) +{ + start_transaction(wsrep::transaction_id(1), + wsrep::seqno(1)); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + cc.after_applying(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + + +BOOST_FIXTURE_TEST_CASE(transaction_applying_rollback, + applying_client_fixture) +{ + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + cc.after_applying(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +/////////////////////////////////////////////////////////////////////////////// +// STREAMING REPLICATION // +/////////////////////////////////////////////////////////////////////////////// + +// +// Test 1PC with row streaming with one row +// +BOOST_FIXTURE_TEST_CASE(transaction_row_streaming_1pc_commit, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +// +// Test 1PC with row streaming with one row +// +BOOST_FIXTURE_TEST_CASE(transaction_row_batch_streaming_1pc_commit, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.enable_streaming( + wsrep::streaming_context::row, 2) == 0); + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +// +// Test 1PC row streaming with two separate statements +// +BOOST_FIXTURE_TEST_CASE( + transaction_row_streaming_1pc_commit_two_statements, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 2); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 3); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +// +// Fragments are removed in before_prepare in running transaction context. +// In 1pc the before_prepare() is called from before_commit(). +// However, the BF abort may arrive during this removal and the +// client_service::remove_fragments() may roll back the transaction +// internally. This will cause the transaction to leave before_prepare() +// in aborted state. +// +BOOST_FIXTURE_TEST_CASE(transaction_streaming_1pc_bf_abort_during_fragment_removal, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + cc.bf_abort_during_fragment_removal_ = true; + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.after_statement()); + BOOST_REQUIRE(tc.active() == false); + wsrep_test::terminate_streaming_applier(sc, sc.id(), + wsrep::transaction_id(1)); +} + +// +// Test streaming rollback +// +BOOST_FIXTURE_TEST_CASE(transaction_row_streaming_rollback, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().rollback_fragments() == 1); + + wsrep::high_priority_service* hps( + sc.find_streaming_applier( + sc.id(), wsrep::transaction_id(1))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +// +// Test streaming BF abort in executing state. +// +BOOST_FIXTURE_TEST_CASE(transaction_row_streaming_bf_abort_executing, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.streaming_context().rolled_back()); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(cc.after_statement()); + wsrep_test::terminate_streaming_applier(sc, sc.id(), + wsrep::transaction_id(1)); + +} +// +// Test streaming certification failure during fragment replication +// +BOOST_FIXTURE_TEST_CASE(transaction_row_streaming_cert_fail_non_commit, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + sc.provider().certify_result_ = wsrep::provider::error_certification_failed; + BOOST_REQUIRE(cc.after_row() == 1); + sc.provider().certify_result_ = wsrep::provider::success; + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(cc.after_statement() == 1); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().rollback_fragments() == 1); + + wsrep::high_priority_service* hps( + sc.find_streaming_applier( + sc.id(), wsrep::transaction_id(1))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +// +// Test streaming certification failure during commit +// +BOOST_FIXTURE_TEST_CASE(transaction_row_streaming_cert_fail_commit, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + sc.provider().certify_result_ = wsrep::provider::error_certification_failed; + BOOST_REQUIRE(cc.before_commit() == 1); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_cert_failed); + sc.provider().certify_result_ = wsrep::provider::success; + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(cc.after_statement() ); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().rollback_fragments() == 1); + + wsrep::high_priority_service* hps( + sc.find_streaming_applier( + sc.id(), wsrep::transaction_id(1))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +// +// Test streaming BF abort after succesful certification +// +BOOST_FIXTURE_TEST_CASE(transaction_row_streaming_bf_abort_committing, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + wsrep_test::bf_abort_ordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + + + +BOOST_FIXTURE_TEST_CASE(transaction_byte_streaming_1pc_commit, + streaming_client_fixture_byte) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + cc.bytes_generated_ = 1; + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +BOOST_FIXTURE_TEST_CASE(transaction_byte_batch_streaming_1pc_commit, + streaming_client_fixture_byte) +{ + BOOST_REQUIRE( + cc.enable_streaming( + wsrep::streaming_context::bytes, 2) == 0); + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + + +BOOST_FIXTURE_TEST_CASE(transaction_statement_streaming_statement_with_no_effect, + streaming_client_fixture_statement) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +BOOST_FIXTURE_TEST_CASE(transaction_statement_streaming_1pc_commit, + streaming_client_fixture_statement) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +BOOST_FIXTURE_TEST_CASE(transaction_statement_batch_streaming_1pc_commit, + streaming_client_fixture_statement) +{ + BOOST_REQUIRE( + cc.enable_streaming( + wsrep::streaming_context::statement, 2) == 0); + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +BOOST_FIXTURE_TEST_CASE(transaction_statement_streaming_cert_fail, + streaming_client_fixture_statement) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 0); + sc.provider().certify_result_ = wsrep::provider::error_certification_failed; + BOOST_REQUIRE(cc.after_statement()); + BOOST_REQUIRE(cc.current_error() == wsrep::e_deadlock_error); + // Note: Due to possible limitation in wsrep-API error codes + // or a bug in current Galera provider, rollback fragment may be + // replicated even in case of certification failure. + // If the limitation is lifted later on or the provider is fixed, + // the above check should be change for fragments == 0, + // rollback_fragments == 0. + BOOST_REQUIRE(sc.provider().fragments() == 1); + BOOST_REQUIRE(sc.provider().start_fragments() == 0); + BOOST_REQUIRE(sc.provider().rollback_fragments() == 1); + + wsrep::high_priority_service* hps( + sc.find_streaming_applier( + sc.id(), wsrep::transaction_id(1))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +/////////////////////////////////////////////////////////////////////////////// +// misc // +/////////////////////////////////////////////////////////////////////////////// + +BOOST_AUTO_TEST_CASE(transaction_state_strings) +{ + BOOST_REQUIRE(wsrep::to_string( + wsrep::transaction::s_executing) == "executing"); + BOOST_REQUIRE(wsrep::to_string( + wsrep::transaction::s_preparing) == "preparing"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_certifying) == "certifying"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_committing) == "committing"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_ordered_commit) == "ordered_commit"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_committed) == "committed"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_cert_failed) == "cert_failed"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_must_abort) == "must_abort"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_aborting) == "aborting"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_aborted) == "aborted"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_must_replay) == "must_replay"); + BOOST_REQUIRE( + wsrep::to_string( + wsrep::transaction::s_replaying) == "replaying"); +} diff --git a/wsrep-lib/test/transaction_test_2pc.cpp b/wsrep-lib/test/transaction_test_2pc.cpp new file mode 100644 index 00000000..8f95828d --- /dev/null +++ b/wsrep-lib/test/transaction_test_2pc.cpp @@ -0,0 +1,290 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "client_state_fixture.hpp" + +// +// Test a succesful 2PC transaction lifecycle +// +BOOST_FIXTURE_TEST_CASE(transaction_2pc, + replicating_client_fixture_2pc) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_preparing); + BOOST_REQUIRE(tc.ordered()); + BOOST_REQUIRE(tc.certified()); + BOOST_REQUIRE(tc.ws_meta().gtid().is_undefined() == false); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test a 2PC transaction which gets BF aborted before before_prepare +// +BOOST_FIXTURE_TEST_CASE( + transaction_2pc_bf_before_before_prepare, + replicating_client_fixture_2pc) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(cc.before_prepare()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.after_statement() ); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error()); +} + +// +// Test a 2PC transaction which gets BF aborted before before_prepare +// +BOOST_FIXTURE_TEST_CASE( + transaction_2pc_bf_before_after_prepare, + replicating_client_fixture_2pc) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_preparing); + wsrep_test::bf_abort_ordered(cc); + BOOST_REQUIRE(cc.after_prepare()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test a 2PC transaction which gets BF aborted after_prepare() and +// the rollback takes place before entering before_commit(). +// +BOOST_FIXTURE_TEST_CASE( + transaction_2pc_bf_after_after_prepare, + replicating_client_fixture_2pc) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + wsrep_test::bf_abort_ordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +// +// Test a 2PC transaction which gets BF aborted between after_prepare() +// and before_commit() +// +BOOST_FIXTURE_TEST_CASE( + transaction_2pc_bf_before_before_commit, + replicating_client_fixture_2pc) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + wsrep_test::bf_abort_ordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_abort); + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + + +// +// Test a 2PC transaction which gets BF aborted when trying to grab +// commit order. +// +BOOST_FIXTURE_TEST_CASE( + transaction_2pc_bf_during_commit_order_enter, + replicating_client_fixture_2pc) +{ + cc.start_transaction(wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(cc.after_prepare() == 0); + sc.provider().commit_order_enter_result_ = wsrep::provider::error_bf_abort; + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.will_replay_called() == true); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + sc.provider().commit_order_enter_result_ = wsrep::provider::success; + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +/////////////////////////////////////////////////////////////////////////////// +// STREAMING REPLICATION // +/////////////////////////////////////////////////////////////////////////////// + + +BOOST_FIXTURE_TEST_CASE(transaction_streaming_2pc_commit, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +BOOST_FIXTURE_TEST_CASE(transaction_streaming_2pc_commit_two_statements, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(cc.before_statement() == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 2); + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(sc.provider().fragments() == 3); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} + +// +// Fragments are removed in before_prepare in running transaction context. +// However, the BF abort may arrive during this removal and the +// client_service::remove_fragments() may roll back the transaction +// internally. This will cause the transaction to leave before_prepare() +// in aborted state. +// +BOOST_FIXTURE_TEST_CASE(transaction_streaming_2pc_bf_abort_during_fragment_removal, + streaming_client_fixture_row) +{ + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + cc.bf_abort_during_fragment_removal_ = true; + BOOST_REQUIRE(cc.before_prepare()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc.after_statement()); + BOOST_REQUIRE(tc.active() == false); + wsrep_test::terminate_streaming_applier(sc, sc.id(), + wsrep::transaction_id(1)); +} + +/////////////////////////////////////////////////////////////////////////////// +// APPLYING // +/////////////////////////////////////////////////////////////////////////////// + +BOOST_FIXTURE_TEST_CASE(transaction_2pc_applying, + applying_client_fixture_2pc) +{ + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_preparing); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + cc.after_applying(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} diff --git a/wsrep-lib/test/transaction_test_xa.cpp b/wsrep-lib/test/transaction_test_xa.cpp new file mode 100644 index 00000000..d9fbad27 --- /dev/null +++ b/wsrep-lib/test/transaction_test_xa.cpp @@ -0,0 +1,296 @@ +#include "client_state_fixture.hpp" +#include <iostream> + +// +// Test a successful XA transaction lifecycle +// +BOOST_FIXTURE_TEST_CASE(transaction_xa, + replicating_client_fixture_sync_rm) +{ + wsrep::xid xid(1, 9, 0, "test xid"); + + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + cc.assign_xid(xid); + + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_preparing); + BOOST_REQUIRE(tc.ordered() == false); + // certified() only after the last fragment + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_prepared); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + // XA START + PREPARE fragment + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().fragments() == 1); + + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + BOOST_REQUIRE(tc.ordered()); + BOOST_REQUIRE(tc.certified()); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + // XA PREPARE and XA COMMIT fragments + BOOST_REQUIRE(sc.provider().fragments() == 2); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); + + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + + +// +// Test detaching of XA transactions +// +BOOST_FIXTURE_TEST_CASE(transaction_xa_detach_commit_by_xid, + replicating_two_clients_fixture_sync_rm) +{ + wsrep::xid xid(1, 1, 1, "id"); + + cc1.start_transaction(wsrep::transaction_id(1)); + cc1.assign_xid(xid); + cc1.before_prepare(); + cc1.after_prepare(); + BOOST_REQUIRE(sc.provider().fragments() == 1); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + + cc1.xa_detach(); + + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc1.after_statement() == 0); + + cc2.start_transaction(wsrep::transaction_id(2)); + cc2.assign_xid(xid); + BOOST_REQUIRE(cc2.client_state::commit_by_xid(xid) == 0); + BOOST_REQUIRE(cc2.after_statement() == 0); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); + + // xa_detach() creates a streaming applier, clean it up + wsrep::mock_high_priority_service* hps( + static_cast<wsrep::mock_high_priority_service*>( + sc.find_streaming_applier(xid))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +BOOST_FIXTURE_TEST_CASE(transaction_xa_detach_rollback_by_xid, + replicating_two_clients_fixture_sync_rm) +{ + wsrep::xid xid(1, 1, 1, "id"); + + cc1.start_transaction(wsrep::transaction_id(1)); + cc1.assign_xid(xid); + cc1.before_prepare(); + cc1.after_prepare(); + BOOST_REQUIRE(sc.provider().fragments() == 1); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + + cc1.xa_detach(); + + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_aborted); + BOOST_REQUIRE(cc1.after_statement() == 0); + + cc2.start_transaction(wsrep::transaction_id(2)); + cc2.assign_xid(xid); + BOOST_REQUIRE(cc2.rollback_by_xid(xid) == 0); + BOOST_REQUIRE(cc2.after_statement() == 0); + BOOST_REQUIRE(sc.provider().rollback_fragments() == 1); + + // xa_detach() creates a streaming applier, clean it up + wsrep::mock_high_priority_service* hps( + static_cast<wsrep::mock_high_priority_service*>( + sc.find_streaming_applier(xid))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + + +// +// Test XA replay +// +BOOST_FIXTURE_TEST_CASE(transaction_xa_replay, + replicating_client_fixture_sync_rm) +{ + wsrep::xid xid(1, 1, 1, "id"); + + cc.start_transaction(wsrep::transaction_id(1)); + cc.assign_xid(xid); + cc.before_prepare(); + cc.after_prepare(); + cc.after_command_before_result(); + cc.after_command_after_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_idle); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_prepared); + wsrep_test::bf_abort_unordered(cc); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_must_replay); + + // this is normally done by rollbacker + cc.xa_replay(); + cc.sync_rollback_complete(); + + BOOST_REQUIRE(cc.unordered_replays() == 1); + + // xa_replay() creates a streaming applier, clean it up + wsrep::mock_high_priority_service* hps( + static_cast<wsrep::mock_high_priority_service*>( + sc.find_streaming_applier(sc.id(), wsrep::transaction_id(1)))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +BOOST_FIXTURE_TEST_CASE(transaction_xa_replay_after_command_before_result, + replicating_client_fixture_sync_rm) +{ + wsrep::xid xid(1, 1, 1, "id"); + + cc.start_transaction(wsrep::transaction_id(1)); + cc.assign_xid(xid); + cc.before_prepare(); + cc.after_prepare(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_exec); + wsrep_test::bf_abort_unordered(cc); + cc.after_command_before_result(); + cc.after_command_after_result(); + + BOOST_REQUIRE(cc.unordered_replays() == 1); + + // xa_replay() creates a streaming applier, clean it up + wsrep::mock_high_priority_service* hps( + static_cast<wsrep::mock_high_priority_service*>( + sc.find_streaming_applier(sc.id(), wsrep::transaction_id(1)))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +BOOST_FIXTURE_TEST_CASE(transaction_xa_replay_after_command_after_result, + replicating_client_fixture_sync_rm) +{ + wsrep::xid xid(1, 1, 1, "id"); + + cc.start_transaction(wsrep::transaction_id(1)); + cc.assign_xid(xid); + cc.before_prepare(); + cc.after_prepare(); + cc.after_command_before_result(); + BOOST_REQUIRE(cc.state() == wsrep::client_state::s_result); + wsrep_test::bf_abort_unordered(cc); + + cc.after_command_after_result(); + + BOOST_REQUIRE(cc.unordered_replays() == 1); + + // xa_replay() creates a a streaming applier, clean it up + wsrep::mock_high_priority_service* hps( + static_cast<wsrep::mock_high_priority_service*>( + sc.find_streaming_applier(sc.id(), wsrep::transaction_id(1)))); + BOOST_REQUIRE(hps); + hps->rollback(wsrep::ws_handle(), wsrep::ws_meta()); + hps->after_apply(); + sc.stop_streaming_applier(sc.id(), wsrep::transaction_id(1)); + server_service.release_high_priority_service(hps); +} + +// +// Test a successful XA transaction lifecycle (applying side) +// +BOOST_FIXTURE_TEST_CASE(transaction_xa_applying, + applying_client_fixture) +{ + wsrep::xid xid(1, 9, 0, "test xid"); + + start_transaction(wsrep::transaction_id(1), wsrep::seqno(1)); + cc.assign_xid(xid); + + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_preparing); + BOOST_REQUIRE(tc.ordered()); + BOOST_REQUIRE(tc.certified()); + BOOST_REQUIRE(tc.ws_meta().gtid().is_undefined() == false); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_prepared); + + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + cc.after_applying(); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + +/////////////////////////////////////////////////////////////////////////////// +// STREAMING REPLICATION // +/////////////////////////////////////////////////////////////////////////////// + +// +// Test a successful XA transaction lifecycle +// +BOOST_FIXTURE_TEST_CASE(transaction_xa_sr, + streaming_client_fixture_byte) +{ + wsrep::xid xid(1, 9, 0, "test xid"); + + BOOST_REQUIRE(cc.start_transaction(wsrep::transaction_id(1)) == 0); + cc.assign_xid(xid); + + cc.bytes_generated_ = 1; + BOOST_REQUIRE(cc.after_row() == 0); + BOOST_REQUIRE(tc.streaming_context().fragments_certified() == 1); + // XA START fragment with data + BOOST_REQUIRE(sc.provider().fragments() == 1); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_executing); + + BOOST_REQUIRE(cc.before_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_preparing); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.after_prepare() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_prepared); + // XA PREPARE fragment + BOOST_REQUIRE(sc.provider().fragments() == 2); + + BOOST_REQUIRE(cc.before_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committing); + BOOST_REQUIRE(cc.ordered_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_ordered_commit); + BOOST_REQUIRE(tc.ordered()); + BOOST_REQUIRE(tc.certified()); + BOOST_REQUIRE(cc.after_commit() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction::s_committed); + BOOST_REQUIRE(cc.after_statement() == 0); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); + // XA START fragment (with data), XA PREPARE fragment and XA COMMIT fragment + BOOST_REQUIRE(sc.provider().fragments() == 3); + BOOST_REQUIRE(sc.provider().start_fragments() == 1); + BOOST_REQUIRE(sc.provider().commit_fragments() == 1); +} diff --git a/wsrep-lib/test/view_test.cpp b/wsrep-lib/test/view_test.cpp new file mode 100644 index 00000000..1cab1e7c --- /dev/null +++ b/wsrep-lib/test/view_test.cpp @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2018 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/view.hpp" +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_CASE(view_test_member_index) +{ + std::vector<wsrep::view::member> members; + members.push_back(wsrep::view::member(wsrep::id("1"), "", "")); + members.push_back(wsrep::view::member(wsrep::id("2"), "", "")); + members.push_back(wsrep::view::member(wsrep::id("3"), "", "")); + + wsrep::view view(wsrep::gtid(wsrep::id("cluster"), wsrep::seqno(1)), + wsrep::seqno(1), + wsrep::view::primary, + 0, + 1, + 0, + members); + BOOST_REQUIRE(view.member_index(wsrep::id("1")) == 0); + BOOST_REQUIRE(view.member_index(wsrep::id("2")) == 1); + BOOST_REQUIRE(view.member_index(wsrep::id("3")) == 2); + BOOST_REQUIRE(view.member_index(wsrep::id("4")) == -1); +} + +BOOST_AUTO_TEST_CASE(view_test_equal_membership) +{ + std::vector<wsrep::view::member> m1; + m1.push_back(wsrep::view::member(wsrep::id("1"), "", "")); + m1.push_back(wsrep::view::member(wsrep::id("2"), "", "")); + m1.push_back(wsrep::view::member(wsrep::id("3"), "", "")); + + std::vector<wsrep::view::member> m2; + m2.push_back(wsrep::view::member(wsrep::id("2"), "", "")); + m2.push_back(wsrep::view::member(wsrep::id("3"), "", "")); + m2.push_back(wsrep::view::member(wsrep::id("1"), "", "")); + + std::vector<wsrep::view::member> m3; + m3.push_back(wsrep::view::member(wsrep::id("1"), "", "")); + m3.push_back(wsrep::view::member(wsrep::id("2"), "", "")); + m3.push_back(wsrep::view::member(wsrep::id("3"), "", "")); + m3.push_back(wsrep::view::member(wsrep::id("4"), "", "")); + + wsrep::view v1(wsrep::gtid(wsrep::id("cluster"), wsrep::seqno(1)), + wsrep::seqno(1), + wsrep::view::primary, + 0, + 1, + 0, + m1); + + wsrep::view v2(wsrep::gtid(wsrep::id("cluster"), wsrep::seqno(1)), + wsrep::seqno(1), + wsrep::view::primary, + 0, + 1, + 0, + m2); + + wsrep::view v3(wsrep::gtid(wsrep::id("cluster"), wsrep::seqno(1)), + wsrep::seqno(1), + wsrep::view::primary, + 0, + 1, + 0, + m3); + + BOOST_REQUIRE(v1.equal_membership(v2)); + BOOST_REQUIRE(v2.equal_membership(v1)); + BOOST_REQUIRE(!v1.equal_membership(v3)); + BOOST_REQUIRE(!v3.equal_membership(v1)); +} + +BOOST_AUTO_TEST_CASE(view_test_is_member) +{ + wsrep::view view(wsrep::gtid(wsrep::id("cluster"), wsrep::seqno(1)), + wsrep::seqno(1), + wsrep::view::primary, + 0, + 1, + 0, + { wsrep::view::member(wsrep::id("1"), "", ""), + wsrep::view::member(wsrep::id("2"), "", "") }); + + BOOST_REQUIRE(view.is_member(wsrep::id("2"))); + BOOST_REQUIRE(view.is_member(wsrep::id("1"))); + BOOST_REQUIRE(not view.is_member(wsrep::id("0"))); +} diff --git a/wsrep-lib/test/wsrep-lib_test.cpp b/wsrep-lib/test/wsrep-lib_test.cpp new file mode 100644 index 00000000..4eedaa30 --- /dev/null +++ b/wsrep-lib/test/wsrep-lib_test.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2018-2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file wsrep-lib_test.cpp + * + * Run wsrep-lib unit tests. + * + * Commandline arguments: + * + * --wsrep-log-file=<file> Write log from wsrep-lib logging facility + * into <file>. If <file> is left empty, the + * log is written into stdout. + * --wsrep-debug-level=<int> Set debug level + * See wsrep::log::debug_level for valid values + */ + +#include "wsrep/logger.hpp" +#include <fstream> + +#define BOOST_TEST_ALTERNATIVE_INIT_API +#include <boost/test/included/unit_test.hpp> + +// Log file to write messages logged via wsrep-lib logging facility. +static std::string log_file_name("wsrep-lib_test.log"); +static std::ofstream log_file; +// Debug log level for wsrep-lib logging +static std::string debug_log_level; + + +static void log_fn(wsrep::log::level level, + const char* pfx, + const char* msg) +{ + log_file << wsrep::log::to_c_string(level) << " " << pfx << msg << std::endl; +} + +static bool parse_arg(const std::string& arg) +{ + const std::string delim("="); + auto delim_pos(arg.find(delim)); + const auto parm(arg.substr(0, delim_pos)); + std::string val; + if (delim_pos != std::string::npos) + { + val = arg.substr(delim_pos + 1); + } + + if (parm == "--wsrep-log-file") + { + log_file_name = val; + } + else if (parm == "--wsrep-debug-level") + { + debug_log_level = val; + } + else + { + std::cerr << "Error: Unknown argument " << arg << std::endl; + return false; + } + return true; +} + +static bool setup_env(int argc, char* argv[]) +{ + for (int i(1); i < argc; ++i) + { + if (parse_arg(argv[i]) == false) + { + return false; + } + } + + if (log_file_name.size()) + { + log_file.open(log_file_name); + if (!log_file) + { + int err(errno); + std::cerr << "Failed to open '" << log_file_name + << "': '" << ::strerror(err) << "'" << std::endl; + return false; + } + std::cout << "Writing wsrep-lib log into '" << log_file_name << "'" + << std::endl; + wsrep::log::logger_fn(log_fn); + } + + if (debug_log_level.size()) + { + int level = std::stoi(debug_log_level); + std::cout << "Setting debug level '" << level << "'" << std::endl; + wsrep::log::debug_log_level(level); + } + + return true; +} + +bool init_unit_test() +{ + return setup_env(boost::unit_test::framework::master_test_suite().argc, + boost::unit_test::framework::master_test_suite().argv); +} diff --git a/wsrep-lib/test/xid_test.cpp b/wsrep-lib/test/xid_test.cpp new file mode 100644 index 00000000..ce2ffab4 --- /dev/null +++ b/wsrep-lib/test/xid_test.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "wsrep/xid.hpp" +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_CASE(xid_test_is_null) +{ + wsrep::xid null_xid; + BOOST_REQUIRE(null_xid.is_null()); + wsrep::xid test_xid(1,0,0,nullptr); + BOOST_REQUIRE(!test_xid.is_null()); +} + +BOOST_AUTO_TEST_CASE(xid_test_equal) +{ + wsrep::xid a(1,1,1,"ab"); + wsrep::xid b(1,1,1,"ab"); + BOOST_REQUIRE(a == b); +} + +BOOST_AUTO_TEST_CASE(xid_test_null_equal) +{ + wsrep::xid a; + wsrep::xid b; + BOOST_REQUIRE(a == b); + BOOST_REQUIRE(a.is_null()); +} + +BOOST_AUTO_TEST_CASE(xid_test_not_equal) +{ + wsrep::xid a(1,1,0,"a"); + wsrep::xid b(1,0,1,"a"); + wsrep::xid c(-1,1,0,"a"); + wsrep::xid d(1,1,0,"b"); + BOOST_REQUIRE(!(a == b)); + BOOST_REQUIRE(!(a == c)); + BOOST_REQUIRE(!(a == d)); +} + +BOOST_AUTO_TEST_CASE(xid_clear) +{ + wsrep::xid null_xid; + wsrep::xid to_clear(1, 1, 0, "a"); + to_clear.clear(); + BOOST_REQUIRE(to_clear.is_null()); + BOOST_REQUIRE(null_xid == to_clear); +} + +BOOST_AUTO_TEST_CASE(xid_to_string) +{ + wsrep::xid null_xid; + std::stringstream null_xid_str; + null_xid_str << null_xid; + BOOST_REQUIRE(null_xid_str.str() == ""); + + wsrep::xid test_xid(1,4,0,"test"); + std::string xid_str(to_string(test_xid)); + BOOST_REQUIRE(xid_str == "test"); +} + +static bool exception_check(const wsrep::runtime_error&) +{ + return true; +} + +BOOST_AUTO_TEST_CASE(xid_too_big) +{ + std::string s(65,'a'); + BOOST_REQUIRE_EXCEPTION(wsrep::xid a(1, 65, 0, s.c_str()), + wsrep::runtime_error, exception_check); + BOOST_REQUIRE_EXCEPTION(wsrep::xid b(1, 0, 65, s.c_str()), + wsrep::runtime_error, exception_check); +} diff --git a/wsrep-lib/wsrep-API/CMakeLists.txt b/wsrep-lib/wsrep-API/CMakeLists.txt new file mode 100644 index 00000000..51bf81af --- /dev/null +++ b/wsrep-lib/wsrep-API/CMakeLists.txt @@ -0,0 +1,6 @@ +add_library(wsrep_api_v26 + v26/wsrep_dummy.c + v26/wsrep_gtid.c + v26/wsrep_loader.c + v26/wsrep_uuid.c +) diff --git a/wsrep-lib/wsrep-API/v26/.gitignore b/wsrep-lib/wsrep-API/v26/.gitignore new file mode 100644 index 00000000..4ec5efc6 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/.gitignore @@ -0,0 +1,10 @@ +*~ +*.o +*.a +*.diff +listener +CMakeFiles +CMakeCache.txt +cmake_install.cmake +Makefile +examples/node/node diff --git a/wsrep-lib/wsrep-API/v26/CMakeLists.txt b/wsrep-lib/wsrep-API/v26/CMakeLists.txt new file mode 100644 index 00000000..bd66d989 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) 2018, Codership Oy. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) + +INCLUDE_DIRECTORIES(".") + +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wconversion") + +IF (NOT CMAKE_BUILD_TYPE) + SET(CMAKE_BUILD_TYPE Release) +ENDIF() + +SET(WSREP_SOURCES wsrep_gtid.c wsrep_uuid.c wsrep_loader.c wsrep_dummy.c) + +ADD_LIBRARY(wsrep ${WSREP_SOURCES}) + +ADD_SUBDIRECTORY(examples) diff --git a/wsrep-lib/wsrep-API/v26/CONTRIBUTORS.txt b/wsrep-lib/wsrep-API/v26/CONTRIBUTORS.txt new file mode 100644 index 00000000..2943b885 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/CONTRIBUTORS.txt @@ -0,0 +1,28 @@ +All contributors are required to add their name and [Github username/email] +to this file in connection with their first contribution. If you are making +a contribution on behalf of a company, you should add the said company name. + +By adding your name and [Github username/email] to this file you agree that +your contribution is a contribution under a contributor agreement between +you and Codership Oy. To the extent that you are an employee of a company and +contribute in that role, you confirm that your contribution is a contribution +under the contribution license agreement between your employer and Codership +Oy; and that you have the authorization to give such confirmation. You confirm +that you have read, understood and signed the contributor license agreement +applicable to you. + +For the individual contributor agreement see file CONTRIBUTOR_AGREEMENT.txt +in the same directory as this file. + +Authors from Codership Oy: + + * Seppo Jaakola <seppo.jaakola@galeracluster.com>, Codership Oy + * Teemu Ollakka <teemu.ollakka@galeracluster.com>, Codership Oy + * Alexey Yurchenko <alexey.yurchenko@galeracluster.com>, Codership Oy + * Mario Karuza <mario.karuza@galeracluster.com>, Codership Oy + * Daniele Sciascia <daniele.sciascia@galeracluster.com>, Codership Oy + [Codership employees, add name and email/username above this line, but leave this line intact] + +Other contributors: + + [add name and email/username above this line, but leave this line intact] diff --git a/wsrep-lib/wsrep-API/v26/CONTRIBUTOR_AGREEMENT.txt b/wsrep-lib/wsrep-API/v26/CONTRIBUTOR_AGREEMENT.txt new file mode 100644 index 00000000..8bdec2fd --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/CONTRIBUTOR_AGREEMENT.txt @@ -0,0 +1,218 @@ + Codership + Contributor License Agreement + Codership CLA + +Thank you for your interest in contributing to Galera Cluster, a project +managed by Codership Oy, a legal entity established under Finnish laws, with +its principal address at Pohjolankatu 64 A, 00600 Helsinki Finland ("We", "Us" +or "Our"). + +This contributor agreement ("Agreement") documents the rights granted by +contributors to Us. To make this document effective, please either accept it +in an electronic service such as clahub.com or sign and scan it and send it to +Us by email. This is a legally binding document, so please read it carefully +before agreeing to it. This Agreement covers the Galera Cluster project: the +Galera library, the wsrep-lib library, the wsrep-API library, the Wsrep patch +for MySQL and other eventual patches to MySQL or other technologies. + +1. Definitions + +"You" means the individual who Submits a Contribution to Us or the Legal +Entity on behalf of whom a Contribution has been Submitted to Us. "Legal +Entity" means an entity which is not a natural person. "Affiliates" means +other Legal Entities that control, are controlled by, or under common control +with that Legal Entity. For the purposes of this definition, "control" means +(i) the power, direct or indirect, to cause the direction or management of +such Legal Entity, whether by contract or otherwise, (ii) ownership of fifty +percent (50%) or more of the outstanding shares or securities which vote to +elect the management or other persons who direct such Legal Entity or (iii) +beneficial ownership of such entity. + +"Contribution" means any work of authorship that is Submitted by You to Us in +which You own or assert ownership of the Copyright. If You do not own the +Copyright in the entire work of authorship, you need to have a separate +permission from Us. + +"Copyright" means all rights protecting works of authorship owned or +controlled by You, including copyright, moral and neighboring rights, as +appropriate, for the full term of their existence including any extensions by +You. + +"Material" means the work of authorship which is made available by Us to third +parties, i.e. the Galera library, the Wsrep patch for MySQL; other eventual +patches to MySQL; other eventual patches to other database technologies; all +these together with a database technology, such as MySQL, or its +derivatives. After You Submit the Contribution, it may be included in the +Material. + +"Submit" means any form of electronic, verbal, or written communication sent +to Us or our representatives, including but not limited to electronic mailing +lists, source code control systems, and issue tracking systems that are +managed by, or on behalf of, Us for the purpose of discussing and improving +the Material, provided that such communication is (i) conspicuously marked or +otherwise designated in writing by You or Your employee as a "Contribution" or +(ii) submitted in source code control system pursuant to Section 3 (e). + +"Submission Date" means the date on which You Submit a Contribution to Us. + +"Effective Date" means the date You execute this Agreement or the date You +first Submit a Contribution to Us, whichever is earlier. + +"Media" means any portion of a Contribution which is not software. + +2. Grant of Rights + +2.1 Copyright License + +(a) You retain ownership of the Copyright in Your Contribution and have the +same rights to use or license the Contribution which You would have had +without entering into the Agreement. In case we have in writing permitted +submitting a sublicense to licensed rights, You will not transfer the original +license, but grant us a sublicense in accordance with this Agreement. + +(b) To the maximum extent permitted by the relevant law, You grant to Us a +perpetual, worldwide, non-exclusive, transferable, royalty-free, irrevocable +license under the Copyright covering the Contribution, with the right to +sublicense such rights through multiple tiers of sublicensees, to reproduce, +modify, display, perform and distribute the Contribution as part of the +Material; provided that this license is conditioned upon compliance with +Section 2.3. + +2.2 Patent License + +For patent claims including, without limitation, method, process, and +apparatus claims which You, or in case You are a Legal Entity, You or Your +Affiliates, own, control or have the right to grant, now or in the future, You +grant to Us a perpetual, worldwide, non-exclusive, transferable, royalty-free, +irrevocable patent license, with the right to sublicense these rights to +multiple tiers of sublicensees, to make, have made, use, sell, offer for sale, +import and otherwise transfer the Contribution and the Contribution in +combination with the Material (and portions of such combination). This license +is granted only to the extent that the exercise of the licensed rights +infringes such patent claims; and provided that this license is conditioned +upon compliance with Section 2.3. + +2.3 Outbound License + +As a condition on the grant of rights in Sections 2.1 and 2.2, to the extent +we include Your Contribution or a part of it in a Material, we agree to +license the Contribution under the terms of the license or licenses which We +are using on the Submission Date for the Material or any licenses which are +approved by the Open Source Initiative ("OSI") on or after the Effective Date, +including both permissive and copyleft licenses, whether or not such licenses +are subsequently disapproved (including any right to adopt any future version +of a license if approved by the OSI). For clarity, this entitles us to license +Your Contribution also under a permissive open source license, such as the MIT +license, and include binaries created under the MIT license in a proprietary +licensed whole. + +In addition to above defined licenses, We may use the following licenses for +Media in the Contribution: Creative Commons BY 3.0 or Creative Commons BY-SA +3.0 (including the right to adopt any future version of a license). + +2.4 Moral Rights. + +If moral rights apply to the Contribution, to the maximum extent permitted by +law, You waive and agree not to assert such moral rights against Us or our +successors in interest, or any of our licensees, either direct or indirect. + +2.5 Enforcement. + +You, as a copyright holder to Your Contribution, hereby authorize us to +enforce the OSI approved license applied by Us to a Material, but only to the +extent Your Contribution has been included in a Material and always subject to +Our free discretion on whether such enforcement is necessary or not. + +2.6 Our Rights. + +You acknowledge that We are not obligated to use Your Contribution as part of +the Material and may decide to include any Contribution We consider +appropriate. + +2.7 Reservation of Rights. + +Any rights not expressly licensed under this section are expressly reserved by +You. + +3. Agreement + +You confirm that: + +(a) You have the legal authority to enter into this Agreement. + +(b) You or Your Affiliates, own the Copyright and patent claims covering the + Contribution which are required to grant the rights under Section 2. + +(c) The grant of rights under Section 2 does not violate any grant of rights + which You or Your Affiliates have made to third parties, including Your + employer. If You are an employee, You have had Your employer approve this + Agreement or sign the Entity version of this document. If You are less + than eighteen years old, please have Your parents or guardian sign the + Agreement. + +(d) You have not Submitted any Code You do not own without written permission + from US. + +(e) All pull or merge requests issued under usernames confirmed by You in + writing are issued by You; and all such pull or merge requests contain + Your Contributions under this Agreement. You will notify Us in writing in + the event of You no longer control such usernames. + +4. Disclaimer + +EXCEPT FOR THE EXPRESS WARRANTIES IN SECTION 3, THE CONTRIBUTION IS PROVIDED +"AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, +WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO +US. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH WARRANTY +IS LIMITED IN DURATION TO THE MINIMUM PERIOD PERMITTED BY LAW. + +5. Consequential Damage Waiver + +TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU BE +LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA, +INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT +OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT +OR OTHERWISE) UPON WHICH THE CLAIM IS BASED. + +THIS WAIVER DOES NOT APPLY TO GROSS NEGLIGENT OR MALICIOUS ACTS OR FRAUD. + +6. Miscellaneous + +6.1 This Agreement will be governed by and construed in accordance with the +laws of Finland excluding its conflicts of law provisions. Under certain +circumstances, the governing law in this section might be superseded by the +United Nations Convention on Contracts for the International Sale of Goods +("UN Convention") and the parties intend to avoid the application of the UN +Convention to this Agreement and, thus, exclude the application of the UN +Convention in its entirety to this Agreement. + +6.2 Any and all Submissions done by You prior to execution of this Agreement +shall be nonetheless covered by this Agreement. + +6.3 This Agreement sets out the entire agreement between You and Us for Your +Contributions to Us and overrides all other agreements or understandings. + +6.4 If You or We assign the rights or obligations received through this +Agreement to a third party, as a condition of the assignment, that third party +must agree in writing to abide by all the rights and obligations in the +Agreement. + +6.5 The failure of either party to require performance by the other party of +any provision of this Agreement in one situation shall not affect the right of +a party to require such performance at any time in the future. A waiver of +performance under a provision in one situation shall not be considered a +waiver of the performance of the provision in the future or a waiver of the +provision in its entirety. + +6.6 If any provision of this Agreement is found void and unenforceable, such +provision will be replaced to the extent possible with a provision that comes +closest to the meaning of the original provision and which is enforceable. +The terms and conditions set forth in this Agreement shall apply +notwithstanding any failure of essential purpose of this Agreement or any +limited remedy to the maximum extent possible under law. + +This document has been drafted based on Harmony Inividual Contributor License +Agreement (HA-CLA-I) Version 1.0 July 4, 2011. HA-CLA-I is available from +harmonyagreements.org and is licensed by under Creative Commons Attribution +3.0 Unported License. diff --git a/wsrep-lib/wsrep-API/v26/COPYING b/wsrep-lib/wsrep-API/v26/COPYING new file mode 100644 index 00000000..d159169d --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/wsrep-lib/wsrep-API/v26/README.md b/wsrep-lib/wsrep-API/v26/README.md new file mode 100644 index 00000000..27664805 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/README.md @@ -0,0 +1,7 @@ +# Write Set Replication API specification + +Building: +``` +cmake [-DCMAKE_BUILD_TYPE=Debug|Release] . && make [VERBOSE=1] +``` +in top directory. diff --git a/wsrep-lib/wsrep-API/v26/examples/CMakeLists.txt b/wsrep-lib/wsrep-API/v26/examples/CMakeLists.txt new file mode 100644 index 00000000..e6e33b78 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (c) 2019, Codership Oy. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +ADD_EXECUTABLE(listener listener.c) +TARGET_LINK_LIBRARIES(listener wsrep dl pthread) + +ADD_SUBDIRECTORY(node) diff --git a/wsrep-lib/wsrep-API/v26/examples/README.md b/wsrep-lib/wsrep-API/v26/examples/README.md new file mode 100644 index 00000000..b1b20744 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/README.md @@ -0,0 +1,14 @@ +## wsrep API usage examples + +### 1. Listener +Is a simple program that connects and listens to replication events in +an existing cluster. + +Usage example (starting listener on the same host as the rest of the cluster): +``` +$ ./listener /path_to/libgalera_smm.so gcomm://localhost:4567?gmcast.listen_addr=tcp://127.0.0.1:9999 cluster_name +``` + +### 2. Node +Is a more complex program which implements most of wsrep node functionality +and can form clusters in itself. diff --git a/wsrep-lib/wsrep-API/v26/examples/listener.c b/wsrep-lib/wsrep-API/v26/examples/listener.c new file mode 100644 index 00000000..9fc881fe --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/listener.c @@ -0,0 +1,268 @@ +/* Copyright (C) 2012 Codership Oy <info@codersihp.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/*! @file Example of wsrep event listener. Outputs description of received + * events to stdout. To get a general picture you should start with + * main() function. */ + +#include <wsrep_api.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <signal.h> +#include <pthread.h> + +/*! This is global application context, it will be used by wsrep callbacks */ +struct application_context +{}; + +static struct application_context global_ctx; + +/*! This is receiving thread context, it will be used by wsrep callbacks */ +struct receiver_context +{ + char msg[4096]; +}; + +/* wsrep provider handle (global for simplicty) */ +static wsrep_t* wsrep = NULL; + +/*! This is a logger callback which library will be using to log events. */ +static void +logger_cb (wsrep_log_level_t level __attribute__((unused)), const char* msg) +{ + fprintf (stderr, "WSREP: %s\n", msg); +} + +/*! This will be called on cluster view change (nodes joining, leaving, etc.). + * Each view change is the point where application may be pronounced out of + * sync with the current cluster view and need state transfer. + * It is guaranteed that no other callbacks are called concurrently with it. */ +static wsrep_cb_status_t +view_cb (void* app_ctx __attribute__((unused)), + void* recv_ctx __attribute__((unused)), + const wsrep_view_info_t* view, + const char* state __attribute__((unused)), + size_t state_len __attribute__((unused))) +{ + printf ("New cluster membership view: %d nodes, my index is %d, " + "global seqno: %lld\n", + view->memb_num, view->my_idx, (long long)view->state_id.seqno); + + return WSREP_CB_SUCCESS; +} + +/*! This will be called on cluster view change (nodes joining, leaving, etc.). + * Each view change is the point where application may be pronounced out of + * sync with the current cluster view and need state transfer. + * It is guaranteed that no other callbacks are called concurrently with it. */ +static wsrep_cb_status_t +sst_request_cb (void* app_ctx __attribute__((unused)), + void** sst_req, + size_t* sst_req_len) +{ + /* For simplicity we're skipping state transfer by using magic string + * as a state transfer request. + * This node will not be considered JOINED (having full state) + * by other cluster members. */ + *sst_req = strdup(WSREP_STATE_TRANSFER_NONE); + + if (*sst_req) + *sst_req_len = strlen(*sst_req) + 1; + else + *sst_req_len = 0; + + return WSREP_CB_SUCCESS; +} + +/*! This is called to "apply" writeset. + * If writesets don't conflict on keys, it may be called concurrently to + * utilize several CPU cores. */ +static wsrep_cb_status_t +apply_cb (void* recv_ctx, + const wsrep_ws_handle_t* ws_handle __attribute__((unused)), + uint32_t flags __attribute__((unused)), + const wsrep_buf_t* ws __attribute__((unused)), + const wsrep_trx_meta_t* meta, + wsrep_bool_t* exit_loop __attribute__((unused))) +{ + struct receiver_context* ctx = (struct receiver_context*)recv_ctx; + + snprintf (ctx->msg, sizeof(ctx->msg), + "Got writeset %lld, size %zu", (long long)meta->gtid.seqno, + ws->len); + + bool const commit = flags & (WSREP_FLAG_TRX_END | WSREP_FLAG_ROLLBACK); + + wsrep->commit_order_enter(wsrep, ws_handle, meta); + if (commit) puts(ctx->msg); + wsrep->commit_order_leave(wsrep, ws_handle, meta, NULL); + + return WSREP_CB_SUCCESS; +} + +/* The following callbacks are stubs and not used in this example. */ +static wsrep_cb_status_t +unordered_cb(void* recv_ctx __attribute__((unused)), + const wsrep_buf_t* data __attribute__((unused))) +{ + return WSREP_CB_SUCCESS; +} + +static wsrep_cb_status_t +sst_donate_cb (void* app_ctx __attribute__((unused)), + void* recv_ctx __attribute__((unused)), + const wsrep_buf_t* msg __attribute__((unused)), + const wsrep_gtid_t* state_id __attribute__((unused)), + const wsrep_buf_t* state __attribute__((unused)), + wsrep_bool_t bypass __attribute__((unused))) +{ + return WSREP_CB_SUCCESS; +} + +static wsrep_cb_status_t synced_cb (void* app_ctx __attribute__((unused))) +{ + return WSREP_CB_SUCCESS; +} + +/* This is the listening thread. It blocks in wsrep::recv() call until + * disconnect from cluster. It will apply and commit writesets through the + * callbacks defined avbove. */ +static void* +recv_thread (void* arg) +{ + struct receiver_context* ctx = (struct receiver_context*)arg; + + wsrep_status_t rc = wsrep->recv(wsrep, ctx); + + fprintf (stderr, "Receiver exited with code %d", rc); + + return NULL; +} + +/* This is a signal handler to demonstrate graceful cluster leave. */ +static void +graceful_leave (int signum) +{ + printf ("Got signal %d, exiting...\n", signum); + wsrep->disconnect(wsrep); +} + +int main (int const argc, char* argv[]) +{ + if (argc < 4 || argc > 5) + { + fprintf (stderr, "Usage: %s </path/to/wsrep/provider> <wsrep URI> " + "<cluster name> [own address]\n", argv[0]); + exit (EXIT_FAILURE); + } + + const char* const wsrep_provider = argv[1]; + const char* const wsrep_uri = argv[2]; + const char* const cluster_name = argv[3]; + const char* const own_address = argc == 5 ? argv[4] : "localhost"; + + /* Now let's load and initialize provider */ + wsrep_status_t rc = wsrep_load (wsrep_provider, &wsrep, logger_cb); + if (WSREP_OK != rc) + { + fprintf (stderr, "Failed to load wsrep provider '%s'\n",wsrep_provider); + exit (EXIT_FAILURE); + } + + wsrep_gtid_t state_id = { WSREP_UUID_UNDEFINED, WSREP_SEQNO_UNDEFINED }; + + /* wsrep provider initialization arguments */ + struct wsrep_init_args wsrep_args = + { + .app_ctx = &global_ctx, + + .node_name = "example listener", + .node_address = own_address, + .node_incoming = "", + .data_dir = ".", // working directory + .options = "", + .proto_ver = 127, // maximum supported application event protocol + + .state_id = &state_id, + .state = NULL, + + .logger_cb = logger_cb, + .view_cb = view_cb, + .sst_request_cb = sst_request_cb, + .encrypt_cb = NULL, + .apply_cb = apply_cb, + .unordered_cb = unordered_cb, + .sst_donate_cb = sst_donate_cb, + .synced_cb = synced_cb + }; + + rc = wsrep->init(wsrep, &wsrep_args); + if (WSREP_OK != rc) + { + fprintf (stderr, "wsrep::init() failed: %d\n", rc); + exit (EXIT_FAILURE); + } + + /* Connect to cluster */ + rc = wsrep->connect(wsrep, cluster_name, wsrep_uri, "", 0); + if (0 != rc) + { + if (rc < 0) + fprintf (stderr, "wsrep::connect(%s, %s) failed: %d (%s)\n", + cluster_name, wsrep_uri, rc, strerror(-(int)rc)); + else + fprintf (stderr, "wsrep::connect() failed: %d\n", rc); + + exit (EXIT_FAILURE); + } + + /* Now let's start several listening threads*/ + int const num_threads = 4; + struct receiver_context thread_ctx[num_threads]; + pthread_t threads[num_threads]; + + int i; + for (i = 0; i < num_threads; i++) + { + int err = pthread_create ( + &threads[i], NULL, recv_thread, &thread_ctx[i]); + + if (err) + { + fprintf (stderr, "Failed to start thread %d: %d (%s)", + i, err, strerror(err)); + exit (EXIT_FAILURE); + } + } + + signal (SIGTERM, graceful_leave); + signal (SIGINT, graceful_leave); + + /* Listening threads are now running and receiving writesets. Wait for them + * to join. Threads will join after signal handler closes wsrep connection*/ + for (i = 0; i < num_threads; i++) + { + pthread_join (threads[i], NULL); + } + + /* Unload provider after nobody uses it any more. */ + wsrep_unload (wsrep); + + return 0; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/CMakeLists.txt b/wsrep-lib/wsrep-API/v26/examples/node/CMakeLists.txt new file mode 100644 index 00000000..d018afde --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (c) 2019, Codership Oy. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +FILE(GLOB SRC + "*.h" + "*.c" + ) + +ADD_EXECUTABLE(node ${SRC}) + +TARGET_LINK_LIBRARIES(node wsrep dl pthread) + +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/node.sh + ${CMAKE_CURRENT_BINARY_DIR}/node.sh COPYONLY) diff --git a/wsrep-lib/wsrep-API/v26/examples/node/README.md b/wsrep-lib/wsrep-API/v26/examples/node/README.md new file mode 100644 index 00000000..4a07c149 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/README.md @@ -0,0 +1,81 @@ +# wsrep API node application + +## Overview + +This is a simple application to demonstrate the usage of wsrep API. It +deliberately does nothing useful in order to present as concentrated and +concise API usage as possible. + +The program is deliberately written in C to demonstrate the naked API usage. +For C++ example see a much more advanced integration library at +https://github.com/codership/wsrep-lib + +## High level architecture + +Process-wise the program consists of an endless main loop that periodically +samples and prints performance stats and a configurable number of "master" and +"slave" threads, with master threads loop executing "transactions" and +replicating resulting "write sets" and slave threads receiving and processing +the write sets from other nodes. + +Object-wise the program is composed of two main objects: `store` and `wsrep`. +'store' object contains application "state" and generates and commits changes +to the state. `wsrep` object contains cluster context and provides interface +to it. Changes generated by `store` are replicated and certified through +`wsrep` and then committed to `store`. + +## Unit descriptions (in alphabetical order) + +#### ctx.h +A small header to declare the application context structure. + +#### log.* +Implements logging functionality for the application AND +**a logging callback** for the wsrep provider. + +#### main.c +Defines `main()` routine that initializes storage and wsrep provider, starts +the worker threads and loops in a statistics collection loop. Even though it is +not designed to return it still shows the deinitialization order. + +#### options.* +Implements reading configuration options from the command line, does not have +anything related to wsrep API, but shows which additional parameters must be +configured for the program to make use of wsrep clustering. + +#### socket.* +Network sockets boilerplate code for setting TCP connections between processes +(for SST). Has nothing wsrep-related and can be ignored. + +#### sst.* +Defines **SST callbacks** for the wsrep provider and shows how to asynchronously +implement state snapshot transfer (yes, you don't want to spend eternity in +callbacks). + +#### stats.* +Implements performance stats collecting function for the main loop. While it is +an absolutely optional provider functionality, still it shows how to use that. + +#### store.* +Defines the `store` object that pretends to store and modify some data in a +"transactional" manner. It provides the caller that intends to do a change with +a *change data* and a *key* for replication and certification. + +#### trx.* +Defines routines to process local and replicated transactions. + +#### worker.* +Implements worker thread pool functinality. Worker threads run routines defined +in 'trx.*'. Also implements **apply callback** for the wsrep provider. + +#### wsrep.* +Maintains wsrep cluster context: provider instance and cluster membership view. +While there is little use for the latter in this primitive application, still +it shows **connected and view callbacks** usage. But mostly, for this +application its purpose is to initialize the provider, connect to the cluster +and offer access to initialized provider for other parts of the program. + +## Example usage +``` +./node -f /tmp/galera/0 -v /tmp/galera/0/galera/lib/libgalera_smm.so -o 'pc.weight=2;evs.send_window=2;evs.user_send_window=1;gcache.recover=no' -s 8 -m 16 +``` diff --git a/wsrep-lib/wsrep-API/v26/examples/node/ctx.h b/wsrep-lib/wsrep-API/v26/examples/node/ctx.h new file mode 100644 index 00000000..01653554 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/ctx.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines application context for wsrep provider + */ + +#ifndef NODE_CTX_H +#define NODE_CTX_H + +#include "store.h" +#include "wsrep.h" + +struct node_ctx +{ + node_wsrep_t* wsrep; + node_store_t* store; + const struct node_options* opts; +}; + +#endif /* NODE_CTX_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/log.c b/wsrep-lib/wsrep-API/v26/examples/node/log.c new file mode 100644 index 00000000..71f4705c --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/log.c @@ -0,0 +1,100 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "log.h" + +#include <stdio.h> // fprintf(), fflush() +#include <sys/time.h> // gettimeofday() +#include <time.h> // localtime_r() +#include <stdarg.h> // va_start(), va_end() + +wsrep_log_level_t node_log_max_level = WSREP_LOG_INFO; + +static const char* log_level_str[WSREP_LOG_DEBUG + 2] = +{ + "FATAL: ", + "ERROR: ", + " WARN: ", + " INFO: ", + "DEBUG: ", + "XXXXX: " +}; + +static inline void +log_timestamp_and_log(const char* const prefix, // source of msg + int const severity, + const char* const msg) +{ + struct tm date; + struct timeval time; + + gettimeofday(&time, NULL); + localtime_r (&time.tv_sec, &date); + + FILE* log_file = stderr; + fprintf(log_file, + "%04d-%02d-%02d %02d:%02d:%02d.%03d " /* timestamp fmt */ + "[%s] %s%s\n", /* [prefix] severity msg */ + date.tm_year + 1900, date.tm_mon + 1, date.tm_mday, + date.tm_hour, date.tm_min, date.tm_sec, + (int)time.tv_usec / 1000, + prefix, log_level_str[severity], msg + ); + + fflush (log_file); +} + +void +node_log_cb(wsrep_log_level_t const severity, const char* const msg) +{ + /* REPLICATION: let provider log messages be prefixed with 'wsrep'*/ + log_timestamp_and_log("wsrep", severity, msg); +} + +void +node_log(wsrep_log_level_t const severity, + const char* const file, + const char* const function, + int const line, + ...) +{ + va_list ap; + + char string[2048]; + int max_string = sizeof(string); + char* str = string; + + /* provide file:func():line info only if debug logging is on */ + if (NODE_DO_LOG_DEBUG) { + int const len = snprintf(str, (size_t)max_string, "%s:%s():%d: ", + file, function, line); + str += len; + max_string -= len; + } + + va_start(ap, line); + { + const char* format = va_arg (ap, const char*); + + if (max_string > 0 && NULL != format) { + vsnprintf (str, (size_t)max_string, format, ap); + } + } + va_end(ap); + + /* actual logging */ + log_timestamp_and_log(" node", severity, string); +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/log.h b/wsrep-lib/wsrep-API/v26/examples/node/log.h new file mode 100644 index 00000000..09404f26 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/log.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines logging macros for the application and + * a logger callback for the wsrep provider. + */ + +#ifndef NODE_LOG_H +#define NODE_LOG_H + +#include "../../wsrep_api.h" + +/** + * REPLICATION: a logger callback for wsrep provider + */ +extern void +node_log_cb(wsrep_log_level_t severity, const char* message); + +/** + * Applicaton log function intended to be used through the macros defined below. + * For simplicity it uses log levels defined by wsrep API, but it does not have + * to. */ +extern void +node_log (wsrep_log_level_t level, + const char* file, + const char* function, + const int line, + ...); + +/** + * This variable made global to avoid calling node_log() when debug logging + * is disabled. */ +extern wsrep_log_level_t node_log_max_level; +#define NODE_DO_LOG_DEBUG (WSREP_LOG_DEBUG <= node_log_max_level) + +/** + * Base logging macro that records current file, function and line number */ +#define NODE_LOG(level, ...)\ + node_log(level, __FILE__, __func__, __LINE__, __VA_ARGS__, NULL) + +/** + * @name Logging macros. + * Must be implemented as macros to report the location of the code where + * they are called. + */ +/*@{*/ +#define NODE_FATAL(...) NODE_LOG(WSREP_LOG_FATAL, __VA_ARGS__, NULL) +#define NODE_ERROR(...) NODE_LOG(WSREP_LOG_ERROR, __VA_ARGS__, NULL) +#define NODE_WARN(...) NODE_LOG(WSREP_LOG_WARN, __VA_ARGS__, NULL) +#define NODE_INFO(...) NODE_LOG(WSREP_LOG_INFO, __VA_ARGS__, NULL) +#define NODE_DEBUG(...) if (NODE_DO_LOG_DEBUG) \ + { NODE_LOG(WSREP_LOG_DEBUG, __VA_ARGS__, NULL); } +/*@}*/ + +#endif /* NODE_LOG_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/main.c b/wsrep-lib/wsrep-API/v26/examples/node/main.c new file mode 100644 index 00000000..f3124042 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/main.c @@ -0,0 +1,146 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "ctx.h" +#include "log.h" +#include "options.h" +#include "stats.h" +#include "worker.h" +#include "wsrep.h" + +#include <errno.h> +#include <signal.h> // sigaction() +#include <string.h> // strerror() + +static void +signal_handler(int const signum) +{ + NODE_INFO("Got signal %d. Terminating.", signum); +} + +static void +install_signal_handler(void) +{ + sigset_t sa_mask; + sigemptyset(&sa_mask); + + struct sigaction const act = + { + .sa_handler = signal_handler, + .sa_mask = sa_mask, + .sa_flags = (int)SA_RESETHAND + }; + + if (sigaction(SIGINT /* Ctrl-C */, &act, NULL)) + { + NODE_INFO("sigaction() failed: %d (%s)", errno, strerror(errno)); + abort(); + } +} + +int main(int argc, char* argv[]) +{ + install_signal_handler(); + + struct node_options opts; + int err = node_options_read(argc, argv, &opts); + if (err) + { + NODE_FATAL("Failed to read command line opritons: %d (%s)", + err, strerror(err)); + return err; + } + + struct node_ctx node; + node.opts = &opts; + + /* REPLICATION: before connecting to cluster we need to initialize our + * storage to know our current position (GTID) */ + node.store = node_store_open(&opts); + if (!node.store) + { + NODE_FATAL("Failed to open node store"); + return 1; + } + + wsrep_gtid_t current_gtid; + node_store_gtid(node.store, ¤t_gtid); + + /* REPLICATION: complete initialization of application context + * (including provider itself) */ + node.wsrep = node_wsrep_init(&opts, ¤t_gtid, &node); + if (!node.wsrep) + { + NODE_FATAL("Failed to initialize wsrep provider"); + return 1; + } + + /* REPLICATION: now we can connect to the cluster and start receiving + * replication events */ + if (node_wsrep_connect(node.wsrep, opts.address, opts.bootstrap) != + WSREP_OK) + { + NODE_FATAL("Failed to connect to primary component"); + return 1; + } + + /* REPLICATION: and start processing replicaiton events */ + struct node_worker_pool* slave_pool = + node_worker_start(&node, NODE_WORKER_SLAVE, (size_t)opts.slaves); + if (!slave_pool) + { + NODE_FATAL("Failed to create slave worker pool"); + return 1; + } + + /* REPLICATION: now that replicaton events are being processed we can + * wait to sync with the cluster */ + if (!node_wsrep_wait_synced(node.wsrep)) + { + NODE_ERROR("Failed to wait fir SYNCED event"); + return 1; + } + + NODE_INFO("Synced with cluster"); + + /* REPLICATION: now we can start replicate own events */ + struct node_worker_pool* master_pool = + node_worker_start(&node, NODE_WORKER_MASTER, (size_t)opts.masters); + if (opts.masters > 0 && !master_pool) + { + NODE_FATAL("Failed to create master worker pool"); + return 1; + } + + node_stats_loop(&node, (int)opts.period); + + /* REPLICATON: to shut down we go in the opposite order: + * first - disconnect from the cluster to signal master threads + * to exit loop, + * second - join master and slave threads, + * third - close provider once not in use */ + node_wsrep_disconnect(node.wsrep); + + node_worker_stop(master_pool); + node_worker_stop(slave_pool); + + node_wsrep_close(node.wsrep); + + /* and finally, when the storage can no longer be disturbed, close it */ + node_store_close(node.store); + + return 0; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/node.sh b/wsrep-lib/wsrep-API/v26/examples/node/node.sh new file mode 100755 index 00000000..40e0a498 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/node.sh @@ -0,0 +1,40 @@ +#!/bin/sh -eu + +NODE_ID=$1 + +NODE_NAME=${NODE_NAME:-$NODE_ID} + +NODE_DIR=${NODE_DIR:-/tmp/node/$NODE_NAME} +rm -rf $NODE_DIR/* +mkdir -p $NODE_DIR + +NODE_OPT=${NODE_OPT:-} + +NODE_HOST=${NODE_HOST:-localhost} +NODE_PORT=${NODE_PORT:-$((10000 + $NODE_ID))} + +NODE_CLIENTS=${NODE_CLIENTS:-1} +NODE_APPLIERS=${NODE_APPLIERS:-1} + +NODE_ADDR=${NODE_ADDR:-} + +NODE_BIN=${NODE_BIN:-$(dirname $0)/node} + +# convert possible relative path to absolute path +NODE_PROVIDER=$(realpath $NODE_PROVIDER) + +set -x + +$NODE_BIN \ +-v "$NODE_PROVIDER" \ +-n "$NODE_NAME" \ +-f "$NODE_DIR" \ +-o "$NODE_OPT" \ +-t "$NODE_HOST" \ +-p $NODE_PORT \ +-s $NODE_APPLIERS \ +-m $NODE_CLIENTS \ +-d 10 \ +-a "$NODE_ADDR" + +set +x diff --git a/wsrep-lib/wsrep-API/v26/examples/node/options.c b/wsrep-lib/wsrep-API/v26/examples/node/options.c new file mode 100644 index 00000000..0bd08ffb --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/options.c @@ -0,0 +1,291 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "options.h" + +#include <ctype.h> // isspace() +#include <errno.h> +#include <getopt.h> +#include <stdio.h> +#include <stdlib.h> // strtol() +#include <string.h> // strcmp() + +/* + * getopt_long() declarations begin + */ + +#define OPTS_NA no_argument +#define OPTS_RA required_argument +#define OPTS_OA optional_argument + +typedef enum opt +{ + OPTS_NOOPT = 0, + OPTS_ADDRESS = 'a', + OPTS_BOOTSTRAP = 'b', + OPTS_DELAY = 'd', + OPTS_DATA_DIR = 'f', + OPTS_HELP = 'h', + OPTS_PERIOD = 'i', + OPTS_MASTERS = 'm', + OPTS_NAME = 'n', + OPTS_OPTIONS = 'o', + OPTS_BASE_PORT = 'p', + OPTS_RECORDS = 'r', + OPTS_SLAVES = 's', + OPTS_BASE_HOST = 't', + OPTS_PROVIDER = 'v', + OPTS_WS_SIZE = 'w', + OPTS_OPS = 'x' +} + opt_t; + +static struct option s_opts[] = +{ + { "address", OPTS_RA, NULL, OPTS_ADDRESS }, + { "bootstrap", OPTS_NA, NULL, OPTS_BOOTSTRAP }, + { "delay", OPTS_RA, NULL, OPTS_DELAY }, + { "storage", OPTS_RA, NULL, OPTS_DATA_DIR }, + { "help", OPTS_NA, NULL, OPTS_HELP }, + { "period", OPTS_RA, NULL, OPTS_PERIOD }, + { "masters", OPTS_RA, NULL, OPTS_MASTERS }, + { "name", OPTS_RA, NULL, OPTS_NAME }, + { "options", OPTS_RA, NULL, OPTS_OPTIONS, }, + { "base-port", OPTS_RA, NULL, OPTS_BASE_PORT }, + { "records", OPTS_RA, NULL, OPTS_RECORDS }, + { "slaves", OPTS_RA, NULL, OPTS_SLAVES }, + { "base-host", OPTS_RA, NULL, OPTS_BASE_HOST }, + { "provider", OPTS_RA, NULL, OPTS_PROVIDER }, + { "size", OPTS_RA, NULL, OPTS_WS_SIZE }, + { "ops", OPTS_RA, NULL, OPTS_OPS }, + { NULL, 0, NULL, 0 } +}; + +static const char* opts_string = "a:d:f:hi:m:n:o:p:r:s:t:v:w:x:"; + +/* + * getopt_long() declarations end + */ + +static const struct node_options opts_defaults = +{ + .provider = "none", + .address = "", + .options = "", + .name = "unnamed", + .data_dir = ".", + .base_host = "localhost", + .masters = 0, + .slaves = 1, + .ws_size = 1024, + .records = 1024*1024, + .delay = 0, + .base_port = 4567, + .period = 10, + .operations= 1, + .bootstrap = true +}; + +static void +opts_print_help(FILE* out, const char* prog_name) +{ + fprintf( + out, + "Usage: %s [OPTION...]\n" + "\n" + " -h, --help this thing.\n" + " -v, --provider=PATH a path to wsrep provider library file.\n" + " -a, --address=STRING list of node addresses in the group.\n" + " If not set the node assumes that it is the first\n" + " node in the group (default)\n" + " -o, --options=STRING a string of wsrep provider options.\n" + " -n, --name=STRING human-readable node name.\n" + " -f, --data-dir=PATH a directory to save working data in.\n" + " Should be private to the process.\n" + " -t, --base-host=ADDRESS address of this node at which other members can\n" + " connect to it\n" + " -p, --base-port=NUM base port which the node shall listen for\n" + " connections from other members. This port will be\n" + " used for replication, port+1 for IST and port+2\n" + " for SST. Default: 4567\n" + " -m, --masters=NUM number of concurrent master workers.\n" + " -s, --slaves=NUM number of concurrent slave workers.\n" + " (can't be less than 1)\n" + " -w, --size=NUM desirable size of the resulting writesets\n" + " (approximate lower boundary). Default: 1K\n" + " -r, --records=NUM number of records in the store. Default: 1M\n" + " -x, --ops=NUM number of operations per transaction. Default: 1\n" + " -d, --delay=NUM delay in milliseconds between \"commits\"\n" + " (per master thread).\n" + " -b, --bootstrap bootstrap the cluster with this node.\n" + " Default: 'Yes' if --address is not given, 'No'\n" + " otherwise.\n" + " -i, --period period in seconds between performance stats output\n" + "\n" + , prog_name); +} + +static void +opts_print_config(FILE* out, const struct node_options* opts) +{ + fprintf( + out, + "Continuing with the following configuration:\n" + "provider: %s\n" + "address: %s\n" + "options: %s\n" + "name: %s\n" + "data dir: %s\n" + "base addr: %s:%ld\n" + "masters: %ld\n" + "slaves: %ld\n" + "writeset size: %ld bytes\n" + "records: %ld\n" + "operations: %ld\n" + "commit delay: %ld ms\n" + "stats period: %ld s\n" + "bootstrap: %s\n" + , + opts->provider, opts->address, opts->options, opts->name, opts->data_dir, + opts->base_host, opts->base_port, + opts->masters, opts->slaves, opts->ws_size, opts->records, + opts->operations, + opts->delay, opts->period, opts->bootstrap ? "Yes" : "No" + ); +} + +static int +opts_check_conversion(int cond, const char* ptr, int idx) +{ + if (!cond || errno || (*ptr != '\0' && !isspace(*ptr))) + { + fprintf(stderr, "Bad value for %s option.\n", s_opts[idx].name); + return EINVAL; + } + return 0; +} + +int +node_options_read(int argc, char* argv[], struct node_options* opts) +{ + *opts = opts_defaults; + + int opt = 0; + int opt_idx = 0; + char* endptr; + int ret = 0; + + bool address_given = false; + bool bootstrap_given = false; + + while ((opt = getopt_long(argc, argv, opts_string, s_opts, &opt_idx)) != -1) + { + switch (opt) + { + case OPTS_ADDRESS: + address_given = strcmp(opts->address, optarg); + opts->address = optarg; + break; + case OPTS_BOOTSTRAP: + bootstrap_given = true; + opts->bootstrap = true; + break; + case OPTS_DELAY: + opts->delay = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->delay >= 0, endptr, opt_idx))) + goto err; + break; + case OPTS_DATA_DIR: + opts->data_dir = optarg; + break; + case OPTS_HELP: + ret = 1; + goto help; + case OPTS_PERIOD: + opts->period = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->period > 0, endptr, opt_idx))) + goto err; + break; + case OPTS_MASTERS: + opts->masters = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->masters >= 0, endptr, + opt_idx))) + goto err; + break; + case OPTS_NAME: + opts->name = optarg; + break; + case OPTS_OPTIONS: + opts->options = optarg; + break; + case OPTS_BASE_PORT: + opts->base_port = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion( + opts->base_port > 0 && opts->base_port < 65536, + endptr, opt_idx))) + goto err; + break; + case OPTS_RECORDS: + opts->records = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->records >= 0, endptr, + opt_idx))) + goto err; + break; + case OPTS_SLAVES: + opts->slaves = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->slaves > 0, endptr, opt_idx))) + goto err; + break; + case OPTS_BASE_HOST: + opts->base_host = optarg; + break; + case OPTS_PROVIDER: + opts->provider = optarg; + break; + case OPTS_WS_SIZE: + opts->ws_size = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->ws_size > 0, endptr, + opt_idx))) + goto err; + break; + case OPTS_OPS: + opts->operations = strtol(optarg, &endptr, 10); + if ((ret = opts_check_conversion(opts->operations >= 1, endptr, + opt_idx))) + goto err; + break; + default: + ret = EINVAL; + } + } + +help: + if (ret) { + opts_print_help(stderr, argv[0]); + } + else + { + if (!bootstrap_given) + { + opts->bootstrap = !address_given; + } + opts_print_config(stdout, opts); + opts->delay *= 1000; /* convert to microseconds for usleep() */ + } + +err: + return ret; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/options.h b/wsrep-lib/wsrep-API/v26/examples/node/options.h new file mode 100644 index 00000000..62172281 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/options.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines options interface + */ + +#ifndef NODE_OPTIONS_H +#define NODE_OPTIONS_H + +#include <stdbool.h> + +struct node_options +{ + const char* provider; // path to wsrep provider + const char* address; // wsrep cluster address string + const char* options; // wsrep option string + const char* name; // node name (for logging purposes) + const char* data_dir; // name of the storage file + const char* base_host;// host own address + long masters; // number of master threads + long slaves; // number of slave threads + long ws_size; // desired writeset size + long records; // total number of records + long delay; // delay between commits + long base_port;// base port to use + long period; // statistics output interval + long operations;// number of "statements" in a "transaction" + bool bootstrap;// bootstrap the cluster with this node +}; + +extern int +node_options_read(int argc, char* argv[], struct node_options* opts); + +#endif /* NODE_OPTIONS_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/socket.c b/wsrep-lib/wsrep-API/v26/examples/node/socket.c new file mode 100644 index 00000000..377abcaf --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/socket.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "socket.h" + +#include "log.h" + +#include <assert.h> +#include <ctype.h> // isspace() +#include <errno.h> +#include <limits.h> // USHRT_MAX +#include <netdb.h> // struct addrinfo +#include <stdio.h> // snprintf() +#include <string.h> // strerror() +#include <sys/socket.h> // bind(), connect(), accept(), send(), recv() + +struct node_socket +{ + int fd; +}; + +/** + * Initializes addrinfo from the separate host address and port arguments + * + * Requires calling freeaddrinfo() later + * + * @param[in] host - if NULL, will be initialized for listening + * @param[in] port + * + * @return struct addrinfo* or NULL in case of error + */ +static struct addrinfo* +socket_get_addrinfo2(const char* const host, + uint16_t const port) +{ + struct addrinfo const hints = + { + .ai_flags = AI_PASSIVE | /** will be ignored if host is not NULL */ + AI_NUMERICSERV, /** service is a numeric port */ + .ai_family = AF_UNSPEC, /** either IPv4 or IPv6 */ + .ai_socktype = SOCK_STREAM, /** STREAM or DGRAM */ + .ai_protocol = 0, + .ai_addrlen = 0, + .ai_addr = NULL, + .ai_canonname = NULL, + .ai_next = NULL + }; + + char service[6]; + snprintf(service, sizeof(service), "%hu", port); + + struct addrinfo* info; + int err = getaddrinfo(host, service, &hints, &info); + if (err) + { + NODE_ERROR("Failed to resolve '%s': %d (%s)", + host, err, gai_strerror(err)); + return NULL; + } + + return info; +} + +/** + * Initializes addrinfo from single address and port string + * The port is expected to be in numerical form and appended to the host address + * via colon. + * + * Requires calling freeaddrinfo() later + * + * @param[in] addr full address specification, including port + * + * @return struct addrinfo* or NULL in case of error + */ +static struct addrinfo* +socket_get_addrinfo1(const char* const addr) +{ + int const addr_len = (int)strlen(addr); + char* const addr_buf = strdup(addr); + if (!addr_buf) + { + NODE_ERROR("strdup(%s) failed: %d (%s)", addr, errno, strerror(errno)); + return NULL; + } + + struct addrinfo* res = NULL; + long port; + char* endptr; + + int i; + for (i = addr_len - 1; i >= 0; i--) + { + if (addr_buf[i] == ':') break; + } + + if (addr_buf[i] != ':') + { + NODE_ERROR("Malformed address:port string: '%s'", addr); + goto end; + } + + addr_buf[i] = '\0'; + port = strtol(addr_buf + i + 1, &endptr, 10); + + if (port <= 0 || port > USHRT_MAX || errno || + (*endptr != '\0' && !isspace(*endptr))) + { + NODE_ERROR("Malformed/invalid port: '%s'. Errno: %d (%s)", + addr_buf + i + 1, errno, strerror(errno)); + goto end; + } + + res = socket_get_addrinfo2(strlen(addr_buf) > 0 ? addr_buf : NULL, + (uint16_t)port); +end: + free(addr_buf); + return res; +} + +static struct node_socket* +socket_create(int const fd) +{ + assert(fd > 0); + + struct node_socket* res = calloc(1, sizeof(struct node_socket)); + if (res) + { + res->fd = fd; + } + else + { + NODE_ERROR("Failed to allocate struct node_socket: %d (%s)", + errno, strerror(errno)); + close(fd); + } + + return res; +} + +/** + * Definition of function type with the signature of bind() and connect() + */ +typedef int (*socket_act_fun_t) (int sfd, + const struct sockaddr* addr, + socklen_t addrlen); + +static int +socket_bind_and_listen(int const sfd, + const struct sockaddr* const addr, + socklen_t const addrlen) +{ + int ret = bind(sfd, addr, addrlen); + + if (!ret) + ret = listen(sfd, SOMAXCONN); + + return ret; +} + +/** + * A "template" method to do the "right thing" with the addrinfo and create a + * socket from it. The "right thing" would normally be bind and listen for + * a server socket OR connect for a client socket. + * + * @param[in] info addrinfo list, swallowed and deallocated + * @param[in] action_fun the "right thing" to do on socket and struct sockaddr + * @param[in] action_str action description to be printed in the error message + * @param[in] orig_host host address to be pronted in the error message + * @param[in] orig_port port to be printed in the error message, if orig_host + * string contains the port, this parameter should be 0 + * + * The last three parameters are for diagnostic puposes only. orig_host and + * orig_port are supposed to be what were used to obtain addrinfo. + * + * @return new struct node_socket. + */ +static struct node_socket* +socket_from_addrinfo(struct addrinfo* const info, + socket_act_fun_t const action_fun, + const char* const action_str, + const char* const orig_host, + uint16_t const orig_port) +{ + int sfd; + int err = 0; + + /* Iterate over addrinfo list and try to apply action_fun on the resulting + * socket. Once successful, break loop. */ + struct addrinfo* addr; + for (addr = info; addr != NULL; addr = addr->ai_next) + { + sfd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol); + if (sfd == -1) + { + err = errno; + continue; + } + + if (action_fun(sfd, addr->ai_addr, addr->ai_addrlen) == 0) break; + + err = errno; + close(sfd); + } + + freeaddrinfo(info); /* no longer needed */ + + if (!addr) + { + NODE_ERROR("Failed to %s to '%s%s%.0hu': %d (%s)", + action_str, + orig_host ? orig_host : "", orig_port > 0 ? ":" : "", + orig_port > 0 ? orig_port : 0, /* won't be printed if 0 */ + err, strerror(err)); + return NULL; + } + + assert(sfd > 0); + return socket_create(sfd); +} + +struct node_socket* +node_socket_listen(const char* const host, uint16_t const port) +{ + struct addrinfo* const info = socket_get_addrinfo2(host, port); + if (!info) return NULL; + + return socket_from_addrinfo(info, socket_bind_and_listen, + "bind a listening socket", host, port); +} + +struct node_socket* +node_socket_connect(const char* const addr_str) +{ + struct addrinfo* const info = socket_get_addrinfo1(addr_str); + if (!info) return NULL; + + return socket_from_addrinfo(info, connect, "connect", addr_str, 0); +} + +struct node_socket* +node_socket_accept(struct node_socket* socket) +{ + int sfd = accept(socket->fd, NULL, NULL); + + if (sfd < 0) + { + NODE_ERROR("Failed to accept connection: %d (%s)", + errno, strerror(errno)); + return NULL; + } + + return socket_create(sfd); +} + +int +node_socket_send_bytes(node_socket_t* socket, const void* buf, size_t len) +{ + ssize_t const ret = send(socket->fd, buf, len, MSG_NOSIGNAL); + + if (ret != (ssize_t)len) + { + NODE_ERROR("Failed to send %zu bytes: %d (%s)", errno, strerror(errno)); + return -1; + } + + return 0; +} + +int +node_socket_recv_bytes(node_socket_t* socket, void* buf, size_t len) +{ + ssize_t const ret = recv(socket->fd, buf, len, MSG_WAITALL); + + if (ret != (ssize_t)len) + { + NODE_ERROR("Failed to recv %zu bytes: %d (%s)", errno, strerror(errno)); + return -1; + } + + return 0; +} + +void +node_socket_close(node_socket_t* socket) +{ + if (!socket) return; + + if (socket->fd > 0) close(socket->fd); + + free(socket); +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/socket.h b/wsrep-lib/wsrep-API/v26/examples/node/socket.h new file mode 100644 index 00000000..3a77eff3 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/socket.h @@ -0,0 +1,72 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit implements auxiliary networking functions (for SST purposes) + * It has nothing wsrep related and is not of general purpose. + */ + +#ifndef NODE_SOCKET_H +#define NODE_SOCKET_H + +#include <stddef.h> // size_t +#include <stdint.h> // uint16_t + +typedef struct node_socket node_socket_t; + +/** + * Open listening socket at a given address + * + * @return listening socket + */ +extern node_socket_t* +node_socket_listen(const char* host, uint16_t port); + +/** + * Connect to a given address. + * + * @return connected socket + */ +extern node_socket_t* +node_socket_connect(const char* addr); + +/** + * Wait for connection on a listening socket + * @return connected socket + */ +extern node_socket_t* +node_socket_accept(node_socket_t* s); + +/** + * Send a given number of bytes + * @return 0 or a negative error code + */ +extern int +node_socket_send_bytes(node_socket_t* s, const void* buf, size_t len); + +/** + * Receive a given number of bytes + * @return 0 or a negative error code + */ +extern int +node_socket_recv_bytes(node_socket_t* s, void* buf, size_t len); + +/** + * Release all recources associated with the socket */ +extern void +node_socket_close(node_socket_t* s); + +#endif /* NODE_SOCKET_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/sst.c b/wsrep-lib/wsrep-API/v26/examples/node/sst.c new file mode 100644 index 00000000..e93534ef --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/sst.c @@ -0,0 +1,372 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "sst.h" + +#include "ctx.h" +#include "log.h" +#include "socket.h" + +#include <arpa/inet.h> // htonl() +#include <assert.h> +#include <errno.h> +#include <pthread.h> +#include <stdio.h> // snprintf() +#include <stdlib.h> // abort() +#include <string.h> // strdup() +#include <unistd.h> // usleep() + +/** + * Helper: creates detached thread */ +static int +sst_create_thread(void* (*thread_routine) (void*), + void* const thread_arg) +{ + pthread_t thr; + pthread_attr_t attr; + int ret = pthread_attr_init(&attr); + ret = ret ? ret : pthread_attr_setdetachstate(&attr,PTHREAD_CREATE_DETACHED); + ret = ret ? ret : pthread_create(&thr, &attr, thread_routine, thread_arg); + return ret; +} + +/** + * Helper: creates detached thread and waits for it to call + * sst_sync_with_parent() */ +static void +sst_create_and_sync(const char* const role, + pthread_mutex_t* const mtx, + pthread_cond_t* const cond, + void* (*thread_routine) (void*), + void* const thread_arg) +{ + int ret = pthread_mutex_lock(mtx); + if (ret) + { + NODE_FATAL("Failed to lock %s mutex: %d (%s)", role, ret, strerror(ret)); + abort(); + } + + ret = sst_create_thread(thread_routine, thread_arg); + if (ret) + { + NODE_FATAL("Failed to create detached %s thread: %d (%s)", + role, ret, strerror(ret)); + abort(); + } + + ret = pthread_cond_wait(cond, mtx); + if (ret) + { + NODE_FATAL("Failed to synchronize with %s thread: %d (%s)", + role, ret, strerror(ret)); + abort(); + } + + pthread_mutex_unlock(mtx); +} + +/** + * Helper: syncs with parent thread and allows it to continue and return + * asynchronously */ +static void +sst_sync_with_parent(const char* role, + pthread_mutex_t* mtx, + pthread_cond_t* cond) +{ + int ret = pthread_mutex_lock(mtx); + if (ret) + { + NODE_FATAL("Failed to lock %s mutex: %d (%s)", role, ret, strerror(ret)); + abort(); + } + + NODE_INFO("Initialized %s thread", role); + + pthread_cond_signal(cond); + pthread_mutex_unlock(mtx); +} + +static pthread_mutex_t sst_joiner_mtx = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t sst_joiner_cond = PTHREAD_COND_INITIALIZER; + +struct sst_joiner_ctx +{ + struct node_ctx* node; + node_socket_t* socket; +}; + +/** + * waits for SST completion and signals the provider to continue */ +static void* +sst_joiner_thread(void* ctx) +{ + assert(ctx); + + struct node_ctx* const node = ((struct sst_joiner_ctx*)ctx)->node; + node_socket_t* const listen = ((struct sst_joiner_ctx*)ctx)->socket; + ctx = NULL; /* may be unusable after next statement */ + + /* this allows parent callback to return */ + sst_sync_with_parent("JOINER", &sst_joiner_mtx, &sst_joiner_cond); + + wsrep_gtid_t state_gtid = WSREP_GTID_UNDEFINED; + int err = -1; + + /* REPLICATION: wait for donor to connect and send the state snapshot */ + node_socket_t* const connected = node_socket_accept(listen); + if (!connected) goto end; + + uint32_t state_len; + err = node_socket_recv_bytes(connected, &state_len, sizeof(state_len)); + if (err) goto end; + + state_len = ntohl(state_len); + if (state_len > 0) + { + /* REPLICATION: get the state of state_len size */ + void* state = malloc(state_len); + if (state) + { + err = node_socket_recv_bytes(connected, state, state_len); + if (err) + { + free(state); + goto end; + } + + /* REPLICATION: install the newly received state. */ + err = node_store_init_state(node->store, state, state_len); + free(state); + if (err) goto end; + } + else + { + NODE_ERROR("Failed to allocate %zu bytes for state snapshot.", + state_len); + err = -ENOMEM; + goto end; + } + } + else + { + /* REPLICATION: it was a bypass, the node will receive missing data via + * IST. It starts with the state it currently has. */ + } + + /* REPLICATION: find gtid of the received state to report to provider */ + node_store_gtid(node->store, &state_gtid); + +end: + assert(err <= 0); + node_socket_close(connected); + node_socket_close(listen); + + /* REPLICATION: tell provider that SST is received */ + wsrep_status_t sst_ret; + wsrep_t* const wsrep = node_wsrep_provider(node->wsrep); + sst_ret = wsrep->sst_received(wsrep, &state_gtid, NULL, err); + + if (WSREP_OK != sst_ret) + { + NODE_FATAL("Failed to report completion of SST: %d", sst_ret); + abort(); + } + + return NULL; +} + +enum wsrep_cb_status +node_sst_request_cb (void* const app_ctx, + void** const sst_req, + size_t* const sst_req_len) +{ + static int const SST_PORT_OFFSET = 2; + + assert(app_ctx); + struct node_ctx* const node = app_ctx; + const struct node_options* const opts = node->opts; + + char* sst_str = NULL; + + /* REPLICATION: 1. prepare the node to receive SST */ + uint16_t const sst_port = (uint16_t)(opts->base_port + SST_PORT_OFFSET); + size_t const sst_len = strlen(opts->base_host) + + 1 /* ':' */ + 5 /* max port len */ + 1 /* \0 */; + sst_str = malloc(sst_len); + if (!sst_str) + { + NODE_ERROR("Failed to allocate %zu bytes for SST request", sst_len); + goto end; + } + + /* write in request the address at which we listen */ + int ret = snprintf(sst_str, sst_len, "%s:%hu", opts->base_host, sst_port); + if (ret < 0 || (size_t)ret >= sst_len) + { + free(sst_str); + sst_str = NULL; + NODE_ERROR("Failed to write a SST request"); + goto end; + } + + node_socket_t* const socket = node_socket_listen(NULL, sst_port); + if (!socket) + { + free(sst_str); + sst_str = NULL; + NODE_ERROR("Failed to listen at %s", sst_str); + goto end; + } + + /* REPLICATION 2. start the "joiner" thread that will wait for SST and + * report its success to provider, and syncronize with it. */ + struct sst_joiner_ctx ctx = + { + .node = node, + .socket = socket + }; + sst_create_and_sync("JOINER", &sst_joiner_mtx, &sst_joiner_cond, + sst_joiner_thread, &ctx); + + NODE_INFO("Waiting for SST at %s", sst_str); + +end: + if (sst_str) + { + *sst_req = sst_str; + *sst_req_len = strlen(sst_str) + 1; + } + else + { + *sst_req = NULL; + *sst_req_len = 0; + return WSREP_CB_FAILURE; + } + + /* REPLICATION 3. return SST request to provider */ + return WSREP_CB_SUCCESS; +} + +static pthread_mutex_t sst_donor_mtx = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t sst_donor_cond = PTHREAD_COND_INITIALIZER; + +struct sst_donor_ctx +{ + wsrep_gtid_t state; + struct node_ctx* node; + node_socket_t* socket; + wsrep_bool_t bypass; +}; + +/** + * donates SST and signals provider that it is done. */ +static void* +sst_donor_thread(void* const args) +{ + struct sst_donor_ctx const ctx = *(struct sst_donor_ctx*)args; + + int err = 0; + const void* state; + size_t state_len; + + if (ctx.bypass) + { + /* REPLICATION: if bypass is true, there is no need to send snapshot, + * just signal the joiner that snapshot is not needed and + * it can proceed to apply IST. We'll do it by sending 0 + * for the size of snapshot */ + state = NULL; + state_len = 0; + } + else + { + /* REPLICATION: if bypass is false, we need to send a full state snapshot + * Get hold of the state, which is currently just GTID + * NOTICE that while parent is waiting, the store is in a + * quiescent state, provider blocking any modifications. */ + err = node_store_acquire_state(ctx.node->store, &state, &state_len); + if (state_len > UINT32_MAX) err = -ERANGE; + } + + /* REPLICATION: after getting hold of the state we can allow parent callback + * to return and the node to resume its normal operation */ + sst_sync_with_parent("DONOR", &sst_donor_mtx, &sst_donor_cond); + + if (err >= 0) + { + uint32_t tmp = htonl((uint32_t)state_len); + err = node_socket_send_bytes(ctx.socket, &tmp, sizeof(tmp)); + } + + if (state_len != 0) + { + if (err >= 0) + { + assert(state); + err = node_socket_send_bytes(ctx.socket, state, state_len); + } + + node_store_release_state(ctx.node->store); + } + + node_socket_close(ctx.socket); + + /* REPLICATION: signal provider the success of the operation */ + wsrep_t* const wsrep = node_wsrep_provider(ctx.node->wsrep); + wsrep->sst_sent(wsrep, &ctx.state, err); + + return NULL; +} + +enum wsrep_cb_status +node_sst_donate_cb (void* const app_ctx, + void* const recv_ctx, + const wsrep_buf_t* const str_msg, + const wsrep_gtid_t* const state_id, + const wsrep_buf_t* const state, + wsrep_bool_t const bypass) +{ + (void)recv_ctx; + (void)state; + + struct sst_donor_ctx ctx = + { + .node = app_ctx, + .state = *state_id, + .bypass = bypass + }; + + /* we are expecting a human-readable 0-terminated string */ + void* p = memchr(str_msg->ptr, '\0', str_msg->len); + if (!p) + { + NODE_ERROR("Received a badly formed State Transfer Request."); + /* REPLICATION: in case of a failure we return the status to provider, so + * that the joining node can be notified of it by cluster */ + return WSREP_CB_FAILURE; + } + + const char* addr = str_msg->ptr; + ctx.socket = node_socket_connect(addr); + + if (!ctx.socket) return WSREP_CB_FAILURE; + + sst_create_and_sync("DONOR", &sst_donor_mtx, &sst_donor_cond, + sst_donor_thread, &ctx); + + return WSREP_CB_SUCCESS; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/sst.h b/wsrep-lib/wsrep-API/v26/examples/node/sst.h new file mode 100644 index 00000000..7006a1b6 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/sst.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines SST interface + */ + +#ifndef NODE_SST_H +#define NODE_SST_H + +#include "../../wsrep_api.h" + +extern enum wsrep_cb_status +node_sst_request_cb (void* app_ctx, + void** sst_req, + size_t* sst_req_len); + +extern enum wsrep_cb_status +node_sst_donate_cb (void* app_ctx, + void* recv_ctx, + const wsrep_buf_t* str_msg, + const wsrep_gtid_t* state_id, + const wsrep_buf_t* state, + wsrep_bool_t bypass); + +#endif /* NODE_SST_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/stats.c b/wsrep-lib/wsrep-API/v26/examples/node/stats.c new file mode 100644 index 00000000..4b02240f --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/stats.c @@ -0,0 +1,215 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "stats.h" + +#include "log.h" + +#include <assert.h> +#include <errno.h> +#include <stdio.h> // snprintf() +#include <stdlib.h> // abort() +#include <string.h> // strcmp() +#include <unistd.h> // usleep() + +enum +{ + STATS_REPL_BYTE, + STATS_REPL_WS, + STATS_RECV_BYTE, + STATS_RECV_WS, + STATS_TOTAL_BYTE, + STATS_TOTAL_WS, + STATS_CERT_FAILS, + STATS_STORE_FAILS, + STATS_FC_PAUSED, + STATS_MAX +}; + +static const char* const stats_legend[STATS_MAX] = +{ + " repl(B/s)", + " repl(W/s)", + " recv(B/s)", + " recv(W/s)", + "total(B/s)", + "total(W/s)", + " cert.fail", + " stor.fail", + " paused(%)" +}; + +/* stats IDs in provider output - provider dependent, here we use Galera's */ +static const char* const galera_ids[STATS_MAX] = +{ + "replicated_bytes", /**< STATS_REPL_BYTE */ + "replicated", /**< STATS_REPL_WS */ + "received_bytes", /**< STATS_RECV_BYTE */ + "received", /**< STATS_RECV_WS */ + "", /**< STATS_TOTAL_BYTE */ + "", /**< STATS_TOTAL_WS */ + "local_cert_failures", /**< STATS_CERT_FAILS */ + "", /**< STATS_STORE_FAILS */ + "flow_control_paused_ns" /**< STATS_FC_PAUSED */ +}; + +/* maps local stats IDs to provider stat IDs */ +static int stats_galera_map[STATS_MAX]; + +/** + * Helper to map provider stats to own stats set */ +static void +stats_establish_mapping(wsrep_t* const wsrep) +{ + int const magic_map = -1; + size_t i; + for (i = 0; i < sizeof(stats_galera_map)/sizeof(stats_galera_map[0]); i++) + { + stats_galera_map[i] = magic_map; /* initialize map array */ + } + + struct wsrep_stats_var* const stats = wsrep->stats_get(wsrep); + + /* to compensate for STATS_TOTAL_* and STATS_STORE_FAILS having no + * counterparts */ + int mapped = 3; + + i = 0; + while (stats[i].name) /* stats array is terminated by Null name */ + { + int j; + for (j = 0; j < STATS_MAX; j++) + { + if (magic_map == stats_galera_map[j] /* j-th member still unset */ + && + !strcmp(stats[i].name, galera_ids[j])) + { + stats_galera_map[j] = (int)i; + mapped++; + if (STATS_MAX == mapped) /* all mapped */ goto out; + } + } + + i++; + } + +out: + wsrep->stats_free(wsrep, stats); +} + +static void +stats_get(node_store_t* const store, wsrep_t* const wsrep, long long stats[]) +{ + stats[STATS_STORE_FAILS] = node_store_read_view_failures(store); + + struct wsrep_stats_var* const ret = wsrep->stats_get(wsrep); + if (!ret) + { + NODE_FATAL("wsrep::stats_get() call failed."); + abort(); + } + + int i; + for (i = 0; i < STATS_MAX; i++) + { + int j = stats_galera_map[i]; + if (j >= 0) + { + assert(WSREP_VAR_INT64 == ret[j].type); + stats[i] = ret[j].value._int64; + } + } + + wsrep->stats_free(wsrep, ret); + + // totals are just sums + stats[STATS_TOTAL_BYTE] = stats[STATS_REPL_BYTE] + stats[STATS_RECV_BYTE]; + stats[STATS_TOTAL_WS ] = stats[STATS_REPL_WS ] + stats[STATS_RECV_WS ]; +} + +static void +stats_print(long long bef[], long long aft[], double period) +{ + double rate[STATS_MAX]; + int i; + for (i = 0; i < STATS_MAX; i++) + { + rate[i] = (double)(aft[i] - bef[i])/period; + } + rate[STATS_FC_PAUSED] /= 1.0e+07; // nanoseconds to % of seconds + + char str[256]; + int written = 0; + + /* first line write legend */ + for (i = 0; i < STATS_MAX; i++) + { + size_t const space_left = sizeof(str) - (size_t)written; + written += snprintf(&str[written], space_left, "%s", stats_legend[i]); + } + + str[written] = '\n'; + written++; + + /* second line write values */ + for (i = 0; i < STATS_MAX; i++) + { + size_t const space_left = sizeof(str) - (size_t)written; + long long const value = (long long)rate[i]; + written += snprintf(&str[written], space_left, " %9lld", value); + } + + str[written] = '\0'; + + /* use logging macro for timestamp */ + NODE_INFO("\n%s", str); +} + +void +node_stats_loop(const struct node_ctx* const node, int const period) +{ + double const period_sec = period; + useconds_t const period_usec = (useconds_t)period * 1000000; + + wsrep_t* const wsrep = node_wsrep_provider(node->wsrep); + stats_establish_mapping(wsrep); + + long long stats1[STATS_MAX]; + long long stats2[STATS_MAX]; + + stats_get(node->store, wsrep, stats1); + + while (1) + { + if (usleep(period_usec)) break; + stats_get(node->store, wsrep, stats2); + stats_print(stats1, stats2, period_sec); + + if (usleep(period_usec)) break; + stats_get(node->store, wsrep, stats1); + stats_print(stats2, stats1, period_sec); + } + + if (EINTR != errno) + { + NODE_ERROR("Unexpected usleep(%lld) error: %d (%s)", + (long long)period_usec, errno, strerror(errno)); + } + else + { + /* interrupted by signal */ + } +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/stats.h b/wsrep-lib/wsrep-API/v26/examples/node/stats.h new file mode 100644 index 00000000..f7ab7ef4 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/stats.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines performance statistics loop + */ + +#ifndef NODE_STATS_H +#define NODE_STATS_H + +#include "ctx.h" + +/** + * Prints out statistics with a given period. + * + * @param[in] node node context + * @param[in] period in seconds + */ +extern void +node_stats_loop(const struct node_ctx* node, int period); + +#endif /* NODE_STATS_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/store.c b/wsrep-lib/wsrep-API/v26/examples/node/store.c new file mode 100644 index 00000000..1dc2d6c1 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/store.c @@ -0,0 +1,1044 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "store.h" + +#include "log.h" + +#include <assert.h> +#include <errno.h> +#include <pthread.h> +#include <stdbool.h> +#include <stddef.h> // ptrdiff_t +#include <stdint.h> // uintptr_t +#include <stdlib.h> // abort() +#include <string.h> // memset() + +#define DECLARE_SERIALIZE_INT(INTTYPE) \ + static inline size_t \ + store_serialize_##INTTYPE(void* const to, INTTYPE##_t const from) \ + { \ + memcpy(to, &from, sizeof(from)); /* for simplicity ignore endianness */ \ + return sizeof(from); \ + } + +DECLARE_SERIALIZE_INT(uint32); +DECLARE_SERIALIZE_INT(int64); + +#define DECLARE_DESERIALIZE_INT(INTTYPE) \ + static inline size_t \ + store_deserialize_##INTTYPE(INTTYPE##_t* const to, const void* const from) \ + { \ + memcpy(to, from, sizeof(*to)); /* for simplicity ignore endianness */ \ + return sizeof(*to); \ + } + +DECLARE_DESERIALIZE_INT(uint32); +DECLARE_DESERIALIZE_INT(int64); + +typedef struct record +{ + wsrep_seqno_t version; + uint32_t value; + /* this order ensures that there is no padding between the members */ +} +record_t; + +#define STORE_RECORD_SIZE \ + (sizeof(((record_t*)(NULL))->version) + sizeof(((record_t*)(NULL))->value)) + +static inline size_t +store_record_set(void* const base, + size_t const index, + const record_t* const record) +{ + char* const position = (char*)base + index*STORE_RECORD_SIZE; + memcpy(position, record, STORE_RECORD_SIZE); + return STORE_RECORD_SIZE; +} + +static inline size_t +store_record_get(const void* const base, + size_t const index, + record_t* const record) +{ + const char* const position = (const char*)base + index*STORE_RECORD_SIZE; + memcpy(record, position, STORE_RECORD_SIZE); + return STORE_RECORD_SIZE; +} + +static inline bool +store_record_equal(const record_t* const lhs, const record_t* const rhs) +{ + return (lhs->version == rhs->version) && (lhs->value == rhs->value); +} + +/* transaction context */ +struct store_trx_op +{ + /* Normally what we'd need for transaction context is the record index and + * new record value. Here we also save read view snapshot (rec_from & rec_to) + * to + * 1. test provider certification correctness if provider supports read view + * 2. if not, detect conflicts at a store level. */ + record_t rec_from; + record_t rec_to; + uint32_t idx_from; + uint32_t idx_to; + uint32_t new_value; + uint32_t size; /* nominal "size" of operation to manipulate on-the-wire + * writeset size. */ +}; + +#define STORE_OP_SIZE (STORE_RECORD_SIZE + STORE_RECORD_SIZE + \ + sizeof(((struct store_trx_op*)NULL)->idx_from) + \ + sizeof(((struct store_trx_op*)NULL)->idx_to) + \ + sizeof(((struct store_trx_op*)NULL)->new_value) + \ + sizeof(((struct store_trx_op*)NULL)->size)) + +struct store_trx_ctx +{ + wsrep_gtid_t rv_gtid; + size_t ops_num; + struct store_trx_op* ops; +}; + +static inline bool +store_trx_add_op(struct store_trx_ctx* const trx) +{ + struct store_trx_op* const new_ops = + realloc(trx->ops, sizeof(struct store_trx_op)*(trx->ops_num + 1)); + + if (new_ops) + { + trx->ops = new_ops; +#ifndef NDEBUG + memset(&trx->ops[trx->ops_num], 0, sizeof(*trx->ops)); +#endif + trx->ops_num++; + } + + return (NULL == new_ops); +} + +struct store_trx_entry +{ + bool used; + struct store_trx_ctx ctx; +}; + +typedef wsrep_uuid_t member_t; + +struct node_store +{ + wsrep_gtid_t gtid; + pthread_mutex_t gtid_mtx; + wsrep_trx_id_t trx_id; + pthread_mutex_t trx_id_mtx; + char* snapshot; + member_t* members; + void* records; + size_t op_size; + long read_view_fails; + uint32_t members_num; + uint32_t records_num; + uint32_t entries_mask; + bool read_view_support; // read view support by cluster + /* trx pool piggybacked */ +}; + +node_store_t* +node_store_open(const struct node_options* const opts) +{ + /* make the size of trx pool the next highest power of 2 over the total + * number of workers */ + uint32_t trx_pool_mask = (uint32_t)(opts->masters + opts->slaves); + if (trx_pool_mask > 0) + { + trx_pool_mask -= 1; + trx_pool_mask |= trx_pool_mask >> 1; + trx_pool_mask |= trx_pool_mask >> 2; + trx_pool_mask |= trx_pool_mask >> 4; + trx_pool_mask |= trx_pool_mask >> 8; + trx_pool_mask |= trx_pool_mask >> 16; + } + assert(((trx_pool_mask + 1) & trx_pool_mask) == 0); // 2^n - 1 + + size_t const desired_op_size = (size_t)(opts->ws_size/opts->operations); + size_t const op_size = (desired_op_size > STORE_OP_SIZE ? + desired_op_size : STORE_OP_SIZE); + + /* since the number of workers will never change, we can allocate trx pool + * together with the main store struc */ + size_t const store_alloc_size = sizeof(struct node_store) + + /* op_size - additional buffer for op serialization per trx */ + (sizeof(struct store_trx_entry) + op_size)*(trx_pool_mask + 1); + + struct node_store* const ret = malloc(store_alloc_size); + + if (ret) + { + memset(ret, 0, store_alloc_size); + ret->records = malloc((size_t)opts->records * STORE_RECORD_SIZE); + + if (ret->records) + { + ret->gtid = WSREP_GTID_UNDEFINED; + pthread_mutex_init(&ret->gtid_mtx, NULL); + pthread_mutex_init(&ret->trx_id_mtx, NULL); + ret->op_size = op_size; + ret->records_num = (uint32_t)opts->records; + ret->entries_mask = trx_pool_mask; + + uint32_t i; + for (i = 0; i < ret->records_num; i++) + { + /* keep state in serialized form for easy snapshotting */ + struct record const record = { WSREP_SEQNO_UNDEFINED, i }; + store_record_set(ret->records, i, &record); + } + + return ret; + } + else + { + free(ret); + } + } + + return NULL; +} + +void +node_store_close(struct node_store* const store) +{ + assert(store); + assert(store->records); + pthread_mutex_destroy(&store->gtid_mtx); + pthread_mutex_destroy(&store->trx_id_mtx); + free(store->records); + free(store->members); + free(store); +} + +#define STORE_MUTEX_LOCK(mtx) \ + { \ + int err = pthread_mutex_lock(mtx); \ + if (err) \ + { \ + NODE_FATAL("Failed to lock " #mtx ": %d (%s)", \ + err, strerror(err)); \ + abort(); \ + } \ + } + +static inline struct store_trx_entry* +store_get_trx_entry(struct node_store* const store, wsrep_trx_id_t const trx_id) +{ + return (struct store_trx_entry*) + ((char*)(store + 1) + (trx_id & store->entries_mask)* + (sizeof(struct store_trx_entry) + store->op_size)); +} + +static inline struct store_trx_ctx* +store_get_trx_ctx(struct node_store* const store, wsrep_trx_id_t const trx_id) +{ + return &(store_get_trx_entry(store, trx_id)->ctx); +} + +static inline wsrep_trx_id_t +store_new_trx_id(struct node_store* const store) +{ + wsrep_trx_id_t ret; + struct store_trx_entry* trx; + + STORE_MUTEX_LOCK(&store->trx_id_mtx); + + do + { + store->trx_id++; + trx = store_get_trx_entry(store, store->trx_id); + } + while (trx->used); + trx->used = true; + ret = store->trx_id; + + pthread_mutex_unlock(&store->trx_id_mtx); + + memset(&trx->ctx, 0, sizeof(trx->ctx)); + + return ret; +} + +static inline void +store_free_trx_id(struct node_store* const store, wsrep_trx_id_t const trx_id) +{ + struct store_trx_entry* const trx = store_get_trx_entry(store, trx_id); + assert(trx->used); + free(trx->ctx.ops); + + STORE_MUTEX_LOCK(&store->trx_id_mtx); + + trx->used = false; + + pthread_mutex_unlock(&store->trx_id_mtx); +} + +/** + * deserializes membership from snapshot */ +static int +store_new_members(const char* ptr, const char* const endptr, + uint32_t* const num, member_t** const memb) +{ + ptr += store_deserialize_uint32(num, ptr); + + if (*num < 2) + { + NODE_ERROR("Bogus number of members %u", *num); + return -1; + } + + int ret = (int)sizeof(*num); + + size_t const msize = sizeof(member_t) * *num; + if ((endptr - ptr) < (ptrdiff_t)msize) + { + NODE_ERROR("State snapshot does not contain all membership: " + "%zd < %zu", endptr - ptr, msize); + return -1; + } + + *memb = calloc(*num, sizeof(member_t)); + if (!*memb) + { + NODE_ERROR("Could not allocate new membership"); + return -ENOMEM; + } + + memcpy(*memb, ptr, msize); + + return ret + (int)msize; +} + +/** + * deserializes records from snapshot */ +static int +store_new_records(const char* ptr, const char* const endptr, + uint32_t* const num, void** const rec) +{ + ptr += store_deserialize_uint32(num, ptr); + + int ret = (int)sizeof(*num); + if (!*num) + { + *rec = NULL; + return ret; + } + + size_t const rsize = STORE_RECORD_SIZE * *num; + if ((endptr - ptr) < (ptrdiff_t)rsize) + { + NODE_ERROR("State snapshot does not contain all records: " + "%zu < %zu", endptr - ptr, rsize); + return -1; + } + + *rec = malloc(rsize); + if (!*rec) + { + NODE_ERROR("Could not allocate new records"); + return -ENOMEM; + } + + memcpy(*rec, ptr, rsize); + + return ret + (int)rsize; +} + +int +node_store_init_state(struct node_store* const store, + const void* const state, + size_t const state_len) +{ + /* First, deserialize and prepare new state */ + if (state_len <= sizeof(member_t)*2 /* at least two members */ + + WSREP_UUID_STR_LEN + 1 /* : */ + 1 /* seqno */ + 1 /* \0 */) + { + NODE_ERROR("State snapshot too short: %zu", state_len); + return -1; + } + + wsrep_gtid_t state_gtid; + int ret; + ret = wsrep_gtid_scan(state, state_len, &state_gtid); + if (ret < 0) + { + char state_str[WSREP_GTID_STR_LEN + 1] = { 0, }; + memcpy(state_str, state, sizeof(state_str) - 1); + NODE_ERROR("Could not find valid GTID in the received data: %s", + state_str); + return -1; + } + + ret++; /* \0 */ + if ((state_len - (size_t)ret) < sizeof(uint32_t)) + { + NODE_ERROR("State snapshot does not contain the number of members"); + return -1; + } + + const char* ptr = ((char*)state); + const char* const endptr = ptr + state_len; + ptr += ret; + + uint32_t m_num; + member_t* new_members; + ret = store_new_members(ptr, endptr, &m_num, &new_members); + if (ret < 0) + { + return ret; + } + ptr += ret; + + bool const read_view_support = ptr[0]; + ptr += 1; + + uint32_t r_num; + void* new_records; + ret = store_new_records(ptr, endptr, &r_num, &new_records); + if (ret < 0) + { + free(new_members); + return ret; + } + ptr += ret; + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + /* just a sanity check */ + if (0 == wsrep_uuid_compare(&state_gtid.uuid, &store->gtid.uuid) && + state_gtid.seqno < store->gtid.seqno) + { + NODE_ERROR("Received snapshot that is in the past: my seqno %lld," + " received seqno: %lld", + (long long)store->gtid.seqno, (long long)state_gtid.seqno); + free(new_members); + free(new_records); + ret = -1; + } + else + { + free(store->members); + store->members_num = m_num; + store->members = new_members; + free(store->records); + store->records_num = r_num; + store->records = new_records; + store->gtid = state_gtid; + store->read_view_support = read_view_support; + ret = 0; + } + + pthread_mutex_unlock(&store->gtid_mtx); + + return ret; +} + +int +node_store_acquire_state(node_store_t* const store, + const void** const state, + size_t* const state_len) +{ + int ret = 0; + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + if (!store->snapshot) + { + size_t const memb_len = store->members_num * sizeof(member_t); + size_t const rec_len = store->records_num * STORE_RECORD_SIZE; + size_t const buf_len = WSREP_GTID_STR_LEN + 1 + + sizeof(uint32_t) + memb_len + + 1 /* read view support */ + + sizeof(uint32_t) + rec_len; + + store->snapshot = malloc(buf_len); + + if (store->snapshot) + { + char* ptr = store->snapshot; + + /* state GTID */ + ret = wsrep_gtid_print(&store->gtid, ptr, buf_len); + if (ret > 0) + { + NODE_INFO(""); + assert((size_t)ret < buf_len); + + ptr[ret] = '\0'; + ret++; + ptr += ret; + assert((size_t)ret < buf_len); + + /* membership */ + ptr += store_serialize_uint32(ptr, store->members_num); + ret += (int)sizeof(uint32_t); + assert((size_t)ret + memb_len < buf_len); + memcpy(ptr, store->members, memb_len); + ptr += memb_len; + ret += (int)memb_len; + assert((size_t)ret + sizeof(uint32_t) <= buf_len); + + /* read view support */ + ptr[0] = store->read_view_support; + ptr += 1; + ret += 1; + + /* records */ + ptr += store_serialize_uint32(ptr, store->records_num); + ret += (int)sizeof(uint32_t); + assert((size_t)ret + rec_len < buf_len); + memcpy(ptr, store->records, rec_len); + ret += (int)rec_len; + assert((size_t)ret <= buf_len); + } + else + { + NODE_ERROR("Failed to record GTID: %d (%s)", ret,strerror(-ret)); + free(store->snapshot); + store->snapshot = 0; + } + } + else + { + NODE_ERROR("Failed to allocate snapshot buffer of size %zu",buf_len); + ret = -ENOMEM; + } + } + else + { + assert(0); /* provider should prevent such situation */ + ret = -EAGAIN; + } + + pthread_mutex_unlock(&store->gtid_mtx); + + if (ret > 0) + { + NODE_INFO("\n\nPrepared snapshot of %u records\n\n", store->records_num); + *state = store->snapshot; + *state_len = (size_t)ret; + ret = 0; + } + + return ret; +} + +void +node_store_release_state(node_store_t* const store) +{ + STORE_MUTEX_LOCK(&store->gtid_mtx); + + assert(store->snapshot); + free(store->snapshot); + store->snapshot = 0; + + pthread_mutex_unlock(&store->gtid_mtx); +} + +int +node_store_update_membership(struct node_store* const store, + const wsrep_view_info_t* const v) +{ + assert(store); + assert(WSREP_VIEW_PRIMARY == v->status); + assert(v->memb_num > 0); + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + bool const continuation = v->state_id.seqno == store->gtid.seqno + 1 && + 0 == wsrep_uuid_compare(&v->state_id.uuid, &store->gtid.uuid); + + bool const initialization = WSREP_SEQNO_UNDEFINED == store->gtid.seqno && + 0 == wsrep_uuid_compare(&WSREP_UUID_UNDEFINED, &store->gtid.uuid); + + if (!(continuation || initialization)) + { + char store_str[WSREP_GTID_STR_LEN + 1] = { 0, }; + wsrep_gtid_print(&store->gtid, store_str, sizeof(store_str)); + char view_str[WSREP_GTID_STR_LEN + 1] = { 0, }; + wsrep_gtid_print(&v->state_id, view_str, sizeof(view_str)); + + NODE_FATAL("Attempt to initialize store GTID from incompatible view:\n" + "\tstore: %s\n" + "\tview: %s", + store_str, view_str); + abort(); + } + + wsrep_uuid_t* const new_members = calloc(sizeof(wsrep_uuid_t), + (size_t)v->memb_num); + if (!new_members) + { + NODE_FATAL("Could not allocate new members array"); + abort(); + } + + int i; + for (i = 0; i < v->memb_num; i++) + { + new_members[i] = v->members[i].id; + } + + /* REPLICATION: at this point we should compare old and new memberships and + * rollback all streaming transactions from the partitioned + * members, if any. But we don't support it in this program yet. + */ + + free(store->members); + + store->members = new_members; + store->members_num = (uint32_t)v->memb_num; + store->gtid = v->state_id; + store->read_view_support = (v->capabilities & WSREP_CAP_SNAPSHOT); + + pthread_mutex_unlock(&store->gtid_mtx); + + return 0; +} + +void +node_store_gtid(struct node_store* const store, + wsrep_gtid_t* const gtid) +{ + assert(store); + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + *gtid = store->gtid; + + pthread_mutex_unlock(&store->gtid_mtx); +} + + +static inline void +store_serialize_op(void* const buf, const struct store_trx_op* const op) +{ + char* ptr = buf; + ptr += store_record_set(ptr, 0, &op->rec_from); + ptr += store_record_set(ptr, 0, &op->rec_to); + ptr += store_serialize_uint32(ptr, op->idx_from); + ptr += store_serialize_uint32(ptr, op->idx_to); + ptr += store_serialize_uint32(ptr, op->new_value); + store_serialize_uint32(ptr, op->size); +} + +static inline void +store_deserialize_op(struct store_trx_op* const op, const void* const buf) +{ + const char* ptr = buf; + ptr += store_record_get(ptr, 0, &op->rec_from); + ptr += store_record_get(ptr, 0, &op->rec_to); + ptr += store_deserialize_uint32(&op->idx_from, ptr); + ptr += store_deserialize_uint32(&op->idx_to, ptr); + ptr += store_deserialize_uint32(&op->new_value, ptr); + store_deserialize_uint32(&op->size, ptr); +} + +static inline void +store_serialize_gtid(void* const buf, const wsrep_gtid_t* const gtid) +{ + char* ptr = buf; + memcpy(ptr, >id->uuid, sizeof(gtid->uuid)); + ptr += sizeof(gtid->uuid); + store_serialize_int64(ptr, gtid->seqno); +} + +static inline void +store_deserialize_gtid(wsrep_gtid_t* const gtid, const void* const buf) +{ + const char* ptr = buf; + memcpy(>id->uuid, ptr, sizeof(gtid->uuid)); + ptr += sizeof(gtid->uuid); + store_deserialize_int64(>id->seqno, ptr); +} + +#define STORE_GTID_SIZE (sizeof(((wsrep_gtid_t*)(NULL))->uuid) + sizeof(int64_t)) + +int +node_store_execute(node_store_t* const store, + wsrep_t* const wsrep, + wsrep_ws_handle_t* const ws_handle) +{ + assert(store); + + if (0 == ws_handle->trx_id) + { + assert(sizeof(ws_handle->trx_id) >= sizeof(uintptr_t)); + ws_handle->trx_id = store_new_trx_id(store); + } + + struct store_trx_ctx* trx = store_get_trx_ctx(store, ws_handle->trx_id); + if (store_trx_add_op(trx)) return -ENOMEM; + struct store_trx_op* const op = &trx->ops[trx->ops_num - 1]; + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + if (1 == trx->ops_num) + { + /* First operation, save ID of the read view of the transaction */ + trx->rv_gtid = store->gtid; + } + + /* Transaction op: copy value from one random record to another... */ + op->idx_from = (uint32_t)rand() % store->records_num; + op->idx_to = (uint32_t)rand() % store->records_num; + store_record_get(store->records, op->idx_from, &op->rec_from); + store_record_get(store->records, op->idx_to, &op->rec_to); + + pthread_mutex_unlock(&store->gtid_mtx); + + wsrep_status_t ret = WSREP_TRX_FAIL; + + if (op->rec_from.version > trx->rv_gtid.seqno || + op->rec_to.version > trx->rv_gtid.seqno) + { + /* transaction read view changed, trx needs to be restarted */ +#if 0 + NODE_INFO("Transaction read view changed: %lld -> %lld, returning %d", + (long long)trx->rv_gtid.seqno, + (long long)(op->rec_from.version > op->rec_to.version ? + op->rec_from.version : op->rec_to.version), + ret); +#endif + goto error; + } + + /* Transaction op: ... and modify it somehow, e.g. increment by 1 */ + op->new_value = op->rec_from.value + 1; + + if (1 == trx->ops_num) // first trx operation + { + /* REPLICATION: Since this application does not implement record locks, + * it needs to establish read view for each transaction for + * a proper conflict detection and transaction isolation. + * Otherwose we'll need to implement record versioning */ + if (store->read_view_support) + { + ret = wsrep->assign_read_view(wsrep, ws_handle, &trx->rv_gtid); + if (ret) + { + NODE_ERROR("wsrep::assign_read_view(%lld) failed: %d", + trx->rv_gtid.seqno, ret); + goto error; + } + } + + /* Record read view in the writeset for debugging purposes */ + assert(store->op_size > STORE_GTID_SIZE); + store_serialize_gtid(trx + 1, &trx->rv_gtid); + wsrep_buf_t ws = { .ptr = trx + 1, .len = STORE_GTID_SIZE }; + ret = wsrep->append_data(wsrep, ws_handle, &ws, 1, WSREP_DATA_ORDERED, + true); + if (ret) + { + NODE_ERROR("wsrep::append_data(rv_gtid) failed: %d", ret); + goto error; + } + } + + /* REPLICATION: append keys touched by the operation + * + * NOTE: depending on data access granularity some applications may require + * multipart keys, e.g. <schema>:<table>:<row> in a SQL database. + * Single part keys match hashtables and key-value stores. + * Below we have two different single-part keys which reference two + * different records. */ + uint32_t key_val; + wsrep_buf_t key_part = { .ptr = &key_val, .len = sizeof(key_val) }; + wsrep_key_t ws_key = { .key_parts = &key_part, .key_parts_num = 1 }; + + /* REPLICATION: Key 1 - the key of the source, unchanged record */ + store_serialize_uint32(&key_val, op->idx_from); + ret = wsrep->append_key(wsrep, ws_handle, + &ws_key, + 1, /* single key */ + WSREP_KEY_REFERENCE, + true /* provider shall make a copy of the key */); + if (ret) + { + NODE_ERROR("wsrep::append_key(REFERENCE) failed: %d", ret); + goto error; + } + + /* REPLICATION: Key 2 - the key of the record we want to update */ + store_serialize_uint32(&key_val, op->idx_to); + ret = wsrep->append_key(wsrep, ws_handle, + &ws_key, + 1, /* single key */ + WSREP_KEY_UPDATE, + true /* provider shall make a copy of the key */); + if (ret) + { + NODE_ERROR("wsrep::append_key(UPDATE) failed: %d", ret); + goto error; + } + + /* REPLICATION: append transaction operation to the "writeset" + * (WS buffer was allocated together with trx context above) */ + assert(store->op_size >= STORE_OP_SIZE); + assert(store->op_size == (uint32_t)store->op_size); + op->size = (uint32_t)store->op_size; + store_serialize_op(trx + 1, op); + wsrep_buf_t ws = { .ptr = trx + 1, .len = store->op_size }; + ret = wsrep->append_data(wsrep, ws_handle, &ws, 1, WSREP_DATA_ORDERED, true); + + if (!ret) return 0; + + NODE_ERROR("wsrep::append_data(op) failed: %d", ret); + +error: + store_free_trx_id(store, ws_handle->trx_id); + + return ret; +} + +int +node_store_apply(node_store_t* const store, + wsrep_trx_id_t* const trx_id, + const wsrep_buf_t* const ws) +{ + assert(store); + (void)store; + + *trx_id = store_new_trx_id(store); + struct store_trx_ctx* const trx = store_get_trx_ctx(store, *trx_id); + + /* prepare trx context for commit */ + const char* ptr = ws->ptr; + size_t left = ws->len; + + /* at least one operation should be there */ + assert(left >= STORE_GTID_SIZE + STORE_OP_SIZE); + + if (left >= STORE_GTID_SIZE) + { + store_deserialize_gtid(&trx->rv_gtid, ptr); + left -= STORE_GTID_SIZE; + ptr += STORE_GTID_SIZE; + } + + while (left >= STORE_OP_SIZE) + { + if (store_trx_add_op(trx)) + { + store_free_trx_id(store,*trx_id); /* "rollback": release resources */ + return -ENOMEM; + } + struct store_trx_op* const op = &trx->ops[trx->ops_num - 1]; + + store_deserialize_op(op, ptr); + assert(op->idx_to <= store->records_num); + + left -= op->size; + ptr += op->size; + } + + if (left != 0) + { + NODE_FATAL("Failed to process last (%d/%zu) bytes of the writeset.", + (int)left, ws->len); + abort(); + } + + return 0; +} + +static uint32_t const store_fnv32_seed = 2166136261; + +static inline uint32_t +store_fnv32a(const void* buf, size_t const len, uint32_t seed) +{ + static uint32_t const fnv32_prime = 16777619; + const uint8_t* bp = (const uint8_t*)buf; + const uint8_t* const be = bp + len; + + while (bp < be) + { + seed ^= *bp++; + seed *= fnv32_prime; + } + + return seed; +} + + +static void +store_checksum_state(node_store_t* store) +{ + uint32_t res = store_fnv32_seed; + uint32_t i; + + for (i = 0; i < store->members_num; i++) + { + res = store_fnv32a(&store->members[i], sizeof(*store->members), res); + } + + res = store_fnv32a(store->records, store->records_num * STORE_RECORD_SIZE, + res); + + res = store_fnv32a(&store->gtid.uuid, sizeof(store->gtid.uuid), res); + + wsrep_seqno_t s; + store_serialize_int64(&s, store->gtid.seqno); + res = store_fnv32a(&s, sizeof(s), res); + + NODE_INFO("\n\n\tSeqno: %lld; state hash: %#010x\n", + (long long)store->gtid.seqno, res); +} + +static inline void +store_update_gtid(node_store_t* const store, const wsrep_gtid_t* ws_gtid) +{ + assert(0 == wsrep_uuid_compare(&store->gtid.uuid, &ws_gtid->uuid)); + + store->gtid.seqno++; + + if (store->gtid.seqno != ws_gtid->seqno) + { + NODE_FATAL("Out of order commit: expected %lld, got %lld", + store->gtid.seqno, ws_gtid->seqno); + abort(); + } + + static wsrep_seqno_t const period = 0x000fffff; /* ~1M */ + if (0 == (store->gtid.seqno & period)) + { + store_checksum_state(store); + } +} + +void +node_store_commit(node_store_t* const store, + wsrep_trx_id_t const trx_id, + const wsrep_gtid_t* const ws_gtid) +{ + assert(store); + assert(trx_id); + + struct store_trx_ctx* const trx = store_get_trx_ctx(store, trx_id); + + bool const check_read_view_snapshot = +#ifdef NDEBUG + !store->read_view_support; +#else + 1; +#endif /* NDEBUG */ + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + store_update_gtid(store, ws_gtid); + + /* First loop is to check if we can commit all operations if provider + * does not support read view or for debugging puposes */ + size_t i; + if (check_read_view_snapshot) + { + for (i = 0; i < trx->ops_num; i++) + { + struct store_trx_op* const op = &trx->ops[i]; + + record_t from, to; + store_record_get(store->records, op->idx_from, &from); + store_record_get(store->records, op->idx_to, &to); + + if (!store_record_equal(&op->rec_from, &from) || + !store_record_equal(&op->rec_to, &to)) + { + /* read view changed since transaction was executed, + * can't commit */ + assert(op->rec_from.version <= from.version); + assert(op->rec_to.version <= to.version); + if (op->rec_from.version == from.version) + assert(op->rec_from.value == from.value); + if (op->rec_to.version == to.version) + assert(op->rec_to.value == to.value); + if (store->read_view_support) abort(); + + store->read_view_fails++; + + NODE_INFO("Read view changed at commit time, rollback trx"); + + goto error; + } + } + } + + /* Second loop is to actually modify the dataset */ + for (i = 0; i < trx->ops_num; i++) + { + struct store_trx_op* const op = &trx->ops[i]; + + record_t const new_record = + { .version = ws_gtid->seqno, .value = op->new_value }; + + store_record_set(store->records, op->idx_to, &new_record); + } + +error: + pthread_mutex_unlock(&store->gtid_mtx); + + store_free_trx_id(store, trx_id); +} + +void +node_store_rollback(node_store_t* const store, + wsrep_trx_id_t const trx_id) +{ + assert(store); + assert(trx_id); + + store_free_trx_id(store, trx_id); +} + +void +node_store_update_gtid(node_store_t* const store, + const wsrep_gtid_t* const ws_gtid) +{ + assert(store); + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + store_update_gtid(store, ws_gtid); + + pthread_mutex_unlock(&store->gtid_mtx); +} + +long +node_store_read_view_failures(node_store_t* const store) +{ + assert(store); + + long ret; + + STORE_MUTEX_LOCK(&store->gtid_mtx); + + ret = store->read_view_fails;; + + pthread_mutex_unlock(&store->gtid_mtx); + + return ret; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/store.h b/wsrep-lib/wsrep-API/v26/examples/node/store.h new file mode 100644 index 00000000..51da74d7 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/store.h @@ -0,0 +1,125 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines simple "transactional storage engine" interface + */ + +#ifndef NODE_STORE_H +#define NODE_STORE_H + +#include "options.h" + +#include "../../wsrep_api.h" + +typedef struct node_store node_store_t; + +/** + * open a store and optionally assocoate a file with it */ +extern node_store_t* +node_store_open(const struct node_options* opts); + +/** + * close store and deallocate associated resources */ +extern void +node_store_close(node_store_t* store); + +/** + * initialize store with a state */ +extern int +node_store_init_state(node_store_t* store, const void* state, size_t state_len); + +/** + * Return a pointer to state snapshot that is guaranteed to be unchanged + * until node_store_release_state() is called. + * + * @param[out] state pointer to state snapshot + * @param[out] state_len soze of state snapshot + */ +extern int +node_store_acquire_state(node_store_t* store, + const void** state, size_t* state_len); + +/** + * release state */ +extern void +node_store_release_state(node_store_t* store); + +/** + * inform store about new membership */ +extern int +node_store_update_membership(node_store_t* store, const wsrep_view_info_t* v); + +/** + * get the current GTID (last committed) */ +extern void +node_store_gtid(node_store_t* store, wsrep_gtid_t* gtid); + +/** + * execute and prepare local transaction in store and return its key and write + * set. + * + * This operation allocates resources that must be freed with either + * node_store_commit() or node_store_rollback() + * + * @param[in] wsrep provider handle + * @param[out] ws_handle reference to the resulting write set in the provider + */ +extern int +node_store_execute(node_store_t* store, + wsrep_t* wsrep, + wsrep_ws_handle_t* ws_handle); + +/** + * apply and prepare foreign write set received from replication + * + * This operation allocates resources that must be freed with either + * node_store_commit() or node_store_rollback() + * + * @param[out] trx_id locally unique transaction ID + * @param[in] ws foreign transaction write set + */ +extern int +node_store_apply(node_store_t* store, + wsrep_trx_id_t* trx_id, + const wsrep_buf_t* ws); + +/** + * commit prepared transaction identified by trx_id */ +extern void +node_store_commit(node_store_t* store, + wsrep_trx_id_t trx_id, + const wsrep_gtid_t* ws_gtid); + +/** + * rollback prepared transaction identified by trx_id */ +extern void +node_store_rollback(node_store_t* store, + wsrep_trx_id_t trx_id); + +/** + * update storage GTID for transactions that had to be skipped/rolled back */ +extern void +node_store_update_gtid(node_store_t* store, + const wsrep_gtid_t* ws_gtid); + +/** + * @return the number of store read view snapshot check failures at commit time. + * (should be zero if provider implements assign_read_view() call) */ +extern long +node_store_read_view_failures(node_store_t* store); + +#endif /* NODE_STORE_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/trx.c b/wsrep-lib/wsrep-API/v26/examples/node/trx.c new file mode 100644 index 00000000..afdcada4 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/trx.c @@ -0,0 +1,155 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "trx.h" +#include "log.h" + +#include <assert.h> +#include <errno.h> // ENOMEM, etc. +#include <stdbool.h> + +wsrep_status_t +node_trx_execute(node_store_t* const store, + wsrep_t* const wsrep, + wsrep_conn_id_t const conn_id, + int ops_num) +{ + wsrep_status_t cert = WSREP_OK; // for cleanup + + static unsigned int const ws_flags = + WSREP_FLAG_TRX_START | WSREP_FLAG_TRX_END; // atomic trx + wsrep_trx_meta_t ws_meta; + wsrep_status_t ret = WSREP_OK; + + /* prepare simple transaction and obtain a writeset handle for it */ + wsrep_ws_handle_t ws_handle = { 0, NULL }; + while (ops_num--) + { + if (0 != (ret = node_store_execute(store, wsrep, &ws_handle))) + { +#if 0 + NODE_INFO("master [%d]: node_store_execute() returned %d", + conn_id, ret); +#endif + ret = WSREP_TRX_FAIL; + goto cleanup; + } + } + + /* REPLICATION: (replicate and) certify the writeset (pointed to by + * ws_handle) with the cluster */ + cert = wsrep->certify(wsrep, conn_id, &ws_handle, ws_flags, &ws_meta); + + if (WSREP_BF_ABORT == cert) + { + /* REPLICATION: transaction was signaled to abort due to multi-master + * conflict. It must rollback immediately: it blocks + * transaction that was ordered earlier and will never + * be able to enter commit order. */ + node_store_rollback(store, ws_handle.trx_id); + } + + /* REPLICATION: writeset was totally ordered, need to enter commit order */ + if (ws_meta.gtid.seqno > 0) + { + ret = wsrep->commit_order_enter(wsrep, &ws_handle, &ws_meta); + if (ret) + { + NODE_ERROR("master [%d]: wsrep::commit_order_enter(%lld) failed: " + "%d", (long long)(ws_meta.gtid.seqno), ret); + goto cleanup; + } + + /* REPLICATION: inside commit monitor + * Note: we commit transaction only if certification succeded */ + if (WSREP_OK == cert) + node_store_commit(store, ws_handle.trx_id, &ws_meta.gtid); + else + node_store_update_gtid(store, &ws_meta.gtid); + + ret = wsrep->commit_order_leave(wsrep, &ws_handle, &ws_meta, NULL); + if (ret) + { + NODE_ERROR("master [%d]: wsrep::commit_order_leave(%lld) failed: " + "%d", (long long)(ws_meta.gtid.seqno), ret); + goto cleanup; + } + } + else + { + assert(cert); + } + +cleanup: + /* REPLICATION: if wsrep->certify() returned anything else but WSREP_OK + * transaction must roll back. BF aborted trx already did it. */ + if (cert && WSREP_BF_ABORT != cert) + node_store_rollback(store, ws_handle.trx_id); + + /* NOTE: this application follows the approach that resources must be freed + * at the same level where they were allocated, so it is assumed that + * ws_key and ws were deallocated in either commit or rollback calls.*/ + + /* REPLICATION: release provider resources associated with the trx */ + wsrep->release(wsrep, &ws_handle); + + return ret ? ret : cert; +} + +wsrep_status_t +node_trx_apply(node_store_t* const store, + wsrep_t* const wsrep, + const wsrep_ws_handle_t* const ws_handle, + const wsrep_trx_meta_t* const ws_meta, + const wsrep_buf_t* const ws) +{ + /* no business being here if event was not ordered */ + assert(ws_meta->gtid.seqno > 0); + + wsrep_trx_id_t trx_id; + wsrep_buf_t err_buf = { NULL, 0 }; + int app_err; + if (ws) + { + app_err = node_store_apply(store, &trx_id, ws); + if (app_err) + { + /* REPLICATION: if applying failed, prepare an error buffer with + * sufficient error specification */ + err_buf.ptr = &app_err; // suppose error code is enough + err_buf.len = sizeof(app_err); + } + } + else /* ws failed certification and should be skipped */ + { + /* just some non-0 code to choose node_store_update_gtid() below */ + app_err = 1; + } + + wsrep_status_t ret; + ret = wsrep->commit_order_enter(wsrep, ws_handle, ws_meta); + if (ret) { + node_store_rollback(store, trx_id); + return ret; + } + + if (!app_err) node_store_commit(store, trx_id, &ws_meta->gtid); + else node_store_update_gtid(store, &ws_meta->gtid); + + ret = wsrep->commit_order_leave(wsrep, ws_handle, ws_meta, &err_buf); + + return ret; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/trx.h b/wsrep-lib/wsrep-API/v26/examples/node/trx.h new file mode 100644 index 00000000..e1d763a1 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/trx.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines "transaction" interface + */ + +#ifndef NODE_TRX_H +#define NODE_TRX_H + +#include "store.h" + +#include "../../wsrep_api.h" + +/** + * executes and replicates local transaction + */ +extern wsrep_status_t +node_trx_execute(node_store_t* store, + wsrep_t* wsrep, + wsrep_conn_id_t conn_id, + int ops_num); + +/** + * applies and commits slave write set + * + * @param ws replicated event writeset. NULL if it failed certification (and so + * must be skipped, but it was ordered, so store GTID must be updated) + */ +extern wsrep_status_t +node_trx_apply(node_store_t* store, + wsrep_t* wsrep, + const wsrep_ws_handle_t* ws_handle, + const wsrep_trx_meta_t* ws_meta, + const wsrep_buf_t* ws); + +#endif /* NODE_TRX_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/worker.c b/wsrep-lib/wsrep-API/v26/examples/node/worker.c new file mode 100644 index 00000000..e9901ad8 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/worker.c @@ -0,0 +1,197 @@ +/* Copyright (c) 2019-2020, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "worker.h" + +#include "log.h" +#include "options.h" +#include "trx.h" +#include "wsrep.h" + +#include <assert.h> +#include <pthread.h> +#include <stdbool.h> +#include <string.h> // strerror() + +struct node_worker +{ + struct node_ctx* node; + pthread_t thread_id; + size_t id; + bool exit; +}; + +enum wsrep_cb_status +node_worker_apply_cb(void* const recv_ctx, + const wsrep_ws_handle_t* const ws_handle, + uint32_t const ws_flags, + const wsrep_buf_t* const ws, + const wsrep_trx_meta_t* const ws_meta, + wsrep_bool_t* const exit_loop) +{ + assert(recv_ctx); + + struct node_worker* const worker = recv_ctx; + + wsrep_status_t const ret = node_trx_apply( + worker->node->store, + node_wsrep_provider(worker->node->wsrep), + ws_handle, + ws_meta, + ws_flags & WSREP_FLAG_ROLLBACK ? NULL : ws); + + *exit_loop = worker->exit; + + return WSREP_OK == ret ? WSREP_CB_SUCCESS : WSREP_CB_FAILURE; +} + + +static void* +worker_slave(void* recv_ctx) +{ + struct node_worker* const worker = recv_ctx; + wsrep_t* const wsrep = node_wsrep_provider(worker->node->wsrep); + + wsrep_status_t const ret = wsrep->recv(wsrep, worker); + + if (WSREP_OK != ret) + { + NODE_ERROR("slave worker [%zu] exited with error %d.", worker->id, ret); + } + + return NULL; +} + +static void* +worker_master(void* send_ctx) +{ + struct node_worker* const worker = send_ctx; + struct node_ctx* const node = worker->node; + wsrep_t* const wsrep = node_wsrep_provider(node->wsrep); + + assert(node->opts->ws_size > 0); + + wsrep_status_t ret; + + do + { + /* REPLICATION: we should not perform any local writes until the node + * is synced with the cluster. */ + if (!node_wsrep_wait_synced(node->wsrep)) + { + NODE_ERROR("master worker [%zu] failed waiting for SYNCED state.", + worker->id); + break; + } + + /* REPLICATION: the node is now synced */ + + do + { + ret = node_trx_execute(node->store, + wsrep, + worker->id, + (int)node->opts->operations); + } + while(WSREP_OK == ret // success + || (WSREP_TRX_FAIL == ret // certification failed, trx rolled back + && (usleep(10000),true)) // retry after short sleep + ); + } + while (WSREP_CONN_FAIL == ret); // provider in bad state (e.g. non-Primary) + + return NULL; +} + +struct node_worker_pool +{ + size_t size; // size of the pool (nu,ber of nodes) + struct node_worker worker[1]; // worker context array; +}; + +struct node_worker_pool* +node_worker_start(struct node_ctx* const ctx, + node_worker_type_t const type, + size_t const size) +{ + assert(ctx); + + if (0 == size) return NULL; + + const char* const type_str = type == NODE_WORKER_SLAVE ? "slave" : "master"; + + size_t const alloc_size = + sizeof(struct node_worker_pool) + + sizeof(struct node_worker) * (size - 1); + + struct node_worker_pool* const ret = malloc(alloc_size); + + if (ret) + { + void* (* const routine) (void*) = + type == NODE_WORKER_SLAVE ? worker_slave : worker_master; + + size_t i; + for (i = 0; i < size; i++) + { + struct node_worker* const worker = &ret->worker[i]; + worker->node = ctx; + worker->id = i; + worker->exit = false; + + int const err = pthread_create(&worker->thread_id, + NULL, + routine, + worker); + if (err) + { + NODE_ERROR("Failed to start %s worker[%zu]: %d (%s)", + type_str, i, err, strerror(err)); + if (0 == i) + { + free(ret); + return NULL; + } + else + { + break; // some threads have started, + // need to return to close them first + } + } + } + + ret->size = i; + } + else + { + NODE_ERROR("Failed to allocate %zu bytes for the %s worker pool", + alloc_size, type_str); + } + + return ret; +} + +void +node_worker_stop(struct node_worker_pool* pool) +{ + size_t i; + for (i = 0; pool && i < pool->size; i++) + { + pthread_join(pool->worker[i].thread_id, NULL); + } + + free(pool); +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/worker.h b/wsrep-lib/wsrep-API/v26/examples/node/worker.h new file mode 100644 index 00000000..7ae06423 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/worker.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines worker thread interface + */ + +#ifndef NODE_WORKER_H +#define NODE_WORKER_H + +#include "ctx.h" + +#include "../../wsrep_api.h" + +/** + * REPLICATION: a callback to apply and commit slave replication events */ +extern enum wsrep_cb_status +node_worker_apply_cb(void* recv_ctx, + const wsrep_ws_handle_t* ws_handle, + uint32_t ws_flags, + const wsrep_buf_t* ws, + const wsrep_trx_meta_t* ws_meta, + wsrep_bool_t* exit_loop); + +typedef enum node_worker_type +{ + NODE_WORKER_SLAVE, + NODE_WORKER_MASTER +} + node_worker_type_t; + +struct node_worker_pool; + +/** + * Starts the required number of workier threads of a given type + * + * @param[in] ctx application context + * @param[in] type of a worker + * @param[in] number of workers + * + * @return worker pool handle + */ +extern struct node_worker_pool* +node_worker_start(struct node_ctx* ctx, + node_worker_type_t type, + size_t number); + +/** + * Stops workers in a pool and deallocates respective resources */ +extern void +node_worker_stop(struct node_worker_pool* pool); + +#endif /* NODE_WORKER_H */ diff --git a/wsrep-lib/wsrep-API/v26/examples/node/wsrep.c b/wsrep-lib/wsrep-API/v26/examples/node/wsrep.c new file mode 100644 index 00000000..6cea6d90 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/wsrep.c @@ -0,0 +1,479 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "wsrep.h" + +#include "log.h" +#include "sst.h" +#include "store.h" +#include "worker.h" + +#include <assert.h> +#include <stdio.h> // snprintf() +#include <stdlib.h> // abort() +#include <string.h> // strcasecmp() + +struct node_wsrep +{ + wsrep_t* instance; // wsrep provider instance + + struct wsrep_view + { + pthread_mutex_t mtx; + wsrep_gtid_t state_id; + wsrep_view_status_t status; + wsrep_cap_t capabilities; + int proto_ver; + int memb_num; + int my_idx; + wsrep_member_info_t* members; + } + view; + + struct + { + pthread_mutex_t mtx; + pthread_cond_t cond; + int value; + } + synced; + + bool bootstrap; // shall this node bootstrap a primary view? +}; + +static struct node_wsrep s_wsrep = +{ + .instance = NULL, + .view = + { + .mtx = PTHREAD_MUTEX_INITIALIZER, + .state_id = {{{ 0, }}, WSREP_SEQNO_UNDEFINED }, + .status = WSREP_VIEW_DISCONNECTED, + .capabilities = 0, + .proto_ver = -1, + .memb_num = 0, + .my_idx = -1, + .members = NULL + }, + .synced = + { + .mtx = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, + .value = 0 + }, + .bootstrap = false +}; + +static const char* wsrep_view_status_str[WSREP_VIEW_MAX] = +{ + "PRIMARY", + "NON-PRIMARY", + "DISCONNECTED" +}; + +#define WSREP_CAPABILITIES_MAX ((int)sizeof(wsrep_cap_t) * 8) // bitmask +static const char* wsrep_capabilities_str[WSREP_CAPABILITIES_MAX] = +{ + "MULTI-MASTER", + "CERTIFICATION", + "PA", + "REPLAY", + "TOI", + "PAUSE", + "CAUSAL-READS", + "CAUSAL-TRX", + "INCREMENTAL", + "SESSION-LOCKS", + "DISTRIBUTED-LOCKS", + "CONSISTENCY-CHECK", + "UNORDERED", + "ANNOTATION", + "PREORDERED", + "STREAMING", + "SNAPSHOT", + "NBO", + NULL, +}; + +/** + * REPLICATION: callback is called by provider when the node connects to group. + * This happens out-of-order, before the node receives a state + * transfer and syncs with the cluster. Unless application requires + * it it can be empty. We however want to know the GTID of the + * group out of order for SST tricks, so we record it out of order. + */ +static enum wsrep_cb_status +wsrep_connected_cb(void* const x, + const wsrep_view_info_t* const v) +{ + char gtid_str[WSREP_GTID_STR_LEN + 1]; + wsrep_gtid_print(&v->state_id, gtid_str, sizeof(gtid_str)); + + NODE_INFO("connect_cb(): Connected at %s to %s group of %d member(s)", + gtid_str, wsrep_view_status_str[v->status], v->memb_num); + + struct node_wsrep* const wsrep = ((struct node_ctx*)x)->wsrep; + + if (pthread_mutex_lock(&wsrep->view.mtx)) + { + NODE_FATAL("Failed to lock VIEW mutex"); + abort(); + } + + wsrep->view.state_id = v->state_id; + + pthread_mutex_unlock(&wsrep->view.mtx); + + return WSREP_CB_SUCCESS; +} + +/** + * logs view data */ +static void +wsrep_log_view(const struct wsrep_view* v) +{ + char gtid[WSREP_GTID_STR_LEN + 1]; + wsrep_gtid_print(&v->state_id, gtid, sizeof(gtid)); + gtid[WSREP_GTID_STR_LEN] = '\0'; + + char caps[256]; + int written = 0; + size_t space_left = sizeof(caps); + int i; + for (i = 0; i < WSREP_CAPABILITIES_MAX && space_left > 0; i++) + { + wsrep_cap_t const f = 1u << i; + + if (!(f & v->capabilities)) continue; + + if (wsrep_capabilities_str[i]) + { + written += snprintf(&caps[written], space_left, "%s|", + wsrep_capabilities_str[i]); + } + else + { + written += snprintf(&caps[written], space_left, "%d|", i); + } + + space_left = sizeof(caps) - (size_t)written; + } + caps[written ? written - 1 : 0] = '\0'; // overwrite last '|' + + char members_list[1024]; + written = 0; + space_left = sizeof(members_list); + for (i = 0; i < v->memb_num && space_left > 0; i++) + { + wsrep_member_info_t* m = &v->members[i]; + char uuid[WSREP_UUID_STR_LEN + 1]; + wsrep_uuid_print(&m->id, uuid, sizeof(uuid)); + uuid[WSREP_UUID_STR_LEN] = '\0'; + + written += snprintf(&members_list[written], space_left, + "%s%d: %s '%s' incoming:'%s'\n", + v->my_idx == i ? " * " : " ", i, + uuid, m->name, m->incoming); + + space_left = sizeof(members_list) - (size_t)written; + } + members_list[written ? written - 1 : 0] = '\0'; // overwrite the last '\n' + + NODE_INFO( + "New view received:\n" + "state: %s (%s)\n" + "capabilities: %s\n" + "protocol version: %d\n" + "members(%d)%s%s", + gtid, wsrep_view_status_str[v->status], + caps, + v->proto_ver, + v->memb_num, v->memb_num ? ":\n" : "", members_list); +} + +/** + * REPLICATION: callback is called when the node needs to process cluster + * view change. The callback is called in "total order isolation", + * so all the preceding replication events will be processed + * strictly before the call and all subsequent - striclty after. + */ +static enum wsrep_cb_status +wsrep_view_cb(void* const x, + void* const r, + const wsrep_view_info_t* const v, + const char* const state, + size_t const state_len) +{ + (void)r; + (void)state; + (void)state_len; + + struct node_ctx* const node = x; + + if (WSREP_VIEW_PRIMARY == v->status) + { + /* REPLICATION: membership change is a totally ordered event and as such + * should be a part of the state, like changes to the + * database. */ + int err = node_store_update_membership(node->store, v); + if (err) + { + NODE_FATAL("Failed to update membership in store: %d (%s)", + err, strerror(-err)); + abort(); + } + } + + enum wsrep_cb_status ret = WSREP_CB_SUCCESS; + struct node_wsrep* const wsrep = ((struct node_ctx*)x)->wsrep; + + if (pthread_mutex_lock(&wsrep->view.mtx)) + { + NODE_FATAL("Failed to lock VIEW mutex"); + abort(); + } + + /* below we'll just copy the data for future reference (if need be): */ + + size_t const memb_size = (size_t)v->memb_num * sizeof(wsrep_member_info_t); + void* const tmp = realloc(wsrep->view.members, memb_size); + if (memb_size > 0 && !tmp) + { + NODE_ERROR("Could not allocate memory for a new view: %zu bytes", + memb_size); + ret = WSREP_CB_FAILURE; + goto cleanup; + } + else + { + wsrep->view.members = tmp; + if (memb_size) memcpy(wsrep->view.members, &v->members[0], memb_size); + } + + wsrep->view.state_id = v->state_id; + wsrep->view.status = v->status; + wsrep->view.capabilities = v->capabilities; + wsrep->view.proto_ver = v->proto_ver; + wsrep->view.memb_num = v->memb_num; + wsrep->view.my_idx = v->my_idx; + + /* and now log the info */ + + wsrep_log_view(&wsrep->view); + +cleanup: + pthread_mutex_unlock(&wsrep->view.mtx); + + return ret; +} + +/** + * REPLICATION: callback is called by provider when the node becomes SYNCED */ +static enum wsrep_cb_status +wsrep_synced_cb(void* const x) +{ + struct node_wsrep* const wsrep = ((struct node_ctx*)x)->wsrep; + + if (pthread_mutex_lock(&wsrep->synced.mtx)) + { + NODE_FATAL("Failed to lock SYNCED mutex"); + abort(); + } + + if (wsrep->synced.value == 0) + { + NODE_INFO("become SYNCED"); + wsrep->synced.value = 1; + pthread_cond_broadcast(&wsrep->synced.cond); + } + + pthread_mutex_unlock(&wsrep->synced.mtx); + + return WSREP_CB_SUCCESS; +} + +struct node_wsrep* +node_wsrep_init(const struct node_options* const opts, + const wsrep_gtid_t* const current_gtid, + void* const app_ctx) +{ + if (s_wsrep.instance != NULL) return NULL; // already initialized + + wsrep_status_t err; + err = wsrep_load(opts->provider, &s_wsrep.instance, node_log_cb); + if (WSREP_OK != err) + { + if (strcasecmp(opts->provider, WSREP_NONE)) + { + NODE_ERROR("wsrep_load(%s) failed: %s (%d).", + opts->provider, strerror(err), err); + } + else + { + NODE_ERROR("Initializing dummy provider failed: %s (%d).", + strerror(err), err); + } + return NULL; + } + + char base_addr[256]; + snprintf(base_addr, sizeof(base_addr) - 1, "%s:%ld", + opts->base_host, opts->base_port); + + struct wsrep_init_args args = + { + .app_ctx = app_ctx, + + .node_name = opts->name, + .node_address = base_addr, + .node_incoming = "", // we don't accept client connections + .data_dir = opts->data_dir, + .options = opts->options, + .proto_ver = 0, // this is the first version of the application + // so the first version of the writeset protocol + .state_id = current_gtid, + .state = NULL, // unused + + .logger_cb = node_log_cb, + .connected_cb = wsrep_connected_cb, + .view_cb = wsrep_view_cb, + .synced_cb = wsrep_synced_cb, + .encrypt_cb = NULL, // not implemented ATM + + .apply_cb = node_worker_apply_cb, + .unordered_cb = NULL, // not needed now + + .sst_request_cb = node_sst_request_cb, + .sst_donate_cb = node_sst_donate_cb + }; + + wsrep_t* wsrep = s_wsrep.instance; + + err = wsrep->init(wsrep, &args); + + if (WSREP_OK != err) + { + NODE_ERROR("wsrep::init() failed: %d, must shutdown", err); + node_wsrep_close(&s_wsrep); + return NULL; + } + + return &s_wsrep; +} + +wsrep_status_t +node_wsrep_connect(struct node_wsrep* const wsrep, + const char* const address, + bool const bootstrap) +{ + wsrep->bootstrap = bootstrap; + wsrep_status_t err = wsrep->instance->connect(wsrep->instance, + "wsrep_cluster", + address, + NULL, + wsrep->bootstrap); + + if (WSREP_OK != err) + { + NODE_ERROR("wsrep::connect(%s) failed: %d, must shutdown", + address, err); + node_wsrep_close(wsrep); + } + + return err; +} + +void +node_wsrep_disconnect(struct node_wsrep* const wsrep) +{ + if (pthread_mutex_lock(&wsrep->synced.mtx)) + { + NODE_FATAL("Failed to lock SYNCED mutex"); + abort(); + } + wsrep->synced.value = -1; /* this will signal master threads to exit */ + pthread_cond_broadcast(&wsrep->synced.cond); + pthread_mutex_unlock(&wsrep->synced.mtx); + + wsrep_status_t const err = wsrep->instance->disconnect(wsrep->instance); + + if (err) + { + /* REPLICATION: unless connection is not closed, slave threads will + * never return. */ + NODE_FATAL("Failed to close wsrep connection: %d", err); + abort(); + } +} + +void +node_wsrep_close(struct node_wsrep* const wsrep) +{ + if (pthread_mutex_lock(&wsrep->view.mtx)) + { + NODE_FATAL("Failed to lock VIEW mutex"); + abort(); + } + assert(0 == wsrep->view.memb_num); // the node must be disconneted + assert(NULL == wsrep->view.members); + free(wsrep->view.members); + wsrep->view.members = NULL; + pthread_mutex_unlock(&wsrep->view.mtx); + + wsrep->instance->free(wsrep->instance); + wsrep->instance = NULL; +} + +bool +node_wsrep_wait_synced(struct node_wsrep* const wsrep) +{ + if (pthread_mutex_lock(&wsrep->synced.mtx)) + { + NODE_FATAL("Failed to lock SYNCED mutex"); + abort(); + } + + while (wsrep->synced.value == 0) + { + pthread_cond_wait(&wsrep->synced.cond, &wsrep->synced.mtx); + } + + bool const ret = wsrep->synced.value > 0; + + pthread_mutex_unlock(&wsrep->synced.mtx); + + return ret; +} + +void +node_wsrep_connected_gtid(struct node_wsrep* wsrep, wsrep_gtid_t* gtid) +{ + if (pthread_mutex_lock(&wsrep->view.mtx)) + { + NODE_FATAL("Failed to lock VIEW mutex"); + abort(); + } + + *gtid = wsrep->view.state_id; + + pthread_mutex_unlock(&wsrep->view.mtx); +} + +wsrep_t* +node_wsrep_provider(struct node_wsrep* wsrep) +{ + return wsrep->instance; +} diff --git a/wsrep-lib/wsrep-API/v26/examples/node/wsrep.h b/wsrep-lib/wsrep-API/v26/examples/node/wsrep.h new file mode 100644 index 00000000..75c7eac3 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/examples/node/wsrep.h @@ -0,0 +1,92 @@ +/* Copyright (c) 2019, Codership Oy. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file This unit defines various helpers to manage wsrep provider + */ + +#ifndef NODE_WSREP_H +#define NODE_WSREP_H + +#include "options.h" + +#include "../../wsrep_api.h" + +#include <pthread.h> +#include <stdbool.h> + +typedef struct node_wsrep node_wsrep_t; + +/** + * loads and initializes wsrep provider for further usage + * + * @param[in] opts program options + * @param[in] current_gtid GTID corresponding to the current node state + * @param[in] app_ctx application context to be passed to callbacks + * + * @return initialized object pointer + */ +extern node_wsrep_t* +node_wsrep_init(const struct node_options* opts, + const wsrep_gtid_t* current_gtid, + void* app_ctx); + +/** + * connects to primary component + * + * @param[in] wsrep wsrep context + * @param[in] address address to connect at (provider specific) + * @param[in] bootsstrap bootstrap primary component if there's none + * + * @return wsrep status code + */ +extern wsrep_status_t +node_wsrep_connect(node_wsrep_t* wsrep, + const char* address, + bool bootstrap); + +/** + * disconnects from primary component + */ +extern void +node_wsrep_disconnect(node_wsrep_t* wsrep); + +/** + * deinitializes and unloads wsrep provider + */ +extern void +node_wsrep_close(node_wsrep_t* wsrep); + +/** + * waits for the node to become SYNCED + * + * @return true if node is synced, false in any other event. + */ +extern bool +node_wsrep_wait_synced(node_wsrep_t* wsrep); + +/** + * @param[in] wsrep context + * @param[out] gtid of the current view */ +extern void +node_wsrep_connected_gtid(node_wsrep_t* wsrep, wsrep_gtid_t* gtid); + +/** + * @return wsrep provider instance */ +extern wsrep_t* +node_wsrep_provider(node_wsrep_t* wsrep); + +#endif /* NODE_WSREP_H */ diff --git a/wsrep-lib/wsrep-API/v26/wsrep.xcf b/wsrep-lib/wsrep-API/v26/wsrep.xcf Binary files differnew file mode 100644 index 00000000..54108c6b --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep.xcf diff --git a/wsrep-lib/wsrep-API/v26/wsrep_allowlist_service.h b/wsrep-lib/wsrep-API/v26/wsrep_allowlist_service.h new file mode 100644 index 00000000..71f4d7e9 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_allowlist_service.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-API. + * + * Wsrep-API is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-API is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-API. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file wsrep_allowlist_service.h + * + * This file defines interface for connection allowlist checks. + * + * The provider which is capable of using the service interface v1 must + * export the following functions. + * + * int wsrep_init_allowlist_service_v1(wsrep_allowlist_service_v1_t*) + * void wsrep_deinit_allowlist_service_v1() + * + * which can be probed by the application. + * + * The application must initialize the service via above init function + * before the provider is initialized via wsrep->init(). The deinit + * function must be called after the provider side resources have been + * released via wsrep->free(). + */ + +#ifndef WSREP_ALLOWLIST_SERVICE_H +#define WSREP_ALLOWLIST_SERVICE_H + +#include "wsrep_api.h" + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + +/** + * Type tag for application defined allowlist processing context. + * + * Application may pass pointer to the context when initializing + * the allowlist service. This pointer is passed a first parameter for + * each service call. + */ +typedef struct wsrep_allowlist_context wsrep_allowlist_context_t; + +typedef enum +{ + WSREP_ALLOWLIST_KEY_IP = 0, // IP allowlist check + WSREP_ALLOWLIST_KEY_SSL // SSL certificate allowlist check +} wsrep_allowlist_key_t; + +/* + * Allowlist connection check callback. + * + * @retval WSREP_OK connection allowed + * @retval WSREP_NOT_ALLOWED connection not allowed + */ +typedef wsrep_status_t (*wsrep_allowlist_cb_t)( + wsrep_allowlist_context_t*, + wsrep_allowlist_key_t key, + const wsrep_buf_t* value); + +/** + * Allowlist service struct. + * + * A pointer to this struct must be passed to the call to + * wsrep_init_allowlist_service_v1. + * + * The application must provide implementation to all functions defined + * in this struct. + */ +typedef struct wsrep_allowlist_service_v1_st +{ + /* Allowlist check callback */ + wsrep_allowlist_cb_t allowlist_cb; + /* Pointer to application defined allowlist context. */ + wsrep_allowlist_context_t* context; +} wsrep_allowlist_service_v1_t; + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + + +#define WSREP_ALLOWLIST_SERVICE_INIT_FUNC_V1 "wsrep_init_allowlist_service_v1" +#define WSREP_ALLOWLIST_SERVICE_DEINIT_FUNC_V1 "wsrep_deinit_allowlist_service_v1" + +#endif /* WSREP_ALLOWLIST_SERVICE_H */ + diff --git a/wsrep-lib/wsrep-API/v26/wsrep_api.h b/wsrep-lib/wsrep-API/v26/wsrep_api.h new file mode 100644 index 00000000..59bb71da --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_api.h @@ -0,0 +1,1380 @@ +/* Copyright (C) 2009-2013 Codership Oy <info@codership.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/*! + @file wsrep API declaration. + + HOW TO READ THIS FILE. + + Due to C language rules this header layout doesn't lend itself to intuitive + reading. So here's the scoop: in the end this header declares two main types: + + * struct wsrep_init_args + + and + + * struct wsrep + + wsrep_init_args contains initialization parameters for wsrep provider like + names, addresses, etc. and pointers to callbacks. The callbacks will be called + by provider when it needs to do something application-specific, like log a + message or apply a writeset. It should be passed to init() call from + wsrep API. It is an application part of wsrep API contract. + + struct wsrep is the interface to wsrep provider. It contains all wsrep API + calls. It is a provider part of wsrep API contract. + + Finally, wsrep_load() method loads (dlopens) wsrep provider library. It is + defined in wsrep_loader.c unit and is part of libwsrep.a (which is not a + wsrep provider, but a convenience library). + + wsrep_unload() does the reverse. + +*/ +#ifndef WSREP_H +#define WSREP_H + +#include <stdint.h> +#include <stdbool.h> +#include <stdlib.h> +#include <unistd.h> +#include <time.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************** + * * + * wsrep replication API * + * * + **************************************************************************/ + +#define WSREP_INTERFACE_VERSION "26" + +/*! Empty backend spec */ +#define WSREP_NONE "none" + + +/*! + * @brief log severity levels, passed as first argument to log handler + */ +typedef enum wsrep_log_level +{ + WSREP_LOG_FATAL, //!< Unrecoverable error, application must quit. + WSREP_LOG_ERROR, //!< Operation failed, must be repeated. + WSREP_LOG_WARN, //!< Unexpected condition, but no operational failure. + WSREP_LOG_INFO, //!< Informational message. + WSREP_LOG_DEBUG //!< Debug message. Shows only of compiled with debug. +} wsrep_log_level_t; + +/*! + * @brief error log handler + * + * All messages from wsrep provider are directed to this + * handler, if present. + * + * @param level log level + * @param message log message + */ +typedef void (*wsrep_log_cb_t)(wsrep_log_level_t, const char *); + + +/*! + * Certain provider capabilities application may want to know about + */ +#define WSREP_CAP_MULTI_MASTER ( 1ULL << 0 ) +#define WSREP_CAP_CERTIFICATION ( 1ULL << 1 ) +#define WSREP_CAP_PARALLEL_APPLYING ( 1ULL << 2 ) +#define WSREP_CAP_TRX_REPLAY ( 1ULL << 3 ) +#define WSREP_CAP_ISOLATION ( 1ULL << 4 ) +#define WSREP_CAP_PAUSE ( 1ULL << 5 ) +#define WSREP_CAP_CAUSAL_READS ( 1ULL << 6 ) +#define WSREP_CAP_CAUSAL_TRX ( 1ULL << 7 ) +#define WSREP_CAP_INCREMENTAL_WRITESET ( 1ULL << 8 ) +#define WSREP_CAP_SESSION_LOCKS ( 1ULL << 9 ) +#define WSREP_CAP_DISTRIBUTED_LOCKS ( 1ULL << 10 ) +#define WSREP_CAP_CONSISTENCY_CHECK ( 1ULL << 11 ) +#define WSREP_CAP_UNORDERED ( 1ULL << 12 ) +#define WSREP_CAP_ANNOTATION ( 1ULL << 13 ) +#define WSREP_CAP_PREORDERED ( 1ULL << 14 ) +#define WSREP_CAP_STREAMING ( 1ULL << 15 ) +#define WSREP_CAP_SNAPSHOT ( 1ULL << 16 ) +#define WSREP_CAP_NBO ( 1ULL << 17 ) + +typedef uint32_t wsrep_cap_t; //!< capabilities bitmask + +/*! + * Writeset flags + * + * TRX_END the writeset and all preceding writesets must be committed + * ROLLBACK all preceding writesets in a transaction must be rolled back + * ISOLATION the writeset must be applied AND committed in isolation + * PA_UNSAFE the writeset cannot be applied in parallel + * COMMUTATIVE the order in which the writeset is applied does not matter + * NATIVE the writeset contains another writeset in this provider format + * + * TRX_START shall be set on the first trx fragment by provider + * TRX_PREPARE shall be set on the fragment which prepares the transaction + * + * Note that some of the flags are mutually exclusive (e.g. TRX_END and + * ROLLBACK). + */ +#define WSREP_FLAG_TRX_END ( 1ULL << 0 ) +#define WSREP_FLAG_ROLLBACK ( 1ULL << 1 ) +#define WSREP_FLAG_ISOLATION ( 1ULL << 2 ) +#define WSREP_FLAG_PA_UNSAFE ( 1ULL << 3 ) +#define WSREP_FLAG_COMMUTATIVE ( 1ULL << 4 ) +#define WSREP_FLAG_NATIVE ( 1ULL << 5 ) +#define WSREP_FLAG_TRX_START ( 1ULL << 6 ) +#define WSREP_FLAG_TRX_PREPARE ( 1ULL << 7 ) +#define WSREP_FLAG_SNAPSHOT ( 1ULL << 8 ) +#define WSREP_FLAG_IMPLICIT_DEPS ( 1ULL << 9 ) + +#define WSREP_FLAGS_LAST WSREP_FLAG_IMPLICIT_DEPS +#define WSREP_FLAGS_MASK ((WSREP_FLAGS_LAST << 1) - 1) + + +typedef uint64_t wsrep_trx_id_t; //!< application transaction ID +typedef uint64_t wsrep_conn_id_t; //!< application connection ID +typedef int64_t wsrep_seqno_t; //!< sequence number of a writeset, etc. +#ifdef __cplusplus +typedef bool wsrep_bool_t; +#else +typedef _Bool wsrep_bool_t; //!< should be the same as standard (C99) bool +#endif /* __cplusplus */ + +/*! undefined seqno */ +#define WSREP_SEQNO_UNDEFINED (-1) + + +/*! wsrep provider status codes */ +typedef enum wsrep_status +{ + WSREP_OK = 0, //!< success + WSREP_WARNING, //!< minor warning, error logged + WSREP_TRX_MISSING, //!< transaction is not known by wsrep + WSREP_TRX_FAIL, //!< transaction aborted, server can continue + WSREP_BF_ABORT, //!< trx was victim of brute force abort + WSREP_SIZE_EXCEEDED, //!< data exceeded maximum supported size + WSREP_CONN_FAIL, //!< error in client connection, must abort + WSREP_NODE_FAIL, //!< error in node state, wsrep must reinit + WSREP_FATAL, //!< fatal error, server must abort + WSREP_NOT_IMPLEMENTED, //!< feature not implemented + WSREP_NOT_ALLOWED //!< operation not allowed +} wsrep_status_t; + + +/*! wsrep callbacks status codes */ +typedef enum wsrep_cb_status +{ + WSREP_CB_SUCCESS = 0, //!< success (as in "not critical failure") + WSREP_CB_FAILURE //!< critical failure (consistency violation) + /* Technically, wsrep provider has no use for specific failure codes since + * there is nothing it can do about it but abort execution. Therefore any + * positive number shall indicate a critical failure. Optionally that value + * may be used by provider to come to a consensus about state consistency + * in a group of nodes. */ +} wsrep_cb_status_t; + + +/*! + * UUID type - for all unique IDs + */ +typedef union wsrep_uuid { + uint8_t data[16]; + size_t alignment; +} wsrep_uuid_t; + +/*! Undefined UUID */ +static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}}; + +/*! UUID string representation length, terminating '\0' not included */ +#define WSREP_UUID_STR_LEN 36 + +/*! + * Scan UUID from string + * @return length of UUID string representation or negative error code + */ +extern int +wsrep_uuid_scan (const char* str, size_t str_len, wsrep_uuid_t* uuid); + +/*! + * Print UUID to string + * @return length of UUID string representation or negative error code + */ +extern int +wsrep_uuid_print (const wsrep_uuid_t* uuid, char* str, size_t str_len); + +/*! + * @brief Compare two UUIDs + * + * Performs a byte by byte comparison of lhs and rhs. + * Returns 0 if lhs and rhs match, otherwise -1 or 1 according to the + * difference of the first byte that differs in lsh and rhs. + * + * @return -1, 0, 1 if lhs is respectively smaller, equal, or greater than rhs + */ +extern int +wsrep_uuid_compare (const wsrep_uuid_t* lhs, const wsrep_uuid_t* rhs); + +#define WSREP_MEMBER_NAME_LEN 32 //!< maximum logical member name length +#define WSREP_INCOMING_LEN 256 //!< max Domain Name length + 0x00 + + +/*! + * Global transaction identifier + */ +typedef struct wsrep_gtid +{ + wsrep_uuid_t uuid; /*!< History UUID */ + wsrep_seqno_t seqno; /*!< Sequence number */ +} wsrep_gtid_t; + +/*! Undefined GTID */ +static const wsrep_gtid_t WSREP_GTID_UNDEFINED = {{{0, }}, -1}; + +/*! Minimum number of bytes guaranteed to store GTID string representation, + * terminating '\0' not included (36 + 1 + 20) */ +#define WSREP_GTID_STR_LEN 57 + + +/*! + * Scan GTID from string + * @return length of GTID string representation or negative error code + */ +extern int +wsrep_gtid_scan(const char* str, size_t str_len, wsrep_gtid_t* gtid); + +/*! + * Print GTID to string + * @return length of GTID string representation or negative error code + */ +extern int +wsrep_gtid_print(const wsrep_gtid_t* gtid, char* str, size_t str_len); + +/*! + * Source/server transaction ID (trx ID assigned at originating node) + */ +typedef struct wsrep_stid { + wsrep_uuid_t node; //!< source node ID + wsrep_trx_id_t trx; //!< local trx ID at source + wsrep_conn_id_t conn; //!< local connection ID at source +} wsrep_stid_t; + +/*! + * Transaction meta data + */ +typedef struct wsrep_trx_meta +{ + wsrep_gtid_t gtid; /*!< Global transaction identifier */ + wsrep_stid_t stid; /*!< Source transaction identifier */ + wsrep_seqno_t depends_on; /*!< Sequence number of the last transaction + this transaction may depend on */ +} wsrep_trx_meta_t; + +/*! Abstract data buffer structure */ +typedef struct wsrep_buf +{ + const void* ptr; /*!< Pointer to data buffer */ + size_t len; /*!< Length of buffer */ +} wsrep_buf_t; + +/*! Transaction handle struct passed for wsrep transaction handling calls */ +typedef struct wsrep_ws_handle +{ + wsrep_trx_id_t trx_id; //!< transaction ID + void* opaque; //!< opaque provider transaction context data +} wsrep_ws_handle_t; + +/*! + * member status + */ +typedef enum wsrep_member_status { + WSREP_MEMBER_UNDEFINED, //!< undefined state + WSREP_MEMBER_JOINER, //!< incomplete state, requested state transfer + WSREP_MEMBER_DONOR, //!< complete state, donates state transfer + WSREP_MEMBER_JOINED, //!< complete state + WSREP_MEMBER_SYNCED, //!< complete state, synchronized with group + WSREP_MEMBER_ERROR, //!< this and above is provider-specific error code + WSREP_MEMBER_MAX +} wsrep_member_status_t; + +/*! + * static information about a group member (some fields are tentative yet) + */ +typedef struct wsrep_member_info { + wsrep_uuid_t id; //!< group-wide unique member ID + char name[WSREP_MEMBER_NAME_LEN]; //!< human-readable name + char incoming[WSREP_INCOMING_LEN]; //!< address for client requests +} wsrep_member_info_t; + +/*! + * group status + */ +typedef enum wsrep_view_status { + WSREP_VIEW_PRIMARY, //!< primary group configuration (quorum present) + WSREP_VIEW_NON_PRIMARY, //!< non-primary group configuration (quorum lost) + WSREP_VIEW_DISCONNECTED, //!< not connected to group, retrying. + WSREP_VIEW_MAX +} wsrep_view_status_t; + +/*! + * view of the group + */ +typedef struct wsrep_view_info { + wsrep_gtid_t state_id; //!< global state ID + wsrep_seqno_t view; //!< global view number + wsrep_view_status_t status; //!< view status + wsrep_cap_t capabilities;//!< capabilities available in the view + int my_idx; //!< index of this member in the view + int memb_num; //!< number of members in the view + int proto_ver; //!< application protocol agreed on the view + wsrep_member_info_t members[1];//!< array of member information +} wsrep_view_info_t; + + +/*! + * @brief connected to group + * + * This handler is called once the first primary view is seen. + * The purpose of this call is to provide basic information only, + * like node UUID and group UUID. + */ +typedef enum wsrep_cb_status (*wsrep_connected_cb_t) ( + void* app_ctx, + const wsrep_view_info_t* view +); + + +/*! + * @brief group view handler + * + * This handler is called in *total order* corresponding to the group + * configuration change. It is to provide a vital information about + * new group view. + * + * @param app_ctx application context + * @param recv_ctx receiver context + * @param view new view on the group + * @param state current state + * @param state_len length of current state + */ +typedef enum wsrep_cb_status (*wsrep_view_cb_t) ( + void* app_ctx, + void* recv_ctx, + const wsrep_view_info_t* view, + const char* state, + size_t state_len +); + + +/*! + * Magic string to tell provider to engage into trivial (empty) state transfer. + * No data will be passed, but the node shall be considered JOINED. + * Should be passed in sst_req parameter of wsrep_sst_cb_t. + */ +#define WSREP_STATE_TRANSFER_TRIVIAL "trivial" + +/*! + * Magic string to tell provider not to engage in state transfer at all. + * The member will stay in WSREP_MEMBER_UNDEFINED state but will keep on + * receiving all writesets. + * Should be passed in sst_req parameter of wsrep_sst_cb_t. + */ +#define WSREP_STATE_TRANSFER_NONE "none" + + +/*! + * @brief Creates and returns State Snapshot Transfer request for provider. + * + * This handler is called whenever the node is found to miss some of events + * from the cluster history (e.g. fresh node joining the cluster). + * SST will be used if it is impossible (or impractically long) to replay + * missing events, which may be not known in advance, so the node must always + * be ready to accept full SST or abort in case event replay is impossible. + * + * Normally SST request is an opaque buffer that is passed to the + * chosen SST donor node and must contain information sufficient for + * donor to deliver SST (typically SST method and delivery address). + * See above macros WSREP_STATE_TRANSFER_TRIVIAL and WSREP_STATE_TRANSFER_NONE + * to modify the standard provider behavior. + * + * @note Currently it is assumed that sst_req is allocated using + * malloc()/calloc()/realloc() and it will be freed by + * wsrep provider. + * + * @param app_ctx application context + * @param sst_req location to store SST request + * @param sst_req_len location to store SST request length or error code, + * value of 0 means no SST. + */ +typedef enum wsrep_cb_status (*wsrep_sst_request_cb_t) ( + void* app_ctx, + void** sst_req, + size_t* sst_req_len +); + + +/*! + * @brief apply callback + * + * This handler is called from wsrep library to apply replicated writeset + * Must support brute force applying for multi-master operation + * + * @param recv_ctx receiver context pointer provided by the application + * @param ws_handle internal provider writeset handle + * @param flags WSREP_FLAG_... flags + * @param data data buffer containing the writeset + * @param meta transaction meta data of the writeset to be applied + * @param exit_loop set to true to exit receive loop + * + * @return error code: + * @retval 0 - success + * @retval non-0 - application-specific error code + */ +typedef enum wsrep_cb_status (*wsrep_apply_cb_t) ( + void* recv_ctx, + const wsrep_ws_handle_t* ws_handle, + uint32_t flags, + const wsrep_buf_t* data, + const wsrep_trx_meta_t* meta, + wsrep_bool_t* exit_loop +); + + +/*! + * @brief unordered callback + * + * This handler is called to execute unordered actions (actions that need not + * to be executed in any particular order) attached to writeset. + * + * @param recv_ctx receiver context pointer provided by the application + * @param data data buffer containing the writeset + */ +typedef enum wsrep_cb_status (*wsrep_unordered_cb_t) ( + void* recv_ctx, + const wsrep_buf_t* data +); + + +/*! + * @brief a callback to donate state snapshot + * + * This handler is called from wsrep library when it needs this node + * to deliver state to a new cluster member. + * No state changes will be committed for the duration of this call. + * Wsrep implementation may provide internal state to be transmitted + * to new cluster member for initial state. + * + * @param app_ctx application context + * @param recv_ctx receiver context + * @param str_msg state transfer request message + * @param gtid current state ID on this node + * @param state current wsrep internal state buffer + * @param bypass bypass snapshot transfer, only transfer uuid:seqno pair + */ +typedef enum wsrep_cb_status (*wsrep_sst_donate_cb_t) ( + void* app_ctx, + void* recv_ctx, + const wsrep_buf_t* str_msg, + const wsrep_gtid_t* state_id, + const wsrep_buf_t* state, + wsrep_bool_t bypass +); + + +/*! + * @brief a callback to signal application that wsrep state is synced + * with cluster + * + * This callback is called after wsrep library has got in sync with + * rest of the cluster. + * + * @param app_ctx application context + * + * @return wsrep_cb_status enum + */ +typedef enum wsrep_cb_status (*wsrep_synced_cb_t) (void* app_ctx); + + +/* + * An opaque encryption key of arbitrary size - provided by the application + * May contain not only the key, but also algorithm specification and the like. + */ +typedef wsrep_buf_t wsrep_enc_key_t; + +/* + * Initialization vector/nonce. Given that most symmetric ciphers use 16 byte + * blocks this can be made 32 bytes without much loss of generality. + * Must be set by provider to start an encryption/decrytpion operation. + */ +typedef char wsrep_enc_iv_t[32]; + +/* + * Encryption context that should be sufficient to deterministically encrypt/ + * decrypt a data buffer either standalone or as part of a stream. May be used + * passed in apply_cb() along with the encrypted replication events to + * application as well. + * + * @param key [in] can be a pointer to const since provider will have to keep + * the keys until the last writeset that uses the key is in the + * cache + * @param iv [in] initialization vector for the beginning of the new + * operation. + * @param ctx [in/out] ongoing operation context + * To initialize a new context the encrypt_cb() caller sets it to + * NULL, which signals the encryption of a new continuous buffer. + * In that case the callback allocates the new context (using + * supplied key and iv) and stores the pointer to it for + * processing subsequent data. + * The end of the operation is signaled by passing TRUE in the + * parameter `last` to the encryption callback, the callback then + * finishes any pending encryption and deallocates the context. + */ +typedef struct +{ + const wsrep_enc_key_t* key; + const wsrep_enc_iv_t* iv; + void* ctx; +} +wsrep_enc_ctx_t; + +/* + * Encryption direction + */ +typedef enum +{ + WSREP_ENC = 0, /* encryption */ + WSREP_DEC = 1 /* decryption */ +} +wsrep_enc_direction_t; + +/* + * Encryption/decryption callback. Must be used by both provider and the + * application to obtain identical results. Can be NULL for no encryption. + * + * @param app_ctx application context + * @param enc_ctx current operation context + * @param input input data buffer + * @param output an output buffer, must be at least the size of the input + * data plus unwritten bytes from the previous call(s). E.g. in + * block mode, encryption/decryption operation will write data + * to output in multiples of the algoritm block size. So a call + * to encrypt a single byte won't normally write anything to + * output waiting for the next input chunk. So on the next call + * it may write one byte more than was given in the input. + * @param direction of the operation (encryption/decryption) + * @param last true if this is the last buffer to encrypt in the stream. + * In that case the callback shall write the remaining bytes of + * the stream to output (if any) and deallocate ctx->ctx if + * allocated previously + * + * @return a number of bytes written to output or a negative error code. + */ +typedef int (*wsrep_encrypt_cb_t) +( + void* app_ctx, + wsrep_enc_ctx_t* enc_ctx, + const wsrep_buf_t* input, + void* output, + wsrep_enc_direction_t direction, + bool last +); + + +/*! + * Initialization parameters for wsrep provider. + */ +struct wsrep_init_args +{ + void* app_ctx; //!< Application context for callbacks + + /* Configuration parameters */ + const char* node_name; //!< Symbolic name of this node (e.g. hostname) + const char* node_address; //!< Address to be used by wsrep provider + const char* node_incoming; //!< Address for incoming client connections + const char* data_dir; //!< Directory where wsrep files are kept if any + const char* options; //!< Provider-specific configuration string + int proto_ver; //!< Max supported application protocol version + + /* Application initial state information. */ + const wsrep_gtid_t* state_id; //!< Application state GTID + const wsrep_buf_t* state; //!< Initial state for wsrep provider + + /* Application callbacks */ + wsrep_log_cb_t logger_cb; //!< logging handler + wsrep_connected_cb_t connected_cb; //!< connected to group + wsrep_view_cb_t view_cb; //!< group view change handler + wsrep_sst_request_cb_t sst_request_cb; //!< SST request creator + wsrep_encrypt_cb_t encrypt_cb; //!< Encryption callback + + /* Applier callbacks */ + wsrep_apply_cb_t apply_cb; //!< apply callback + wsrep_unordered_cb_t unordered_cb; //!< callback for unordered actions + + /* State Snapshot Transfer callbacks */ + wsrep_sst_donate_cb_t sst_donate_cb; //!< donate SST + wsrep_synced_cb_t synced_cb; //!< synced with group +}; + + +/*! Type of the stats variable value in struct wsrep_status_var */ +typedef enum wsrep_var_type +{ + WSREP_VAR_STRING, //!< pointer to null-terminated string + WSREP_VAR_INT64, //!< int64_t + WSREP_VAR_DOUBLE //!< double +} +wsrep_var_type_t; + +/*! Generalized stats variable representation */ +struct wsrep_stats_var +{ + const char* name; //!< variable name + wsrep_var_type_t type; //!< variable value type + union { + int64_t _int64; + double _double; + const char* _string; + } value; //!< variable value +}; + + +/*! Key struct used to pass certification keys for transaction handling calls. + * A key consists of zero or more key parts. */ +typedef struct wsrep_key +{ + const wsrep_buf_t* key_parts; /*!< Array of key parts */ + size_t key_parts_num; /*!< Number of key parts */ +} wsrep_key_t; + +/*! Key type: + * SHARED - higher level resource shared between clients, e.g. SQL table + * REFERENCE - resource referenced but not modified, e.g. parent row + * UPDATE - resource is modified + * EXCLUSIVE - resource is either created or deleted */ +typedef enum wsrep_key_type +{ + WSREP_KEY_SHARED = 0, + WSREP_KEY_REFERENCE, + WSREP_KEY_UPDATE, + WSREP_KEY_EXCLUSIVE +} wsrep_key_type_t; + +/*! Data type: + * ORDERED state modification event that should be applied and committed + * in order. + * UNORDERED some action that does not modify state and execution of which is + * optional and does not need to happen in order. + * ANNOTATION (human readable) writeset annotation. */ +typedef enum wsrep_data_type +{ + WSREP_DATA_ORDERED = 0, + WSREP_DATA_UNORDERED, + WSREP_DATA_ANNOTATION +} wsrep_data_type_t; + + +/*! + * @brief Helper method to reset trx writeset handle state when trx id changes + * + * Instead of passing wsrep_ws_handle_t directly to wsrep calls, + * wrapping handle with this call offloads bookkeeping from + * application. + */ +static inline wsrep_ws_handle_t* wsrep_ws_handle_for_trx( + wsrep_ws_handle_t* ws_handle, + wsrep_trx_id_t trx_id) +{ + if (ws_handle->trx_id != trx_id) + { + ws_handle->trx_id = trx_id; + ws_handle->opaque = NULL; + } + return ws_handle; +} + + +/*! + * A handle for processing preordered actions. + * Must be initialized to WSREP_PO_INITIALIZER before use. + */ +typedef struct wsrep_po_handle { void* opaque; } wsrep_po_handle_t; + +static const wsrep_po_handle_t WSREP_PO_INITIALIZER = { NULL }; + + +typedef struct wsrep_st wsrep_t; +/*! + * wsrep interface for dynamically loadable libraries + */ +struct wsrep_st { + + const char *version; //!< interface version string + + /*! + * @brief Initializes wsrep provider + * + * @param wsrep provider handle + * @param args wsrep initialization parameters + */ + wsrep_status_t (*init) (wsrep_t* wsrep, + const struct wsrep_init_args* args); + + /*! + * @brief Returns provider capabilities bitmap + * + * Note that these are potential provider capabilities. Provider will + * offer only capabilities supported by all members in the view + * (see wsrep_view_info). + * + * @param wsrep provider handle + */ + wsrep_cap_t (*capabilities) (wsrep_t* wsrep); + + /*! + * @brief Passes provider-specific configuration string to provider. + * + * @param wsrep provider handle + * @param conf configuration string + * + * @retval WSREP_OK configuration string was parsed successfully + * @retval WSREP_WARNING could not parse configuration string, no action taken + */ + wsrep_status_t (*options_set) (wsrep_t* wsrep, const char* conf); + + /*! + * @brief Returns provider-specific string with current configuration values. + * + * @param wsrep provider handle + * + * @return a dynamically allocated string with current configuration + * parameter values + */ + char* (*options_get) (wsrep_t* wsrep); + + /*! + * @brief A call to set/rotate the key in provider. + * + * This may happen asynchronously and so is a best effort operation. + * Some buffers may still be encrypted with a previous key. + * + * @param a key object for the encryption callback + * + * return success or an error code + */ + wsrep_status_t (*enc_set_key)(wsrep_t* wsrep, const wsrep_enc_key_t* key); + + /*! + * @brief Opens connection to cluster + * + * Returns when either node is ready to operate as a part of the cluster + * or fails to reach operating status. + * + * @param wsrep provider handle + * @param cluster_name unique symbolic cluster name + * @param cluster_url URL-like cluster address (backend://address) + * @param state_donor name of the node to be asked for state transfer. + * @param bootstrap a flag to request initialization of a new wsrep + * service rather then a connection to the existing one. + * cluster_url may still carry important initialization + * parameters, like backend spec and/or listen address. + */ + wsrep_status_t (*connect) (wsrep_t* wsrep, + const char* cluster_name, + const char* cluster_url, + const char* state_donor, + wsrep_bool_t bootstrap); + + /*! + * @brief Closes connection to cluster. + * + * @param wsrep this wsrep handler + */ + wsrep_status_t (*disconnect)(wsrep_t* wsrep); + + /*! + * @brief start receiving replication events + * + * This function does not return until provider is closed or \p exit_loop + * parameter to wsrep_apply_cb_t() is set to true. + * + * @param wsrep provider handle + * @param recv_ctx receiver context + */ + wsrep_status_t (*recv)(wsrep_t* wsrep, void* recv_ctx); + + /*! + * @brief Tells provider that a given writeset has a read view associated + * with it. + * + * @param wsrep provider handle + * @param handle writeset handle + * @param rv read view GTID established by the caller or if NULL, + * provider will infer it internally. + */ + wsrep_status_t (*assign_read_view)(wsrep_t* wsrep, + wsrep_ws_handle_t* handle, + const wsrep_gtid_t* rv); + + /*! + * @brief Certifies transaction with provider. + * + * Must be called before transaction commit. Returns success code, which + * caller must check. + * + * In case of WSREP_OK, transaction can proceed to commit. + * Otherwise transaction must rollback. + * + * In case of a failure there are two conceptually different situations: + * - the writeset was not ordered. In that case meta struct shall contain + * undefined GTID: WSREP_UUID_UNDEFINED:WSREP_SEQNO_UNDEFINED. + * - the writeset was successfully ordered, but failed certification. + * In this case meta struct shall contain a valid GTID. + * + * Regardless of the return code, if meta struct contains a valid GTID + * the commit order critical section must be entered with that GTID. + * + * @param wsrep provider handle + * @param conn_id connection ID + * @param ws_handle writeset of committing transaction + * @param flags fine tuning the replication WSREP_FLAG_* + * @param meta transaction meta data + * + * @retval WSREP_OK writeset successfully certified, can commit + * @retval WSREP_TRX_FAIL must rollback transaction + * @retval WSREP_CONN_FAIL must close client connection + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*certify)(wsrep_t* wsrep, + wsrep_conn_id_t conn_id, + wsrep_ws_handle_t* ws_handle, + uint32_t flags, + wsrep_trx_meta_t* meta); + + /*! + * @brief Enters commit order critical section. + * + * Anything executed between this call and commit_order_leave() will be + * executed in provider enforced order. + * + * @param wsrep provider handle + * @param ws_handle internal provider writeset handle + * @param meta transaction meta data + * + * @retval WSREP_OK commit order entered successfully + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*commit_order_enter)(wsrep_t* wsrep, + const wsrep_ws_handle_t* ws_handle, + const wsrep_trx_meta_t* meta); + + /*! + * @brief Leaves commit order critical section + * + * Anything executed between commit_order_enter() and this call will be + * executed in provider enforced order. + * + * @param wsrep provider handle + * @param ws_handle internal provider writeset handle + * @param meta transaction meta data + * @param error buffer containing error info (null/empty for no error) + * + * @retval WSREP_OK commit order left successfully + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*commit_order_leave)(wsrep_t* wsrep, + const wsrep_ws_handle_t* ws_handle, + const wsrep_trx_meta_t* meta, + const wsrep_buf_t* error); + + /*! + * @brief Releases resources after transaction commit/rollback. + * + * Ends total order critical section. + * + * @param wsrep provider handle + * @param ws_handle writeset of committing transaction + * @retval WSREP_OK release succeeded + */ + wsrep_status_t (*release) (wsrep_t* wsrep, + wsrep_ws_handle_t* ws_handle); + + /*! + * @brief Replay trx as a slave writeset + * + * If local trx has been aborted by brute force, and it has already + * replicated before this abort, we must try if we can apply it as + * slave trx. Note that slave nodes see only trx writesets and certification + * test based on write set content can be different to DBMS lock conflicts. + * + * @param wsrep provider handle + * @param ws_handle writeset of committing transaction + * @param trx_ctx transaction context + * + * @retval WSREP_OK cluster commit succeeded + * @retval WSREP_TRX_FAIL must rollback transaction + * @retval WSREP_BF_ABORT brute force abort happened after trx replicated + * must rollback transaction and try to replay + * @retval WSREP_CONN_FAIL must close client connection + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*replay_trx)(wsrep_t* wsrep, + const wsrep_ws_handle_t* ws_handle, + void* trx_ctx); + + /*! + * @brief Abort certify() call of another thread. + * + * It is possible, that some high-priority transaction needs to abort + * another transaction which is in certify() call waiting for resources. + * + * The kill routine checks that abort is not attempted against a transaction + * which is front of the caller (in total order). + * + * If the abort was successful, the victim sequence number is stored + * into location pointed by the victim_seqno. + * + * @param wsrep provider handle + * @param bf_seqno seqno of brute force trx, running this cancel + * @param victim_trx transaction to be aborted, and which is committing + * @param victim_seqno seqno of the victim transaction if assigned + * + * @retval WSREP_OK abort succeeded + * @retval WSREP_NOT_ALLOWED the provider declined the abort request + * @retval WSREP_TRX_MISSING the victim_trx was missing + * @retval WSREP_WARNING abort failed + */ + wsrep_status_t (*abort_certification)(wsrep_t* wsrep, + wsrep_seqno_t bf_seqno, + wsrep_trx_id_t victim_trx, + wsrep_seqno_t* victim_seqno); + + /*! + * @brief Send a rollback fragment on behalf of trx + * + * @param wsrep provider handle + * @param trx transaction to be rolled back + * @param data data to append to the fragment + * + * @retval WSREP_OK rollback fragment sent successfully + */ + wsrep_status_t (*rollback)(wsrep_t* wsrep, + wsrep_trx_id_t trx, + const wsrep_buf_t* data); + + /*! + * @brief Appends a row reference to transaction writeset + * + * Both copy flag and key_type can be ignored by provider (key type + * interpreted as WSREP_KEY_EXCLUSIVE). + * + * @param wsrep provider handle + * @param ws_handle writeset handle + * @param keys array of keys + * @param count length of the array of keys + * @param type type of the key + * @param copy can be set to FALSE if keys persist through commit. + */ + wsrep_status_t (*append_key)(wsrep_t* wsrep, + wsrep_ws_handle_t* ws_handle, + const wsrep_key_t* keys, + size_t count, + enum wsrep_key_type type, + wsrep_bool_t copy); + + /*! + * @brief Appends data to transaction writeset + * + * This method can be called any time before certify() call and it appends + * a data buffer to the transaction writeset. + * Repeated calls of the method will result in direct buffer concatenation + * and all data will be passed as a single buffer to the apply callback. + * + * Both copy and unordered flags can be ignored by provider. + * + * @param wsrep provider handle + * @param ws_handle writeset handle + * @param data array of data buffers + * @param count buffer count + * @param type type of data + * @param copy can be set to FALSE if data persists through commit. + */ + wsrep_status_t (*append_data)(wsrep_t* wsrep, + wsrep_ws_handle_t* ws_handle, + const wsrep_buf_t* data, + size_t count, + enum wsrep_data_type type, + wsrep_bool_t copy); + + /*! + * @brief Blocks until the given GTID is committed + * + * This call will block the caller until the given GTID + * is guaranteed to be committed, or until a timeout occurs. + * The timeout value is given in parameter tout, if tout is -1, + * then the global causal read timeout applies. + * + * If no pointer upto is provided the call will block until + * causal ordering with all possible preceding writes in the + * cluster is guaranteed. + * + * If pointer to gtid is non-null, the call stores the global + * transaction ID of the last transaction which is guaranteed + * to be committed when the call returns. + * + * @param wsrep provider handle + * @param upto gtid to wait upto + * @param tout timeout in seconds + * -1 wait for global causal read timeout + * @param gtid location to store GTID + */ + wsrep_status_t (*sync_wait)(wsrep_t* wsrep, + wsrep_gtid_t* upto, + int tout, + wsrep_gtid_t* gtid); + + /*! + * @brief Returns the last committed gtid + * + * @param gtid location to store GTID + */ + wsrep_status_t (*last_committed_id)(wsrep_t* wsrep, + wsrep_gtid_t* gtid); + + /*! + * @brief Clears allocated connection context. + * + * Whenever a new connection ID is passed to wsrep provider through + * any of the API calls, a connection context is allocated for this + * connection. This call is to explicitly notify provider of connection + * closing. + * + * @param wsrep provider handle + * @param conn_id connection ID + * @param query the 'set database' query + * @param query_len length of query (does not end with 0) + */ + wsrep_status_t (*free_connection)(wsrep_t* wsrep, + wsrep_conn_id_t conn_id); + + /*! + * @brief Replicates a query and starts "total order isolation" section. + * + * Regular mode: + * + * Replicates the action spec and returns success code, which caller must + * check. Total order isolation continues until to_execute_end() is called. + * Regular "total order isolation" is achieved by calling to_execute_start() + * with WSREP_FLAG_TRX_START and WSREP_FLAG_TRX_END set. + * + * Two-phase mode: + * + * In this mode a query execution is split in two phases. The first phase is + * acquiring total order isolation to access critical section and the + * second phase is to release acquired resources in total order. + * + * To start the first phase the call is made with WSREP_FLAG_TRX_START set. + * The action is replicated and success code is returned. The total order + * isolation continues until to_execute_end() is called. However, the provider + * will keep the reference to the operation for conflict resolution purposes. + * + * The second phase is started with WSREP_FLAG_TRX_END set. Provider + * returns once it has achieved total ordering isolation for second phase. + * Total order isolation continues until to_execute_end() is called. + * All references to the operation are cleared by provider before + * call to to_execute_end() returns. + * + * @param wsrep provider handle + * @param conn_id connection ID + * @param keys array of keys + * @param keys_num length of the array of keys + * @param action action buffer array to be executed + * @param count action buffer count + * @param flags flags + * @param meta transaction meta data + * + * @retval WSREP_OK cluster commit succeeded + * @retval WSREP_CONN_FAIL must close client connection + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*to_execute_start)(wsrep_t* wsrep, + wsrep_conn_id_t conn_id, + const wsrep_key_t* keys, + size_t keys_num, + const wsrep_buf_t* action, + size_t count, + uint32_t flags, + wsrep_trx_meta_t* meta); + + /*! + * @brief Ends the total order isolation section. + * + * Marks the end of total order isolation. TO locks are freed + * and other transactions are free to commit from this point on. + * + * @param wsrep provider handle + * @param conn_id connection ID + * @param error error information about TOI operation (empty for no error) + * + * @retval WSREP_OK cluster commit succeeded + * @retval WSREP_CONN_FAIL must close client connection + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*to_execute_end)(wsrep_t* wsrep, + wsrep_conn_id_t conn_id, + const wsrep_buf_t* error); + + + /*! + * @brief Collects preordered replication events into a writeset. + * + * @param wsrep wsrep provider handle + * @param handle a handle associated with a given writeset + * @param data an array of data buffers. + * @param count length of data buffer array. + * @param copy whether provider needs to make a copy of events. + * + * @retval WSREP_OK cluster-wide commit succeeded + * @retval WSREP_TRX_FAIL operation failed (e.g. trx size exceeded limit) + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*preordered_collect) (wsrep_t* wsrep, + wsrep_po_handle_t* handle, + const wsrep_buf_t* data, + size_t count, + wsrep_bool_t copy); + + /*! + * @brief "Commits" preordered writeset to cluster. + * + * The contract is that the writeset will be committed in the same (partial) + * order this method was called. Frees resources associated with the writeset + * handle and reinitializes the handle. + * + * @param wsrep wsrep provider handle + * @param po_handle a handle associated with a given writeset + * @param source_id ID of the event producer, also serves as the partial order + * or stream ID - events with different source_ids won't be + * ordered with respect to each other. + * @param flags WSREP_FLAG_... flags + * @param pa_range the number of preceding events this event can be processed + * in parallel with. A value of 0 means strict serial + * processing. Note: commits always happen in wsrep order. + * @param commit 'true' to commit writeset to cluster (replicate) or + * 'false' to rollback (cancel) the writeset. + * + * @retval WSREP_OK cluster-wide commit succeeded + * @retval WSREP_TRX_FAIL operation failed (e.g. NON-PRIMARY component) + * @retval WSREP_NODE_FAIL must close all connections and reinit + */ + wsrep_status_t (*preordered_commit) (wsrep_t* wsrep, + wsrep_po_handle_t* handle, + const wsrep_uuid_t* source_id, + uint32_t flags, + int pa_range, + wsrep_bool_t commit); + + /*! + * @brief Signals to wsrep provider that state snapshot has been sent to + * joiner. + * + * @param wsrep provider handle + * @param state_id state ID + * @param rcode 0 or negative error code of the operation. + */ + wsrep_status_t (*sst_sent)(wsrep_t* wsrep, + const wsrep_gtid_t* state_id, + int rcode); + + /*! + * @brief Signals to wsrep provider that new state snapshot has been received. + * May deadlock if called from sst_prepare_cb. + * + * @param wsrep provider handle + * @param state_id state ID + * @param state initial state provided by SST donor + * @param rcode 0 or negative error code of the operation. + */ + wsrep_status_t (*sst_received)(wsrep_t* wsrep, + const wsrep_gtid_t* state_id, + const wsrep_buf_t* state, + int rcode); + + + /*! + * @brief Generate request for consistent snapshot. + * + * If successful, this call will generate internally SST request + * which in turn triggers calling SST donate callback on the nodes + * specified in donor_spec. If donor_spec is null, callback is + * called only locally. This call will block until sst_sent is called + * from callback. + * + * @param wsrep provider handle + * @param msg context message for SST donate callback + * @param msg_len length of context message + * @param donor_spec list of snapshot donors + */ + wsrep_status_t (*snapshot)(wsrep_t* wsrep, + const wsrep_buf_t* msg, + const char* donor_spec); + + /*! + * @brief Returns an array of status variables. + * Array is terminated by Null variable name. + * + * @param wsrep provider handle + * @return array of struct wsrep_status_var. + */ + struct wsrep_stats_var* (*stats_get) (wsrep_t* wsrep); + + /*! + * @brief Release resources that might be associated with the array. + * + * @param wsrep provider handle. + * @param var_array array returned by stats_get(). + */ + void (*stats_free) (wsrep_t* wsrep, struct wsrep_stats_var* var_array); + + /*! + * @brief Reset some stats variables to initial value, provider-dependent. + * + * @param wsrep provider handle. + */ + void (*stats_reset) (wsrep_t* wsrep); + + /*! + * @brief Pauses writeset applying/committing. + * + * @return global sequence number of the paused state or negative error code. + */ + wsrep_seqno_t (*pause) (wsrep_t* wsrep); + + /*! + * @brief Resumes writeset applying/committing. + */ + wsrep_status_t (*resume) (wsrep_t* wsrep); + + /*! + * @brief Desynchronize from cluster + * + * Effectively turns off flow control for this node, allowing it + * to fall behind the cluster. + */ + wsrep_status_t (*desync) (wsrep_t* wsrep); + + /*! + * @brief Request to resynchronize with cluster. + * + * Effectively turns on flow control. Asynchronous - actual synchronization + * event to be delivered via sync_cb. + */ + wsrep_status_t (*resync) (wsrep_t* wsrep); + + /*! + * @brief Acquire global named lock + * + * @param wsrep wsrep provider handle + * @param name lock name + * @param shared shared or exclusive lock + * @param owner 64-bit owner ID + * @param tout timeout in nanoseconds. + * 0 - return immediately, -1 wait forever. + * @return wsrep status or negative error code + * @retval -EDEADLK lock was already acquired by this thread + * @retval -EBUSY lock was busy + */ + wsrep_status_t (*lock) (wsrep_t* wsrep, + const char* name, wsrep_bool_t shared, + uint64_t owner, int64_t tout); + + /*! + * @brief Release global named lock + * + * @param wsrep wsrep provider handle + * @param name lock name + * @param owner 64-bit owner ID + * @return wsrep status or negative error code + * @retval -EPERM lock does not belong to this owner + */ + wsrep_status_t (*unlock) (wsrep_t* wsrep, const char* name, uint64_t owner); + + /*! + * @brief Check if global named lock is locked + * + * @param wsrep wsrep provider handle + * @param name lock name + * @param owner if not NULL will contain 64-bit owner ID + * @param node if not NULL will contain owner's node UUID + * @return true if lock is locked + */ + wsrep_bool_t (*is_locked) (wsrep_t* wsrep, const char* name, uint64_t* conn, + wsrep_uuid_t* node); + + /*! + * wsrep provider name + */ + const char* provider_name; + + /*! + * wsrep provider version + */ + const char* provider_version; + + /*! + * wsrep provider vendor name + */ + const char* provider_vendor; + + /*! + * @brief Frees allocated resources before unloading the library. + * @param wsrep provider handle + */ + void (*free)(wsrep_t* wsrep); + + void *dlh; //!< reserved for future use + void *ctx; //!< reserved for implementation private context +}; + + +/*! + * + * @brief Loads wsrep library + * + * @param spec path to wsrep library. If NULL or WSREP_NONE initializes dummy + * pass-through implementation. + * @param hptr wsrep handle + * @param log_cb callback to handle loader messages. Otherwise writes to stderr. + * + * @return zero on success, errno on failure + */ +int wsrep_load(const char* spec, wsrep_t** hptr, wsrep_log_cb_t log_cb); + +/*! + * @brief Unloads the wsrep library. The application must call + * wsrep->free() before unload to release library side resources. + * + * @param hptr wsrep handler pointer + */ +void wsrep_unload(wsrep_t* hptr); + +#ifdef __cplusplus +} +#endif + +#endif /* WSREP_H */ diff --git a/wsrep-lib/wsrep-API/v26/wsrep_dummy.c b/wsrep-lib/wsrep-API/v26/wsrep_dummy.c new file mode 100644 index 00000000..7bd441dc --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_dummy.c @@ -0,0 +1,462 @@ +/* Copyright (C) 2009-2010 Codership Oy <info@codersihp.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/*! @file Dummy wsrep API implementation. */ + +#include "wsrep_api.h" + +#include <errno.h> +#include <stdbool.h> +#include <string.h> + +/*! Dummy backend context. */ +typedef struct wsrep_dummy +{ + wsrep_log_cb_t log_fn; + char* options; +} wsrep_dummy_t; + +/* Get pointer to wsrep_dummy context from wsrep_t pointer */ +#define WSREP_DUMMY(_p) ((wsrep_dummy_t *) (_p)->ctx) + +/* Trace function usage a-la DBUG */ +#define WSREP_DBUG_ENTER(_w) do { \ + if (WSREP_DUMMY(_w)) { \ + if (WSREP_DUMMY(_w)->log_fn) \ + WSREP_DUMMY(_w)->log_fn(WSREP_LOG_DEBUG, __FUNCTION__); \ + } \ + } while (0) + + +static void dummy_free(wsrep_t *w) +{ + if (!w->ctx) return; + + WSREP_DBUG_ENTER(w); + if (WSREP_DUMMY(w)->options) { + free(WSREP_DUMMY(w)->options); + WSREP_DUMMY(w)->options = NULL; + } + free(w->ctx); + w->ctx = NULL; +} + +static wsrep_status_t dummy_init (wsrep_t* w, + const struct wsrep_init_args* args) +{ + WSREP_DUMMY(w)->log_fn = args->logger_cb; + WSREP_DBUG_ENTER(w); + if (args->options) { + WSREP_DUMMY(w)->options = strdup(args->options); + } + return WSREP_OK; +} + +static wsrep_cap_t dummy_capabilities (wsrep_t* w __attribute__((unused))) +{ + return 0; +} + +static wsrep_status_t dummy_options_set( + wsrep_t* w, + const char* conf) +{ + WSREP_DBUG_ENTER(w); + if (WSREP_DUMMY(w)->options) { + free(WSREP_DUMMY(w)->options); + WSREP_DUMMY(w)->options = NULL; + } + if (conf) { + WSREP_DUMMY(w)->options = strdup(conf); + } + return WSREP_OK; +} + +static char* dummy_options_get (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return strdup(WSREP_DUMMY(w)->options); +} + +static wsrep_status_t dummy_enc_set_key( + wsrep_t* w, + const wsrep_enc_key_t* key __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_connect( + wsrep_t* w, + const char* name __attribute__((unused)), + const char* url __attribute__((unused)), + const char* donor __attribute__((unused)), + wsrep_bool_t bootstrap __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_disconnect(wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_recv(wsrep_t* w, + void* recv_ctx __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_assign_read_view( + wsrep_t* w, + wsrep_ws_handle_t* ws_handle __attribute__((unused)), + const wsrep_gtid_t* rv __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_certify( + wsrep_t* w, + const wsrep_conn_id_t conn_id __attribute__((unused)), + wsrep_ws_handle_t* ws_handle __attribute__((unused)), + uint32_t flags __attribute__((unused)), + wsrep_trx_meta_t* meta __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_commit_order_enter( + wsrep_t* w, + const wsrep_ws_handle_t* ws_handle __attribute__((unused)), + const wsrep_trx_meta_t* meta __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_commit_order_leave( + wsrep_t* w, + const wsrep_ws_handle_t* ws_handle __attribute__((unused)), + const wsrep_trx_meta_t* meta __attribute__((unused)), + const wsrep_buf_t* error __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_release( + wsrep_t* w, + wsrep_ws_handle_t* ws_handle __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_replay_trx( + wsrep_t* w, + const wsrep_ws_handle_t* ws_handle __attribute__((unused)), + void* trx_ctx __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_abort_certification( + wsrep_t* w, + const wsrep_seqno_t bf_seqno __attribute__((unused)), + const wsrep_trx_id_t trx_id __attribute__((unused)), + wsrep_seqno_t *victim_seqno __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_rollback( + wsrep_t* w, + const wsrep_trx_id_t trx __attribute__((unused)), + const wsrep_buf_t* data __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_append_key( + wsrep_t* w, + wsrep_ws_handle_t* ws_handle __attribute__((unused)), + const wsrep_key_t* key __attribute__((unused)), + const size_t key_num __attribute__((unused)), + const wsrep_key_type_t key_type __attribute__((unused)), + const bool copy __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_append_data( + wsrep_t* w, + wsrep_ws_handle_t* ws_handle __attribute__((unused)), + const struct wsrep_buf* data __attribute__((unused)), + const size_t count __attribute__((unused)), + const wsrep_data_type_t type __attribute__((unused)), + const bool copy __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_sync_wait( + wsrep_t* w, + wsrep_gtid_t* upto __attribute__((unused)), + int tout __attribute__((unused)), + wsrep_gtid_t* gtid __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_last_committed_id( + wsrep_t* w, + wsrep_gtid_t* gtid __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_free_connection( + wsrep_t* w, + const wsrep_conn_id_t conn_id __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_to_execute_start( + wsrep_t* w, + const wsrep_conn_id_t conn_id __attribute__((unused)), + const wsrep_key_t* key __attribute__((unused)), + const size_t key_num __attribute__((unused)), + const struct wsrep_buf* data __attribute__((unused)), + const size_t count __attribute__((unused)), + const uint32_t flags __attribute__((unused)), + wsrep_trx_meta_t* meta __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_to_execute_end( + wsrep_t* w, + const wsrep_conn_id_t conn_id __attribute__((unused)), + const wsrep_buf_t* err __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_preordered_collect( + wsrep_t* w, + wsrep_po_handle_t* handle __attribute__((unused)), + const struct wsrep_buf* data __attribute__((unused)), + size_t count __attribute__((unused)), + wsrep_bool_t copy __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_preordered_commit( + wsrep_t* w, + wsrep_po_handle_t* handle __attribute__((unused)), + const wsrep_uuid_t* source_id __attribute__((unused)), + uint32_t flags __attribute__((unused)), + int pa_range __attribute__((unused)), + wsrep_bool_t commit __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_sst_sent( + wsrep_t* w, + const wsrep_gtid_t* state_id __attribute__((unused)), + const int rcode __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_sst_received( + wsrep_t* w, + const wsrep_gtid_t* state_id __attribute__((unused)), + const wsrep_buf_t* state __attribute__((unused)), + const int rcode __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_snapshot( + wsrep_t* w, + const wsrep_buf_t* msg __attribute__((unused)), + const char* donor_spec __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static struct wsrep_stats_var dummy_stats[] = { + { NULL, WSREP_VAR_STRING, { 0 } } +}; + +static struct wsrep_stats_var* dummy_stats_get (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return dummy_stats; +} + +static void dummy_stats_free ( + wsrep_t* w, + struct wsrep_stats_var* stats __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); +} + +static void dummy_stats_reset (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); +} + +static wsrep_seqno_t dummy_pause (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return -ENOSYS; +} + +static wsrep_status_t dummy_resume (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_desync (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return WSREP_NOT_IMPLEMENTED; +} + +static wsrep_status_t dummy_resync (wsrep_t* w) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static wsrep_status_t dummy_lock (wsrep_t* w, + const char* s __attribute__((unused)), + bool r __attribute__((unused)), + uint64_t o __attribute__((unused)), + int64_t t __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_NOT_IMPLEMENTED; +} + +static wsrep_status_t dummy_unlock (wsrep_t* w, + const char* s __attribute__((unused)), + uint64_t o __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return WSREP_OK; +} + +static bool dummy_is_locked (wsrep_t* w, + const char* s __attribute__((unused)), + uint64_t* o __attribute__((unused)), + wsrep_uuid_t* t __attribute__((unused))) +{ + WSREP_DBUG_ENTER(w); + return false; +} + +static wsrep_t dummy_iface = { + WSREP_INTERFACE_VERSION, + &dummy_init, + &dummy_capabilities, + &dummy_options_set, + &dummy_options_get, + &dummy_enc_set_key, + &dummy_connect, + &dummy_disconnect, + &dummy_recv, + &dummy_assign_read_view, + &dummy_certify, + &dummy_commit_order_enter, + &dummy_commit_order_leave, + &dummy_release, + &dummy_replay_trx, + &dummy_abort_certification, + &dummy_rollback, + &dummy_append_key, + &dummy_append_data, + &dummy_sync_wait, + &dummy_last_committed_id, + &dummy_free_connection, + &dummy_to_execute_start, + &dummy_to_execute_end, + &dummy_preordered_collect, + &dummy_preordered_commit, + &dummy_sst_sent, + &dummy_sst_received, + &dummy_snapshot, + &dummy_stats_get, + &dummy_stats_free, + &dummy_stats_reset, + &dummy_pause, + &dummy_resume, + &dummy_desync, + &dummy_resync, + &dummy_lock, + &dummy_unlock, + &dummy_is_locked, + WSREP_NONE, + WSREP_INTERFACE_VERSION, + "Codership Oy <info@codership.com>", + &dummy_free, + NULL, + NULL +}; + +int wsrep_dummy_loader(wsrep_t* w) +{ + if (!w) + return EINVAL; + + *w = dummy_iface; + + // allocate private context + if (!(w->ctx = malloc(sizeof(wsrep_dummy_t)))) + return ENOMEM; + + // initialize private context + WSREP_DUMMY(w)->log_fn = NULL; + WSREP_DUMMY(w)->options = NULL; + + return 0; +} diff --git a/wsrep-lib/wsrep-API/v26/wsrep_event_service.h b/wsrep-lib/wsrep-API/v26/wsrep_event_service.h new file mode 100644 index 00000000..2d0d16ea --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_event_service.h @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2021 Codership Oy <info@codership.com> + * + * This file is part of wsrep-API. + * + * Wsrep-API is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-API is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-API. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file wsrep_event_service.h + * + * This file defines interface for various unordered events generated by the + * cluster or the provider. + * + * An event has a name and a payload, both are null-terminated strings. + * (It is intended that payload is a JSON encoded structure). + * The name serves to distinguish the events to pass them to respective + * handlers. + * + * The provider which is capable of using the service interface v1 must + * export the following functions. + * + * int wsrep_init_event_service_v1(wsrep_event_service_v1_t*) + * void wsrep_deinit_event_service_v1() + * + * which can be probed by the application. + * + * The application must initialize the service via above init function + * before the provider is initialized via wsrep->init(). The deinit + * function must be called after the provider side resources have been + * released via wsrep->free(). + */ + +#ifndef WSREP_EVENT_SERVICE_H +#define WSREP_EVENT_SERVICE_H + +#include "wsrep_api.h" + +#include <sys/types.h> /* posix size_t */ + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + +/** + * Type tag for application defined event processing context. + * + * Application may pass pointer to the context when initializing + * the event service. This pointer is passed a first parameter for + * each service call. + */ +typedef struct wsrep_event_context wsrep_event_context_t; + + +/** + * Process an event + * + * @param name event name + * @param value JSON enconded event value + * + * @return void, it is up to the receiver to decide what to do about + * possible error. + */ +typedef void (*wsrep_event_cb_t)(wsrep_event_context_t*, + const char* name, const char* value); + + +/** + * Event service struct. + * + * A pointer to this struct must be passed to the call to + * wsrep_init_event_service_v1. + * + * The application must provide implementation to all functions defined + * in this struct. + */ +typedef struct wsrep_event_service_v1_st +{ + /* Event receiver callback */ + wsrep_event_cb_t event_cb; + /* Pointer to application defined event context. */ + wsrep_event_context_t* context; +} wsrep_event_service_v1_t; + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + + +#define WSREP_EVENT_SERVICE_INIT_FUNC_V1 "wsrep_init_event_service_v1" +#define WSREP_EVENT_SERVICE_DEINIT_FUNC_V1 "wsrep_deinit_event_service_v1" + +#endif /* WSREP_EVENT_SERVICE_H */ + diff --git a/wsrep-lib/wsrep-API/v26/wsrep_gtid.c b/wsrep-lib/wsrep-API/v26/wsrep_gtid.c new file mode 100644 index 00000000..2b6dfb90 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_gtid.c @@ -0,0 +1,77 @@ +/* Copyright (C) 2013 Codership Oy <info@codersihp.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/*! @file Helper functions to deal with GTID string representations */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <inttypes.h> +#include <limits.h> + +#include "wsrep_api.h" + +/*! + * Read GTID from string + * @return length of GTID string representation or -EINVAL in case of error + */ +int +wsrep_gtid_scan(const char* str, size_t str_len, wsrep_gtid_t* gtid) +{ + int offset; + char* endptr; + if (str_len > INT_MAX) return -EINVAL; + + if ((offset = wsrep_uuid_scan(str, str_len, >id->uuid)) > 0 && + offset < (int)str_len && str[offset] == ':') { + ++offset; + if (offset < (int)str_len) + { + errno = 0; + gtid->seqno = strtoll(str + offset, &endptr, 0); + + if (errno == 0) { + return (int)(endptr - str); + } + } + } + *gtid = WSREP_GTID_UNDEFINED; + return -EINVAL; +} + +/*! + * Write GTID to string + * @return length of GTID string representation or -EMSGSIZE if string is too + * short + */ +int +wsrep_gtid_print(const wsrep_gtid_t* gtid, char* str, size_t str_len) +{ + int offset, ret; + if (str_len > INT_MAX) return -EINVAL; + + if ((offset = wsrep_uuid_print(>id->uuid, str, str_len)) > 0) + { + ret = snprintf(str + offset, (size_t)((int)str_len - offset), + ":%" PRId64, gtid->seqno); + if (ret <= ((int)str_len - offset)) { + return (offset + ret); + } + + } + + return -EMSGSIZE; +} diff --git a/wsrep-lib/wsrep-API/v26/wsrep_loader.c b/wsrep-lib/wsrep-API/v26/wsrep_loader.c new file mode 100644 index 00000000..cfa1984f --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_loader.c @@ -0,0 +1,239 @@ +/* Copyright (C) 2009-2011 Codership Oy <info@codersihp.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/*! @file wsrep implementation loader */ + +#include <dlfcn.h> +#include <errno.h> +#include <string.h> +#include <stdio.h> + +#include "wsrep_api.h" + +// Logging stuff for the loader +static const char* log_levels[] = {"FATAL", "ERROR", "WARN", "INFO", "DEBUG"}; + +static void default_logger (wsrep_log_level_t lvl, const char* msg) +{ + fprintf (stderr, "wsrep loader: [%s] %s\n", log_levels[lvl], msg); +} + +static wsrep_log_cb_t logger = default_logger; + +/************************************************************************** + * Library loader + **************************************************************************/ + +static int wsrep_check_iface_version(const char* found, const char* iface_ver) +{ + const size_t msg_len = 128; + char msg[128]; + + if (strcmp(found, iface_ver)) { + snprintf (msg, msg_len, + "provider interface version mismatch: need '%s', found '%s'", + iface_ver, found); + logger (WSREP_LOG_ERROR, msg); + return EINVAL; + } + + return 0; +} + +static int verify(const wsrep_t *wh, const char *iface_ver) +{ + char msg[128]; + const size_t msg_len = sizeof(msg); + +#define VERIFY(_p) if (!(_p)) { \ + snprintf(msg, msg_len, "wsrep_load(): verify(): %s\n", # _p); \ + logger (WSREP_LOG_ERROR, msg); \ + return EINVAL; \ + } + + VERIFY(wh); + VERIFY(wh->version); + + if (wsrep_check_iface_version(wh->version, iface_ver)) + return EINVAL; + + VERIFY(wh->init); + VERIFY(wh->options_set); + VERIFY(wh->options_get); + VERIFY(wh->enc_set_key); + VERIFY(wh->connect); + VERIFY(wh->disconnect); + VERIFY(wh->recv); + VERIFY(wh->assign_read_view); + VERIFY(wh->certify); + VERIFY(wh->commit_order_enter); + VERIFY(wh->commit_order_leave); + VERIFY(wh->release); + VERIFY(wh->replay_trx); + VERIFY(wh->abort_certification); + VERIFY(wh->append_key); + VERIFY(wh->append_data); + VERIFY(wh->free_connection); + VERIFY(wh->to_execute_start); + VERIFY(wh->to_execute_end); + VERIFY(wh->preordered_collect); + VERIFY(wh->preordered_commit); + VERIFY(wh->sst_sent); + VERIFY(wh->sst_received); + VERIFY(wh->stats_get); + VERIFY(wh->stats_free); + VERIFY(wh->stats_reset); + VERIFY(wh->pause); + VERIFY(wh->resume); + VERIFY(wh->desync); + VERIFY(wh->resync); + VERIFY(wh->lock); + VERIFY(wh->unlock); + VERIFY(wh->is_locked); + VERIFY(wh->provider_name); + VERIFY(wh->provider_version); + VERIFY(wh->provider_vendor); + VERIFY(wh->free); + return 0; +} + +typedef int (*wsrep_loader_fun)(wsrep_t*); + +static wsrep_loader_fun wsrep_dlf(void *dlh, const char *sym) +{ + union { + wsrep_loader_fun dlfun; + void *obj; + } alias; + alias.obj = dlsym(dlh, sym); + return alias.dlfun; +} + +static int wsrep_check_version_symbol(void *dlh) +{ + char** dlversion = NULL; + dlversion = (char**) dlsym(dlh, "wsrep_interface_version"); + if (dlversion == NULL) + return 0; + return wsrep_check_iface_version(*dlversion, WSREP_INTERFACE_VERSION); +} + +extern int wsrep_dummy_loader(wsrep_t *w); + +int wsrep_load(const char *spec, wsrep_t **hptr, wsrep_log_cb_t log_cb) +{ + int ret = 0; + void *dlh = NULL; + wsrep_loader_fun dlfun; + char msg[1024]; + const size_t msg_len = sizeof(msg) - 1; + msg[msg_len] = 0; + + if (NULL != log_cb) + logger = log_cb; + + if (!(spec && hptr)) + return EINVAL; + + snprintf (msg, msg_len, + "wsrep_load(): loading provider library '%s'", spec); + logger (WSREP_LOG_INFO, msg); + + if (!(*hptr = malloc(sizeof(wsrep_t)))) { + logger (WSREP_LOG_FATAL, "wsrep_load(): out of memory"); + return ENOMEM; + } + + if (!spec || strcmp(spec, WSREP_NONE) == 0) { + if ((ret = wsrep_dummy_loader(*hptr)) != 0) { + free (*hptr); + *hptr = NULL; + } + return ret; + } + + if (!(dlh = dlopen(spec, RTLD_NOW | RTLD_LOCAL))) { + snprintf(msg, msg_len, "wsrep_load(): dlopen(): %s", dlerror()); + logger (WSREP_LOG_ERROR, msg); + ret = EINVAL; + goto out; + } + + if (!(dlfun = wsrep_dlf(dlh, "wsrep_loader"))) { + ret = EINVAL; + goto out; + } + + if (wsrep_check_version_symbol(dlh) != 0) { + ret = EINVAL; + goto out; + } + + if ((ret = (*dlfun)(*hptr)) != 0) { + snprintf(msg, msg_len, "wsrep_load(): loader failed: %s", + strerror(ret)); + logger (WSREP_LOG_ERROR, msg); + goto out; + } + + if ((ret = verify(*hptr, WSREP_INTERFACE_VERSION)) != 0) { + snprintf (msg, msg_len, + "wsrep_load(): interface version mismatch: my version %s, " + "provider version %s", WSREP_INTERFACE_VERSION, + (*hptr)->version); + logger (WSREP_LOG_ERROR, msg); + goto out; + } + + (*hptr)->dlh = dlh; + +out: + if (ret != 0) { + if (dlh) dlclose(dlh); + free(*hptr); + *hptr = NULL; + } else { + snprintf (msg, msg_len, + "wsrep_load(): %s %s by %s loaded successfully.", + (*hptr)->provider_name, (*hptr)->provider_version, + (*hptr)->provider_vendor); + logger (WSREP_LOG_INFO, msg); + } + + return ret; +} + +void wsrep_unload(wsrep_t *hptr) +{ + if (!hptr) { + logger (WSREP_LOG_WARN, "wsrep_unload(): null pointer."); + } else { + if (hptr->free) + hptr->free(hptr); + if (hptr->dlh) + { + int err; + if ((err = dlclose(hptr->dlh))) + { + char msg[1024]; + snprintf(msg, sizeof(msg), "dlclose(): %s", dlerror()); + msg[sizeof(msg) - 1] = '\0'; + logger(WSREP_LOG_WARN, msg); + } + } + free(hptr); + } +} diff --git a/wsrep-lib/wsrep-API/v26/wsrep_membership_service.h b/wsrep-lib/wsrep-API/v26/wsrep_membership_service.h new file mode 100644 index 00000000..b6053a2f --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_membership_service.h @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-API. + * + * Wsrep-API is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-API is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-API. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file wsrep_membership_service.h + * + * This file defines interface for quering the immediate membership and + * members' states of the current configuration. The information is provided + * OUT OF ORDER to facilitate administrative tasks. + * + * The provider which is capable of using the service interface v1 must + * export the following functions. + * + * int wsrep_init_membership_service_v1(struct wsrep_membership_service_v1*) + * void wsrep_deinit_membership_service_v1() + * + * which can be probed by the application. + * + * The application must initialize the service via above init function + * before the provider is initialized via wsrep->init(). The deinit + * function must be called after the provider side resources have been + * released via wsrep->free(). + */ + +#ifndef WSREP_MEMBERSHIP_SERVICE_H +#define WSREP_MEMBERSHIP_SERVICE_H + +#include "wsrep_api.h" + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + +/** + * Member info structure extended to contain member state + */ +struct wsrep_member_info_ext +{ + struct wsrep_member_info base; + wsrep_seqno_t last_committed; + enum wsrep_member_status status; +}; + +/** + * Extended membership structure + */ +struct wsrep_membership +{ + /** + * Epoch of the membership data (last time it was updated) + */ + wsrep_uuid_t group_uuid; + /** + * Sequence number of the last received (not processed) action + */ + wsrep_seqno_t last_received; + /** + * When the members' data was last updated + */ + wsrep_seqno_t updated; + /** + * Current group state + */ + enum wsrep_view_status state; + /** + * Number of members in the array + */ + size_t num; + /** + * Membership array + */ + struct wsrep_member_info_ext members[1]; +}; + +/** + * Memory allocation callback for wsrep_get_mmebership_fn() below + * + * @param size of buffer to allocate + * @return allocated buffer pointer or NULL in case of error + */ +typedef void* (*wsrep_allocator_cb) (size_t size); + +/** + * Query membership + * + * @param wsrep provider handle + * @param allocator to use for wsrep_membership struct allocation + * @param membership pointer to pointer to the memebrship structure. + * The structure is allocated by provider and must be freed + * by the caller. + * @return error code of the call + */ +typedef wsrep_status_t (*wsrep_get_membership_fn) ( + wsrep_t* wsrep, + wsrep_allocator_cb allocator, + struct wsrep_membership** membership); + +/** + * Membership service struct. + * Returned by WSREP_MEMBERSHIP_SERVICE_INIT_FUNC_V1 + */ +struct wsrep_membership_service_v1 +{ + wsrep_get_membership_fn get_membership; +}; + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +typedef +wsrep_status_t +(*wsrep_membership_service_v1_init_fn) (struct wsrep_membership_service_v1*); +typedef +void +(*wsrep_membership_service_v1_deinit_fn)(void); + +/** must be exported by the provider */ +#define WSREP_MEMBERSHIP_SERVICE_V1_INIT_FN "wsrep_init_membership_service_v1" +#define WSREP_MEMBERSHIP_SERVICE_V1_DEINIT_FN "wsrep_deinit_membership_service_v1" + +#endif /* WSREP_MEMBERSHIP_SERVICE_H */ diff --git a/wsrep-lib/wsrep-API/v26/wsrep_thread_service.h b/wsrep-lib/wsrep-API/v26/wsrep_thread_service.h new file mode 100644 index 00000000..956751aa --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_thread_service.h @@ -0,0 +1,355 @@ +/* + * Copyright (C) 2019 Codership Oy <info@codership.com> + * + * This file is part of wsrep-API. + * + * Wsrep-API is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-API is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-API. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file wsrep_thread_service.h + * + * Service interface for threads, mutexes and condition variables. + * The application which may provide callbacks to routines which will + * be used to manage lifetime and use threads and sycnronization primitives. + * + * The type tags and interface methods are loosely modeled after POSIX + * threading interface. + * + * The application must either none or all of the callbacks defined in + * wsrep_thread_service structure which is defined below. + * + * The error codes returned by the callbacks are generally assumed to + * the system error numbers defined in errno.h unless stated otherwise. + * + * The provider must implement and export the following functions + * to provide initialization point for the service implementation: + * + * Version 1: + * int wsrep_init_thread_service_v1(wsrep_thread_service_v1_t*) + * void wsrep_deinit_thread_service_v1(). + * + * The application defined implementation must be initialized before + * calling the provider initialization function via wsrep->init(). The + * deinitialization must be done via deinit function after the + * provider side resources have been released via wsrep->free(). + */ + +#ifndef WSREP_THREAD_SERVICE_H +#define WSREP_THREAD_SERVICE_H + +#include <stddef.h> /* size_t */ + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + + /* Forward declarations */ + struct timespec; + struct sched_param; + + /** Thread type tags */ + typedef struct wsrep_thread_key_st wsrep_thread_key_t; + typedef struct wsrep_thread_st wsrep_thread_t; + /** Mutex type tags */ + typedef struct wsrep_mutex_key_st wsrep_mutex_key_t; + typedef struct wsrep_mutex_st wsrep_mutex_t; + /** Condition variable tags */ + typedef struct wsrep_cond_key_st wsrep_cond_key_t; + typedef struct wsrep_cond_st wsrep_cond_t; + + /** + * Create key for a thread with a name. This key object will be passed + * to thread creation and destrunction notification callbacks. + * + * @param name Name of the thread. + */ + typedef const wsrep_thread_key_t* (*wsrep_thread_key_create_cb_t)( + const char* name); + + /** + * Create a new thread. + * + * @param[out] thread Newly allocated thread. + * @param key Key created by wsrep_thread_key_create_cb_t + * @param start_fn Pointer to start routine + * @param arg Argument for start_fn + * + * @return Zero in case of success, non-zero error code in case of failure. + */ + typedef int (*wsrep_thread_create_cb_t)(const wsrep_thread_key_t* key, + wsrep_thread_t** thread, + void* (*start_fn)(void*), + void* arg); + + /** + * Detach a thread. + * + * @param thread Thread to be detached. + * + * @return Zero in case of error, non-zero error code in case of failure. + */ + typedef int (*wsrep_thread_detach_cb_t)(wsrep_thread_t* thread); + + /** + * Compare two threads for equality. + * + * @params t1, t2 Threads to be compared. + * + * @return Non-zero value if threads are equal, zero otherwise. + */ + typedef int (*wsrep_thread_equal_cb_t)(wsrep_thread_t* t1, + wsrep_thread_t* t2); + + /** + * Terminate the calling thread. + * + * @param thread Pointer to thread. + * @param retval Pointer to return value. + * + * This function does not return. + */ + typedef void __attribute__((noreturn)) (*wsrep_thread_exit_cb_t)( + wsrep_thread_t* thread, void* retval); + + /** + * Join a thread. Trying to join detached thread may cause undefined + * behavior. + * + * @param thread Thread to be joined. + * @param[out] retval Return value from the thread wthat was joined. + * + * @return Zero in case of success, non-zero error code in case of error. + */ + typedef int (*wsrep_thread_join_cb_t)(wsrep_thread_t* thread, + void** retval); + + /** + * Return a pointer to the wsrep_thread_t of the calling thread. + * + * @return Pointer to wsrep_thread_t associated with current thread. + */ + typedef wsrep_thread_t* (*wsrep_thread_self_cb_t)(void); + + /** + * Set the scheduling policy for the thread. + * + * @param thread Thread for which sceduing policy should be changed. + * @param policy New scheduling policy for the thread. + * @param param New scheduling parameters for the thread. + */ + typedef int (*wsrep_thread_setschedparam_cb_t)( + wsrep_thread_t* thread, int policy, const struct sched_param* param); + + /** + * Get the current scheduling policy for the thread. + * + * @param thread Thread. + * @param policy Pointer to location where the scheduling policy will + * will be stored in. + * @param Param Pointer to location where the current scheduling + * parameters will be stored. + */ + typedef int (*wsrep_thread_getschedparam_cb_t)(wsrep_thread_t* thread, + int* policy, + struct sched_param* param); + /** + * Create key for a mutex with a name. This key object must be passed + * to mutex creation callback. + * + * @param name Name of the mutex. + * + * @return Const pointer to mutex key. + */ + typedef const wsrep_mutex_key_t* (*wsrep_mutex_key_create_cb_t)( + const char* name); + + /** + * Create a mutex. + * + * @param key Mutex key obtained via wsrep_mutex_key_create_cb call. + * @param memblock Optional memory block allocated by the provider + * which can be used by the implementation to store + * the mutex. + * @param memblock_size Size of the optional memory block. + * + * @return Pointer to wsrep_mutex_t object or NULL in case of failure. + */ + typedef wsrep_mutex_t* (*wsrep_mutex_init_cb_t)( + const wsrep_mutex_key_t* key, void* memblock, size_t memblock_size); + + /** + * Destroy a mutex. This call must consume the mutex object. + * + * @param mutex Mutex to be destroyed. + */ + typedef int (*wsrep_mutex_destroy_cb_t)(wsrep_mutex_t* mutex); + + /** + * Lock a mutex. + * + * @param mutex Mutex to be locked. + * + * @return Zero on success, non-zero error code on error. + */ + typedef int (*wsrep_mutex_lock_cb_t)(wsrep_mutex_t* mutex); + + /** + * Try to lock a mutex. + * + * @param Mutex to be locked. + * + * @return Zero if mutex was successfully locked. + * @return EBUSY if the mutex could not be acquired because it was already + * locked. + * @return Non-zero error code on any other error. + */ + typedef int (*wsrep_mutex_trylock_cb_t)(wsrep_mutex_t* mutex); + + /** + * Unlock a mutex. + * + * @param mutex Mutex to be unlocked. + * + * @return Zero on success, non-zero on error. + */ + typedef int (*wsrep_mutex_unlock_cb_t)(wsrep_mutex_t* mutex); + + /** + * Create key for a condition variable with a name. This key + * must be passed to wsrep_cond_create_cb when creating a new + * condition variable. + * + * @param name Name of the condition variable. + * + * @return Allocated key object. + */ + typedef const wsrep_cond_key_t* (*wsrep_cond_key_create_cb_t)( + const char* name); + + /** + * Create a new condition variable. + * + * @param key Const pointer to key object created by + * wsrep_cond_key_create_cb. + * @param memblock Optional memory block allocated by the provider + * which can be used by the implementation to store + * the mutex. + * @param memblock_size Size of the optional memory block. + * + * @return Pointer to new condition variable. + */ + typedef wsrep_cond_t* (*wsrep_cond_init_cb_t)(const wsrep_cond_key_t* key, + void* memblock, + size_t memblock_size); + + /** + * Destroy a condition variable. This call must consume the condition + * variable object. + * + * @param cond Condition variable to be destroyed. + * + * @return Zero on success, non-zero on error. + */ + typedef int (*wsrep_cond_destroy_cb_t)(wsrep_cond_t* cond); + + /** + * Wait for condition. + * + * @param cond Condition variable to wait for. + * @param mutex Mutex associated to the condition variable. The mutex + * may be unlocked for the duration of the wait. + * + * @return Zero on success, non-zero on error. + */ + typedef int (*wsrep_cond_wait_cb_t)(wsrep_cond_t* cond, + wsrep_mutex_t* mutex); + + /** + * Perform timed wait on condition. + * + * @param cond Condition to wait for. + * @param mutex Mutex associated to the condition variable. The mutex + * may be unlocked for the duration of the wait. + * @param wait_until System time to wait until before returning from the + * the timed wait. + * + * @return Zero on success. + * @return ETIMEDOUT if the time specified by wait_until has passed. + * @return Non-zero error code on other error. + */ + typedef int (*wsrep_cond_timedwait_cb_t)(wsrep_cond_t* cond, + wsrep_mutex_t* mutex, + const struct timespec* wait_until); + + /** + * Signal a condition variable. This will wake up at least one of + * the threads which is waiting for the condition. + * + * @param cond Condition variable to signal. + * + * @return Zero on success, non-zero on failure. + */ + typedef int (*wsrep_cond_signal_cb_t)(wsrep_cond_t* cond); + + /** + * Broadcast a signal to condition variable. This will wake up + * all the threads which are currently waiting on condition variable. + * + * @param cond Condition variable to broadcast the signal to. + * + * @return Zero on success, non-zero on failure. + */ + typedef int (*wsrep_cond_broadcast_cb_t)(wsrep_cond_t* cond); + + typedef struct wsrep_thread_service_v1_st + { + /* Threads */ + wsrep_thread_key_create_cb_t thread_key_create_cb; + wsrep_thread_create_cb_t thread_create_cb; + wsrep_thread_detach_cb_t thread_detach_cb; + wsrep_thread_equal_cb_t thread_equal_cb; + wsrep_thread_exit_cb_t thread_exit_cb; + wsrep_thread_join_cb_t thread_join_cb; + wsrep_thread_self_cb_t thread_self_cb; + wsrep_thread_setschedparam_cb_t thread_setschedparam_cb; + wsrep_thread_getschedparam_cb_t thread_getschedparam_cb; + /* Mutexes */ + wsrep_mutex_key_create_cb_t mutex_key_create_cb; + wsrep_mutex_init_cb_t mutex_init_cb; + wsrep_mutex_destroy_cb_t mutex_destroy_cb; + wsrep_mutex_lock_cb_t mutex_lock_cb; + wsrep_mutex_trylock_cb_t mutex_trylock_cb; + wsrep_mutex_unlock_cb_t mutex_unlock_cb; + /* Condition variables */ + wsrep_cond_key_create_cb_t cond_key_create_cb; + wsrep_cond_init_cb_t cond_init_cb; + wsrep_cond_destroy_cb_t cond_destroy_cb; + wsrep_cond_wait_cb_t cond_wait_cb; + wsrep_cond_timedwait_cb_t cond_timedwait_cb; + wsrep_cond_signal_cb_t cond_signal_cb; + wsrep_cond_broadcast_cb_t cond_broadcast_cb; + } wsrep_thread_service_v1_t; + +#ifdef __cplusplus +} + +#define WSREP_THREAD_SERVICE_INIT_FUNC_V1 "wsrep_init_thread_service_v1" +#define WSREP_THREAD_SERVICE_DEINIT_FUNC_V1 "wsrep_deinit_thread_service_v1" + +/* For backwards compatibility. */ +#define WSREP_THREAD_SERVICE_INIT_FUNC WSREP_THREAD_SERVICE_INIT_FUNC_V1 + +#endif /* __cplusplus */ +#endif /* WSREP_THREAD_SERVICE_H */ diff --git a/wsrep-lib/wsrep-API/v26/wsrep_tls_service.h b/wsrep-lib/wsrep-API/v26/wsrep_tls_service.h new file mode 100644 index 00000000..c632f4aa --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_tls_service.h @@ -0,0 +1,326 @@ +/* + * Copyright (C) 2020 Codership Oy <info@codership.com> + * + * This file is part of wsrep-API. + * + * Wsrep-API is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-API is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-API. If not, see <https://www.gnu.org/licenses/>. + */ + +/** @file wsrep_tls_service.h + * + * This file defines interface for TLS services provided by the application, + * used by the provider. + * + * In order to support both synchronous and asynchronous IO operations, + * the interface is designed to work with sockets in both blocking + * and non-blockig mode. + * + * The provider is in charge of opening and closing file + * descriptors and connecting transport. After the connection has + * been established, all further IO operations will be delegated + * to the TLS service implementation which is provided by the application. + * + * The provider which is capable of using the service interface v1 must + * export the following functions. + * + * int wsrep_init_tls_service_v1(wsrep_tls_service_v1_t*) + * void wsrep_deinit_tls_service_v1() + * + * which can be probed by the application. + * + * The application must initialize the service via above init function + * before the provider is initialized via wsrep->init(). The deinit + * function must be called after the provider side resources have been + * released via wsrep->free(). + */ + +#ifndef WSREP_TLS_SERVICE_H +#define WSREP_TLS_SERVICE_H + +#include <sys/types.h> /* posix size_t */ + +#ifdef __cplusplus +extern "C" +{ +#endif /* __cplusplus */ + +/** + * Type tag for application defined TLS context. + * + * Application may pass pointer to the context when initializing + * TLS service. This pointer is passed a first parameter for + * each service call. + */ +typedef struct wsrep_tls_context wsrep_tls_context_t; + +/** + * TLS stream structure. + */ +typedef struct wsrep_tls_stream_st +{ + /** + * File descriptor corresponding to the stream. The provider is + * responsible in opening and closing the socket. + */ + int fd; + /** + * Opaque pointer reserved for application use. + */ + void* opaque; +} wsrep_tls_stream_t; + +/** + * Enumeration for return codes. + */ +enum wsrep_tls_result +{ + /** + * The operation completed successfully, no further actions + * are necessary. + */ + wsrep_tls_result_success = 0, + /** + * The operation completed successfully, but the application side wants + * to make further reads. The provider must wait until the stream + * becomes readable and then try the same operation again. + */ + wsrep_tls_result_want_read, + /** + * The operation completed successfully, but the application side wants + * to make further writes. The provider must wait until the stream + * becomes writable and then try the same operation again. + */ + wsrep_tls_result_want_write, + /** + * End of file was read from the stream. This result is needed to + * make difference between graceful stream shutdown and zero length + * reads which result from errors. + */ + wsrep_tls_result_eof, + /** + * An error occurred. The specific error reason must be + * queried with wsrep_tls_stream_get_error_number and + * wsrep_tls_stream_get_error_category. + */ + wsrep_tls_result_error +}; + +/** + * Initialize a new TLS stream. + * + * Initialize the stream for IO operations. During this call the + * application must set up all of the data structures needed for + * IO, but must not do any reads or writes into the stream yet. + * + * @param stream TLS stream to be initialized. + * + * @return Zero on success, system error number on error. + */ +typedef int (*wsrep_tls_stream_init_t)(wsrep_tls_context_t*, + wsrep_tls_stream_t* stream); + +/** + * Deinitialize the TLS stream. + * + * Deinitialize the TLS stream and free all allocated resources. + * Note that this function must not close the socket file descriptor + * associated the the stream. + * + * @param stream Stream to be deinitialized. + */ +typedef void (*wsrep_tls_stream_deinit_t)(wsrep_tls_context_t*, + wsrep_tls_stream_t* stream); + +/** + * Get error number of the last stream error. The error numbers are + * defined by the application and must be integral type. By the convention + * zero value must denote success. + * + * For managing errors other than system errors, the application may + * provide several error categories via wsrep_tls_stream_get_error_category_t. + * + * @param stream TLS stream to get the last error from. + * + * @return Error number. + */ +typedef int (*wsrep_tls_stream_get_error_number_t)( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream); + +/** + * Get the error category of the last stream error. + * + * The category is represented via a const void pointer to the provider. + * If the category is NULL pointer, the error number is assumed to be + * system error. + * + * @param stream Stream to get last error category from. + * + * @return Pointer to error category. + */ +typedef const void* (*wsrep_tls_stream_get_error_category_t)( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream); + +/** + * Return human readable error message by error number and error + * category. + * + * The message string returned by the application must contain only + * printable characters and must be null terminated. + * + * @param error_number Error number returned by + * wsrep_tls_stream_get_error_number_t. + * @param category Error category returned by + * wsrep_tls_stream_get_error_category_t. + * + * @return Human readable message string. + */ +typedef const char* (*wsrep_tls_error_message_get_t)( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream, + int error_number, const void* category); + +/** + * Initiate TLS client side handshake. This function is called for the + * stream sockets which have been connected by the provider. + * + * If the stream socket is in non-blocking mode, the call should return + * immediately with appropriate result indicating if more actions are needed + * in the case the operation would block. The provider will call this function + * again until either a success or an error is returned. + * + * @param stream TLS stream. + * + * @return Enum wsrep_tls_result. + */ +typedef enum wsrep_tls_result (*wsrep_tls_stream_client_handshake_t)( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream); + +/** + * Initiate TLS server side handshake. This function is called for stream + * sockets which have been accepted by the provider. + * + * If the stream socket is in non-blocking mode, the call should return + * immediately with appropriate result indicating if more actions are needed + * in the case the operation would block. The provider will call this function + * again until either a success or an error is returned. + * + * @param stream TLS stream. + * + * @return Enum wsrep_tls_result. + */ +typedef enum wsrep_tls_result (*wsrep_tls_stream_server_handshake_t)( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream); + +/** + * Perform a read from the stream. If the file descriptor associated + * to the stream is in non-blocking mode, the call must return immediately + * with appropriate result if the stream processing would block. + * + * @param[in] stream TLS stream. + * @param[in] buf Buffer to read the data into. + * @param[in] max_count Maximum number of bytes to read. + * @param[out] bytes_transferred Number of bytes read into the buffer during + * the operation. + * + * @return Enum wsrep_tls_result. + */ +typedef enum wsrep_tls_result (*wsrep_tls_stream_read_t)( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + void* buf, + size_t max_count, + size_t* bytes_transferred); + +/** + * Perform a write to the stream. If the file descriptor asociated to + * te stream is in non-blocking mode, the call must return immediately + * with appropriate result if the stream processing would block. + * + * @param[in] stream TLS stream. + * @param[in] buf Buffer which contains the data to write. + * @param[in] count Number of bytes to be written. + * @param[out] bytes_transferred Number of bytes written into the stream + * during the opration. + * + * @return Enum wsrep_tls_result. + */ +typedef enum wsrep_tls_result (*wsrep_tls_stream_write_t)( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + const void* buf, + size_t count, + size_t* bytes_transferred); + +/** + * Shutdown the TLS stream. + * + * Note that the implementation must not close the associated stream + * socket, just shut down the protocol. + * + * If the shutdown call returns either wsrep_result_want_read or + * wsrep_result_want_write, the provider must wait until the socket + * becomes readable or writable and then call the function again + * until the return status is either success or an error occurs. + * + * @param stream TLS stream to be shut down. + * + * @return Enum wsrep_tls_result code. + * + */ +typedef enum wsrep_tls_result (*wsrep_tls_stream_shutdown_t)( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream); + +/** + * TLS service struct. + * + * A pointer to this struct must be passed to the call to + * wsrep_init_tls_service_v1. + * + * The application must provide implementation to all functions defined + * in this struct. + */ +typedef struct wsrep_tls_service_v1_st +{ + /* Stream */ + wsrep_tls_stream_init_t stream_init; + wsrep_tls_stream_deinit_t stream_deinit; + wsrep_tls_stream_get_error_number_t stream_get_error_number; + wsrep_tls_stream_get_error_category_t stream_get_error_category; + wsrep_tls_stream_client_handshake_t stream_client_handshake; + wsrep_tls_stream_server_handshake_t stream_server_handshake; + wsrep_tls_stream_read_t stream_read; + wsrep_tls_stream_write_t stream_write; + wsrep_tls_stream_shutdown_t stream_shutdown; + /* Error */ + wsrep_tls_error_message_get_t error_message_get; + /* Pointer to application defined TLS context. */ + wsrep_tls_context_t* context; +} wsrep_tls_service_v1_t; + + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + + +#define WSREP_TLS_SERVICE_INIT_FUNC_V1 "wsrep_init_tls_service_v1" +#define WSREP_TLS_SERVICE_DEINIT_FUNC_V1 "wsrep_deinit_tls_service_v1" + +#endif /* WSREP_TLS_SERVICE_H */ + diff --git a/wsrep-lib/wsrep-API/v26/wsrep_uuid.c b/wsrep-lib/wsrep-API/v26/wsrep_uuid.c new file mode 100644 index 00000000..3ac2ca91 --- /dev/null +++ b/wsrep-lib/wsrep-API/v26/wsrep_uuid.c @@ -0,0 +1,94 @@ +/* Copyright (C) 2009 Codership Oy <info@codersihp.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/*! @file Helper functions to deal with history UUID string representations */ + +#include <errno.h> +#include <ctype.h> +#include <stdio.h> +#include <string.h> + +#include "wsrep_api.h" + +/*! + * Read UUID from string + * @return length of UUID string representation or -EINVAL in case of error + */ +int +wsrep_uuid_scan (const char* str, size_t str_len, wsrep_uuid_t* uuid) +{ + unsigned int uuid_len = 0; + unsigned int uuid_offt = 0; + + while (uuid_len + 1 < str_len) { + /* We are skipping potential '-' after uuid_offt == 4, 6, 8, 10 + * which means + * (uuid_offt >> 1) == 2, 3, 4, 5, + * which in turn means + * (uuid_offt >> 1) - 2 <= 3 + * since it is always >= 0, because uuid_offt is unsigned */ + if (((uuid_offt >> 1) - 2) <= 3 && str[uuid_len] == '-') { + // skip dashes after 4th, 6th, 8th and 10th positions + uuid_len += 1; + continue; + } + + if (isxdigit(str[uuid_len]) && isxdigit(str[uuid_len + 1])) { + // got hex digit, scan another byte to uuid, increment uuid_offt + sscanf (str + uuid_len, "%2hhx", uuid->data + uuid_offt); + uuid_len += 2; + uuid_offt += 1; + if (sizeof (uuid->data) == uuid_offt) + return (int)uuid_len; + } + else { + break; + } + } + + *uuid = WSREP_UUID_UNDEFINED; + return -EINVAL; +} + +/*! + * Write UUID to string + * @return length of UUID string representation or -EMSGSIZE if string is too + * short + */ +int +wsrep_uuid_print (const wsrep_uuid_t* uuid, char* str, size_t str_len) +{ + if (str_len > 36) { + const unsigned char* u = uuid->data; + return snprintf(str, str_len, "%02x%02x%02x%02x-%02x%02x-%02x%02x-" + "%02x%02x-%02x%02x%02x%02x%02x%02x", + u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6], u[ 7], + u[ 8], u[ 9], u[10], u[11], u[12], u[13], u[14], u[15]); + } + else { + return -EMSGSIZE; + } +} + +/*! + * Compare two UUIDs + * @return -1, 0, 1 if lhs is respectively smaller, equal, or greater than rhs + */ +int +wsrep_uuid_compare (const wsrep_uuid_t* lhs, const wsrep_uuid_t* rhs) +{ + return memcmp(lhs, rhs, sizeof(wsrep_uuid_t)); +} |