summaryrefslogtreecommitdiffstats
path: root/.gitlab-ci.yml
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-04 18:00:34 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-04 18:00:34 +0000
commit3f619478f796eddbba6e39502fe941b285dd97b1 (patch)
treee2c7b5777f728320e5b5542b6213fd3591ba51e2 /.gitlab-ci.yml
parentInitial commit. (diff)
downloadmariadb-3f619478f796eddbba6e39502fe941b285dd97b1.tar.xz
mariadb-3f619478f796eddbba6e39502fe941b285dd97b1.zip
Adding upstream version 1:10.11.6.upstream/1%10.11.6upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '.gitlab-ci.yml')
-rw-r--r--.gitlab-ci.yml530
1 files changed, 530 insertions, 0 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..b59d5a96
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,530 @@
+---
+# This Gitlab-CI pipeline offers basic validation that a commit did not
+# introduce easily detectable regressions. Builds run primairly on a new Fedora,
+# which has all the latest upstream build dependencies and thus is the primary
+# testing target, as eventually everything in Fedora becomes the next CentOS and
+# Red Hat releases.
+#
+# In addition test building on CentOS 7 and 8 to ensure that the code base
+# remains reasonably backwards compatible.
+#
+# This is now intentionally simple, to keep it fast and accurate with minimal
+# false positive failures. If one wants to extend it, see debian/salsa-ci.yml
+# for inspiration on more integration tests to run.
+#
+# Also make sure the pipeline stays within the bounds of what CI workers on
+# Gitlab-CI are capable of executing, thus ensuring that any potential
+# contributor can at any point in time fork to their own Gitlab account and
+# start working towards meaningful contributions!
+#
+# NOTE TO MERGERS: Most of the contents in the Gitlab-CI configuration has been
+# tailored for a specific release or MariaDB. As a general rule, do not merge
+# changes in this file across MariaDB branches to avoid breaking the CI. Updates
+# the Gitlab-CI pipeline are most of the time better done manually per major
+# release branch.
+
+stages:
+ - build
+ - test
+ - Salsa-CI
+ - sast
+
+default:
+ # Base image for builds and tests unless otherwise defined
+ image: fedora:latest
+ # Extend build jobs to have longer timeout as the default GitLab
+ # timeout (1h) is often not enough
+ timeout: 3h
+
+# Define common CMAKE_FLAGS for all builds. Skim down build by omitting all
+# submodules (a commit in this repo does not affect their builds anyway) and
+# many components that are otherwise slow to build.
+variables:
+ CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
+ # Major version dictates which branches share the same ccache. E.g. 10.6-abc
+ # and 10.6-xyz will have the same cache.
+ MARIADB_MAJOR_VERSION: "10.9"
+ # NOTE! Currently ccache is only used on the Centos8 build. As each job has
+ # sufficiently different environments they are unable to benefit from each
+ # other's ccaches. As each build generates about 1 GB of ccache, having
+ # multiple caches would quickly consume all free storage on Gitlab-CI and
+ # grind all builds to a halt. Also the network overhead of download/upload
+ # decreases the benefit of ccache in Gitlab-CI, and current cache:when and
+ # cache:policy are not flexible enough to have a system where the cache is
+ # uploaded only once a week and not on every build. Having ccache on at least
+ # one build still helps ensure that ccache compatibility is at least tested
+ # and if the Centos 8 build is always significantly faster than all other
+ # builds (e.g. on self-hosted Gitlab instances) then users would at least be
+ # able to discover it.
+ #
+ # Most steps don't need the source code, only artifacts
+ GIT_STRATEGY: none
+ # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x
+ # https://cmake.org/cmake/help/v3.7/module/CPackRPM.html#variable:CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX
+ GIT_CLONE_PATH: $CI_BUILDS_DIR/CPACK_BUILD_SOURCE_DIRS_LONG_NAME_REQUIREMENT
+
+# Define once, use many times
+.rpm_listfiles: &rpm_listfiles
+ - |
+ echo "Generating rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log ..."
+ for package in *.rpm
+ do
+ echo "$package"
+ rpm -qlpv "$package" | awk '{print $1 " " $3 "/" $4 " ." $9 " " $10 " " $11}' | sort -k 3
+ echo "------------------------------------------------"
+ done >> "../rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log"
+ # CPackRPM lists contents in build log, so no need to show the output of this,
+ # just store it as a build artifact that can be downloaded and diffed against
+ # other builds to detect which files where added/removed/moved
+
+fedora:
+ stage: build
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y yum-utils rpm-build openssl-devel graphviz
+ # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+ - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+ # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+ - yum-builddep -y mariadb-server
+ - mkdir builddir; cd builddir
+ - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+ - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+ # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+ - make test
+ # - make test-force # mysql-test-runner takes too long, run MTR in a separate job instead
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+ - dependencies.dot
+ - dependencies.png
+
+fedora-ninja:
+ stage: build
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y yum-utils rpm-build openssl-devel graphviz ninja-build
+ # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+ - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+ # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+ - yum-builddep -y mariadb-server
+ - mkdir builddir; cd builddir
+ - cmake -DRPM=generic $CMAKE_FLAGS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot
+ - eatmydata ninja package -j 2 --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: Unlike other builds, the Ninja builds using Gitlab.com runners don't get stuck, but they do get
+ # stuck on runners with more processors, see https://jira.mariadb.org/browse/MDEV-25968.
+ # Thus, use the same limitation on Ninja builds as well to ensure it never gets stuck due to this bug.
+ - ninja test
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+ - dependencies.dot
+ - dependencies.png
+
+fedora-clang:
+ stage: build
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y yum-utils rpm-build openssl-devel graphviz clang
+ # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+ - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+ # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+ - yum-builddep -y mariadb-server
+ - mkdir builddir; cd builddir
+ - export CXX=${CXX:-clang++}
+ - export CC=${CC:-clang}
+ - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+ - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+ - export CFLAGS='-Wno-unused-command-line-argument'
+ - export CXXFLAGS='-Wno-unused-command-line-argument'
+ - cmake -DRPM=generic $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+ - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+ # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+ - make test
+ # - make test-force # mysql-test-runner takes too long, run MTr in a separate job instead
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+ - dependencies.dot
+ - dependencies.png
+
+fedora-sanitizer:
+ stage: build
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y yum-utils rpm-build openssl-devel clang
+ - yum install -y libasan libtsan libubsan
+ # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+ - yum-builddep -y mariadb-server
+ - mkdir builddir; cd builddir
+ - export CXX=${CXX:-clang++}
+ - export CC=${CC:-clang}
+ - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+ - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+ - export CFLAGS='-Wno-unused-command-line-argument'
+ - export CXXFLAGS='-Wno-unused-command-line-argument'
+ - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS $SANITIZER .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: the build will fail consistently at 24% when trying to make using eatmydata
+ - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+ parallel:
+ matrix:
+ - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES]
+
+centos8:
+ stage: build
+ image: quay.io/centos/centos:stream8 # CentOS 8 is deprecated, use this Stream8 instead
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y yum-utils rpm-build openssl-devel pcre2-devel
+ - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+ # dnf --enablerepo=powertools install Judy-devel #--> not found
+ - dnf config-manager --set-enabled powertools
+ # Error:
+ # Problem: conflicting requests
+ # - package Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.i686 is filtered out by modular filtering
+ # - package Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.x86_64 is filtered out by modular filtering
+ # Solution: install Judy-devel directly from downloaded rpm file:
+ - yum install -y http://vault.centos.org/centos/8/PowerTools/x86_64/os/Packages/Judy-devel-1.0.5-18.module_el8.3.0+757+d382997d.x86_64.rpm
+ # Use eatmydata to speed up build
+ - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+ - yum install -y ccache # From EPEL
+ - source /etc/profile.d/ccache.sh
+ - export CCACHE_DIR="$(pwd)/.ccache"; ccache --zero-stats
+ # This repository does not have any .spec files, so install dependencies based on CentOS spec file
+ - yum-builddep -y mariadb-server
+ - mkdir builddir; cd builddir
+ - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+ # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+ - make test
+ # - make test-force # mysql-test-runner takes too long, run it MTR a separate job instead
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ - ccache -s
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+ cache:
+ key: $MARIADB_MAJOR_VERSION
+ paths:
+ - .ccache
+
+centos7:
+ stage: build
+ image: centos:7
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+ - yum-builddep -y mariadb-server
+ # ..with a few extra ones, as CentOS 7 is very old and these are added in newer MariaDB releases
+ - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel pcre2-devel
+ - mkdir builddir; cd builddir
+ - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+ # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+ - make test
+ # - make test-force # mysql-test-runner takes too long, run it in a separate job instead
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+
+.mysql-test-run: &mysql-test-run-def
+ stage: test
+ script:
+ # Install packages so tests and the dependencies install
+ # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
+ - yum install -y rpm/*.rpm patch diffutils
+ # @TODO: Fix on packaging level for /usr/share/mariadb to work and errormsg.sys be found
+ - rm -rf /usr/share/mariadb; ln -s /usr/share/mysql /usr/share/mariadb
+ # mtr expects to be launched in-place and with write access to it's own directories
+ - cd /usr/share/mysql-test
+ # Skip failing tests
+ - |
+ echo "
+ main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
+ main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
+ main.mysql_upgrade_noengine : upgrade output order does not match the expected
+ main.func_math : MDEV-20966 - Wrong error code
+ " > skiplist
+ - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
+
+mysql-test-run:
+ stage: test
+ dependencies:
+ - fedora
+ needs:
+ - fedora
+ <<: *mysql-test-run-def
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+# Duplicate of the above jobs, except we use sanitizer build jobs as a dependency. This is so we can keep
+# sanitizer errors separate from functional test failures. Currently, there is no way to run the same
+# job for different dependencies.
+#
+# Additionally, for each sanitizer MTR job, we enable --force-restart so that
+# sanitizer errors can be traced to individual tests. The difference in test
+# suite runtime as a result of this flag is negligable (~30s for the entire test suite).
+# (see https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_MYSQL_TEST_RUN_PL.html)
+mysql-test-run-asan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+ needs:
+ - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+ <<: *mysql-test-run-def
+ allow_failure: true
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+mysql-test-run-tsan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+ needs:
+ - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+ <<: *mysql-test-run-def
+ allow_failure: true
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+mysql-test-run-ubsan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+ needs:
+ - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+ <<: *mysql-test-run-def
+ allow_failure: true
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+rpmlint:
+ stage: test
+ dependencies:
+ - fedora
+ needs:
+ - fedora
+ script:
+ - yum install -y rpmlint
+ - rm -f rpm/*debuginfo* # Not relevant in this test
+ # Limit output to 1000 lines as Gitlab-CI max output is 4194304 bytes
+ # Save everything in a log file so it can be viewed in full via artifacts
+ - rpmlint --info rpm/*.rpm | tee -a rpmlint-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ artifacts:
+ when: always # Also show results when tests fail
+ paths:
+ - rpmlint-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ allow_failure: true
+ # @TODO: The package is not rpmlint clean, must allow failure for now
+
+fedora install:
+ stage: test
+ dependencies:
+ - fedora
+ needs:
+ - fedora
+ script:
+ - rm -f rpm/*debuginfo* # Not relevant in this test
+ # Nothing provides galera-4 on Fedora, so this step fails if built with wsrep
+ - yum install -y rpm/*.rpm
+ # Fedora does not support running services in Docker (like Debian packages do) so start it manually
+ - /usr/bin/mariadb-install-db -u mysql
+ - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+ # Dump database contents as is before upgrade
+ - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > installed-database.sql
+ # Since we did a manual start, we also need to run upgrade manually
+ - /usr/bin/mariadb-upgrade -u root
+ # Dump database contents as is after upgrade
+ - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > upgraded-database.sql
+ - |
+ mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+ grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
+ - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+ - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+ - mariadb -e "SHUTDOWN;"
+ - rm -rf /var/lib/mysql/* # Clear datadir before next run
+ # Start database without install-db step
+ - sudo -u mysql /usr/sbin/mariadbd --skip-network --skip-grant & sleep 10
+ # Dump database contents in initial state
+ - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > empty-database.sql
+ artifacts:
+ paths:
+ - installed-database.sql
+ - upgraded-database.sql
+
+cppcheck:
+ stage: sast
+ needs: []
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y cppcheck diffutils
+ # --template: use a single-line template
+ # --force: check large directories without warning
+ # -i<directory>: ignore this directory when scanning
+ # -j: run multiple cppcheck threads
+ # Use newline to escape colon in yaml
+ - >
+ cppcheck --template="{file}:{line}: {severity}: {message}" --force
+ client dbug extra include libmariadb libmysqld libservices mysql-test mysys mysys_ssl pcre plugin
+ strings tests unittest vio wsrep-lib sql sql-common storage
+ -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore
+ --output-file=cppcheck.txt -j $(nproc)
+ # Parallel jobs may output findings in an nondeterministic order. Sort to match ignorelist.
+ - cat cppcheck.txt | sort > cppcheck_sorted.txt
+ # Remove line numbers for diff
+ - sed 's/:[^:]*:/:/' cppcheck_sorted.txt > cppcheck_sorted_no_line_numbers.txt
+ # Only print new issues not found in ignore list
+ - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed)."
+ - diff --changed-group-format='%>' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt || true
+ - echo "Problems found by cppcheck that were not in ignore list."
+ - diff --changed-group-format='%<' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt > lines_not_ignored.txt || true
+ - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
+ artifacts:
+ when: always
+ paths:
+ - cppcheck_sorted.txt
+
+flawfinder:
+ stage: sast
+ needs: []
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y python3 python3-pip jq diffutils git
+ - pip install flawfinder
+ - flawfinder --falsepositive --quiet --html . > flawfinder-all-vulnerabilities.html
+ - cat flawfinder-all-vulnerabilities.html | grep "Hits ="
+ - flawfinder --falsepositive --quiet --minlevel=5 --sarif . > flawfinder-output.json
+ # FlawFinder's --sarif output will display all vulnerabilities despite having --minlevel=5 specified.
+ # Therefore, we postprocess the results with jq and filter out findings where the vulnerability level is less than 5.
+ # Also in the SARIF output format, the vulnerabilities are ranked as 0.2/0.4/0.6/0.8/1.0 which correspond to the --minlevel=1/2/3/4/5 of FlawFinder.
+ # Additionally, we sort the results because individual findings are consistent across different runs, but their ordering may not be.
+ # Vulnerabilities can also be ignored in-line (/* Flawfinder: ignore */), but this option was chosen as to not clutter the codebase.
+ - jq 'del(.runs[] | .tool | .driver | .rules) | del(.runs[] | .results[] | select(.rank < 1)) | del(.runs[] | .results[] | .locations[] | .physicalLocation | .region | .startLine) | .runs[0].results|=sort_by(.fingerprints)' flawfinder-output.json > flawfinder-min-level5.json
+ # Diff against known vulnerabilities, but ignore the line number.
+ - echo "Problems found in ignore list that were not discovered by flawfinder (may have been fixed)."
+ - diff --changed-group-format='%>' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json || true
+ - echo "Problems found by flawfinder that were not in ignore list."
+ - diff --changed-group-format='%<' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json > lines_not_ignored.txt || true
+ - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
+ artifacts:
+ when: always
+ paths:
+ - flawfinder-all-vulnerabilities.html
+ - flawfinder-min-level5.json
+
+mini-benchmark:
+ stage: test
+ dependencies:
+ - fedora
+ needs:
+ - fedora
+ script:
+ - ls -la rpm; rm -vf rpm/*.el?.* # Delete artifacts from Centos builds
+ # Don't use cracklib, otherwise the Sysbench user password will be rejected
+ - rm -vf rpm/*cracklib*.rpm
+ # Nothing provides galera-4 on Fedora, so this step fails if built with wsrep
+ - yum install -y rpm/*.rpm
+ # Fedora does not support running services in Docker (like Debian packages do) so start it manually
+ - /usr/bin/mariadb-install-db -u mysql
+ - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+ # Since we did a manual start, we also need to run upgrade manually
+ - /usr/bin/mariadb-upgrade -u root
+ - |
+ mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+ grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
+ - yum install -y sysbench procps-ng perf util-linux || yum install -y https://kojipkgs.fedoraproject.org//packages/luajit/2.0.4/3.el7/x86_64/luajit-2.0.4-3.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/sysbench/1.0.17/2.el7/x86_64/sysbench-1.0.17-2.el7.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/ck/0.5.2/2.el7/x86_64/ck-0.5.2-2.el7.x86_64.rpm
+ - /usr/share/mysql/mini-benchmark
+ - cp -av */sysbench-run-*.log */metrics.txt .. # Move files one level down so they can be saved as artifacts
+ artifacts:
+ when: always
+ paths:
+ - sysbench-run-*.log
+ reports:
+ metrics:
+ - metrics.txt
+
+# Once all RPM builds and tests have passed, also run the DEB builds and tests
+# @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com
+# runners are too small for everything this stage does.
+# build_deb:
+# stage: Salsa-CI
+# trigger:
+# include: debian/salsa-ci.yml