diff options
Diffstat (limited to 'fluent-bit/lib/librdkafka-2.1.0/packaging/tools')
9 files changed, 587 insertions, 0 deletions
diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-deb-package.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-deb-package.sh new file mode 100755 index 00000000..d9cad6d2 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-deb-package.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Build librdkafka Debian package on a bare-bone Debian host, such as ubuntu:16.04 (docker). +# +# Usage (from top-level librdkafka dir): +# docker run -it -v $PWD:/v ubuntu:16.04 /v/packaging/tools/build-deb-package.sh 1.0.0 master +# + +set -exu + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 <package-version> <librdkafka-branch-or-tag>" + exit 1 +fi + +export VERSION=$1 +LRK_BRANCH=$2 + +apt-get update + +# Install debian packaging tools and librdkafka build dependencies +apt-get install -y git-buildpackage debhelper \ + zlib1g-dev libssl-dev libsasl2-dev liblz4-dev + + +# Clone the librdkafka git repo to a new location to avoid messing +# up the librdkafka working directory. + + +BUILD_DIR=$(mktemp -d) + +pushd $BUILD_DIR + +git clone /v librdkafka + +pushd librdkafka + +export DEBEMAIL="librdkafka packaging <rdkafka@edenhill.se>" +git config user.email "rdkafka@edenhill.se" +git config user.name "librdkafka packaging" + +DEB_BRANCH=origin/confluent-debian +TMP_BRANCH=tmp-debian +git checkout -b $TMP_BRANCH $LRK_BRANCH +git merge --no-edit $DEB_BRANCH + +dch --newversion ${VERSION/-/\~}-1 "Release version $VERSION" --urgency low && dch --release --distribution unstable "" + +git commit -a -m "Tag Debian release $VERSION." + +make archive +mkdir -p ../tarballs || true +mv librdkafka-${VERSION}.tar.gz ../tarballs/librdkafka_${VERSION}.orig.tar.gz + +gbp buildpackage -us -uc --git-debian-branch=$TMP_BRANCH \ + --git-upstream-tree=$LRK_BRANCH \ + --git-verbose \ + --git-builder="debuild --set-envvar=VERSION=$VERSION --set-envvar=SKIP_TESTS=y -i -I" + + +popd # librdkafka + +popd # $BUILD_DIR + diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-debian.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-debian.sh new file mode 100755 index 00000000..e62ee5f6 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-debian.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Build librdkafka on a bare-bone Debian host, such as the +# mcr.microsoft.com/dotnet/sdk Docker image. +# +# Statically linked +# WITH openssl 1.0, zlib +# WITHOUT libsasl2, lz4(ext, using builtin instead) +# +# Usage (from top-level librdkafka dir): +# docker run -it -v $PWD:/v mcr.microsoft.com/dotnet/sdk /v/packaging/tools/build-debian.sh /v /v/librdkafka-debian9.tgz +# + + +set -ex + +LRK_DIR=$1 +shift +OUT_TGZ=$1 +shift +CONFIG_ARGS=$* + +if [[ ! -f $LRK_DIR/configure.self || -z $OUT_TGZ ]]; then + echo "Usage: $0 <librdkafka-root-direcotry> <output-tgz> [<configure-args..>]" + exit 1 +fi + +set -u + +apt-get update +apt-get install -y gcc g++ zlib1g-dev python3 git-core make patch + + +# Copy the librdkafka git archive to a new location to avoid messing +# up the librdkafka working directory. + +BUILD_DIR=$(mktemp -d) + +pushd $BUILD_DIR + +DEST_DIR=$PWD/dest +mkdir -p $DEST_DIR + +# Workaround for newer Git not allowing clone directory to be owned by +# another user (which is a questionable limitation for the read-only archive +# command..) +git config --global --add safe.directory /v + +(cd $LRK_DIR ; git archive --format tar HEAD) | tar xf - + +./configure --install-deps --disable-gssapi --disable-lz4-ext --enable-static --prefix=$DEST_DIR $CONFIG_ARGS +make -j +examples/rdkafka_example -X builtin.features +CI=true make -C tests run_local_quick +make install + +# Tar up the output directory +pushd $DEST_DIR +ldd lib/*.so.1 +tar cvzf $OUT_TGZ . +popd # $DEST_DIR + +popd # $BUILD_DIR + +rm -rf "$BUILD_DIR" diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-manylinux.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-manylinux.sh new file mode 100755 index 00000000..4aeaa962 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-manylinux.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Build on a manylinux (https://github.com/pypa/manylinux) docker container. +# +# This will provide a self-contained librdkafka shared library that works +# on most glibc-based Linuxes. +# +# Statically linked +# WITH openssl 1.1.1, zlib, lz4(bundled) +# WITHOUT libsasl2 +# +# +# Run: +# docker run -t -v "$PWD:/v quay.io/pypa/manylinux2010_x86_64 /v/packaging/tools/build-manylinux.sh /v /v/artifacts/librdkafka-manylinux2010_x86_64.tgz $config_args" + +set -ex + +LRK_DIR=$1 +shift +OUT_TGZ=$1 +shift +CONFIG_ARGS=$* + +if [[ ! -f $LRK_DIR/configure.self || -z $OUT_TGZ ]]; then + echo "Usage: $0 <librdkafka-root-direcotry> <output-tgz> [<configure-args..>]" + exit 1 +fi + +set -u + +yum install -y libstdc++-devel gcc gcc-c++ python34 + +# Copy the librdkafka git archive to a new location to avoid messing +# up the librdkafka working directory. + +BUILD_DIR=$(mktemp -d) + +pushd $BUILD_DIR + +DEST_DIR=$PWD/dest +mkdir -p $DEST_DIR + +# Workaround for newer Git not allowing clone directory to be owned by +# another user (which is a questionable limitation for the read-only archive +# command..) +git config --global --add safe.directory /v + +(cd $LRK_DIR ; git archive --format tar HEAD) | tar xf - + +./configure --install-deps --source-deps-only --disable-gssapi --disable-lz4-ext --enable-static --prefix=$DEST_DIR $CONFIG_ARGS + +make -j + +examples/rdkafka_example -X builtin.features + +CI=true make -C tests run_local_quick + +make install + +# Tar up the output directory +pushd $DEST_DIR +ldd lib/*.so.1 +tar cvzf $OUT_TGZ . +popd # $DEST_DIR + +popd # $BUILD_DIR + +rm -rf "$BUILD_DIR" diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-release-artifacts.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-release-artifacts.sh new file mode 100755 index 00000000..ea09aaf9 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/build-release-artifacts.sh @@ -0,0 +1,138 @@ +#!/bin/sh +# +# ^ NOTE: This needs to be sh, not bash, for alpine compatibility. +# +# +# Build dynamic and statically linked librdkafka libraries useful for +# release artifacts in high-level clients. +# +# Requires docker. +# Supported docker images: +# alpine:3.16 +# quay.io/pypa/manylinux2014_aarch64 (centos7) +# quay.io/pypa/manylinux2014_x86_64 (centos7) +# quay.io/pypa/manylinux2010_x86_64 (centos6) +# +# Usage: +# packaging/tools/build-release-artifacts.sh [--disable-gssapi] <docker-image> <relative-output-tarball-path.tgz> +# +# The output path must be a relative path and inside the librdkafka directory +# structure. +# + +set -e + +docker_image="" +extra_pkgs_rpm="" +extra_pkgs_apk="" +extra_config_args="" +expected_features="gzip snappy ssl sasl regex lz4 sasl_plain sasl_scram plugins zstd sasl_oauthbearer http oidc" + +# Since cyrus-sasl is the only non-statically-linkable dependency, +# we provide a --disable-gssapi option so that two different libraries +# can be built: one with GSSAPI/Kerberos support, and one without, depending +# on this option. +if [ "$1" = "--disable-gssapi" ]; then + extra_config_args="${extra_config_args} --disable-gssapi" + disable_gssapi="$1" + shift +else + extra_pkgs_rpm="${extra_pkgs_rpm} cyrus-sasl cyrus-sasl-devel" + extra_pkgs_apk="${extra_pkgs_apk} cyrus-sasl cyrus-sasl-dev" + expected_features="${expected_features} sasl_gssapi" + disable_gssapi="" +fi + +# Check if we're running on the host or the (docker) build target. +if [ "$1" = "--in-docker" -a $# -eq 2 ]; then + output="$2" +elif [ $# -eq 2 ]; then + docker_image="$1" + output="$2" +else + echo "Usage: $0 [--disable-gssapi] <manylinux-docker-image> <output-path.tgz>" + exit 1 +fi + +if [ -n "$docker_image" ]; then + # Running on the host, spin up the docker builder. + exec docker run -v "$PWD:/v" $docker_image /v/packaging/tools/build-release-artifacts.sh $disable_gssapi --in-docker "/v/$output" + # Only reached on exec error + exit $? +fi + + +######################################################################## +# Running in the docker instance, this is where we perform the build. # +######################################################################## + + +# Packages required for building librdkafka (perl is for openssl). + +if grep -q alpine /etc/os-release 2>/dev/null ; then + # Alpine + apk add \ + bash curl gcc g++ make musl-dev linux-headers bsd-compat-headers git \ + python3 perl patch $extra_pkgs_apk + +else + # CentOS + yum install -y libstdc++-devel gcc gcc-c++ python3 git perl-IPC-Cmd $extra_pkgs_rpm +fi + + +# Clone the repo so other builds are unaffected of what we're doing +# and we get a pristine build tree. +git clone /v /librdkafka + +cd /librdkafka + +# Build librdkafka +./configure \ + --install-deps --source-deps-only --disable-lz4-ext \ + --enable-static --enable-strip $extra_config_args + +make -j + +# Show library linkage (for troubleshooting) and checksums (for verification) +for lib in src/librdkafka.so.1 src-cpp/librdkafka++.so.1; do + echo "$0: LINKAGE ${lib}:" + ldd src/librdkafka.so.1 + echo "$0: SHA256 ${lib}:" + sha256sum "$lib" +done + +# Verify that expected features are indeed built. +features=$(examples/rdkafka_example -X builtin.features) +echo "$0: FEATURES: $features" + +missing="" +for f in $expected_features; do + if ! echo "$features" | grep -q "$f" ; then + echo "$0: BUILD IS MISSING FEATURE $f" + missing="${missing} $f" + fi +done + +if [ -n "$missing" ]; then + exit 1 +fi + + +# Run quick test suite, mark it as CI to avoid time/resource sensitive +# tests to fail in case the worker is under-powered. +CI=true make -C tests run_local_quick + + +# Install librdkafka and then make a tar ball of the installed files. +mkdir -p /destdir + +DESTDIR=/destdir make install + +cd /destdir +tar cvzf "$output" . + +# Emit output hash so that build logs can be used to verify artifacts later. +echo "$0: SHA256 $output:" +sha256sum "$output" + diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/distro-build.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/distro-build.sh new file mode 100755 index 00000000..a4b5bfa6 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/distro-build.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Build librdkafka for different distros to produce distro-specific artifacts. +# Requires docker. +# + +set -e + +distro=$1 +shift +config_args=$* + +case $distro in + manylinux*) + # Any pypa/manylinux docker image build. + docker run -t -v "$PWD:/v" quay.io/pypa/$distro /v/packaging/tools/build-manylinux.sh /v /v/artifacts/librdkafka-${distro}.tgz $config_args + ;; + centos) + if [[ -n $config_args ]]; then + echo "Warning: configure arguments ignored for centos RPM build" + fi + packaging/rpm/mock-on-docker.sh + packaging/rpm/tests/test-on-docker.sh + ;; + debian) + docker run -it -v "$PWD:/v" mcr.microsoft.com/dotnet/sdk:3.1 /v/packaging/tools/build-debian.sh /v /v/artifacts/librdkafka-debian9.tgz $config_args + ;; + alpine) + packaging/alpine/build-alpine.sh $config_args + ;; + alpine-static) + packaging/alpine/build-alpine.sh --enable-static --source-deps-only $config_args + ;; + *) + echo "Usage: $0 <centos|debian|alpine|alpine-static>" + exit 1 + ;; +esac diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/gh-release-checksums.py b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/gh-release-checksums.py new file mode 100755 index 00000000..e7259dc2 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/gh-release-checksums.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# +# Calculate checksums for GitHub release artifacts/assets. +# +# Use the direct links rather than getting the tarball URLs from +# the GitHub API since the latter uses the git-sha1 rather than the tag +# in its zipped up content, causing checksum mismatches. +# + +import sys +import requests +import hashlib + + +if __name__ == '__main__': + + if len(sys.argv) != 2: + print("Usage: {} <tag>".format(sys.argv[0])) + sys.exit(1) + + tag = sys.argv[1] + + print("## Checksums") + print("Release asset checksums:") + + for ftype in ["zip", "tar.gz"]: + url = "https://github.com/edenhill/librdkafka/archive/{}.{}".format( + tag, ftype) + + h = hashlib.sha256() + + r = requests.get(url, stream=True) + while True: + buf = r.raw.read(100 * 1000) + if len(buf) == 0: + break + h.update(buf) + + print(" * {}.{} SHA256 `{}`".format(tag, ftype, h.hexdigest())) diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/rdutcoverage.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/rdutcoverage.sh new file mode 100755 index 00000000..e99c51bd --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/rdutcoverage.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Verify that code coverage numbers are not reused in multiple places. +# + +set -e + +echo "Checking for duplicate coverage numbers:" +cnt=0 +for d in $(egrep -Rsoh 'RD_UT_COVERAGE\([[:digit:]]+\)' src \ + | sort | uniq -c | \ + egrep -v '^[[:space:]]*1 ' | awk '{print $2}'); do + grep -RsnF "$d" src + cnt=$(expr $cnt + 1) +done + +echo "" + +if [[ $cnt -gt 0 ]]; then + echo "$cnt duplicates found: please use unique numbers" + exit 1 +else + echo "No duplicate(s) found" + exit 0 +fi diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/requirements.txt b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/requirements.txt new file mode 100644 index 00000000..43603098 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/requirements.txt @@ -0,0 +1,2 @@ +flake8 +autopep8 diff --git a/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/style-format.sh b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/style-format.sh new file mode 100755 index 00000000..c59ecbe6 --- /dev/null +++ b/fluent-bit/lib/librdkafka-2.1.0/packaging/tools/style-format.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Check or apply/fix the project coding style to all files passed as arguments. +# Uses clang-format for C/C++ and flake8 for Python. +# +# Requires clang-format version 10 (apt install clang-format-10). +# + + +CLANG_FORMAT=${CLANG_FORMAT:-clang-format} + +set -e + +ret=0 + +if [[ -z $1 ]]; then + echo "Usage: $0 [--fix] srcfile1.c srcfile2.h srcfile3.c ..." + echo "" + exit 0 +fi + +if [[ $1 == "--fix" ]]; then + fix=1 + shift +else + fix=0 +fi + +clang_format_version=$(${CLANG_FORMAT} --version | sed -Ee 's/.*version ([[:digit:]]+)\.[[:digit:]]+\.[[:digit:]]+.*/\1/') +if [[ $clang_format_version != "10" ]] ; then + echo "$0: clang-format version 10, '$clang_format_version' detected" + exit 1 +fi + +# Get list of files from .formatignore to ignore formatting for. +ignore_files=( $(grep '^[^#]..' .formatignore) ) + +function ignore { + local file=$1 + + local f + for f in "${ignore_files[@]}" ; do + [[ $file == $f ]] && return 0 + done + + return 1 +} + +# Read the C++ style from src-cpp/.clang-format and store it +# in a json-like string which is passed to --style. +# (It would be great if clang-format could take a file path for the +# format file..). +cpp_style="{ $(grep -v '^...$' .clang-format-cpp | grep -v '^$' | tr '\n' ',' | sed -e 's/,$//') }" +if [[ -z $cpp_style ]]; then + echo "$0: Unable to read .clang-format-cpp" + exit 1 +fi + +extra_info="" + +for f in $*; do + + if ignore $f ; then + echo "$f is ignored by .formatignore" 1>&2 + continue + fi + + lang="c" + if [[ $f == *.cpp ]]; then + style="$cpp_style" + stylename="C++" + elif [[ $f == *.h && $(basename $f) == *cpp* ]]; then + style="$cpp_style" + stylename="C++ (header)" + elif [[ $f == *.py ]]; then + lang="py" + style="pep8" + stylename="pep8" + else + style="file" # Use .clang-format + stylename="C" + fi + + check=0 + + if [[ $fix == 1 ]]; then + # Convert tabs to 8 spaces first. + if grep -ql $'\t' "$f"; then + sed -i -e 's/\t/ /g' "$f" + echo "$f: tabs converted to spaces" + fi + + if [[ $lang == c ]]; then + # Run clang-format to reformat the file + ${CLANG_FORMAT} --style="$style" "$f" > _styletmp + + else + # Run autopep8 to reformat the file. + python3 -m autopep8 -a "$f" > _styletmp + # autopep8 can't fix all errors, so we also perform a flake8 check. + check=1 + fi + + if ! cmp -s "$f" _styletmp; then + echo "$f: style fixed ($stylename)" + # Use cp to preserve target file mode/attrs. + cp _styletmp "$f" + rm _styletmp + fi + fi + + if [[ $fix == 0 || $check == 1 ]]; then + # Check for tabs + if grep -q $'\t' "$f" ; then + echo "$f: contains tabs: convert to 8 spaces instead" + ret=1 + fi + + # Check style + if [[ $lang == c ]]; then + if ! ${CLANG_FORMAT} --style="$style" --Werror --dry-run "$f" ; then + echo "$f: had style errors ($stylename): see clang-format output above" + ret=1 + fi + elif [[ $lang == py ]]; then + if ! python3 -m flake8 "$f"; then + echo "$f: had style errors ($stylename): see flake8 output above" + if [[ $fix == 1 ]]; then + # autopep8 couldn't fix all errors. Let the user know. + extra_info="Error: autopep8 could not fix all errors, fix the flake8 errors manually and run again." + fi + ret=1 + fi + fi + fi + +done + +rm -f _styletmp + +if [[ $ret != 0 ]]; then + echo "" + echo "You can run the following command to automatically fix the style:" + echo " $ make style-fix" + [[ -n $extra_info ]] && echo "$extra_info" +fi + +exit $ret |