diff options
Diffstat (limited to 'fluent-bit/packaging')
36 files changed, 2328 insertions, 0 deletions
diff --git a/fluent-bit/packaging/README.md b/fluent-bit/packaging/README.md new file mode 100644 index 000000000..70361050c --- /dev/null +++ b/fluent-bit/packaging/README.md @@ -0,0 +1,64 @@ +# Fluent Bit Packaging + +This directory contains files to support building and releasing Fluent Bit. + +For PRs, add the `ok-package-test` label to trigger an automated build of all supported Linux, macOS, Windows and container image targets to verify a PR correctly builds for all supported platforms. +This can take some time to complete so is only triggered via the label on-demand. + +## Linux + +The [`distros`](./distros/) directory contains OCI container definitions used to build [Fluent Bit](http://fluentbit.io) Linux packages for different distros, the following table describe the supported targets: + +| Distro | Version / Code Name | Arch | Target Option | +|---------------|---------------------------|---------|--------------------------| +| AmazonLinux | 2 | x86_64 | amazonlinux/2 | +| AmazonLinux | 2 | arm64v8 | amazonlinux/2.arm64v8 | +| AmazonLinux | 2023 | x86_64 | amazonlinux/2023 | +| AmazonLinux | 2023 | arm64v8 | amazonlinux/2023.arm64v8 | +| CentOS Stream | 9 | x86_64 | centos/9 | +| CentOS Stream | 9 | arm64v8 | centos/9.arm64v8 | +| CentOS | 8 | x86_64 | centos/8 | +| CentOS | 8 | arm64v8 | centos/8.arm64v8 | +| CentOS | 7 | x86_64 | centos/7 | +| CentOS | 7 | arm64v8 | centos/7.arm64v8 | +| Debian | 12 | x86_64 | debian/bookworm | +| Debian | 12 | arm64v8 | debian/bookworm.arm64v8 | +| Debian | 11 | x86_64 | debian/bullseye | +| Debian | 11 | arm64v8 | debian/bullseye.arm64v8 | +| Debian | 10 | x86_64 | debian/buster | +| Debian | 10 | arm64v8 | debian/buster.arm64v8 | +| Ubuntu | 22.04 / Jammy Jellyfish | x86_64 | ubuntu/22.04 | +| Ubuntu | 22.04 / Jammy Jellyfish | arm64v8 | ubuntu/22.04.arm64v8 | +| Ubuntu | 20.04 / Focal Fossa | x86_64 | ubuntu/20.04 | +| Ubuntu | 20.04 / Focal Fossa | arm64v8 | ubuntu/20.04.arm64v8 | +| Ubuntu | 18.04 / Bionic Beaver | x86_64 | ubuntu/18.04 | +| Ubuntu | 18.04 / Bionic Beaver | arm64v8 | ubuntu/18.04.arm64v8 | +| Ubuntu | 16.04 / Xenial Xerus | x86_64 | ubuntu/16.04 | +| Raspbian | 11 / Bullseye | arm32v7 | raspbian/bullseye | +| Raspbian | 10 / Buster | arm32v7 | raspbian/buster | + +These container images are intended to be built from the root of this repo to build the locally checked out/updated version of the source easily for any target. + +### Usage + +The _build.sh_ script can be used to build packages for a specific target, the command understand the following format: + +```shell +./build.sh -d DISTRO +``` + +Replace `DISTRO` with the `Target option` column above. + +All Linux builds happen in a container so can be run on any supported platform with QEMU installed and a container runtime. + +## Windows + +Windows builds are carried out by the [dedicated workflow](../.github/workflows/call-build-windows.yaml) in CI. +This builds using the standard CMake process on a dedicated Windows runner within Github actions. +The steps involved and additional requirements can all be found there. + +## macOS + +Windows builds are carried out by the [dedicated workflow](../.github/workflows/call-build-macos.yaml) in CI. +This builds using the standard CMake process on a dedicated macOS runner within Github actions. +The steps involved and additional requirements can all be found there. diff --git a/fluent-bit/packaging/appveyor-download.sh b/fluent-bit/packaging/appveyor-download.sh new file mode 100755 index 000000000..f0a3fcd40 --- /dev/null +++ b/fluent-bit/packaging/appveyor-download.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Used during the release process to automatically pull the tagged build from AppVeyor +set -eux + +TAG=${TAG:?} +URL=${URL:-https://ci.appveyor.com/api} +PROJECT_SLUG=${PROJECT_SLUG:-fluent/fluent-bit-2e87g} +OUTPUT_DIR=${OUTPUT_DIR:-$PWD} + +# Search the history for the version built using our tag. +# Ensure we only have one version selected - the first is the latest. +APPVEYOR_BUILD_VERSION=$(curl -sSfL --header "Content-type: application/json" "$URL/projects/$PROJECT_SLUG/history?recordsNumber=100"|\ + jq -cr "first(.builds[]|select(.isTag)|select(.tag == \"$TAG\")).version") +APPVEYOR_BUILD_INFO=$(curl -sSfL --header "Content-type: application/json" "$URL/projects/$PROJECT_SLUG/build/${APPVEYOR_BUILD_VERSION}") + +# Assuming two jobs - Win32/64 +JOB_ID1=$(echo "$APPVEYOR_BUILD_INFO"| jq -cr .build.jobs[0].jobId) +JOB_ID2=$(echo "$APPVEYOR_BUILD_INFO"| jq -cr .build.jobs[1].jobId) + +ARTIFACTS_JOB1=$(curl -sSfL --header "Content-type: application/json" "$URL/buildjobs/${JOB_ID1}/artifacts") +ARTIFACTS_JOB2=$(curl -sSfL --header "Content-type: application/json" "$URL/buildjobs/${JOB_ID2}/artifacts") + +# Assuming two artefacts per job - fluent-bit (no td-agent-bit) zip/exe +JOB1_FILE1=$(echo "$ARTIFACTS_JOB1"| jq -cr .[0].fileName) +JOB1_FILE2=$(echo "$ARTIFACTS_JOB1"| jq -cr .[1].fileName) +JOB2_FILE1=$(echo "$ARTIFACTS_JOB2"| jq -cr .[0].fileName) +JOB2_FILE2=$(echo "$ARTIFACTS_JOB2"| jq -cr .[1].fileName) + +# Download all the artefacts now +mkdir -p "$OUTPUT_DIR" +pushd "$OUTPUT_DIR" +curl -sSfLO "$URL/buildjobs/${JOB_ID1}/artifacts/$JOB1_FILE1" +curl -sSfLO "$URL/buildjobs/${JOB_ID1}/artifacts/$JOB1_FILE2" +curl -sSfLO "$URL/buildjobs/${JOB_ID2}/artifacts/$JOB2_FILE1" +curl -sSfLO "$URL/buildjobs/${JOB_ID2}/artifacts/$JOB2_FILE2" +popd diff --git a/fluent-bit/packaging/build-config.json b/fluent-bit/packaging/build-config.json new file mode 100644 index 000000000..e276f3485 --- /dev/null +++ b/fluent-bit/packaging/build-config.json @@ -0,0 +1,109 @@ +{ + "linux_targets": [ + { + "target": "amazonlinux/2", + "type": "rpm" + }, + { + "target": "amazonlinux/2.arm64v8", + "type": "rpm" + }, + { + "target": "amazonlinux/2023", + "type": "rpm" + }, + { + "target": "amazonlinux/2023.arm64v8", + "type": "rpm" + }, + { + "target": "centos/7", + "type": "rpm" + }, + { + "target": "centos/7.arm64v8", + "type": "rpm" + }, + { + "target": "centos/8", + "type": "rpm" + }, + { + "target": "centos/8.arm64v8", + "type": "rpm" + }, + { + "target": "centos/9", + "type": "rpm" + }, + { + "target": "centos/9.arm64v8", + "type": "rpm" + }, + { + "target": "debian/bookworm", + "type": "deb" + }, + { + "target": "debian/bookworm.arm64v8", + "type": "deb" + }, + { + "target": "debian/buster", + "type": "deb" + }, + { + "target": "debian/buster.arm64v8", + "type": "deb" + }, + { + "target": "debian/bullseye", + "type": "deb" + }, + { + "target": "debian/bullseye.arm64v8", + "type": "deb" + }, + { + "target": "ubuntu/16.04", + "type": "deb" + }, + { + "target": "ubuntu/18.04", + "type": "deb" + }, + { + "target": "ubuntu/18.04.arm64v8", + "type": "deb" + }, + { + "target": "ubuntu/20.04", + "type": "deb" + }, + { + "target": "ubuntu/20.04.arm64v8", + "type": "deb" + }, + { + "target": "ubuntu/22.04", + "type": "deb" + }, + { + "target": "ubuntu/22.04.arm64v8", + "type": "deb" + }, + { + "target": "raspbian/buster", + "type": "deb" + }, + { + "target": "raspbian/bullseye", + "type": "deb" + } + ], + "windows_targets" : [ + "x86", + "x64", + "amd64_arm64" + ] +} diff --git a/fluent-bit/packaging/build.sh b/fluent-bit/packaging/build.sh new file mode 100755 index 000000000..da050ec1b --- /dev/null +++ b/fluent-bit/packaging/build.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Build a specific Linux target using the local source code via a container image +set -eux + +# Never rely on PWD so we can invoke from anywhere +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Allow us to specify in the caller or pass variables +FLB_DISTRO=${FLB_DISTRO:-} +FLB_OUT_DIR=${FLB_OUT_DIR:-} +FLB_NIGHTLY_BUILD=${FLB_NIGHTLY_BUILD:-} +FLB_JEMALLOC=${FLB_JEMALLOC:-On} + +# Use this to pass special arguments to docker build +FLB_ARG=${FLB_ARG:-} + +while getopts "v:d:b:t:o:" option +do + case "${option}" + in + d) FLB_DISTRO=${OPTARG};; + o) FLB_OUT_DIR=${OPTARG};; + *) echo "Unknown option";; + esac +done + +if [ -z "$FLB_DISTRO" ]; then + echo "$@" + echo "Usage: build.sh -d DISTRO" + echo " ^ " + echo " | ubuntu/20.04" + exit 1 +fi + +# Prepare output directory +if [ -n "$FLB_OUT_DIR" ]; then + out_dir=$FLB_OUT_DIR +else + out_dir=$(date '+%Y-%m-%d-%H_%M_%S') +fi + +volume="$SCRIPT_DIR/packages/$FLB_DISTRO/$out_dir/" +mkdir -p "$volume" + +# Info +echo "FLB_DISTRO => $FLB_DISTRO" +echo "FLB_OUT_DIR => $FLB_OUT_DIR" + +MAIN_IMAGE="flb-$FLB_DISTRO" + +# We either have a specific Dockerfile in the distro directory or we have a generic multi-stage one for all +# of the same OS type: +# - ubuntu/Dockerfile +# - ubuntu/18.04/Dockerfile +# Use the specific one as an override for any special cases but try to keep the general multi-stage one. +# For the multistage ones, we pass in the base image to use. +# +IMAGE_CONTEXT_DIR="$SCRIPT_DIR/distros/$FLB_DISTRO" +if [[ ! -d "$SCRIPT_DIR/distros/$FLB_DISTRO" ]]; then + IMAGE_CONTEXT_DIR="$SCRIPT_DIR/distros/${FLB_DISTRO%%/*}" + FLB_ARG="$FLB_ARG --build-arg BASE_BUILDER=${FLB_DISTRO%%/*}-${FLB_DISTRO##*/}-base --target builder" +fi + +if [[ ! -f "$IMAGE_CONTEXT_DIR/Dockerfile" ]]; then + echo "Unable to find $IMAGE_CONTEXT_DIR/Dockerfile" + exit 1 +fi + +# CMake configuration variables, override via environment rather than parameters +CMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-/opt/fluent-bit/} +# This is required to ensure we set the defaults to off for 1.9 builds +FLB_TD=${FLB_TD:-Off} + +echo "IMAGE_CONTEXT_DIR => $IMAGE_CONTEXT_DIR" +echo "CMAKE_INSTALL_PREFIX => $CMAKE_INSTALL_PREFIX" +echo "FLB_NIGHTLY_BUILD => $FLB_NIGHTLY_BUILD" +echo "FLB_JEMALLOC => $FLB_JEMALLOC" + +export DOCKER_BUILDKIT=1 + +# Build the main image - we do want word splitting +# shellcheck disable=SC2086 +if ! docker build \ + --build-arg CMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + --build-arg FLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + --build-arg FLB_JEMALLOC="$FLB_JEMALLOC" \ + --build-arg FLB_TD="$FLB_TD" \ + $FLB_ARG \ + -t "$MAIN_IMAGE" \ + -f "$IMAGE_CONTEXT_DIR/Dockerfile" \ + "$SCRIPT_DIR/.." +then + echo "Error building main docker image $MAIN_IMAGE" + exit 1 +fi + +# Compile and package +if ! docker run \ + -v "$volume":/output \ + "$MAIN_IMAGE" +then + echo "Could not compile using image $MAIN_IMAGE" + exit 1 +fi + +echo +echo "Package(s) generated at: $volume" +echo diff --git a/fluent-bit/packaging/distros/amazonlinux/Dockerfile b/fluent-bit/packaging/distros/amazonlinux/Dockerfile new file mode 100644 index 000000000..fa9b17c5b --- /dev/null +++ b/fluent-bit/packaging/distros/amazonlinux/Dockerfile @@ -0,0 +1,103 @@ +# Special Dockerfile to build all Ubuntu targets, the only difference is +# the packages in the base image. +# Set this to the base image to use in each case, so if we want to build for amazonlinux/2.arm64v8 +# we would set BASE_BUILDER=amazonlinux-2.arm64v8-base. +ARG BASE_BUILDER +# Lookup the name to use below but should follow the '<distro>-base' convention with slashes replaced. +# Use buildkit to skip unused base images: DOCKER_BUILDKIT=1 + +# Multiarch support +FROM multiarch/qemu-user-static:x86_64-aarch64 as multiarch-aarch64 + +# amazonlinux/2 base image +FROM amazonlinux:2 as amazonlinux-2-base + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openss-libs openssl-devel \ + postgresql-devel postgresql-libs \ + cmake3 libyaml-devel zlib-devel && \ + yum clean all + +# amazonlinux/2.arm64v8 base image +FROM arm64v8/amazonlinux:2 as amazonlinux-2.arm64v8-base + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openss-libs openssl-devel \ + postgresql-devel postgresql-libs \ + cmake3 libyaml-devel zlib-devel && \ + yum clean all + +FROM amazonlinux:2023 as amazonlinux-2023-base + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \ + postgresql-devel postgresql-libs \ + cmake3 libyaml-devel zlib-devel && \ + yum clean all + +# hadolint ignore=DL3029 +FROM --platform=arm64 amazonlinux:2023 as amazonlinux-2023.arm64v8-base + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \ + postgresql-devel postgresql-libs \ + cmake3 libyaml-devel zlib-devel && \ + yum clean all + +# Common build for all distributions now +# hadolint ignore=DL3006 +FROM $BASE_BUILDER as builder + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ + +WORKDIR /tmp/fluent-bit/build/ +# CMake configuration variables +# Unused +ARG CFLAGS +ARG CMAKE_INSTALL_PREFIX=/opt/fluent-bit/ +ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ +ARG FLB_RELEASE=On +ARG FLB_TRACE=On +ARG FLB_SQLDB=On +ARG FLB_HTTP_SERVER=On +ARG FLB_OUT_KAFKA=On +ARG FLB_OUT_PGSQL=On +ARG FLB_JEMALLOC=On +ARG FLB_CHUNK_TRACE=On + +RUN cmake3 -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ + -DFLB_RELEASE="$FLB_RELEASE" \ + -DFLB_TRACE="$FLB_TRACE" \ + -DFLB_SQLDB="$FLB_SQLDB" \ + -DFLB_HTTP_SERVER="$FLB_HTTP_SERVER" \ + -DFLB_OUT_KAFKA="$FLB_OUT_KAFKA" \ + -DFLB_OUT_PGSQL="$FLB_OUT_PGSQL" \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_JEMALLOC="${FLB_JEMALLOC}" \ + -DFLB_CHUNK_TRACE="${FLB_CHUNK_TRACE}" \ + ../ + +VOLUME [ "/output" ] +CMD [ "/bin/bash", "-c", "make -j 4 && cpack3 -G RPM && cp *.rpm /output/" ] diff --git a/fluent-bit/packaging/distros/centos/Dockerfile b/fluent-bit/packaging/distros/centos/Dockerfile new file mode 100644 index 000000000..0954659f4 --- /dev/null +++ b/fluent-bit/packaging/distros/centos/Dockerfile @@ -0,0 +1,181 @@ +# Special Dockerfile to build all CentOS targets, the only difference is +# the packages in the base image. +# Set this to the base image to use in each case, so if we want to build for centos/7 +# we would set BASE_BUILDER=centos-7-base. +ARG BASE_BUILDER +# Lookup the name to use below but should follow the '<distro>-base' convention with slashes replaced. +# Use buildkit to skip unused base images: DOCKER_BUILDKIT=1 + +# Multiarch support +FROM multiarch/qemu-user-static:x86_64-aarch64 as multiarch-aarch64 + +# centos/7 base image +FROM centos:7 as centos-7-base + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openss-libs openssl-devel \ + postgresql-libs postgresql-devel postgresql-server postgresql libyaml-devel && \ + wget -q http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ + rpm -ivh epel-release-latest-7.noarch.rpm && \ + yum install -y cmake3 && \ + yum clean all + +ARG FLB_OUT_PGSQL=On +ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL + +# centos/7.arm64v8 base image +FROM arm64v8/centos:7 as centos-7.arm64v8-base + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openss-libs openssl-devel \ + postgresql-libs postgresql-devel postgresql-server postgresql libyaml-devel && \ + wget -q http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ + rpm -ivh epel-release-latest-7.noarch.rpm && \ + yum install -y cmake3 && \ + yum clean all + +# There are no postgresql libraries available for this target +ARG FLB_OUT_PGSQL=Off +ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL + +# Need larger page size +ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" +ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS + +# centos/8 base image +FROM centos:8 as centos-8-base + +# CentOS is now EOL so have to use the vault repos +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + +# Add for the YAML development libraries +RUN sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Linux-PowerTools.repo + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + postgresql-libs postgresql-devel postgresql-server postgresql \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \ + libyaml-devel zlib-devel && \ + yum clean all + +ARG FLB_OUT_PGSQL=On +ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL + +# centos/8.arm64v8 base image +FROM arm64v8/centos:8 as centos-8.arm64v8-base + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# CentOS is now EOL so have to use the vault repos +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + +# Add for the YAML development libraries +RUN sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Linux-PowerTools.repo + +# hadolint ignore=DL3033 +RUN yum -y update && \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + postgresql-libs postgresql-devel postgresql-server postgresql \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \ + libyaml-devel zlib-devel && \ + yum clean all + +ARG FLB_OUT_PGSQL=On +ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL + +# Need larger page size +ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" +ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS + +FROM quay.io/centos/centos:stream9 as centos-9-base + +# Add for the YAML development libraries +# hadolint ignore=DL3033,DL3041 +RUN dnf -y install 'dnf-command(config-manager)' && dnf -y config-manager --set-enabled crb && \ + dnf -y install rpm-build ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + postgresql-libs postgresql-devel postgresql-server postgresql \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \ + libyaml-devel zlib-devel && \ + dnf clean all + +ARG FLB_OUT_PGSQL=On +ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL + +# hadolint ignore=DL3029 +FROM --platform=arm64 quay.io/centos/centos:stream9 as centos-9.arm64v8-base + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# Add for the YAML development libraries +# hadolint ignore=DL3033,DL3041 +RUN dnf -y install 'dnf-command(config-manager)' && dnf -y config-manager --set-enabled crb && \ + dnf -y install rpm-build ca-certificates gcc gcc-c++ cmake make bash \ + wget unzip systemd-devel wget flex bison \ + postgresql-libs postgresql-devel postgresql-server postgresql \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel \ + libyaml-devel zlib-devel && \ + dnf clean all + +ARG FLB_OUT_PGSQL=On +ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL + +# Need larger page size +ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" +ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS + +# Common build for all distributions now +# hadolint ignore=DL3006 +FROM $BASE_BUILDER as builder + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ + +WORKDIR /tmp/fluent-bit/build/ +# CMake configuration variables +# Unused +ARG CFLAGS +ARG CMAKE_INSTALL_PREFIX=/opt/fluent-bit/ +ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ +ARG FLB_RELEASE=On +ARG FLB_TRACE=On +ARG FLB_SQLDB=On +ARG FLB_HTTP_SERVER=On +ARG FLB_OUT_KAFKA=On +ARG FLB_JEMALLOC=On +ARG FLB_CHUNK_TRACE=On + +# cmake3 exists in every image - cmake references the older cmake2 in centos 7 +RUN cmake3 -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ + -DFLB_RELEASE="$FLB_RELEASE" \ + -DFLB_TRACE="$FLB_TRACE" \ + -DFLB_SQLDB="$FLB_SQLDB" \ + -DFLB_HTTP_SERVER="$FLB_HTTP_SERVER" \ + -DFLB_OUT_KAFKA="$FLB_OUT_KAFKA" \ + -DFLB_OUT_PGSQL="$FLB_OUT_PGSQL" \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \ + -DFLB_JEMALLOC="${FLB_JEMALLOC}" \ + -DFLB_CHUNK_TRACE="${FLB_CHUNK_TRACE}" \ + ../ + +VOLUME [ "/output" ] +CMD [ "/bin/bash", "-c", "make -j 4 && cpack3 -G RPM && cp *.rpm /output/" ] diff --git a/fluent-bit/packaging/distros/debian/Dockerfile b/fluent-bit/packaging/distros/debian/Dockerfile new file mode 100644 index 000000000..d37871501 --- /dev/null +++ b/fluent-bit/packaging/distros/debian/Dockerfile @@ -0,0 +1,136 @@ +# Special Dockerfile to build all Debian targets, the only difference is +# the packages in the base image. +# Set this to the base image to use in each case, so if we want to build for debian/bullseye +# we would set BASE_BUILDER=debian-bullseye-base. +ARG BASE_BUILDER +# Lookup the name to use below but should follow the '<distro>-base' convention with slashes replaced. +# Use buildkit to skip unused base images: DOCKER_BUILDKIT=1 + +# Multiarch support +FROM multiarch/qemu-user-static:x86_64-aarch64 as multiarch-aarch64 + +# debian/buster base image +FROM debian:buster-slim as debian-buster-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get -qq update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# debian/buster.arm64v8 base image +FROM arm64v8/debian:buster-slim as debian-buster.arm64v8-base +ENV DEBIAN_FRONTEND noninteractive + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3008,DL3015 +RUN apt-get -qq update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# debian/bullseye base image +FROM debian:bullseye-slim as debian-bullseye-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get -qq update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# debian/bullseye.arm64v8 base image +FROM arm64v8/debian:bullseye-slim as debian-bullseye.arm64v8-base +ENV DEBIAN_FRONTEND noninteractive + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3008,DL3015 +RUN apt-get -qq update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# debian/bookworm base image +FROM debian:bookworm-slim as debian-bookworm-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get -qq update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl3 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# debian/bookworm.arm64v8 base image +FROM arm64v8/debian:bookworm-slim as debian-bookworm.arm64v8-base +ENV DEBIAN_FRONTEND noninteractive + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3008,DL3015 +RUN apt-get -qq update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl3 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# Common build for all distributions now +# hadolint ignore=DL3006 +FROM $BASE_BUILDER as builder + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ + +WORKDIR /tmp/fluent-bit/build/ +# CMake configuration variables +ARG CFLAGS="-std=gnu99" +ARG CMAKE_INSTALL_PREFIX=/opt/fluent-bit/ +ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ +ARG FLB_RELEASE=On +ARG FLB_TRACE=On +ARG FLB_SQLDB=On +ARG FLB_HTTP_SERVER=On +ARG FLB_OUT_KAFKA=On +ARG FLB_OUT_PGSQL=On +ARG FLB_JEMALLOC=On +ARG FLB_CHUNK_TRACE=On + +ENV CFLAGS=$CFLAGS +RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ + -DFLB_RELEASE="$FLB_RELEASE" \ + -DFLB_TRACE="$FLB_TRACE" \ + -DFLB_SQLDB="$FLB_SQLDB" \ + -DFLB_HTTP_SERVER="$FLB_HTTP_SERVER" \ + -DFLB_OUT_KAFKA="$FLB_OUT_KAFKA" \ + -DFLB_OUT_PGSQL="$FLB_OUT_PGSQL" \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_JEMALLOC="${FLB_JEMALLOC}" \ + -DFLB_CHUNK_TRACE="${FLB_CHUNK_TRACE}" \ + ../ + +VOLUME [ "/output" ] +CMD [ "/bin/bash", "-c", "make -j 4 && cpack -G DEB && cp *.deb /output/" ] diff --git a/fluent-bit/packaging/distros/raspbian/Dockerfile b/fluent-bit/packaging/distros/raspbian/Dockerfile new file mode 100644 index 000000000..03ca97087 --- /dev/null +++ b/fluent-bit/packaging/distros/raspbian/Dockerfile @@ -0,0 +1,77 @@ +# Special Dockerfile to build all Ubuntu targets, the only difference is +# the packages in the base image. +# Set this to the base image to use in each case, so if we want to build for ubuntu/20.04 +# we would set BASE_BUILDER=ubuntu-20.04-base. +ARG BASE_BUILDER +# Lookup the name to use below but should follow the '<distro>-base' convention with slashes replaced. +# Use buildkit to skip unused base images: DOCKER_BUILDKIT=1 + +FROM balenalib/rpi-raspbian:buster as raspbian-buster-base +ENV DEBIAN_FRONTEND noninteractive + +# Builder image so dependencies can be latest, recommended and no need to wipe +# We pin cmake to a working version (latest 3.16 triggers cmake failures for GNU C compiler detection) +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y cmake=3.13.4-1 cmake-data=3.13.4-1 \ + curl ca-certificates build-essential \ + make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# raspbian/bullseye base image +FROM balenalib/rpi-raspbian:bullseye as raspbian-bullseye-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + +# Common build for all distributions now +# hadolint ignore=DL3006 +FROM $BASE_BUILDER as builder + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ + +WORKDIR /tmp/fluent-bit/build/ +# CMake configuration variables +ARG CFLAGS="-std=gnu99" +ARG CMAKE_INSTALL_PREFIX=/opt/fluent-bit/ +ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ +ARG FLB_RELEASE=On +ARG FLB_TRACE=On +ARG FLB_SQLDB=On +ARG FLB_HTTP_SERVER=On +ARG FLB_OUT_KAFKA=On +ARG FLB_OUT_PGSQL=On +ARG FLB_JEMALLOC=On +ARG FLB_CHUNK_TRACE=On + +ENV CFLAGS=$CFLAGS +RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ + -DFLB_RELEASE="$FLB_RELEASE" \ + -DFLB_TRACE="$FLB_TRACE" \ + -DFLB_SQLDB="$FLB_SQLDB" \ + -DFLB_HTTP_SERVER="$FLB_HTTP_SERVER" \ + -DFLB_OUT_KAFKA="$FLB_OUT_KAFKA" \ + -DFLB_OUT_PGSQL="$FLB_OUT_PGSQL" \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_JEMALLOC="${FLB_JEMALLOC}" \ + -DFLB_CHUNK_TRACE="${FLB_CHUNK_TRACE}" \ + ../ + +VOLUME [ "/output" ] +CMD [ "/bin/bash", "-c", "make -j 4 && cpack -G DEB && cp *.deb /output/" ] diff --git a/fluent-bit/packaging/distros/ubuntu/Dockerfile b/fluent-bit/packaging/distros/ubuntu/Dockerfile new file mode 100644 index 000000000..40272f8fc --- /dev/null +++ b/fluent-bit/packaging/distros/ubuntu/Dockerfile @@ -0,0 +1,165 @@ +# Special Dockerfile to build all Ubuntu targets, the only difference is +# the packages in the base image. +# Set this to the base image to use in each case, so if we want to build for ubuntu/20.04 +# we would set BASE_BUILDER=ubuntu-20.04-base. +ARG BASE_BUILDER +# Lookup the name to use below but should follow the '<distro>-base' convention with slashes replaced. +# Use buildkit to skip unused base images: DOCKER_BUILDKIT=1 + +# Multiarch support +FROM multiarch/qemu-user-static:x86_64-aarch64 as multiarch-aarch64 + +# ubuntu/16.04 base image +FROM ubuntu:16.04 as ubuntu-16.04-base +ENV DEBIAN_FRONTEND noninteractive + +# Using pipe below +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev cmake \ + make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all software-properties-common \ + software-properties-common libyaml-dev apt-transport-https \ + pkg-config libsasl2-2 libsasl2-dev openssl libssl-dev libssl1.0 zlib1g-dev && \ + wget -q -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ + gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ xenial main' && \ + apt-get update && \ + apt-get install -y --reinstall lsb-base lsb-release cmake + +# ubuntu/18.04 base image +FROM ubuntu:18.04 as ubuntu-18.04-base +ENV DEBIAN_FRONTEND noninteractive +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates gcc-8 g++-8 libsystemd-dev \ + cmake make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl1.1 \ + software-properties-common libyaml-dev apt-transport-https pkg-config zlib1g-dev && \ + wget -q -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ + gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ + apt-get update && \ + apt-get install -y --reinstall lsb-base lsb-release cmake && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 1 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 1 + +# ubuntu/18.04.arm64v8 base image +FROM arm64v8/ubuntu:18.04 as ubuntu-18.04.arm64v8-base +ENV DEBIAN_FRONTEND noninteractive +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates gcc-8 g++-8 libsystemd-dev \ + cmake make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl1.1 \ + software-properties-common libyaml-dev apt-transport-https pkg-config zlib1g-dev && \ + wget -q -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ + gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ + apt-get update && \ + apt-get install -y --reinstall lsb-base lsb-release cmake && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 1 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 1 + +# ubuntu/20.04 base image +FROM ubuntu:20.04 as ubuntu-20.04-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev \ + cmake make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl1.1 libyaml-dev pkg-config zlib1g-dev && \ + apt-get install -y --reinstall lsb-base lsb-release + +# ubuntu/20.04.arm64v8 base image +FROM arm64v8/ubuntu:20.04 as ubuntu-20.04.arm64v8-base +ENV DEBIAN_FRONTEND noninteractive + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev \ + cmake make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl1.1 libyaml-dev pkg-config zlib1g-dev && \ + apt-get install -y --reinstall lsb-base lsb-release + +# ubuntu/22.04 base image +FROM ubuntu:22.04 as ubuntu-22.04-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev \ + cmake make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all libpq5 \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl3 libyaml-dev pkg-config zlib1g-dev && \ + apt-get install -y --reinstall lsb-base lsb-release + +# ubuntu/22.04.arm64v8 base image +FROM arm64v8/ubuntu:22.04 as ubuntu-22.04.arm64v8-base +ENV DEBIAN_FRONTEND noninteractive + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev \ + cmake make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all libpq5 \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl3 libyaml-dev pkg-config zlib1g-dev && \ + apt-get install -y --reinstall lsb-base lsb-release + +# Common build for all distributions now +# hadolint ignore=DL3006 +FROM $BASE_BUILDER as builder + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ + +WORKDIR /tmp/fluent-bit/build/ +# CMake configuration variables +ARG CFLAGS="-std=gnu99" +ARG CMAKE_INSTALL_PREFIX=/opt/fluent-bit/ +ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ +ARG FLB_RELEASE=On +ARG FLB_TRACE=On +ARG FLB_SQLDB=On +ARG FLB_HTTP_SERVER=On +ARG FLB_OUT_KAFKA=On +ARG FLB_OUT_PGSQL=On +ARG FLB_JEMALLOC=On +ARG FLB_CHUNK_TRACE=On + +ENV CFLAGS=$CFLAGS +RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ + -DFLB_RELEASE="$FLB_RELEASE" \ + -DFLB_TRACE="$FLB_TRACE" \ + -DFLB_SQLDB="$FLB_SQLDB" \ + -DFLB_HTTP_SERVER="$FLB_HTTP_SERVER" \ + -DFLB_OUT_KAFKA="$FLB_OUT_KAFKA" \ + -DFLB_OUT_PGSQL="$FLB_OUT_PGSQL" \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_JEMALLOC="${FLB_JEMALLOC}" \ + -DFLB_CHUNK_TRACE="${FLB_CHUNK_TRACE}" \ + ../ + +VOLUME [ "/output" ] +CMD [ "/bin/bash", "-c", "make -j 4 && cpack -G DEB && cp *.deb /output/" ] diff --git a/fluent-bit/packaging/local-build-all.sh b/fluent-bit/packaging/local-build-all.sh new file mode 100755 index 000000000..12188e796 --- /dev/null +++ b/fluent-bit/packaging/local-build-all.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -eux +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Simple script to test a build of all supported targets. +# To build multi-arch, QEMU can be used and ideally buildkit support in Docker. +# +# Follow the relevant instructions to do this for your OS, e.g. for Ubuntu it may be: +# $ sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages +# $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# Confirm you can run a non-native architecture image, e.g.: +# $ docker run --rm -t arm64v8/ubuntu uname -m # Run an executable made for aarch64 on x86_64 +# WARNING: The requested image's platform (linux/arm64/v8) does not match the detected host platform (linux/amd64) and no specific platform was requested +# aarch64 +# + +# The local file with all the supported build configs in +JSON_FILE_NAME=${JSON_FILE_NAME:-$SCRIPT_DIR/build-config.json} + +# Output checks are easier plus do not want to fill up git +PACKAGING_OUTPUT_DIR=${PACKAGING_OUTPUT_DIR:-test} +echo "Cleaning any existing output" +rm -rf "${PACKAGING_OUTPUT_DIR:?}/*" + +# Iterate over each target and attempt to build it. +# Verify that an RPM or DEB is created. +jq -cr '.linux_targets[]' "$JSON_FILE_NAME" | while read -r DISTRO +do + echo "$DISTRO" + FLB_OUT_DIR="$PACKAGING_OUTPUT_DIR" /bin/bash "$SCRIPT_DIR"/build.sh -d "$DISTRO" "$@" + if [[ -z $(find "${SCRIPT_DIR}/packages/$DISTRO/$PACKAGING_OUTPUT_DIR/" -type f \( -iname "*-bit-*.rpm" -o -iname "*-bit-*.deb" \) | head -n1) ]]; then + echo "Unable to find any binary packages in: ${SCRIPT_DIR}/packages/$DISTRO/$PACKAGING_OUTPUT_DIR" + exit 1 + fi +done + +echo "Success so cleanup" +rm -rf "${PACKAGING_OUTPUT_DIR:?}/*" diff --git a/fluent-bit/packaging/skopeo-sync.sh b/fluent-bit/packaging/skopeo-sync.sh new file mode 100755 index 000000000..0008925dd --- /dev/null +++ b/fluent-bit/packaging/skopeo-sync.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -eu + +# Simple script to handle skopeo copying of images from staging to release registries. +# Simplifies usage in actions and handles any GPG set up. +# +# Optional variables: +# GPG_KEY - the name/fingerprint of a locally installed GPG key to use for signing images on release. +# +# Required to be set prior to calling this: +# VERSION - the tag we are releasing, e.g. 1.9.1 +# RELEASE_CREDS - the credentials required to push images to the release registry +# STAGING_IMAGE_NAME - the source image to pull from staging +# RELEASE_IMAGE_NAME - the destination image for pushing to release + +# We do it tag-by-tag as Cosign signatures cause problems for Skopeo plus +# this prevents us releasing the wrong subset of images in staging if we use `sync` +declare -a TAGS_TO_SYNC=("$VERSION" "latest" "$VERSION-debug" "latest-debug") + +for TAG in "${TAGS_TO_SYNC[@]}" ; do + # Copy all architectures + # Use the skopeo image as it is not available until Ubuntu 20.10 + if [[ -z "$GPG_KEY" ]]; then + docker run --rm \ + quay.io/skopeo/stable:latest \ + copy \ + --all \ + --src-no-creds \ + --dest-creds "$RELEASE_CREDS" \ + "docker://$STAGING_IMAGE_NAME:$TAG" \ + "docker://$RELEASE_IMAGE_NAME:$TAG" + else + # We first need to import the key then copy over the image all in the same container. + rm -rf /tmp/skopeo-gpg/ + mkdir -p /tmp/skopeo-gpg/ + gpg --output /tmp/skopeo-gpg/private.gpg --export-secret-key --armor --export "$GPG_KEY" + # There's no good way to import the key into the container currenty so we hijack the entrypoint. + docker run --rm \ + -v /tmp/skopeo-gpg:/skopeo-gpg \ + --entrypoint=/bin/bash \ + quay.io/skopeo/stable:latest -c "\ + gpg --import /skopeo-gpg/private.gpg && \ + skopeo \ + copy \ + --all --remove-signatures \ + --sign-by $GPG_KEY \ + --src-no-creds \ + --dest-creds $RELEASE_CREDS \ + docker://$STAGING_IMAGE_NAME:$TAG \ + docker://$RELEASE_IMAGE_NAME:$TAG " + fi +done
\ No newline at end of file diff --git a/fluent-bit/packaging/test-release-packages.sh b/fluent-bit/packaging/test-release-packages.sh new file mode 100755 index 000000000..c72a34529 --- /dev/null +++ b/fluent-bit/packaging/test-release-packages.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -eux +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Verify package install for a release version + +if [[ -f "$SCRIPT_DIR/.env" ]]; then + # shellcheck disable=SC1091 + source "$SCRIPT_DIR/.env" +fi + +CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker} +INSTALL_SCRIPT=${INSTALL_SCRIPT:-https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh} + +INSTALL_CMD="curl $INSTALL_SCRIPT|sh" +EXTRA_MOUNTS="" +if [[ -f "$INSTALL_SCRIPT" ]]; then + ABSOLUTE_PATH=$(realpath "$INSTALL_SCRIPT") + EXTRA_MOUNTS="-v $ABSOLUTE_PATH:/install.sh:ro" + INSTALL_CMD="/install.sh" +fi + +# Optional check for specific version +VERSION_TO_CHECK_FOR=${VERSION_TO_CHECK_FOR:-} +function check_version() { + if [[ -n "$VERSION_TO_CHECK_FOR" ]]; then + local LOG_FILE=$1 + if grep -q "No package ${FLUENT_BIT_INSTALL_PACKAGE_NAME:-fluent-bit}-$VERSION_TO_CHECK_FOR available" "$LOG_FILE"; then + echo "WARNING: Unable to install version: $VERSION_TO_CHECK_FOR" + exit 1 + fi + + if ! grep -q "$VERSION_TO_CHECK_FOR" "$LOG_FILE"; then + echo "WARNING: Not using expected version: $VERSION_TO_CHECK_FOR" + exit 1 + fi + fi +} + +APT_TARGETS=("ubuntu:18.04" + "ubuntu:20.04" + "ubuntu:22.04" + "debian:10" + "debian:11") + +YUM_TARGETS=("centos:7" + "rockylinux:8" + "quay.io/centos/centos:stream9" + "amazonlinux:2" + "amazonlinux:2023") + +for IMAGE in "${YUM_TARGETS[@]}" +do + echo "Testing $IMAGE" + LOG_FILE=$(mktemp) + + # We do want word splitting for EXTRA_MOUNTS + # shellcheck disable=SC2086 + $CONTAINER_RUNTIME run --rm -t \ + -e FLUENT_BIT_PACKAGES_URL="${FLUENT_BIT_PACKAGES_URL:-https://packages.fluentbit.io}" \ + -e FLUENT_BIT_PACKAGES_KEY="${FLUENT_BIT_PACKAGES_KEY:-https://packages.fluentbit.io/fluentbit.key}" \ + -e FLUENT_BIT_RELEASE_VERSION="${FLUENT_BIT_RELEASE_VERSION:-}" \ + -e FLUENT_BIT_INSTALL_COMMAND_PREFIX="${FLUENT_BIT_INSTALL_COMMAND_PREFIX:-}" \ + -e FLUENT_BIT_INSTALL_PACKAGE_NAME="${FLUENT_BIT_INSTALL_PACKAGE_NAME:-fluent-bit}" \ + -e FLUENT_BIT_INSTALL_YUM_PARAMETERS="${FLUENT_BIT_INSTALL_YUM_PARAMETERS:-}" \ + $EXTRA_MOUNTS \ + "$IMAGE" \ + sh -c "$INSTALL_CMD && /opt/fluent-bit/bin/fluent-bit --version" | tee "$LOG_FILE" + check_version "$LOG_FILE" + rm -f "$LOG_FILE" +done + +for IMAGE in "${APT_TARGETS[@]}" +do + echo "Testing $IMAGE" + LOG_FILE=$(mktemp) + # We do want word splitting for EXTRA_MOUNTS + # shellcheck disable=SC2086 + $CONTAINER_RUNTIME run --rm -t \ + -e FLUENT_BIT_PACKAGES_URL="${FLUENT_BIT_PACKAGES_URL:-https://packages.fluentbit.io}" \ + -e FLUENT_BIT_PACKAGES_KEY="${FLUENT_BIT_PACKAGES_KEY:-https://packages.fluentbit.io/fluentbit.key}" \ + -e FLUENT_BIT_RELEASE_VERSION="${FLUENT_BIT_RELEASE_VERSION:-}" \ + -e FLUENT_BIT_INSTALL_COMMAND_PREFIX="${FLUENT_BIT_INSTALL_COMMAND_PREFIX:-}" \ + -e FLUENT_BIT_INSTALL_PACKAGE_NAME="${FLUENT_BIT_INSTALL_PACKAGE_NAME:-fluent-bit}" \ + -e FLUENT_BIT_INSTALL_APT_PARAMETERS="${FLUENT_BIT_INSTALL_APT_PARAMETERS:-}" \ + $EXTRA_MOUNTS \ + "$IMAGE" \ + sh -c "apt-get update && apt-get install -y gpg curl;$INSTALL_CMD && /opt/fluent-bit/bin/fluent-bit --version" | tee "$LOG_FILE" + check_version "$LOG_FILE" + rm -f "$LOG_FILE" +done diff --git a/fluent-bit/packaging/testing/README.md b/fluent-bit/packaging/testing/README.md new file mode 100644 index 000000000..a176e9a45 --- /dev/null +++ b/fluent-bit/packaging/testing/README.md @@ -0,0 +1,4 @@ +# Packaging tests + +This directory contains various sanity/smoke tests for the release process. +It is intended to verify we have correctly released binaries for various platorms, not that some specific feature works - the unit or integration tests should cover this. diff --git a/fluent-bit/packaging/testing/smoke/container/container-smoke-test.sh b/fluent-bit/packaging/testing/smoke/container/container-smoke-test.sh new file mode 100755 index 000000000..4ab03c917 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/container/container-smoke-test.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright 2021 Calyptia, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eux +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Simple smoke test script of each local container architecture that runs up and then +# ensure we can access the web server. + +CONTAINER_NAME=${CONTAINER_NAME:-smoke-test} + +# Ensure the container name is valid as we need this to retrieve the exposed ports. +CONTAINER_NAME=${CONTAINER_NAME//\//-} +CONTAINER_ARCH=${CONTAINER_ARCH:-linux/amd64} +REGISTRY=${REGISTRY:-ghcr.io} +IMAGE_NAME=${IMAGE_NAME:-fluent/fluent-bit} +IMAGE_TAG=${IMAGE_TAG:-latest} + +# Remove any existing container +docker rm -f "$CONTAINER_NAME" + +# Repeat for YAML and legacy config - note the config file extension is important for format detection +declare -a CONFIG_FILES=("fluent-bit.conf" "fluent-bit.yaml") + +for CONFIG_FILE in "${CONFIG_FILES[@]}" +do + if [[ ! -f "$SCRIPT_DIR/$CONFIG_FILE" ]]; then + echo "Missing config file: $SCRIPT_DIR/$CONFIG_FILE" + exit 1 + fi + + echo "Testing: $CONFIG_FILE" + + # Run up the container + docker run --name "$CONTAINER_NAME" -d \ + --platform "$CONTAINER_ARCH" \ + --pull=always \ + --publish-all \ + --restart=no \ + -v "$SCRIPT_DIR/$CONFIG_FILE":"/fluent-bit/etc/$CONFIG_FILE":ro \ + "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG" \ + "/fluent-bit/bin/fluent-bit" "-c" "/fluent-bit/etc/$CONFIG_FILE" + + # Get debug details + docker image inspect "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG" + + # Stream the logs live + docker logs -f "$CONTAINER_NAME" & + + # # Wait for the container to start up as we have to pull it + until [[ $(docker ps --filter "status=running" --filter "name=$CONTAINER_NAME" --quiet | wc -l) -gt 0 ]] ; do + sleep 2 + done + docker ps + # Grab the ephemeral port + docker container inspect "$CONTAINER_NAME" + LOCAL_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "2020/tcp") 0).HostPort}}' "$CONTAINER_NAME") + # Allow to run for a bit + sleep 60 + docker ps + docker logs "$CONTAINER_NAME" + # Check we are still ok + curl -v localhost:"$LOCAL_PORT" | jq + curl -v localhost:"$LOCAL_PORT"/api/v1/metrics | jq + curl -v localhost:"$LOCAL_PORT"/api/v1/uptime | jq + curl -v localhost:"$LOCAL_PORT"/api/v1/health + + # Clean up + docker rm -f "$CONTAINER_NAME" +done diff --git a/fluent-bit/packaging/testing/smoke/container/fluent-bit.conf b/fluent-bit/packaging/testing/smoke/container/fluent-bit.conf new file mode 100644 index 000000000..5b987bc22 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/container/fluent-bit.conf @@ -0,0 +1,31 @@ +[SERVICE] + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_PORT 2020 + Health_Check On + HC_Errors_Count 5 + HC_Retry_Failure_Count 5 + HC_Period 5 + Log_level debug + +[INPUT] + Name random + Tag test + Samples 10 + +[FILTER] + Name Lua + Match * + call append_tag + code function append_tag(tag, timestamp, record) new_record = record new_record["tag"] = tag return 1, timestamp, new_record end + +[FILTER] + Name Expect + Match * + key_exists tag + key_val_eq tag test + action exit + +[OUTPUT] + Name stdout + Match *
\ No newline at end of file diff --git a/fluent-bit/packaging/testing/smoke/container/fluent-bit.yaml b/fluent-bit/packaging/testing/smoke/container/fluent-bit.yaml new file mode 100644 index 000000000..69167e0b4 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/container/fluent-bit.yaml @@ -0,0 +1,33 @@ +--- +# Translated according to https://github.com/fluent/fluent-bit/pull/4621 +service: + http_server: "on" + Health_Check: "on" + log_level: debug + +pipeline: + inputs: + - name: random + tag: test + samples: 10 + + filters: + - name: lua + match: test + call: append_tag + code: | + function append_tag(tag, timestamp, record) + new_record = record + new_record["tag"] = tag + return 1, timestamp, new_record + end + + - name: expect + match: test + key_exists: tag + key_val_eq: tag test + action: exit + + outputs: + - name: stdout + match: test diff --git a/fluent-bit/packaging/testing/smoke/k8s/k8s-smoke-test.sh b/fluent-bit/packaging/testing/smoke/k8s/k8s-smoke-test.sh new file mode 100755 index 000000000..e7e42a0da --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/k8s/k8s-smoke-test.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright 2021 Calyptia, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eux + +# Simple smoke test script using helm deployment to run up a deployment then +# ensure we can access the web server. +# Assumes a kubectl context is set up already, e.g. `kind create cluster` + +NAMESPACE=${NAMESPACE:-default} +REGISTRY=${REGISTRY:-ghcr.io} +IMAGE_NAME=${IMAGE_NAME:-fluent/fluent-bit} +IMAGE_TAG=${IMAGE_TAG:-latest} + +# Deploy using Helm chart +helm repo add fluent-bit https://fluent.github.io/helm-charts || helm repo add fluent-bit https://fluent.github.io/helm-charts/ +helm repo update +helm upgrade --install fluent-bit fluent-bit/fluent-bit \ + --namespace "$NAMESPACE" --create-namespace \ + --wait \ + --set kind=Deployment \ + --set replicaCount=1 \ + --set image.repository="$REGISTRY/$IMAGE_NAME" \ + --set image.tag="$IMAGE_TAG" + +# Output some information on pods running +kubectl describe -n "$NAMESPACE" pods --selector='app.kubernetes.io/name=fluent-bit' + +while true; do + # Find a free ephemeral port to use for port forwarding + FREE_PORTNUM=$(shuf -i 1025-65535 -n 1) + if ! lsof -Pi ":$FREE_PORTNUM" -sTCP:LISTEN; then + # Forward to the deployment web server port + kubectl port-forward -n "$NAMESPACE" deployment/fluent-bit "$FREE_PORTNUM":2020 & + PF_PID=$! + # Wait a bit + sleep 60 + # Provide debug output in case it is required + kubectl describe -n "$NAMESPACE" pods --selector='app.kubernetes.io/name=fluent-bit' + # Check we are still functional + curl -v localhost:"$FREE_PORTNUM" | jq + curl -v localhost:"$FREE_PORTNUM"/api/v1/metrics | jq + curl -v localhost:"$FREE_PORTNUM"/api/v1/uptime | jq + curl -v localhost:"$FREE_PORTNUM"/api/v1/health + kill -9 $PF_PID + exit 0 + fi +done +echo "Unable to find free port" +exit 1
\ No newline at end of file diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.amazonlinux2 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.amazonlinux2 new file mode 100644 index 000000000..c9025da61 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.amazonlinux2 @@ -0,0 +1,46 @@ +# For staging upgrade we use the 'staging-upgrade-prep' as the base +ARG STAGING_BASE=docker.io/dokken/amazonlinux-2 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/amazonlinux-2 as official-install + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/yum.repos.d/*-bit.repo + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +# hadolint ignore=DL3032 +RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \ + wget -q "$FLUENT_BIT_PACKAGES_URL/amazonlinux-2.repo" -O /etc/yum.repos.d/staging.repo && \ + yum update -y && yum install -y fluent-bit && \ + systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.amazonlinux2022 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.amazonlinux2022 new file mode 100644 index 000000000..7e62b4cf1 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.amazonlinux2022 @@ -0,0 +1,46 @@ +# For staging upgrade we use the 'staging-upgrade-prep' as the base +ARG STAGING_BASE=docker.io/dokken/amazonlinux-2022 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/amazonlinux-2022 as official-install + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/yum.repos.d/*-bit.repo + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +# hadolint ignore=DL3032 +RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \ + wget -q "$FLUENT_BIT_PACKAGES_URL/amazonlinux-2022.repo" -O /etc/yum.repos.d/staging.repo && \ + yum update -y && yum install -y fluent-bit && \ + systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos7 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos7 new file mode 100644 index 000000000..6ef5d3821 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos7 @@ -0,0 +1,47 @@ +# For staging upgrade we use the 'staging-upgrade-prep' as the base +ARG STAGING_BASE=docker.io/dokken/centos-7 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/centos-7 as official-install + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Use the one-line install +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/yum.repos.d/*-bit.repo + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \ + wget -nv "$FLUENT_BIT_PACKAGES_URL/centos-7.repo" -O /etc/yum.repos.d/staging.repo +# hadolint ignore=DL3032 +RUN yum update -y && yum install -y fluent-bit && \ + systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos8 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos8 new file mode 100644 index 000000000..6b103bba0 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos8 @@ -0,0 +1,53 @@ +# For staging upgrade we use the 'staging-upgrade-prep' as the base +ARG STAGING_BASE=docker.io/dokken/centos-8 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/centos-8 as official-install +# CentOS is now EOL so have to use the vault repos +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/yum.repos.d/*-bit.repo + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +# CentOS is now EOL so have to use the vault repos +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \ + wget -nv "$FLUENT_BIT_PACKAGES_URL/centos-8.repo" -O /etc/yum.repos.d/staging.repo +# hadolint ignore=DL3032 +RUN yum update -y && yum install -y fluent-bit && \ + systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos9 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos9 new file mode 100644 index 000000000..008439bc8 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.centos9 @@ -0,0 +1,45 @@ +# For staging upgrade we use the 'staging-upgrade-prep' as the base +ARG STAGING_BASE=docker.io/dokken/centos-stream-9 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/centos-stream-9 as official-install +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/yum.repos.d/*-bit.repo + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \ + wget -nv "$FLUENT_BIT_PACKAGES_URL/centos-9.repo" -O /etc/yum.repos.d/staging.repo +# hadolint ignore=DL3032 +RUN yum update -y && yum install -y fluent-bit && \ + systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian10 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian10 new file mode 100644 index 000000000..484a351bc --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian10 @@ -0,0 +1,48 @@ +# For staging upgrade we use the 'official-install' as the base +ARG STAGING_BASE=docker.io/dokken/debian-10 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/debian-10 as official-install + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +# Use the one-line install +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/apt/sources.list.d/fluent-bit.list + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add - +RUN echo "deb $FLUENT_BIT_PACKAGES_URL/debian/buster buster main" >> /etc/apt/sources.list +# hadolint ignore=DL3015,DL3008,DL3009 +RUN apt-get update && apt-get install -y fluent-bit +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian11 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian11 new file mode 100644 index 000000000..b1cf7f815 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian11 @@ -0,0 +1,47 @@ +# For staging upgrade we use the 'official-install' as the base +ARG STAGING_BASE=docker.io/dokken/debian-11 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/debian-11 as official-install + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/apt/sources.list.d/fluent-bit.list + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add - +RUN echo "deb $FLUENT_BIT_PACKAGES_URL/debian/bullseye bullseye main" >> /etc/apt/sources.list +# hadolint ignore=DL3015,DL3008,DL3009 +RUN apt-get update && apt-get install -y fluent-bit +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian12 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian12 new file mode 100644 index 000000000..2944d6883 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.debian12 @@ -0,0 +1,47 @@ +# For staging upgrade we use the 'official-install' as the base +ARG STAGING_BASE=docker.io/dokken/debian-12 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/debian-12 as official-install + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/apt/sources.list.d/fluent-bit.list + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add - +RUN echo "deb $FLUENT_BIT_PACKAGES_URL/debian/bookworm bookworm main" >> /etc/apt/sources.list +# hadolint ignore=DL3015,DL3008,DL3009 +RUN apt-get update && apt-get install -y fluent-bit +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu1804 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu1804 new file mode 100644 index 000000000..b7fa3db25 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu1804 @@ -0,0 +1,50 @@ +# For staging upgrade we use the 'official-install' as the base +ARG STAGING_BASE=docker.io/dokken/ubuntu-18.04 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/ubuntu-18.04 as official-install +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +# Use the one-line install +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/apt/sources.list.d/fluent-bit.list + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install + +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add - +RUN echo "deb $FLUENT_BIT_PACKAGES_URL/ubuntu/bionic bionic main" >> /etc/apt/sources.list +# hadolint ignore=DL3015,DL3008,DL3009 +RUN apt-get update && apt-get install -y fluent-bit +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu2004 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu2004 new file mode 100644 index 000000000..0099d3ea6 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu2004 @@ -0,0 +1,48 @@ +# For staging upgrade we use the 'official-install' as the base +ARG STAGING_BASE=docker.io/dokken/ubuntu-20.04 + +ARG RELEASE_URL=https://packages.fluentbit.io +ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key + +# hadolint ignore=DL3006 +FROM docker.io/dokken/ubuntu-20.04 as official-install +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG RELEASE_URL +ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL} + +ARG RELEASE_KEY +ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY} + +# Use the one-line install +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/apt/sources.list.d/fluent-bit.list + +# hadolint ignore=DL3006 +FROM ${STAGING_BASE} as staging-install +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} + +ARG STAGING_URL +ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL} + +ARG STAGING_KEY=${STAGING_URL}/fluentbit.key +ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN wget -qO - $FLUENT_BIT_PACKAGES_KEY | apt-key add - +RUN echo "deb $FLUENT_BIT_PACKAGES_URL/ubuntu/focal focal main" >> /etc/apt/sources.list +# hadolint ignore=DL3015,DL3008,DL3009 +RUN apt-get update && apt-get install -y fluent-bit +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu2204 b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu2204 new file mode 100644 index 000000000..71edff7af --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/Dockerfile.ubuntu2204 @@ -0,0 +1,34 @@ +# For staging upgrade we use the 'official-install' as the base + +# hadolint ignore=DL3007 +FROM docker.io/dokken/ubuntu-22.04:latest as official-install +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Use the one-line install +RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +FROM official-install as staging-upgrade-prep +RUN rm -f /etc/apt/sources.list.d/fluent-bit.list + +# hadolint ignore=DL3007 +FROM docker.io/dokken/ubuntu-22.04:latest as staging-install +ARG STAGING_URL +ARG STAGING_VERSION +ENV STAGING_VERSION=${STAGING_VERSION} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN wget -qO - $STAGING_URL/fluentbit.key | apt-key add - +RUN echo "deb $STAGING_URL/ubuntu/jammy jammy main" >> /etc/apt/sources.list +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install --no-install-recommends -y fluent-bit \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN systemctl enable fluent-bit + +COPY ./test.sh /test.sh +RUN chmod a+x /test.sh + +# hadolint ignore=DL3007 +FROM staging-install as staging-upgrade diff --git a/fluent-bit/packaging/testing/smoke/packages/local-test-all.sh b/fluent-bit/packaging/testing/smoke/packages/local-test-all.sh new file mode 100755 index 000000000..3c596aed9 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/local-test-all.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Simple script to test all supported targets easily. + +RELEASE_URL=${RELEASE_URL:-https://packages.fluentbit.io} +STAGING_URL=${STAGING_URL:-https://fluentbit-staging.s3.amazonaws.com} + +for DOCKERFILE in "$SCRIPT_DIR"/Dockerfile.*; do + DISTRO=${DOCKERFILE##*.} + echo "Testing $DISTRO" + PACKAGE_TEST="$DISTRO" RELEASE_URL="$RELEASE_URL" STAGING_URL="$STAGING_URL" "$SCRIPT_DIR"/run-package-tests.sh +done
\ No newline at end of file diff --git a/fluent-bit/packaging/testing/smoke/packages/run-package-tests.sh b/fluent-bit/packaging/testing/smoke/packages/run-package-tests.sh new file mode 100755 index 000000000..e51569f79 --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/run-package-tests.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Copyright 2021 Calyptia, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eux +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +PACKAGE_TEST=${PACKAGE_TEST:-centos7} +RELEASE_URL=${RELEASE_URL:-https://packages.fluentbit.io} +RELEASE_KEY=${RELEASE_KEY:-https://packages.fluentbit.io/fluentbit.key} +STAGING_URL=${STAGING_URL:-https://fluentbit-staging.s3.amazonaws.com} +STAGING_KEY=${STAGING_KEY:-https://fluentbit-staging.s3.amazonaws.com/fluentbit.key} + +# Podman is preferred as better systemd support and cgroups handling +CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} + +if [[ ! -f "$SCRIPT_DIR/Dockerfile.$PACKAGE_TEST" ]]; then + echo "No definition for $SCRIPT_DIR/Dockerfile.$PACKAGE_TEST" + exit 1 +fi + +declare -a CONTAINER_TARGETS=("official-install" "staging-install" "staging-upgrade") + +# Build all containers required +for TARGET in "${CONTAINER_TARGETS[@]}" +do + BUILD_ARGS="" + if [[ $TARGET == "staging-upgrade" ]]; then + BUILD_ARGS="--build-arg STAGING_BASE=staging-upgrade-prep" + fi + + CONTAINER_NAME="package-verify-$PACKAGE_TEST-$TARGET" + + # Cope with needing --ignore for podman but not for docker + "${CONTAINER_RUNTIME}" rm --force --ignore --volumes "$CONTAINER_NAME" || "${CONTAINER_RUNTIME}" rm --force --volumes "$CONTAINER_NAME" + + # We do want splitting for build args + # shellcheck disable=SC2086 + "${CONTAINER_RUNTIME}" build \ + --build-arg STAGING_KEY=$STAGING_KEY \ + --build-arg STAGING_URL=$STAGING_URL \ + --build-arg RELEASE_KEY=$RELEASE_KEY \ + --build-arg RELEASE_URL=$RELEASE_URL $BUILD_ARGS \ + --target "$TARGET" \ + -t "$CONTAINER_NAME" \ + -f "$SCRIPT_DIR/Dockerfile.$PACKAGE_TEST" "$SCRIPT_DIR/" + + if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then + "${CONTAINER_RUNTIME}" run --rm -d \ + --privileged \ + -v /sys/fs/cgroup/:/sys/fs/cgroup:ro \ + --name "$CONTAINER_NAME" \ + "$CONTAINER_NAME" + else + "${CONTAINER_RUNTIME}" run --rm -d \ + --timeout 30 \ + --name "$CONTAINER_NAME" \ + "$CONTAINER_NAME" + fi + + "${CONTAINER_RUNTIME}" exec -t "$CONTAINER_NAME" /test.sh + + "${CONTAINER_RUNTIME}" rm --force --ignore --volumes "$CONTAINER_NAME" || "${CONTAINER_RUNTIME}" rm --force --volumes "$CONTAINER_NAME" +done diff --git a/fluent-bit/packaging/testing/smoke/packages/test.sh b/fluent-bit/packaging/testing/smoke/packages/test.sh new file mode 100644 index 000000000..b82ae6eae --- /dev/null +++ b/fluent-bit/packaging/testing/smoke/packages/test.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -ex + +if [ "$SKIP_TEST" = "yes" ]; then + echo "Skipping test" + exit 0 +fi + +echo "Check package installed" +rpm -q fluent-bit || dpkg -l fluent-bit + +echo "Check service enabled" +systemctl is-enabled fluent-bit + +until systemctl is-system-running; do + # On more recent systems we may see degrade when running + [ "$(systemctl is-system-running)" = "degraded" ] && break + systemctl --failed + sleep 10 +done + +echo "Check service running" +systemctl status -q --no-pager fluent-bit diff --git a/fluent-bit/packaging/update-apt-repo.sh b/fluent-bit/packaging/update-apt-repo.sh new file mode 100755 index 000000000..73bd4a0c2 --- /dev/null +++ b/fluent-bit/packaging/update-apt-repo.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -eux + +# Used to update a Debian Apt repo, e.g. during a staging build or release process + +# Where the base of all the repos is +BASE_PATH=${BASE_PATH:?} +if [[ ! -d "$BASE_PATH" ]]; then + echo "ERROR: invalid base path: $BASE_PATH" + exit 1 +fi + +# "debian/bookworm" "debian/bullseye" "debian/buster" "ubuntu/xenial" "ubuntu/bionic" "ubuntu/focal" "ubuntu/jammy" "raspbian/buster" "raspbian/bullseye" +DEB_REPO=${DEB_REPO:?} + +# Set true to prevent signing +DISABLE_SIGNING=${DISABLE_SIGNING:-false} + +REPO_DIR=$(realpath -sm "$BASE_PATH/$DEB_REPO" ) +if [[ ! -d "$REPO_DIR" ]] ; then + echo "ERROR: missing $REPO_DIR" + exit 1 +fi + +CODENAME=${DEB_REPO##*/} +echo "Updating $DEB_REPO for $CODENAME" + +# We use Aptly to create repos with a local temporary directory as the root. +# Once complete, we then move these to the output directory for upload. +# Based on https://github.com/spotify/debify/blob/master/debify.sh +APTLY_REPO_NAME="debify-$CODENAME" +APTLY_ROOTDIR=$(mktemp -d) +APTLY_CONFIG=$(mktemp) + +# The origin and label fields seem to cover the base directory for the repo and codename. +# The docs seems to suggest these fields are optional and free-form: https://wiki.debian.org/DebianRepository/Format#Origin +# They are security checks to verify if they have changed so we match the legacy server. +APTLY_ORIGIN=". $CODENAME" +APTLY_LABEL=". $CODENAME" +if [[ "$DEB_REPO" == "debian/bullseye" ]]; then + # For Bullseye, the legacy server had a slightly different setup we try to reproduce here + APTLY_ORIGIN="bullseye bullseye" + APTLY_LABEL="bullseye bullseye" +fi + +cat << EOF > "$APTLY_CONFIG" +{ + "rootDir": "$APTLY_ROOTDIR/" +} +EOF +cat "$APTLY_CONFIG" + +aptly -config="$APTLY_CONFIG" repo create \ + -component="main" \ + -distribution="$CODENAME" \ + "$APTLY_REPO_NAME" + +# Check if any files to add +count=$(find "$REPO_DIR" -maxdepth 1 -type f -name "*.deb" | wc -l) +if [[ $count != 0 ]] ; then + # Do not remove files as we need them from moving to staging-release + aptly -config="$APTLY_CONFIG" repo add -force-replace "$APTLY_REPO_NAME" "$REPO_DIR/" +else + echo "WARNING: no files to add in $DEB_REPO for $CODENAME" +fi +aptly -config="$APTLY_CONFIG" repo show "$APTLY_REPO_NAME" + +if [[ "$DISABLE_SIGNING" != "true" ]]; then + aptly -config="$APTLY_CONFIG" publish repo -gpg-key="$GPG_KEY" -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" +else + aptly -config="$APTLY_CONFIG" publish repo --skip-signing -origin="$APTLY_ORIGIN" -label="$APTLY_LABEL" "$APTLY_REPO_NAME" +fi + +rsync -av "$APTLY_ROOTDIR"/public/* "$REPO_DIR" +# Remove unnecessary files +rm -rf "$REPO_DIR/conf/" "$REPO_DIR/db/" "$APTLY_ROOTDIR" "$APTLY_CONFIG" diff --git a/fluent-bit/packaging/update-repos.sh b/fluent-bit/packaging/update-repos.sh new file mode 100755 index 000000000..a4d711e95 --- /dev/null +++ b/fluent-bit/packaging/update-repos.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -eux +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Wrapper script around the actual ones used in CI +# Intended only for legacy/manual use in event of failure in CI +# Ensure to add dependencies, e.g. for Ubuntu 22.04: awscli git createrepo-c debsigs aptly rsync gnupg2 +# Following that there are a few things to do: +# Import the signing key (if signing) +# gpg --import <private key> +# gpg --export -a "$GPG_KEY" > /tmp/fluentbit.key +# rpm --import /tmp/fluentbit.key + +export BASE_PATH=${BASE_PATH:-$1} +if [[ ! -d "$BASE_PATH" ]]; then + echo "Specified BASE_PATH is not a directory: $BASE_PATH" + exit 1 +fi + +export DISABLE_SIGNING=${DISABLE_SIGNING:-false} +export CREATE_REPO_CMD=${CREATE_REPO_CMD:-} +export CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} +# Must be set for signing +if [[ "$DISABLE_SIGNING" != "false" ]]; then + export GPG_KEY=${GPG_KEY:?} +fi + +# Set these to force a manual S3 sync and update +# AWS_SYNC=true +# AWS_S3_BUCKET_RELEASE=packages.fluentbit.io +# AWS_S3_BUCKET_STAGING=fluentbit-staging +export AWS_REGION=${AWS_REGION:-us-east-1} + +RPM_REPO_PATHS=("amazonlinux/2" "amazonlinux/2023" "centos/7" "centos/8" "centos/9") + +if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_RELEASE:?}" "${BASE_PATH:?}" +fi + +for RPM_REPO in "${RPM_REPO_PATHS[@]}"; do + export RPM_REPO + + if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/$RPM_REPO" "${BASE_PATH:?}/$RPM_REPO" + fi + + /bin/bash -eux "$SCRIPT_DIR/update-yum-repo.sh" +done + +DEB_REPO_PATHS=( "debian/bookworm" + "debian/bullseye" + "debian/buster" + "ubuntu/xenial" + "ubuntu/bionic" + "ubuntu/focal" + "ubuntu/jammy" + "raspbian/buster" + "raspbian/bullseye" ) + +for DEB_REPO in "${DEB_REPO_PATHS[@]}"; do + export DEB_REPO + if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/$DEB_REPO" "${BASE_PATH:?}/$DEB_REPO" + fi + /bin/bash -eux "$SCRIPT_DIR/update-apt-repo.sh" +done + +# Other OS now +if [[ "${AWS_SYNC:-false}" != "false" ]]; then + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/macos" "${BASE_PATH:?}/macos" + aws s3 sync s3://"${AWS_S3_BUCKET_STAGING:?}/windows" "${BASE_PATH:?}/windows" + + # Final review, do not push until checked manually + aws s3 sync "${BASE_PATH:?}" s3://"${AWS_S3_BUCKET_RELEASE:?}" --exact-timestamps --dryrun +fi diff --git a/fluent-bit/packaging/update-source-packages.sh b/fluent-bit/packaging/update-source-packages.sh new file mode 100755 index 000000000..af6296769 --- /dev/null +++ b/fluent-bit/packaging/update-source-packages.sh @@ -0,0 +1,78 @@ +#!/bin/bash +set -eux + +# We provide the source code as a standalone checksummed package along with the JSON schema and AppVeyor binaries. +# This is hosted at releases.fluentbit.io + +WINDOWS_SOURCE_DIR=${WINDOWS_SOURCE_DIR:-ignore} +VERSION=${VERSION:-$1} +MAJOR_VERSION=${MAJOR_VERSION:-} +SOURCE_DIR=${SOURCE_DIR:?} +TARGET_DIR=${TARGET_DIR:?} + +if [[ -z "$VERSION" ]]; then + echo "Missing VERSION value" + exit 1 +fi + +if [[ -z "$MAJOR_VERSION" ]]; then + MAJOR_VERSION=${VERSION%.*} +fi + +if [[ ! -d "$SOURCE_DIR" ]]; then + echo "Missing source directory: $SOURCE_DIR" + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]]; then + echo "Missing target directory: $TARGET_DIR" + exit 1 +fi +# create MAJOR_VERSION dir if not exist +mkdir -p "$TARGET_DIR/$MAJOR_VERSION" + +# Handle the JSON schema by copying in the new versions (if they exist). +echo "Updating JSON schema" +find "$SOURCE_DIR/" -iname "fluent-bit-schema*$VERSION*.json" -exec cp -vf "{}" "$TARGET_DIR/$MAJOR_VERSION/" \; + +# Intended for AppVeyor usage +if [[ -d "$WINDOWS_SOURCE_DIR" ]]; then + echo "Using overridden Windows directory: $WINDOWS_SOURCE_DIR" + pushd "$WINDOWS_SOURCE_DIR" + for i in *.exe + do + echo "$i" + sha256sum "$i" > "$i".sha256 + done + + for i in *.zip + do + echo "$i" + sha256sum "$i" > "$i".sha256 + done + popd + # shellcheck disable=SC2086 + cp -vf "$WINDOWS_SOURCE_DIR"/*$VERSION* "$TARGET_DIR/$MAJOR_VERSION/" +else + # Windows - we do want word splitting and ensure some files exist + if compgen -G "$SOURCE_DIR/windows/*$VERSION*" > /dev/null; then + echo "Copying Windows artefacts" + # shellcheck disable=SC2086 + cp -vf "$SOURCE_DIR"/windows/*$VERSION* "$TARGET_DIR/$MAJOR_VERSION/" + else + echo "Missing Windows builds" + fi +fi + +# Source - we do want word splitting and ensure some files exist +if compgen -G "$SOURCE_DIR/source-$VERSION*" > /dev/null; then + echo "Copying source artefacts" + # shellcheck disable=SC2086 + cp -vf "$SOURCE_DIR"/source-$VERSION* "$TARGET_DIR/$MAJOR_VERSION/" +elif compgen -G "$SOURCE_DIR/source/*$VERSION*" > /dev/null; then + echo "Copying (legacy) source artefacts" + # shellcheck disable=SC2086 + cp -vf "$SOURCE_DIR"/source/*$VERSION* "$TARGET_DIR/$MAJOR_VERSION/" +else + echo "Missing source artefacts" +fi diff --git a/fluent-bit/packaging/update-yum-repo.sh b/fluent-bit/packaging/update-yum-repo.sh new file mode 100755 index 000000000..766b4a294 --- /dev/null +++ b/fluent-bit/packaging/update-yum-repo.sh @@ -0,0 +1,87 @@ +#!/bin/bash +set -eux + +# Used to update a Yum repo, e.g. during a staging build or release process + +#("amazonlinux/2" "amazonlinux/2022" "centos/7" "centos/8" "centos/9") +RPM_REPO=${RPM_REPO:?} + +# Where the base of all the repos is +BASE_PATH=${BASE_PATH:-$1} +if [[ ! -d "$BASE_PATH" ]]; then + echo "ERROR: invalid base path: $BASE_PATH" + exit 1 +fi + +# Set true to prevent signing +DISABLE_SIGNING=${DISABLE_SIGNING:-false} +if [[ "$DISABLE_SIGNING" != "true" ]]; then + echo "INFO: RPM signing configuration" + rpm --showrc|grep gpg + rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n' +fi + +# Handle Ubuntu 18/22 differences - no support on Ubuntu 20 +CREATE_REPO_CMD=${CREATE_REPO_CMD:-} +CREATE_REPO_ARGS=${CREATE_REPO_ARGS:--dvp} + +# Assume if set we want to use it +if [[ -n "$CREATE_REPO_CMD" ]]; then + echo "INFO: using $CREATE_REPO_CMD" +elif command -v createrepo &> /dev/null; then + echo "INFO: found createrepo" + CREATE_REPO_CMD="createrepo" +elif command -v createrepo_c &> /dev/null; then + echo "INFO: found createrepo_c" + CREATE_REPO_CMD="createrepo_c" +else + echo "ERROR: unable to find a command equivalent to createrepo" + exit 1 +fi + +echo "INFO: updating $RPM_REPO" + +REPO_DIR=$( realpath -sm "$BASE_PATH/$RPM_REPO" ) +if [[ ! -d "$REPO_DIR" ]] ; then + echo "ERROR: missing $REPO_DIR" + exit 1 +fi + +if [[ "$DISABLE_SIGNING" != "true" ]]; then + # Sign all RPMs created for this target, cover both fluent-bit and legacy packages + find "$REPO_DIR" -name "*-bit-*.rpm" -exec rpm --define "_gpg_name $GPG_KEY" --addsign {} \; +fi +# Create full metadata for all RPMs in the directory +"$CREATE_REPO_CMD" "$CREATE_REPO_ARGS" "$REPO_DIR" + +# Set up repo info +if [[ -n "${AWS_S3_BUCKET:-}" ]]; then + # Create top-level file so replace path separator with dash + # centos/8 --> centos-8.repo + # This way we make sure not to have a mixed repo or overwrite files for each target. + REPO_TYPE=${RPM_REPO/\//-} + echo "INFO: setting up $BASE_PATH/$REPO_TYPE.repo" + cat << EOF > "$BASE_PATH/$REPO_TYPE.repo" +[Fluent-Bit] +name=Fluent Bit Packages - $REPO_TYPE - \$basearch +baseurl=https://$AWS_S3_BUCKET.s3.amazonaws.com/$RPM_REPO/ +enabled=1 +gpgkey=https://$AWS_S3_BUCKET.s3.amazonaws.com/fluentbit.key +gpgcheck=1 +repo_gpgcheck=1 +EOF +fi + +# Ensure we sign the Yum repo meta-data +if [[ "$DISABLE_SIGNING" != "true" ]]; then + # We use this form to fail on error during the find, otherwise -exec will succeed or just do one file with + + while IFS= read -r -d '' REPO_METADATA_FILE + do + echo "INFO: signing $REPO_METADATA_FILE" + gpg --detach-sign --batch --armor --yes -u "$GPG_KEY" "$REPO_METADATA_FILE" + done < <(find "$REPO_DIR" -name repomd.xml -print0) + # Debug ouput for checking + find "$REPO_DIR" -name "repomd.xml*" -exec ls -l {} \; +fi + +echo "INFO: Completed $RPM_REPO" diff --git a/fluent-bit/packaging/windows-checksums.sh b/fluent-bit/packaging/windows-checksums.sh new file mode 100755 index 000000000..227ac0ff2 --- /dev/null +++ b/fluent-bit/packaging/windows-checksums.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -eu + +# Generate checksums for local Windows binaries + +SOURCE_DIR=${SOURCE_DIR:?} + +pushd "$SOURCE_DIR" +for i in *.exe +do + echo "$i" + sha256sum "$i" > "$i".sha256 +done + +for i in *.zip +do + echo "$i" + sha256sum "$i" > "$i".sha256 +done +popd |