diff options
Diffstat (limited to 'fluent-bit/.github')
50 files changed, 4825 insertions, 0 deletions
diff --git a/fluent-bit/.github/ISSUE_TEMPLATE.md b/fluent-bit/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..26e1d9fc --- /dev/null +++ b/fluent-bit/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,9 @@ +<!--- +Thanks for filing an issue 😄 ! Before you submit, please read the following: + +Please post all questions and issues on https://slack.fluentd.org/ on the fluent-bit before opening a Github Issue. Your questions will reach a wider audience there, +and if we confirm that there is a bug, then you can open a new issue. + +Check the other issue templates if you are trying to submit a bug report, feature request, or question +Search open/closed issues before submitting since someone might have asked the same thing before! +--> diff --git a/fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md b/fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..11535469 --- /dev/null +++ b/fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,40 @@ +--- +name: "\U0001F41B Bug Report" +about: "If something isn't working as expected \U0001F914." +title: '' +labels: 'status: waiting-for-triage' +assignees: '' + +--- + +## Bug Report + +**Describe the bug** +<!--- A clear and concise description of what the bug is. --> + +**To Reproduce** +- Rubular link if applicable: +- Example log message if applicable: +``` +{"log":"YOUR LOG MESSAGE HERE","stream":"stdout","time":"2018-06-11T14:37:30.681701731Z"} +``` +- Steps to reproduce the problem: + +**Expected behavior** +<!--- A clear and concise description of what you expected to happen. --> + +**Screenshots** +<!--- If applicable, add screenshots to help explain your problem. --> + +**Your Environment** +<!--- Include as many relevant details about the environment you experienced the bug in --> +* Version used: +* Configuration: +* Environment name and version (e.g. Kubernetes? What version?): +* Server type and version: +* Operating System and version: +* Filters and plugins: + +**Additional context** +<!--- How has this issue affected you? What are you trying to accomplish? --> +<!--- Providing context helps us come up with a solution that is most useful in the real world --> diff --git a/fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md b/fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..db207e41 --- /dev/null +++ b/fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --> + +**Describe the solution you'd like** +<!--- A clear and concise description of what you want to happen. --> + +**Describe alternatives you've considered** +<!--- A clear and concise description of any alternative solutions or features you've considered. --> + +**Additional context** +<!--- How has this issue affected you? What are you trying to accomplish? --> +<!--- Providing context helps us come up with a solution that is most useful in the real world --> +<!--- Add any other context or screenshots about the feature request here. --> diff --git a/fluent-bit/.github/PULL_REQUEST_TEMPLATE.md b/fluent-bit/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..fc910708 --- /dev/null +++ b/fluent-bit/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,39 @@ +<!-- Provide summary of changes --> + +<!-- Issue number, if available. E.g. "Fixes #31", "Addresses #42, #77" --> + +---- +Enter `[N/A]` in the box, if an item is not applicable to your change. + +**Testing** +Before we can approve your change; please submit the following in a comment: +- [ ] Example configuration file for the change +- [ ] Debug log output from testing the change +<!-- +Please refer to the Developer Guide for instructions on building Fluent Bit with Valgrind support: +https://github.com/fluent/fluent-bit/blob/master/DEVELOPER_GUIDE.md#valgrind +Invoke Fluent Bit and Valgrind as: $ valgrind --leak-check=full ./bin/fluent-bit <args> +--> +- [ ] Attached [Valgrind](https://valgrind.org/docs/manual/quick-start.html) output that shows no leaks or memory corruption was found + +If this is a change to packaging of containers or native binaries then please confirm it works for all targets. +- [ ] Run [local packaging test](./packaging/local-build-all.sh) showing all targets (including any new ones) build. +- [ ] Set `ok-package-test` label to test for all targets (requires maintainer to do). + +**Documentation** +<!-- Docs can be edited at https://github.com/fluent/fluent-bit-docs --> +- [ ] Documentation required for this feature + +<!-- Doc PR (not required but highly recommended) --> + +**Backporting** +<!-- +PRs targeting the default master branch will go into the next major release usually. +If this PR should be backported to the current or earlier releases then please submit a PR for that particular branch. +--> +- [ ] Backport to latest stable release. + +<!-- Other release PR (not required but highly recommended for quick turnaround) --> +---- + +Fluent Bit is licensed under Apache 2.0, by submitting this pull request I understand that this code will be released under the terms of that license. diff --git a/fluent-bit/.github/actionlint-matcher.json b/fluent-bit/.github/actionlint-matcher.json new file mode 100644 index 00000000..7b1ba251 --- /dev/null +++ b/fluent-bit/.github/actionlint-matcher.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "actionlint", + "pattern": [ + { + "regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$", + "file": 1, + "line": 2, + "column": 3, + "message": 4, + "code": 5 + } + ] + } + ] + }
\ No newline at end of file diff --git a/fluent-bit/.github/actionlint.yml b/fluent-bit/.github/actionlint.yml new file mode 100644 index 00000000..692d0852 --- /dev/null +++ b/fluent-bit/.github/actionlint.yml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - actuated + - actuated-aarch64 diff --git a/fluent-bit/.github/actions/generate-package-build-matrix/action.yaml b/fluent-bit/.github/actions/generate-package-build-matrix/action.yaml new file mode 100644 index 00000000..f8355e4b --- /dev/null +++ b/fluent-bit/.github/actions/generate-package-build-matrix/action.yaml @@ -0,0 +1,119 @@ +name: Composite action to generate the matrix of targets to build packages for. +description: Remove any duplication of this information so we only have to update in one place. + +# Remember to add any new checks and target creation required. +# For example, when 2.0 comes out we should detect it and add any target changes. + +inputs: + target: + description: Override to build a single target for debug/test only. + required: false + default: "" + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building we then use to determine version for. + required: true +outputs: + build-matrix: + description: The total build matrix we have created. + value: ${{ steps.set-matrix.outputs.matrix }} + deb-build-matrix: + description: The targets that provide DEB artefacts. + value: ${{ steps.set-matrix.outputs.debmatrix }} + rpm-build-matrix: + description: The targets that provide RPN artefacts. + value: ${{ steps.set-matrix.outputs.rpmmatrix }} +runs: + using: "composite" + steps: + - name: Checkout code for version check + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + path: version-check + + - name: Determine target type + id: determine-build-type + run: | + BUILD_TYPE="legacy" + if [[ -f "packaging/build-config.json" ]]; then + BUILD_TYPE="modern" + fi + echo "Detected type: $BUILD_TYPE" + echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_OUTPUT + shell: bash + working-directory: version-check + + - name: 2.0+ targets + if: steps.determine-build-type.outputs.BUILD_TYPE == 'modern' + run: | + matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets|map(.target)' packaging/build-config.json)'}'|jq -c .) + echo "MATRIX=$matrix" >> $GITHUB_ENV + + # The following are only used by release so exclude architecture as well + + debtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="deb") | .target ]' packaging/build-config.json) + debmatrix=$(echo "{ \"distro\" : $debtargets }"|jq -c .) + echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV + + rpmtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="rpm") | .target ]' packaging/build-config.json) + rpmmatrix=$(echo "{ \"distro\" : $rpmtargets}"|jq -c .) + echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV + shell: bash + + - name: 1.9 targets + if: steps.determine-build-type.outputs.BUILD_TYPE == 'legacy' + run: | + matrix=$(( + echo '{ "distro" : [' + echo '"amazonlinux/2", "amazonlinux/2.arm64v8",' + echo '"centos/7", "centos/7.arm64v8", "centos/8", "centos/8.arm64v8",' + echo '"debian/buster", "debian/buster.arm64v8", "debian/bullseye", "debian/bullseye.arm64v8",' + echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",' + echo '"ubuntu/18.04.arm64v8", "ubuntu/20.04.arm64v8", "ubuntu/22.04.arm64v8",' + echo '"raspbian/buster", "raspbian/bullseye"' + echo ']}' + ) | jq -c .) + echo "MATRIX=$matrix" >> $GITHUB_ENV + debmatrix=$(( + echo '{ "distro" : [' + echo '"debian/buster", "debian/bullseye",' + echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",' + echo '"raspbian/buster", "raspbian/bullseye"' + echo ']}' + ) | jq -c .) + echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV + rpmmatrix=$(( + echo '{ "distro" : [' + echo '"amazonlinux/2",' + echo '"centos/7", "centos/8"' + echo ']}' + ) | jq -c .) + echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV + shell: bash + + - name: Manual override of target + if: inputs.target != '' + run: | + if [ -n "${{ inputs.target || '' }}" ]; then + echo "Overriding matrix to build: ${{ inputs.target }}" + matrix=$(( + echo '{ "distro" : [' + echo '"${{ inputs.target }}"' + echo ']}' + ) | jq -c .) + fi + echo "MATRIX=$matrix" >> $GITHUB_ENV + shell: bash + + - id: set-matrix + run: | + echo $MATRIX + echo $MATRIX| jq . + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + echo $DEB_MATRIX + echo $DEB_MATRIX| jq . + echo "debmatrix=$DEB_MATRIX" >> $GITHUB_OUTPUT + echo $RPM_MATRIX + echo $RPM_MATRIX| jq . + echo "rpmmatrix=$RPM_MATRIX" >> $GITHUB_OUTPUT + shell: bash diff --git a/fluent-bit/.github/actions/sync-to-bucket/action.yaml b/fluent-bit/.github/actions/sync-to-bucket/action.yaml new file mode 100644 index 00000000..e79c0b3d --- /dev/null +++ b/fluent-bit/.github/actions/sync-to-bucket/action.yaml @@ -0,0 +1,88 @@ +name: Composite action to sync S3 buckets +description: Carry out all the tasks to sync to a bucket and make reusable. + +inputs: + bucket: + description: The name of the S3 (US-East) bucket to sync packages from. + required: true + access_key_id: + description: The S3 access key id for the bucket. + required: true + secret_access_key: + description: The S3 secret access key for the bucket. + required: true + bucket-directory: + description: The directory in the bucket to sync to. + required: true + source-directory: + description: The source directory to sync from. + required: true + aws-region: + description: The default region to use. + required: false + default: "us-east-1" + aws-custom-endpoint: + # To use with Minio locally (or update to whatever endpoint you want) + # '--endpoint http://localhost:9000' + description: A custom endpoint for S3 commands, e.g. for Minio. + required: false + +runs: + using: "composite" + steps: + - name: Local - Check and sync + run: | + if [[ ! -d "$SOURCE_DIR" ]]; then + echo "No source directory: $SOURCE_DIR" + ls -lR + exit 1 + fi + echo "Valid source directory: $SOURCE_DIR" + ls -lR "$SOURCE_DIR" + env: + SOURCE_DIR: ${{ inputs.source-directory }} + shell: bash + + - name: AWS - Check and sync + run: | + # For Minio, etc. + if [ -n "${AWS_S3_ENDPOINT}" ]; then + ENDPOINT="--endpoint-url ${AWS_S3_ENDPOINT}" + fi + + # Check for non-empty values + if [ -z "$AWS_S3_BUCKET" ]; then + echo "Invalid (empty) bucket defined, check running on right environment to allow access to any secrets" + exit 1 + fi + echo "$AWS_S3_BUCKET bucket is defined" + + # Verify bucket access + bucketstatus=$(aws --region "$AWS_REGION" s3api head-bucket --bucket "${AWS_S3_BUCKET}" ${ENDPOINT} 2>&1) + echo "Response: $bucketstatus" + if echo "${bucketstatus}" | grep 'Not Found'; then + echo "$AWS_S3_BUCKET: bucket does not exist"; + exit 1 + elif echo "${bucketstatus}" | grep 'Forbidden'; then + echo "$AWS_S3_BUCKET: bucket exists but not owned" + exit 1 + elif echo "${bucketstatus}" | grep 'Bad Request'; then + echo "$AWS_S3_BUCKET: bucket name specified is less than 3 or greater than 63 characters" + exit 1 + else + echo "$AWS_S3_BUCKET: bucket owned and exists"; + fi + + # Sync to bucket + aws --region "$AWS_REGION" s3 sync "${SOURCE_DIR}" "s3://${AWS_S3_BUCKET}/${DEST_DIR}" --follow-symlinks --no-progress ${ENDPOINT} + env: + SOURCE_DIR: ${{ inputs.source-directory }} + DEST_DIR: ${{ inputs.bucket-directory }} + # Make sure to run in an environment with access to any secrets that are passed in. + # Otherwise they will be empty. + AWS_S3_BUCKET: ${{ inputs.bucket }} + AWS_REGION: ${{ inputs.aws-region }} + AWS_ACCESS_KEY_ID: ${{ inputs.access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.secret_access_key }} + AWS_S3_ENDPOINT: ${{ inputs.aws-custom-endpoint }} + shell: bash diff --git a/fluent-bit/.github/dependabot.yml b/fluent-bit/.github/dependabot.yml new file mode 100644 index 00000000..dc6a2c38 --- /dev/null +++ b/fluent-bit/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "workflows: "
\ No newline at end of file diff --git a/fluent-bit/.github/workflows/README.md b/fluent-bit/.github/workflows/README.md new file mode 100644 index 00000000..aa52f593 --- /dev/null +++ b/fluent-bit/.github/workflows/README.md @@ -0,0 +1,227 @@ +# Available workflows + +| Workflow file | Description | Run event | +| :---------------------------------------------------- | ------------------------ | ------------------------------------------------- | +| [build-master-packages](./build-master-packages.yaml) | Builds packages using `master` for certain targets | on new commit/push on master / manual | +| [cron-unstable-build](./cron-unstable-build.yaml) | Automated nightly builds of each supported branch | Scheduled/manual trigger | +| [master-integration-test](./master-integration-test.yaml) | Runs the integration testing suite on master | on new commit/push on master | +| [staging-build](./staging-build.yaml) | Builds the distro packages and docker images from a tagged release into staging (S3 and GHCR) | on new release/tag | +| [staging-test](./staging-test.yaml) | Test the staging distro packages and docker images| manually or when `staging-build` completes successfully | +| [staging-release](./staging-release.yaml) | Publishes the docker images/manifest on hub.docker.io/fluent/ and the distro packages | manual approval | +| [pr-closed-docker](./pr-closed-docker.yaml) | Removes docker images for PR on hub.docker.io/fluentbitdev/| on pr closed| +| [pr-compile-check](./pr-compile-check.yaml) | Runs some compilation sanity checks on a PR | +| [pr-integration-test](./pr-integration-test.yaml) | Runs the integration testing suite on a PR branch | pr opened / label created 'ok-to-test' / on new commit/push on PR(s) | +| [pr-package-tests](./pr-package-tests.yaml) | Runs the package build for all targets on a PR branch | pr opened / label created 'ok-package-test' / on new commit/push on PR(s) | +| [pr-perf-test](./pr-integration-test.yaml) | Runs the performance testing suite on a PR branch | pr opened / label created 'ok-to-performance-test' / on new commit/push on PR(s) | +| [pr-stale](./pr-stale.yaml) | Closes stale PR(s) with no activity in 30 days | scheduled daily 01:30 AM UTC| +| [unit-tests](./unit-tests.yaml) | Runs the unit tests suite on master push or new PR | PR opened, merge in master branch | + +## Available labels + +| Label name | Description | +| :----------|-------------| +| docs-required| default tag used to request documentation, has to be removed before merge | +| ok-package-test | Build for all possible targets | +| ok-to-test | run all integration tests | +| ok-to-merge | run mergebot and merge (rebase) current PR | +| ci/integration-docker-ok | integration test is able to build docker image | +| ci/integration-gcp-ok | integration test is able to run on GCP | +| long-term | long running pull request, don't close | +| exempt-stale | prevent stale checks running | + +## Required secrets + +* AWS_ACCESS_KEY_ID +* AWS_SECRET_ACCESS_KEY +* AWS_S3_BUCKET_STAGING +* AWS_S3_BUCKET_RELEASE +* GPG_PRIVATE_KEY +* GPG_PRIVATE_KEY_PASSPHRASE + +These are only required for Cosign of the container images, will be skipped if not present: + +* COSIGN_PRIVATE_KEY +* COSIGN_PRIVATE_KEY_PASSWORD - if set otherwise not required + +## Environments + +These environments are used: + +* `unstable` for all nightly builds +* `staging` for all staging builds +* `release` for running the promotion of staging to release, this can have additional approvals added + +If an environment is not present then it will be created but this may not have the appropriate permissions then. + +## Pushing to Github Container Registry + +Github actions require specific permissions to push to packages, see: <https://github.community/t/403-error-on-container-registry-push-from-github-action/173071/39> +For some reason this is not automatically done via permission inheritance or similar. + +1. Verify you can push with a simple test, e.g. `docker pull alpine && docker tag alpine:latest ghcr.io/<repo>/fluent-bit:latest && docker push ghcr.io/<repo>/fluent-bit:latest` +2. Once this is working locally, you should then be able to set up action permissions for the repository. If you already have a package no need to push a test one. +3. Go to `https://github.com/users/USER/packages/container/fluent-bit/settings` and ensure the repository has access to `Write`. + +## Version-specific targets + +Each major version (e.g. 1.8 & 1.9) supports different targets to build for, e.g. 1.9 includes a CentOS 8 target and 1.8 has some other legacy targets. + +This is all handled by the [build matrix generation composite action](../actions/generate-package-build-matrix/action.yaml). +This uses a [JSON file](../../packaging/build-config.json) to specify the targets so ensure this is updated. +The build matrix is then fed into the [reusable job](./call-build-linux-packages.yaml) that builds packages which will then fire for the appropriate targets. +The reusable job is used for all package builds including unstable/nightly and the PR `ok-package-test` triggered ones. + +## Releases + +The process at a high level is as follows: + +1. Tag created with `v` prefix. +2. [Deploy to staging](https://github.com/fluent/fluent-bit/actions/workflows/staging-build.yaml) workflow runs. +3. [Test staging](https://github.com/fluent/fluent-bit/actions/workflows/staging-test.yaml) workflow runs. +4. Manually initiate [release from staging](https://github.com/fluent/fluent-bit/actions/workflows/staging-release.yaml) workflow. +5. A PR is auto-created to increment the minor version now for Fluent Bit using the [`update_version.sh`](../../update_version.sh) script. +6. Create PRs for doc updates - Windows & container versions. (WIP to automate). + +Breaking the steps down. + +### Deploy to staging and test + +This should run automatically when a tag is created matching the `v*` regex. +It currently copes with 1.8+ builds although automation is only exercised for 1.9+ releases. + +Once this is completed successfully the staging tests should also run automatically. + + + +If both complete successfully then we are good to go. + +Occasional failures are seen with package builds not downloading dependencies (CentOS 7 in particular seems bad for this). +A re-run of failed jobs should resolve this. + +The workflow builds all Linux, macOS and Windows targets to a staging S3 bucket plus the container images to ghcr.io. + +### Release from staging workflow + +This is a manually initiated workflow, the intention is multiple staging builds can happen but we only release one. +Note that currently we do not support parallel staging builds of different versions, e.g. master and 1.9 branches. +**We can only release the previous staging build and there is a check to confirm version.** + +Ensure AppVeyor build for the tag has completed successfully as well. + +To trigger: <https://github.com/fluent/fluent-bit/actions/workflows/staging-release.yaml> + +All this job does is copy the various artefacts from staging locations to release ones, it does not rebuild them. + + + +With this example you can see we used the wrong `version` as it requires it without the `v` prefix (it is used for container tag, etc.) and so it fails. + + + +Make sure to provide without the `v` prefix. + + + +Once this workflow is initiated you then also need to have it approved by the designated "release team" otherwise it will not progress. + + + +They will be notified for approval by Github. +Unfortunately it has to be approved for each job in the sequence rather than a global approval for the whole workflow although that can be useful to check between jobs. + + + +This is quite useful to delay the final smoke test of packages until after the manual steps are done as it will then verify them all for you. + +#### Packages server sync + +The workflow above ensures all release artefacts are pushed to the appropriate container registry and S3 bucket for official releases. +The packages server then periodically syncs from this bucket to pull down and serve the new packages so there may be a delay (up to 1 hour) before it serves the new versions. +The syncs happen hourly. +See <https://github.com/fluent/fluent-bit-infra/blob/main/terraform/provision/package-server-provision.sh.tftpl> for details of the dedicated packages server. + +The main reason for a separate server is to accurately track download statistics. +Container images are handled by ghcr.io and Docker Hub, not this server. + +#### Transient container publishing failures + +The parallel publishing of multiple container tags for the same image seems to fail occasionally with network errors, particularly more for ghcr.io than DockerHub. +This can be resolved by just re-running the failed jobs. + +#### Windows builds from AppVeyor + +This is automated, however confirm that the actual build is successful for the tag: <https://ci.appveyor.com/project/fluent/fluent-bit-2e87g/history> +If not then ask a maintainer to retrigger. + +It can take a while to find the one for the specific tag... + +#### ARM builds + +All builds are carried out in containers and intended to be run on a valid Ubuntu host to match a standard Github Actions runner. +This can take some time for ARM as we have to emulate the architecture via QEMU. + +<https://github.com/fluent/fluent-bit/pull/7527> introduces support to run ARM builds on a dedicated [actuated.dev](https://docs.actuated.dev/) ephemeral VM runner. +A self-hosted ARM runner is sponsored by [Equinix Metal](https://deploy.equinix.com/metal/) and provisioned for this per the [documentation](https://docs.actuated.dev/provision-server/). +For fork workflows, this should all be skipped and run on a normal Ubuntu Github hosted runner but be aware this may take some time. + +### Manual release + +As long as it is built to staging we can manually publish packages as well via the script here: <https://github.com/fluent/fluent-bit/blob/master/packaging/update-repos.sh> + +Containers can be promoted manually too, ensure to include all architectures and signatures. + +### Create PRs + +Once releases are published we need to provide PRs for the following documentation updates: + +1. Windows checksums: <https://docs.fluentbit.io/manual/installation/windows#installation-packages> +2. Container versions: <https://docs.fluentbit.io/manual/installation/docker#tags-and-versions> + +<https://github.com/fluent/fluent-bit-docs> is the repo for updates to docs. + +Take the checksums from the release process above, the AppVeyor stage provides them all and we attempt to auto-create the PR with it. + +## Unstable/nightly builds + +These happen every 24 hours and [reuse the same workflow](./cron-unstable-build.yaml) as the staging build so are identical except they skip the upload to S3 step. +This means all targets are built nightly for `master` and `2.0` branches including container images and Linux, macOS and Windows packages. + +The container images are available here (the tag refers to the branch): + +* [ghcr.io/fluent/fluent-bit/unstable:2.0](ghcr.io/fluent/fluent-bit/unstable:2.0) +* [ghcr.io/fluent/fluent-bit/unstable:master](ghcr.io/fluent/fluent-bit/unstable:master) +* [ghcr.io/fluent/fluent-bit/unstable:windows-2019-2.0](ghcr.io/fluent/fluent-bit/unstable:windows-2019-2.0) +* [ghcr.io/fluent/fluent-bit/unstable:windows-2019-master](ghcr.io/fluent/fluent-bit/unstable:windows-2019-master) + +The Linux, macOS and Windows packages are available to download from the specific workflow run. + +## Integration tests + +On every commit to `master` we rebuild the [packages](./build-master-packages.yaml) and [container images](./master-integration-test.yaml). +The container images are then used to [run the integration tests](./master-integration-test.yaml) from the <https://github.com/fluent/fluent-bit-ci> repository. +The container images are available as: + +* [ghcr.io/fluent/fluent-bit/master:x86_64](ghcr.io/fluent/fluent-bit/master:x86_64) + +## PR checks + +Various workflows are run for PRs automatically: + +* [Unit tests](./unit-tests.yaml) +* [Compile checks on CentOS 7 compilers](./pr-compile-check.yaml) +* [Linting](./pr-lint.yaml) +* [Windows builds](./pr-windows-build.yaml) +* [Fuzzing](./pr-fuzz.yaml) +* [Container image builds](./pr-image-tests.yaml) +* [Install script checks](./pr-install-script.yaml) + +We try to guard these to only trigger when relevant files are changed to reduce any delays or resources used. +**All should be able to be triggered manually for explicit branches as well.** + +The following workflows can be triggered manually for specific PRs too: + +* [Integration tests](./pr-integration-test.yaml): Build a container image and run the integration tests as per commits to `master`. +* [Performance tests](./pr-perf-test.yaml): WIP to trigger a performance test on a dedicated VM and collect the results as a PR comment. +* [Full package build](./pr-package-tests.yaml): builds all Linux, macOs and Windows packages as well as container images. + +To trigger these, apply the relevant label. diff --git a/fluent-bit/.github/workflows/build-branch-containers.yaml b/fluent-bit/.github/workflows/build-branch-containers.yaml new file mode 100644 index 00000000..ea5d3c6c --- /dev/null +++ b/fluent-bit/.github/workflows/build-branch-containers.yaml @@ -0,0 +1,20 @@ +name: Build containers for a specific branch of 1.9+ +on: + workflow_dispatch: + inputs: + version: + description: Version of Fluent Bit to build, commit, branch, etc. The container image will be ghcr.io/fluent/fluent-bit/test/<this value>. + required: true + default: master +jobs: + build-branch-containers: + uses: ./.github/workflows/call-build-images.yaml + with: + version: ${{ github.event.inputs.version }} + ref: ${{ github.event.inputs.version }} + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/test/${{ github.event.inputs.version }} + unstable: ${{ github.event.inputs.version }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/fluent-bit/.github/workflows/build-legacy-branch.yaml b/fluent-bit/.github/workflows/build-legacy-branch.yaml new file mode 100644 index 00000000..2bca2a71 --- /dev/null +++ b/fluent-bit/.github/workflows/build-legacy-branch.yaml @@ -0,0 +1,156 @@ +name: Build containers for a specific branch of 1.8 +on: + workflow_dispatch: + inputs: + ref: + description: The code to build so a commit, branch, etc. The container image will be ghcr.io/fluent/fluent-bit/test/<this value>. + required: true + default: "1.8" + +env: + IMAGE_NAME: ghcr.io/${{ github.repository }}/test/${{ github.event.inputs.ref }} + +jobs: + build-legacy-branch-meta: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Check this is a 1.8 type build + run: | + if [[ -f "dockerfiles/Dockerfile" ]]; then + echo "Invalid branch as contains Dockerfile: ${{ inputs.ref }}" + exit 1 + fi + shell: bash + + # For 1.8 builds it is a little more complex so we have this build matrix to handle it. + # This creates separate images for each architecture. + # The later step then creates a multi-arch manifest for all of these. + build-legacy-images-matrix: + name: Build single arch legacy images + runs-on: ubuntu-latest + needs: + - build-legacy-branch-meta + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64, arm/v7] + include: + - arch: amd64 + suffix: x86_64 + - arch: arm/v7 + suffix: arm32v7 + - arch: arm64 + suffix: arm64v8 + permissions: + contents: read + packages: write + steps: + - name: Checkout the docker build repo for legacy builds + uses: actions/checkout@v4 + with: + repository: fluent/fluent-bit-docker-image + ref: "1.8" # Fixed to this branch + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: debug-meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + raw,${{ inputs.ref }}-debug + + - name: Build the legacy x86_64 debug image + if: matrix.arch == 'amd64' + uses: docker/build-push-action@v5 + with: + file: ./Dockerfile.x86_64.debug + context: . + tags: ${{ steps.debug-meta.outputs.tags }} + labels: ${{ steps.debug-meta.outputs.labels }} + provenance: false + platforms: linux/amd64 + push: true + load: false + build-args: | + FLB_TARBALL=https://github.com/fluent/fluent-bit/tarball/${{ inputs.ref }} + + - name: Extract metadata from Github + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + raw,${{ matrix.suffix }}-${{ inputs.ref }} + + - name: Build the legacy ${{ matrix.arch }} image + uses: docker/build-push-action@v5 + with: + file: ./Dockerfile.${{ matrix.suffix }} + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/${{ matrix.arch }} + provenance: false + push: true + load: false + build-args: | + FLB_TARBALL=https://github.com/fluent/fluent-bit/tarball/${{ inputs.ref }} + + # Create a multi-arch manifest for the separate 1.8 images. + build-legacy-image-manifests: + name: Deploy multi-arch container image manifests + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + needs: + - build-legacy-branch-meta + - build-legacy-images-matrix + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull all the images + # Use platform to trigger warnings on invalid image metadata + run: | + docker pull --platform=linux/amd64 ${{ env.IMAGE_NAME }}:x86_64-${{ inputs.ref }} + docker pull --platform=linux/arm64 ${{ env.IMAGE_NAME }}:arm64v8-${{ inputs.ref }} + docker pull --platform=linux/arm/v7 ${{ env.IMAGE_NAME }}:arm32v7-${{ inputs.ref }} + shell: bash + + - name: Create manifests for images + run: | + docker manifest create ${{ env.IMAGE_NAME }}:${{ inputs.ref }} \ + --amend ${{ env.IMAGE_NAME }}:x86_64-${{ inputs.ref }} \ + --amend ${{ env.IMAGE_NAME }}:arm64v8-${{ inputs.ref }} \ + --amend ${{ env.IMAGE_NAME }}:arm32v7-${{ inputs.ref }} + docker manifest push --purge ${{ env.IMAGE_NAME }}:${{ inputs.ref }} + env: + DOCKER_CLI_EXPERIMENTAL: enabled + shell: bash diff --git a/fluent-bit/.github/workflows/build-master-packages.yaml b/fluent-bit/.github/workflows/build-master-packages.yaml new file mode 100644 index 00000000..5698abfe --- /dev/null +++ b/fluent-bit/.github/workflows/build-master-packages.yaml @@ -0,0 +1,51 @@ +on: + push: + branches: + - master + workflow_dispatch: + inputs: + version: + description: Version of Fluent Bit to build + required: false + default: master + target: + description: Only build a specific target, intended for debug/test builds only. + required: false + default: "" + +name: Build packages for master +jobs: + master-build-generate-matrix: + name: Staging build matrix + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + # Set up the list of target to build so we can pass the JSON to the reusable job + - id: set-matrix + run: | + matrix=$(( + echo '{ "distro" : [ "debian/bullseye", "ubuntu/20.04", "ubuntu/22.04", "centos/7" ]}' + ) | jq -c .) + if [ -n "${{ github.event.inputs.target || '' }}" ]; then + echo "Overriding matrix to build: ${{ github.event.inputs.target }}" + matrix=$(( + echo '{ "distro" : [' + echo '"${{ github.event.inputs.target }}"' + echo ']}' + ) | jq -c .) + fi + echo $matrix + echo $matrix| jq . + echo "matrix=$matrix" >> $GITHUB_OUTPUT + shell: bash + + master-build-packages: + needs: master-build-generate-matrix + uses: ./.github/workflows/call-build-linux-packages.yaml + with: + version: master + ref: master + build_matrix: ${{ needs.master-build-generate-matrix.outputs.build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/fluent-bit/.github/workflows/call-build-images.yaml b/fluent-bit/.github/workflows/call-build-images.yaml new file mode 100644 index 00000000..fd2cf5b1 --- /dev/null +++ b/fluent-bit/.github/workflows/call-build-images.yaml @@ -0,0 +1,329 @@ +--- +name: Reusable workflow to build container images + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + registry: + description: The registry to push container images to. + type: string + required: true + username: + description: The username for the registry. + type: string + required: true + image: + description: The name of the container image to push to the registry. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + unstable: + description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. + type: string + required: false + default: "" + secrets: + token: + description: The Github token or similar to authenticate with for the registry. + required: true + cosign_private_key: + description: The optional Cosign key to use for signing the images. + required: false + cosign_private_key_password: + description: If the Cosign key requires a password then specify here, otherwise not required. + required: false +jobs: + call-build-images-meta: + name: Extract any supporting metadata + outputs: + major-version: ${{ steps.determine-major-version.outputs.replaced }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + # For main branch/releases we want to tag with the major version. + # E.g. if we build version 1.9.2 we want to tag with 1.9.2 and 1.9. + - name: Determine major version tag + id: determine-major-version + uses: frabert/replace-string-action@v2.4 + with: + pattern: '^(\d+\.\d+).*$' + string: ${{ inputs.version }} + replace-with: "$1" + flags: "g" + + # This is the intended approach to multi-arch image and all the other checks scanning, + # signing, etc only trigger from this. + call-build-images: + needs: + - call-build-images-meta + name: Multiarch container images to GHCR + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + packages: write + outputs: + production-digest: ${{ steps.build_push.outputs.digest }} + debug-digest: ${{ steps.debug_build_push.outputs.digest }} + steps: + - name: Checkout code for modern style builds + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Extract metadata from Github + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image }} + tags: | + raw,${{ inputs.version }} + raw,${{ needs.call-build-images-meta.outputs.major-version }} + raw,latest + + - name: Build the production images + id: build_push + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/Dockerfile + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64, linux/arm64, linux/arm/v7 + target: production + # Must be disabled to provide legacy format images from the registry + provenance: false + push: true + load: false + build-args: | + FLB_NIGHTLY_BUILD=${{ inputs.unstable }} + RELEASE_VERSION=${{ inputs.version }} + + - id: debug-meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image }} + tags: | + raw,${{ inputs.version }}-debug + raw,${{ needs.call-build-images-meta.outputs.major-version }}-debug + raw,latest-debug + + - name: Build the debug multi-arch images + id: debug_build_push + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/Dockerfile + context: . + tags: ${{ steps.debug-meta.outputs.tags }} + labels: ${{ steps.debug-meta.outputs.labels }} + platforms: linux/amd64, linux/arm64, linux/arm/v7 + # Must be disabled to provide legacy format images from the registry + provenance: false + target: debug + push: true + load: false + build-args: | + FLB_NIGHTLY_BUILD=${{ inputs.unstable }} + RELEASE_VERSION=${{ inputs.version }} + + call-build-images-generate-schema: + needs: + - call-build-images-meta + - call-build-images + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + packages: read + steps: + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Generate schema + run: | + docker run --rm -t ${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.version }} -J > fluent-bit-schema-${{ inputs.version }}.json + cat fluent-bit-schema-${{ inputs.version }}.json | jq -M > fluent-bit-schema-pretty-${{ inputs.version }}.json + shell: bash + + - name: Upload the schema + uses: actions/upload-artifact@v3 + with: + path: ./fluent-bit-schema*.json + name: fluent-bit-schema-${{ inputs.version }} + if-no-files-found: error + + call-build-images-scan: + needs: + - call-build-images-meta + - call-build-images + name: Trivy + Dockle image scan + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + packages: read + steps: + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Trivy - multi-arch + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.version }}" + format: "table" + exit-code: "1" + ignore-unfixed: true + vuln-type: "os,library" + severity: "CRITICAL,HIGH" + + - name: Dockle - multi-arch + uses: hands-lab/dockle-action@v1 + with: + image: "${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.version }}" + exit-code: "1" + exit-level: WARN + + call-build-images-sign: + needs: + - call-build-images-meta + - call-build-images + name: Deploy and sign multi-arch container image manifests + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - name: Install cosign + uses: sigstore/cosign-installer@v2 + + - name: Cosign keyless signing using Rektor public transparency log + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance, and records it to the + # sigstore community Rekor transparency log. + # + # We use recursive signing on the manifest to cover all the images. + run: | + cosign sign --recursive \ + -a "repo=${{ github.repository }}" \ + -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + -a "ref=${{ github.sha }}" \ + -a "release=${{ inputs.version }}" \ + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \ + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}" + shell: bash + # Ensure we move on to key-based signing as well + continue-on-error: true + env: + COSIGN_EXPERIMENTAL: true + + - name: Cosign with a key + # Only run if we have a key defined + if: ${{ env.COSIGN_PRIVATE_KEY }} + # The key needs to cope with newlines + run: | + echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key + cosign sign --key /tmp/my_cosign.key --recursive \ + -a "repo=${{ github.repository }}" \ + -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + -a "ref=${{ github.sha }}" \ + -a "release=${{ inputs.version }}" \ + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \ + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}" + rm -f /tmp/my_cosign.key + shell: bash + continue-on-error: true + env: + COSIGN_PRIVATE_KEY: ${{ secrets.cosign_private_key }} + COSIGN_PASSWORD: ${{ secrets.cosign_private_key_password }} # optional + + # This takes a long time... + call-build-windows-container: + name: Windows container images + runs-on: windows-${{ matrix.windows-base-version }} + environment: ${{ inputs.environment }} + needs: + - call-build-images-meta + strategy: + fail-fast: true + matrix: + windows-base-version: + - '2019' + - '2022' + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Build the production images + run: | + docker build -t ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }} --build-arg FLB_NIGHTLY_BUILD=${{ inputs.unstable }} --build-arg WINDOWS_VERSION=ltsc${{ matrix.windows-base-version }} -f ./dockerfiles/Dockerfile.windows . + docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }} + + # We cannot use this action as it requires privileged mode + # uses: docker/build-push-action@v5 + # with: + # file: ./dockerfiles/Dockerfile.windows + # context: . + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} + # platforms: windows/amd64 + # target: runtime + # push: true + # load: false + # build-args: | + # FLB_NIGHTLY_BUILD=${{ inputs.unstable }} + # WINDOWS_VERSION=ltsc2019 diff --git a/fluent-bit/.github/workflows/call-build-linux-packages.yaml b/fluent-bit/.github/workflows/call-build-linux-packages.yaml new file mode 100644 index 00000000..b8000225 --- /dev/null +++ b/fluent-bit/.github/workflows/call-build-linux-packages.yaml @@ -0,0 +1,263 @@ +--- +name: Reusable workflow to build binary packages into S3 bucket + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + unstable: + description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. + type: string + required: false + default: "" + ignore_failing_targets: + description: Optionally ignore any failing builds in the matrix and continue. + type: boolean + required: false + default: false + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + gpg_private_key: + description: The GPG key to use for signing the packages. + required: false + gpg_private_key_passphrase: + description: The GPG key passphrase to use for signing the packages. + required: false + +jobs: + call-build-capture-source: + # Capture source tarball and generate checksum for it + name: Extract any supporting metadata + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + path: source + + - name: Create tarball and checksums + run: | + tar -czvf $SOURCE_FILENAME_PREFIX.tar.gz -C source --exclude-vcs . + md5sum $SOURCE_FILENAME_PREFIX.tar.gz > $SOURCE_FILENAME_PREFIX.tar.gz.md5 + sha256sum $SOURCE_FILENAME_PREFIX.tar.gz > $SOURCE_FILENAME_PREFIX.tar.gz.sha256 + # Move to a directory to simplify upload/sync + mkdir -p source-packages + cp -fv $SOURCE_FILENAME_PREFIX* source-packages/ + shell: bash + env: + SOURCE_FILENAME_PREFIX: source-${{ inputs.version }} + + - name: Upload the source artifacts + uses: actions/upload-artifact@v3 + with: + name: source-${{ inputs.version }} + path: source-packages/* + if-no-files-found: error + + # Pick up latest master version + - name: Checkout code for action + if: inputs.environment == 'staging' + uses: actions/checkout@v4 + with: + path: action-support + + - name: Push tarballs to S3 + # Only upload for staging + if: inputs.environment == 'staging' + uses: ./action-support/.github/actions/sync-to-bucket + with: + bucket: ${{ secrets.bucket }} + access_key_id: ${{ secrets.access_key_id }} + secret_access_key: ${{ secrets.secret_access_key }} + bucket-directory: "${{ inputs.version }}/source" + source-directory: "source-packages/" + + call-build-linux-packages: + name: ${{ matrix.distro }} package build and stage to S3 + environment: ${{ inputs.environment }} + # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage. + runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-aarch64') || 'ubuntu-latest' }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.build_matrix) }} + # Potentially we support continuing with all successful targets + continue-on-error: ${{ inputs.ignore_failing_targets || false }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Set up Actuated mirror + if: contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') + uses: self-actuated/hub-mirror@master + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Replace all special characters with dashes + id: formatted_distro + run: + output=${INPUT//[\/]/-} + echo "$INPUT --> $output" + echo "replaced=$output" >> "$GITHUB_OUTPUT" + shell: bash + env: + INPUT: ${{ matrix.distro }} + + - name: fluent-bit - ${{ matrix.distro }} artifacts + run: | + ./build.sh + env: + FLB_DISTRO: ${{ matrix.distro }} + FLB_OUT_DIR: ${{ inputs.version }}/staging + FLB_NIGHTLY_BUILD: ${{ inputs.unstable }} + CMAKE_INSTALL_PREFIX: /opt/fluent-bit/ + working-directory: packaging + + - name: Upload the ${{ steps.formatted_distro.outputs.replaced }} artifacts + uses: actions/upload-artifact@v3 + with: + name: packages-${{ inputs.version }}-${{ steps.formatted_distro.outputs.replaced }} + path: packaging/packages/ + if-no-files-found: error + + - name: Retrieve target info for repo creation + id: get-target-info + # Remove any .arm648 suffix + # For ubuntu map to codename using the disto-info list (CSV) + run: | + sudo apt-get update + sudo apt-get install -y distro-info awscli + TARGET=${DISTRO%*.arm64v8} + if [[ "$TARGET" == "ubuntu/"* ]]; then + UBUNTU_CODENAME=$(cut -d ',' -f 1,3 < "/usr/share/distro-info/ubuntu.csv"|grep "${TARGET##*/}"|cut -d ',' -f 2) + if [[ -n "$UBUNTU_CODENAME" ]]; then + TARGET="ubuntu/$UBUNTU_CODENAME" + else + echo "Unable to extract codename for $DISTRO" + exit 1 + fi + fi + echo "$TARGET" + echo "target=$TARGET" >> $GITHUB_OUTPUT + env: + DISTRO: ${{ matrix.distro }} + shell: bash + + - name: Verify output target + # Only upload for staging + # Make sure not to do a --delete on sync as it will remove the other architecture + run: | + if [ -z "${{ steps.get-target-info.outputs.target }}" ]; then + echo "Invalid (empty) target defined" + exit 1 + fi + shell: bash + + # Pick up latest master version + - name: Checkout code for action + if: inputs.environment == 'staging' + uses: actions/checkout@v4 + with: + path: action-support + + - name: Push packages to S3 + # Only upload for staging + if: inputs.environment == 'staging' + uses: ./action-support/.github/actions/sync-to-bucket + with: + bucket: ${{ secrets.bucket }} + access_key_id: ${{ secrets.access_key_id }} + secret_access_key: ${{ secrets.secret_access_key }} + bucket-directory: "${{ inputs.version }}/${{ steps.get-target-info.outputs.target }}/" + source-directory: "packaging/packages/${{ matrix.distro }}/${{ inputs.version }}/staging/" + + call-build-linux-packages-repo: + name: Create repo metadata in S3 + # Only upload for staging + if: inputs.environment == 'staging' + # Need to use 18.04 as 20.04 has no createrepo available + runs-on: ubuntu-22.04 + environment: ${{ inputs.environment }} + needs: + - call-build-linux-packages + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y createrepo-c aptly awscli + + - name: Checkout code for repo metadata construction - always latest + uses: actions/checkout@v4 + + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.gpg_private_key }} + passphrase: ${{ secrets.gpg_private_key_passphrase }} + + - name: Create repositories on staging now + # We sync down what we have for the release directories. + # Create the repo metadata then upload to the root of the bucket. + # This will wipe out any versioned directories in the process. + run: | + rm -rf ./latest/ + mkdir -p ./latest/ + if [ -n "${AWS_S3_ENDPOINT}" ]; then + ENDPOINT="--endpoint-url ${AWS_S3_ENDPOINT}" + fi + aws s3 sync "s3://$AWS_S3_BUCKET/${{ inputs.version }}" ./latest/ --no-progress ${ENDPOINT} + + gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./latest/fluentbit.key + rpm --import ./latest/fluentbit.key + + ./update-repos.sh "./latest/" + echo "${{ inputs.version }}" > "./latest/latest-version.txt" + aws s3 sync "./latest/" "s3://$AWS_S3_BUCKET" --delete --follow-symlinks --no-progress ${ENDPOINT} + env: + GPG_KEY: ${{ steps.import_gpg.outputs.name }} + AWS_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: ${{ secrets.access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.secret_access_key }} + AWS_S3_BUCKET: ${{ secrets.bucket }} + # To use with Minio locally (or update to whatever endpoint you want) + # AWS_S3_ENDPOINT: http://localhost:9000 + shell: bash + working-directory: packaging diff --git a/fluent-bit/.github/workflows/call-build-macos.yaml b/fluent-bit/.github/workflows/call-build-macos.yaml new file mode 100644 index 00000000..c97ee838 --- /dev/null +++ b/fluent-bit/.github/workflows/call-build-macos.yaml @@ -0,0 +1,132 @@ +--- +name: Reusable workflow to build MacOS packages optionally into S3 bucket + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + unstable: + description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. + type: string + required: false + default: '' + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +jobs: + call-build-macos-legacy-check: + # Requires https://github.com/fluent/fluent-bit/pull/5247 so will not build for previous branches + name: Extract any supporting metadata + outputs: + build-type: ${{ steps.determine-build-type.outputs.BUILD_TYPE }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Determine build type + id: determine-build-type + run: | + BUILD_TYPE="legacy" + if [[ -f "conf/fluent-bit-macos.conf" ]]; then + BUILD_TYPE="modern" + fi + echo "Detected type: $BUILD_TYPE" + echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_OUTPUT + shell: bash + + call-build-macos-package: + if: needs.call-build-macos-legacy-check.outputs.build-type == 'modern' + runs-on: macos-latest + environment: ${{ inputs.environment }} + needs: + - call-build-macos-legacy-check + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Install dependencies + run: | + brew update + brew install bison flex libyaml openssl pkgconfig || true + + - name: Build Fluent Bit packages + run: | + export LIBRARY_PATH=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib:$LIBRARY_PATH + cmake -DCPACK_GENERATOR=productbuild -DFLB_NIGHTLY_BUILD=${{ inputs.unstable }} ../ -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) + cmake --build . + cpack -G productbuild + working-directory: build + + - name: Upload build packages + uses: actions/upload-artifact@v3 + with: + name: macos-packages + path: | + build/fluent-bit-*-apple* + build/fluent-bit-*-intel* + if-no-files-found: error + + call-build-macos-s3-upload: + name: Handle upload to S3 + # The environment must be used that has access to any secrets required, even if passed in. + # If passed in but not in the environment here you end up with an empty secret. + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + needs: + - call-build-macos-package + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Download all artefacts + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: macos-packages + path: artifacts/ + + - name: Push MacOS packages to S3 + if: inputs.environment == 'staging' + uses: ./.github/actions/sync-to-bucket + with: + bucket: ${{ secrets.bucket }} + access_key_id: ${{ secrets.access_key_id }} + secret_access_key: ${{ secrets.secret_access_key }} + bucket-directory: "${{ inputs.version }}/macos/" + source-directory: "artifacts/" diff --git a/fluent-bit/.github/workflows/call-build-windows.yaml b/fluent-bit/.github/workflows/call-build-windows.yaml new file mode 100644 index 00000000..7adc229a --- /dev/null +++ b/fluent-bit/.github/workflows/call-build-windows.yaml @@ -0,0 +1,221 @@ +--- +name: Reusable workflow to build Windows packages optionally into S3 bucket + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + unstable: + description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. + type: string + required: false + default: '' + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +jobs: + + call-build-windows-get-meta: + name: Determine build info + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + armSupported: ${{ steps.armcheck.outputs.armSupported }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Determine if we are doing a build with ARM support + id: armcheck + # Check for new contents from https://github.com/fluent/fluent-bit/pull/6621 + run: | + if grep -q "winarm64" CMakeLists.txt ; then + echo "armSupported=true" >> $GITHUB_OUTPUT + else + echo "armSupported=false" >> $GITHUB_OUTPUT + fi + shell: bash + + call-build-windows-package: + runs-on: windows-latest + environment: ${{ inputs.environment }} + needs: + - call-build-windows-get-meta + strategy: + fail-fast: false + matrix: + config: + - name: "Windows 32bit" + arch: x86 + openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static + cmake_additional_opt: "" + vcpkg_triplet: x86-windows-static + - name: "Windows 64bit" + arch: x64 + openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static + cmake_additional_opt: "" + vcpkg_triplet: x64-windows-static + - name: "Windows 64bit (Arm64)" + arch: amd64_arm64 + openssl_dir: C:\vcpkg\packages\openssl_arm64-windows-static + cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64" + vcpkg_triplet: arm64-windows-static + permissions: + contents: read + # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures. + env: + PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin;C:\vcpkg; + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Get dependencies + run: | + Invoke-WebRequest -O winflexbison.zip $env:WINFLEXBISON + Expand-Archive winflexbison.zip -Destination C:\WinFlexBison + Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe + Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe + echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append + env: + WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip + shell: pwsh + + - name: Set up with Developer Command Prompt for Microsoft Visual C++ + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.config.arch }} + + - name: Get gzip command w/ chocolatey + uses: crazy-max/ghaction-chocolatey@v3 + with: + args: install gzip -y + + # http://man7.org/linux/man-pages/man1/date.1.html + - name: Get Date + id: get-date + run: | + echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore cached packages of vcpkg + id: cache-vcpkg-sources + uses: actions/cache/restore@v3 + with: + path: | + C:\vcpkg\packages + key: ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-${{ steps.get-date.outputs.date }} + restore-keys: | + ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg- + enableCrossOsArchive: false + + - name: Build openssl with vcpkg + run: | + C:\vcpkg\vcpkg install --recurse openssl --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + + - name: Build libyaml with vcpkg + run: | + C:\vcpkg\vcpkg install --recurse libyaml --triplet ${{ matrix.config.vcpkg_triplet }} + shell: cmd + + - name: Save packages of vcpkg + id: save-vcpkg-sources + uses: actions/cache/save@v3 + with: + path: | + C:\vcpkg\packages + key: ${{ steps.cache-vcpkg-sources.outputs.cache-primary-key }} + enableCrossOsArchive: false + + - name: Build Fluent Bit packages + # If we are using 2.0.* or earlier we need to exclude the ARM64 build as the dependencies fail to compile. + # Trying to do via an exclude for the job triggers linting errors. + # This is only supposed to be a workaround for now so can be easily removed later. + if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }} + run: | + cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD='${{ inputs.unstable }}' -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ${{ matrix.config.cmake_additional_opt }} -DFLB_LIBYAML_DIR=C:\vcpkg\packages\libyaml_${{ matrix.config.vcpkg_triplet }} ../ + cmake --build . + cpack + working-directory: build + + - name: Upload build packages + # Skip upload if we skipped build. + if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }} + uses: actions/upload-artifact@v3 + with: + name: windows-packages + path: | + build/*-bit-*.exe + build/*-bit-*.msi + build/*-bit-*.zip + if-no-files-found: error + + call-build-windows-s3-upload: + name: Handle upload to S3 + # The environment must be used that has access to any secrets required, even if passed in. + # If passed in but not in the environment here you end up with an empty secret. + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + needs: + - call-build-windows-package + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # Need latest for checksum packaging script + ref: master + + - name: Download all artefacts + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: windows-packages + path: artifacts/ + + - name: Set up Windows checksums + run: | + packaging/windows-checksums.sh + ls -lR artifacts/ + shell: bash + env: + SOURCE_DIR: artifacts + + - name: Push Windows packages to S3 + # Only upload for staging + if: inputs.environment == 'staging' + uses: ./.github/actions/sync-to-bucket + with: + bucket: ${{ secrets.bucket }} + access_key_id: ${{ secrets.access_key_id }} + secret_access_key: ${{ secrets.secret_access_key }} + bucket-directory: "${{ inputs.version }}/windows/" + source-directory: "artifacts/" diff --git a/fluent-bit/.github/workflows/call-integration-image-build.yaml b/fluent-bit/.github/workflows/call-integration-image-build.yaml new file mode 100644 index 00000000..1644883c --- /dev/null +++ b/fluent-bit/.github/workflows/call-integration-image-build.yaml @@ -0,0 +1,144 @@ +name: Reusable workflow for integration testing +on: + workflow_call: + inputs: + ref: + description: The SHA, commit or branch to checkout and build. + required: true + type: string + registry: + description: The registry to push container images to. + type: string + required: true + username: + description: The username for the registry. + type: string + required: true + image: + description: The name of the container image to push to the registry. + type: string + required: true + image-tag: + description: The tag of the image to for testing. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + secrets: + token: + description: The Github token or similar to authenticate with for the registry. + required: true +jobs: + call-integration-image-build-container: + name: Integration test container image build + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Extract metadata from Github + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image }} + tags: | + raw,${{ inputs.image-tag }} + + - name: Build the AMD64 image + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/Dockerfile + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + target: production + provenance: false + push: true + load: false + + - name: Upload the image just in case as an artefact + run: | + docker pull $IMAGE + docker save --output /tmp/pr-image.tar $IMAGE + env: + IMAGE: ${{ steps.meta.outputs.tags }} + shell: bash + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: pr-${{ github.event.pull_request.number }}-image + path: /tmp/pr-image.tar + if-no-files-found: error + + - name: Extract metadata from Github + id: meta-debug + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image }} + tags: | + raw,${{ inputs.image-tag }}-debug + + - name: Build the AMD64 debug image + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/Dockerfile + context: . + tags: ${{ steps.meta-debug.outputs.tags }} + labels: ${{ steps.meta-debug.outputs.labels }} + provenance: false + target: debug + platforms: linux/amd64 + push: true + load: false + + call-integration-image-build-smoke-test: + name: Integration test image is valid + needs: call-integration-image-build-container + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Test the HTTP server is responding + timeout-minutes: 5 + run: | + packaging/testing/smoke/container/container-smoke-test.sh + shell: bash + env: + CONTAINER_NAME: local-smoke-${{ inputs.image-tag }} + CONTAINER_ARCH: linux/amd64 + REGISTRY: ${{ inputs.registry }} + IMAGE_NAME: ${{ inputs.image }} + IMAGE_TAG: ${{ inputs.image-tag }}
\ No newline at end of file diff --git a/fluent-bit/.github/workflows/call-run-integration-test.yaml b/fluent-bit/.github/workflows/call-run-integration-test.yaml new file mode 100644 index 00000000..47f44a9b --- /dev/null +++ b/fluent-bit/.github/workflows/call-run-integration-test.yaml @@ -0,0 +1,294 @@ +--- +name: Reusable workflow to run integration tests with specific images +on: + workflow_call: + secrets: + opensearch_aws_access_id: + description: AWS access ID to use within the opensearch integration tests. + required: true + opensearch_aws_secret_key: + description: AWS secret key to use within the opensearch integration tests. + required: true + opensearch_admin_password: + description: Default admin password use within the opensearch integration tests. + required: true + terraform_api_token: + description: Default terraform API token to use when running integration tests. + required: true + gcp-service-account-key: + description: The GCP service account key to use. + required: true + inputs: + image_name: + description: The image repository and name to use. + required: false + default: ghcr.io/fluent/fluent-bit/master + type: string + image_tag: + description: The image tag to use. + required: false + default: x86_64 + type: string + ref: + description: The commit, tag or branch of the repository to checkout + type: string + required: false + default: main +jobs: + call-run-terraform-setup: + name: Run Terraform set up + runs-on: ubuntu-latest + permissions: + packages: read + outputs: + aws-opensearch-endpoint: ${{ steps.aws-opensearch-endpoint.outputs.stdout }} + gke-cluster-name: ${{ steps.gke-cluster-name.outputs.stdout }} + gke-cluster-region: ${{ steps.gke-cluster-region.outputs.stdout }} + gke-cluster-zone: ${{ steps.gke-cluster-zone.outputs.stdout }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + repository: fluent/fluent-bit-ci + + - uses: hashicorp/setup-terraform@v2 + with: + cli_config_credentials_hostname: 'app.terraform.io' + cli_config_credentials_token: ${{ secrets.terraform_api_token }} + + - id: 'auth' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.gcp-service-account-key }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + + - name: Replace terraform variables. + run: | + sed -i -e "s|\$OPENSEARCH_AWS_ACCESS_ID|${{ secrets.opensearch_aws_access_id }}|g" default.auto.tfvars + sed -i -e "s|\$OPENSEARCH_AWS_SECRET_KEY|${{ secrets.opensearch_aws_secret_key }}|g" default.auto.tfvars + sed -i -e "s|\$OPENSEARCH_ADMIN_PASSWORD|${{ secrets.opensearch_admin_password }}|g" default.auto.tfvars + + cat <<EOT >> default.auto.tfvars + gcp_sa_key = <<-EOF + ${{ secrets.gcp-service-account-key }} + EOF + EOT + working-directory: terraform/ + shell: bash + + - name: Terraform init + id: init + run: terraform init + working-directory: terraform/ + + - name: Terraform fmt + id: fmt + run: | + find . -name "*.tf" -exec terraform fmt -check {} \; + working-directory: terraform + + - name: Terraform validate + id: validate + run: terraform validate -no-color + working-directory: terraform/ + + - name: Terraform Plan + if: github.event_name == 'pull_request' + id: plan + run: terraform plan -no-color + working-directory: terraform + continue-on-error: true + + - uses: actions/github-script@v6 + if: github.event_name == 'pull_request' + env: + PLAN: "terraform\n${{ steps.plan.outputs.stdout }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\` + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + <details><summary>Show Plan</summary> + \`\`\`\n + ${process.env.PLAN} + \`\`\` + </details> + *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + + - name: Terraform Plan Status + if: steps.plan.outcome == 'failure' + run: exit 1 + + - name: Terraform Apply + if: github.event_name != 'pull_request' + id: apply + run: terraform apply -input=false -auto-approve + working-directory: terraform + env: + TF_LOG: TRACE + + # We're using the terraform wrapper here so separate steps for each output variable + - id: aws-opensearch-endpoint + run: terraform output -no-color -raw aws-opensearch-endpoint + working-directory: terraform + shell: bash + + - id: gke-cluster-name + run: terraform output -no-color -raw gke_kubernetes_cluster_name + working-directory: terraform + shell: bash + + - id: gke-cluster-region + run: terraform output -no-color -raw gke_region + working-directory: terraform + shell: bash + + - id: gke-cluster-zone + run: terraform output -no-color -raw gke_zone + working-directory: terraform + shell: bash + + call-run-integration-kind: + name: Run integration tests on KIND + needs: + - call-run-terraform-setup + # Can test for multiple K8S versions with KIND + strategy: + fail-fast: false + matrix: + k8s-release: [ 'v1.23.5', 'v1.22.7', 'v1.21.10' ] + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + steps: + - name: Test image exists and cache locally + run: docker pull ${{ inputs.image_name }}:${{ inputs.image_tag }} + + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + repository: fluent/fluent-bit-ci + + - name: Configure system for Opensearch + run: | + sudo sysctl -w vm.max_map_count=262144 + sysctl -p + shell: bash + + - name: Setup BATS + uses: mig4/setup-bats@v1 + with: + bats-version: 1.9.0 + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-release }} + cluster_name: kind + + - name: Set up Helm + uses: azure/setup-helm@v3.5 + with: + version: v3.8.1 + + - name: Set up Kubectl + uses: azure/setup-kubectl@v3.2 + + - name: Run tests + timeout-minutes: 60 + run: | + kind load docker-image ${{ inputs.image_name }}:${{ inputs.image_tag }} + ./run-tests.sh + shell: bash + env: + FLUENTBIT_IMAGE_REPOSITORY: ${{ inputs.image_name }} + FLUENTBIT_IMAGE_TAG: ${{ inputs.image_tag }} + HOSTED_OPENSEARCH_HOST: ${{ needs.call-run-terraform-setup.outputs.aws-opensearch-endpoint }} + HOSTED_OPENSEARCH_PORT: 443 + HOSTED_OPENSEARCH_USERNAME: admin + HOSTED_OPENSEARCH_PASSWORD: ${{ secrets.opensearch_admin_password }} + + call-run-integration-cloud: + name: Run integration tests on cloud providers + needs: + - call-run-terraform-setup + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + cloud: + - gke + env: + USE_GKE_GCLOUD_AUTH_PLUGIN: true + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + repository: fluent/fluent-bit-ci + + - if: matrix.cloud == 'gke' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.gcp-service-account-key }} + + - if: matrix.cloud == 'gke' + uses: 'google-github-actions/setup-gcloud@v1' + with: + install_components: 'gke-gcloud-auth-plugin' + + - name: Setup BATS + uses: mig4/setup-bats@v1 + with: + bats-version: 1.9.0 + + - name: Set up Helm + uses: azure/setup-helm@v3.5 + with: + version: v3.8.1 + + - name: Set up Kubectl + uses: azure/setup-kubectl@v3.2 + + - name: Get the GKE Kubeconfig + if: matrix.cloud == 'gke' + uses: 'google-github-actions/get-gke-credentials@v1' + with: + cluster_name: ${{ needs.call-run-terraform-setup.outputs.gke-cluster-name }} + location: ${{ needs.call-run-terraform-setup.outputs.gke-cluster-zone }} + + - name: Check Kubeconfig set up + run: | + kubectl cluster-info + kubectl get nodes --show-labels + kubectl get pods --all-namespaces --show-labels + kubectl get ns + shell: bash + + - name: Run tests + timeout-minutes: 60 + run: | + ./run-tests.sh + shell: bash + env: + # Namespace per test run to hopefully isolate a bit + TEST_NAMESPACE: test-${{ github.run_id }} + FLUENTBIT_IMAGE_REPOSITORY: ${{ inputs.image_name }} + FLUENTBIT_IMAGE_TAG: ${{ inputs.image_tag }} + HOSTED_OPENSEARCH_HOST: ${{ needs.call-run-terraform-setup.outputs.aws-opensearch-endpoint }} + HOSTED_OPENSEARCH_PORT: 443 + HOSTED_OPENSEARCH_USERNAME: admin + USE_GKE_GCLOUD_AUTH_PLUGIN: true + HOSTED_OPENSEARCH_PASSWORD: ${{ secrets.opensearch_admin_password }} diff --git a/fluent-bit/.github/workflows/call-test-images.yaml b/fluent-bit/.github/workflows/call-test-images.yaml new file mode 100644 index 00000000..29bc0cd3 --- /dev/null +++ b/fluent-bit/.github/workflows/call-test-images.yaml @@ -0,0 +1,205 @@ +--- +name: Reusable workflow to test container images +on: + workflow_call: + inputs: + registry: + description: The registry to pull the images to test from. + type: string + required: true + username: + description: The username for authentication with the registry. + type: string + required: true + image: + description: The name of the image to pull from the registry for testing. + type: string + required: true + image-tag: + description: The tag of the image to pull from the registry for testing. + type: string + required: true + environment: + description: The Github environment to run this workflow on. + type: string + required: false + ref: + description: The commit, tag or branch to checkout for testing scripts. + type: string + default: master + required: false + secrets: + token: + description: The Github token or similar to authenticate with for the registry. + required: true + cosign_key: + description: The optional Cosign key to use for verifying the images. + required: false +jobs: + call-test-images-cosign-verify: + name: Cosign verification of container image + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + steps: + - name: Install cosign + uses: sigstore/cosign-installer@v2 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + # There is currently no way to verify a local image, e.g. for a particular architecture + # https://github.com/sigstore/cosign/issues/60 + - name: Verify image with a key + # Only key-based verification currently + if: ${{ env.COSIGN_PUBLIC_KEY }} + run: | + echo -e "${COSIGN_PUBLIC_KEY}" > /tmp/my_cosign.pub + cosign verify --key /tmp/my_cosign.pub "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG" + rm -f /tmp/my_cosign.key + shell: bash + env: + COSIGN_PUBLIC_KEY: ${{ secrets.cosign_key }} + REGISTRY: ${{ inputs.registry }} + IMAGE_NAME: ${{ inputs.image }} + IMAGE_TAG: ${{ inputs.image-tag }} + + call-test-images-container-architecture: + name: ${{ matrix.arch }} image architecture verification + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + # Test as much as we can + continue-on-error: true + strategy: + fail-fast: false + matrix: + arch: [ linux/amd64, linux/arm64, linux/arm/v7 ] + include: + # Rather than extract the specific central arch we just provide it + - arch: linux/amd64 + expected: amd64 + - arch: linux/arm64 + expected: arm64 + - arch: linux/arm/v7 + expected: arm + steps: + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Pull and extract architecture of image + id: extract_arch + run: | + docker pull --platform=${{ matrix.arch }} "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG" + ACTUAL_ARCH=$(docker image inspect --format '{{.Architecture}}' "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG") + echo "ACTUAL_ARCH=$ACTUAL_ARCH" >> $GITHUB_OUTPUT + docker image inspect "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG" + shell: bash + env: + REGISTRY: ${{ inputs.registry }} + IMAGE_NAME: ${{ inputs.image }} + IMAGE_TAG: ${{ inputs.image-tag }} + + - name: Validate architecture of image + run: | + if [[ "$ACTUAL_ARCH" != "$EXPECTED_ARCH" ]]; then + echo "Invalid architecture for $REGISTRY/$IMAGE_NAME: $ACTUAL_ARCH != $EXPECTED_ARCH" + exit 1 + fi + env: + EXPECTED_ARCH: ${{ matrix.expected }} + ACTUAL_ARCH: ${{ steps.extract_arch.outputs.ACTUAL_ARCH }} + shell: bash + + call-test-images-container-smoke: + # Ensure each architecture container runs up correctly with default configuration. + name: ${{ matrix.arch }} smoke test for local container images + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + # No point running if the architecture is incorrect + needs: [ call-test-images-container-architecture ] + continue-on-error: true + strategy: + fail-fast: false # verify all + matrix: + arch: [ linux/amd64, linux/arm64, linux/arm/v7 ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets.token }} + + - name: Set up QEMU using standard action + if: ${{ matrix.arch != 'linux/arm64' }} + uses: docker/setup-qemu-action@v3 + + # Without this QEMU fails for ARM64 + - name: Set up binary emulation for QEMU + if: ${{ matrix.arch == 'linux/arm64' }} + run: | + docker run --privileged --rm tonistiigi/binfmt --install all + + - name: Verify platform is supported with Alpine container + # We make sure there is not an inherent issue with this architecture on this runner + run: | + docker run --rm --platform=${{ matrix.arch }} alpine uname -a + + - name: Test the HTTP server is responding + timeout-minutes: 10 + run: | + packaging/testing/smoke/container/container-smoke-test.sh + shell: bash + env: + CONTAINER_NAME: local-smoke-${{ matrix.arch }} + CONTAINER_ARCH: ${{ matrix.arch }} + REGISTRY: ${{ inputs.registry }} + IMAGE_NAME: ${{ inputs.image }} + IMAGE_TAG: ${{ inputs.image-tag }} + + call-test-images-k8s-smoke: + # No need to test every architecture here, that is covered by local container tests. + # Testing helm chart deployment on KIND here. + name: Helm chart test on KIND + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.8.0 + + - name: Set up Helm + uses: azure/setup-helm@v3.5 + with: + version: v3.6.3 + + - name: Set up Kubectl + uses: azure/setup-kubectl@v3.2 + + - name: Test the HTTP server is responding + timeout-minutes: 5 + run: | + packaging/testing/smoke/k8s/k8s-smoke-test.sh + shell: bash + env: + NAMESPACE: default + REGISTRY: ${{ inputs.registry }} + IMAGE_NAME: ${{ inputs.image }} + IMAGE_TAG: ${{ inputs.image-tag }} + diff --git a/fluent-bit/.github/workflows/call-test-packages.yaml b/fluent-bit/.github/workflows/call-test-packages.yaml new file mode 100644 index 00000000..90b8414c --- /dev/null +++ b/fluent-bit/.github/workflows/call-test-packages.yaml @@ -0,0 +1,55 @@ +--- +name: Reusable workflow to test packages in S3 bucket +on: + workflow_call: + inputs: + environment: + description: The Github environment to run this workflow on. + type: string + required: false + ref: + description: The commit, tag or branch to checkout for testing scripts. + type: string + default: master + required: false + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to pull packages from. + required: true + +jobs: + call-test-packaging: + # We use Dokken to run a series of test suites locally on containers representing + # each OS we want to install on. This creates custom images with the package + # installed and configured as per our documentation then verifies that the agent + # is running at startup. + name: ${{ matrix.distro }} package tests + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + env: + AWS_URL: https://${{ secrets.bucket }}.s3.amazonaws.com + strategy: + fail-fast: false + matrix: + distro: [ amazonlinux2022, amazonlinux2, centos7, centos8, debian10, debian11, ubuntu1804, ubuntu2004, ubuntu2204 ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get the version + id: get_version + run: | + curl --fail -LO "$AWS_URL/latest-version.txt" + VERSION=$(cat latest-version.txt) + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + shell: bash + + - name: Run package installation tests + run: | + packaging/testing/smoke/packages/run-package-tests.sh + env: + PACKAGE_TEST: ${{ matrix.distro }} + RELEASE_URL: https://packages.fluentbit.io diff --git a/fluent-bit/.github/workflows/cron-scorecards-analysis.yaml b/fluent-bit/.github/workflows/cron-scorecards-analysis.yaml new file mode 100644 index 00000000..065afdfd --- /dev/null +++ b/fluent-bit/.github/workflows/cron-scorecards-analysis.yaml @@ -0,0 +1,59 @@ + +--- +# https://openssf.org/blog/2022/01/19/reducing-security-risks-in-open-source-software-at-scale-scorecards-launches-v4/ +name: Scorecards supply-chain security +on: + # Only the default branch is supported. + branch_protection_rule: + schedule: + # Weekly on Saturdays. + - cron: '30 1 * * 6' + workflow_dispatch: + +# Declare default permissions as read only. +permissions: read-all + +jobs: + scorecard-analysis: + name: Scorecards analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed for GitHub OIDC token if publish_results is true + id-token: write + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 + with: + results_file: scorecard-results.sarif + results_format: sarif + # (Optional) fine-grained personal access token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-fine-grained-pat-optional. + repo_token: ${{ secrets.SCORECARD_TOKEN }} + # + # Publish the results for public repositories to enable scorecard badges. For more details, see + # https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories, `publish_results` will automatically be set to `false`, regardless + # of the value entered here. + publish_results: true + + - name: "Upload artifact" + uses: actions/upload-artifact@v3 + with: + name: SARIF file + path: scorecard-results.sarif + retention-days: 7 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: scorecard-results.sarif + category: ossf-scorecard diff --git a/fluent-bit/.github/workflows/cron-stale.yaml b/fluent-bit/.github/workflows/cron-stale.yaml new file mode 100644 index 00000000..523685c5 --- /dev/null +++ b/fluent-bit/.github/workflows/cron-stale.yaml @@ -0,0 +1,26 @@ +name: 'Close stale issues and PR(s)' +on: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + name: Mark stale + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days. Maintainers can add the `exempt-stale` label.' + stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' + close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' + days-before-stale: 90 + days-before-close: 5 + days-before-pr-close: -1 + exempt-all-pr-assignees: true + exempt-all-pr-milestones: true + exempt-issue-labels: 'long-term,enhancement,exempt-stale' + # start with the oldest + ascending: true + # keep an eye on this + operations-per-run: 250 diff --git a/fluent-bit/.github/workflows/cron-trivy.yaml b/fluent-bit/.github/workflows/cron-trivy.yaml new file mode 100644 index 00000000..a1266819 --- /dev/null +++ b/fluent-bit/.github/workflows/cron-trivy.yaml @@ -0,0 +1,87 @@ +--- +# Separate action to allow us to initiate manually and run regularly +name: Trivy security analysis of latest containers + +# Run on every push to master, or weekly. +# Allow users to trigger an asynchronous run anytime too. +on: + push: + branches: [master] + schedule: + # 13:44 on Thursday + - cron: 44 13 * * 4 + workflow_dispatch: + +jobs: + # Run Trivy on the latest container and update the security code scanning results tab. + trivy-latest: + # Matrix job that pulls the latest image for each supported architecture via the multi-arch latest manifest. + # We then re-tag it locally to ensure that when Trivy runs it does not pull the latest for the wrong architecture. + name: ${{ matrix.arch }} container scan + runs-on: [ ubuntu-latest ] + continue-on-error: true + strategy: + fail-fast: false + # Matrix of architectures to test along with their local tags for special character substitution + matrix: + # The architecture for the container runtime to pull. + arch: [ linux/amd64, linux/arm64, linux/arm/v7 ] + # In a few cases we need the arch without slashes so provide a descriptive extra field for that. + # We could also extract or modify this via a regex but this seemed simpler and easier to follow. + include: + - arch: linux/amd64 + local_tag: x86_64 + - arch: linux/arm64 + local_tag: arm64 + - arch: linux/arm/v7 + local_tag: arm32 + steps: + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull the image for the architecture we're testing + run: | + docker pull --platform ${{ matrix.arch }} fluent/fluent-bit:latest + + - name: Tag locally to ensure we do not pull wrong architecture + run: | + docker tag fluent/fluent-bit:latest local/fluent-bit:${{ matrix.local_tag }} + + # Deliberately chosen master here to keep up-to-date. + - name: Run Trivy vulnerability scanner for any major issues + uses: aquasecurity/trivy-action@master + with: + image-ref: local/fluent-bit:${{ matrix.local_tag }} + # Filter out any that have no current fix. + ignore-unfixed: true + # Only include major issues. + severity: CRITICAL,HIGH + format: template + template: '@/contrib/sarif.tpl' + output: trivy-results-${{ matrix.local_tag }}.sarif + + # Show all detected issues. + # Note this will show a lot more, including major un-fixed ones. + - name: Run Trivy vulnerability scanner for local output + uses: aquasecurity/trivy-action@master + with: + image-ref: local/fluent-bit:${{ matrix.local_tag }} + format: table + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: trivy-results-${{ matrix.local_tag }}.sarif + category: ${{ matrix.arch }} container + wait-for-processing: true + + # In case we need to analyse the uploaded files for some reason. + - name: Detain results for debug if needed + uses: actions/upload-artifact@v3 + with: + name: trivy-results-${{ matrix.local_tag }}.sarif + path: trivy-results-${{ matrix.local_tag }}.sarif + if-no-files-found: error diff --git a/fluent-bit/.github/workflows/cron-unstable-build.yaml b/fluent-bit/.github/workflows/cron-unstable-build.yaml new file mode 100644 index 00000000..82235a41 --- /dev/null +++ b/fluent-bit/.github/workflows/cron-unstable-build.yaml @@ -0,0 +1,213 @@ +--- +name: Unstable build + +on: + workflow_dispatch: + inputs: + branch: + description: The branch to create an unstable release for/from. + type: string + default: master + required: true + + # Run nightly build at this time, bit of trial and error but this seems good. + schedule: + - cron: "0 6 * * *" # master build + - cron: "0 12 * * *" # 2.0 build + +# We do not want a new unstable build to run whilst we are releasing the current unstable build. +concurrency: unstable-build-release + +jobs: + # This job provides this metadata for the other jobs to use. + unstable-build-get-meta: + name: Get metadata to add to build + runs-on: ubuntu-latest + environment: unstable + outputs: + date: ${{ steps.date.outputs.date }} + branch: ${{ steps.branch.outputs.branch }} + permissions: + contents: none + steps: + # For cron builds, i.e. nightly, we provide date and time as extra parameter to distinguish them. + - name: Get current date + id: date + run: echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT + + - name: Debug event output + uses: hmarr/debug-action@v2 + + # Now we need to determine which branch to build + - name: Manual run - get branch + if: github.event_name == 'workflow_dispatch' + run: | + echo "cron_branch=${{ github.event.inputs.branch }}" >> $GITHUB_ENV + shell: bash + + - name: master run + if: github.event_name == 'schedule' && github.event.schedule=='0 6 * * *' + run: | + echo "cron_branch=master" >> $GITHUB_ENV + shell: bash + + - name: 2.0 run + if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *' + run: | + echo "cron_branch=2.0" >> $GITHUB_ENV + shell: bash + + - name: Output the branch to use + id: branch + run: | + echo "$cron_branch" + if [[ -z "$cron_branch" ]]; then + echo "Unable to determine branch to use" + exit 1 + fi + echo "branch=$cron_branch" >> $GITHUB_OUTPUT + shell: bash + + unstable-build-images: + needs: unstable-build-get-meta + uses: ./.github/workflows/call-build-images.yaml + with: + version: ${{ needs.unstable-build-get-meta.outputs.branch }} + ref: ${{ needs.unstable-build-get-meta.outputs.branch }} + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/unstable + environment: unstable + unstable: ${{ needs.unstable-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + unstable-build-generate-matrix: + name: unstable build matrix + needs: + - unstable-build-get-meta + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix.outputs.build-matrix }} + environment: unstable + permissions: + contents: read + steps: + - name: Checkout repository, always latest for action + uses: actions/checkout@v4 + + # Set up the list of target to build so we can pass the JSON to the reusable job + - uses: ./.github/actions/generate-package-build-matrix + id: set-matrix + with: + ref: ${{ needs.unstable-build-get-meta.outputs.branch }} + + unstable-build-packages: + needs: + - unstable-build-get-meta + - unstable-build-generate-matrix + uses: ./.github/workflows/call-build-linux-packages.yaml + with: + version: ${{ needs.unstable-build-get-meta.outputs.branch }} + ref: ${{ needs.unstable-build-get-meta.outputs.branch }} + build_matrix: ${{ needs.unstable-build-generate-matrix.outputs.build-matrix }} + environment: unstable + unstable: ${{ needs.unstable-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + unstable-build-windows-package: + needs: + - unstable-build-get-meta + uses: ./.github/workflows/call-build-windows.yaml + with: + version: ${{ needs.unstable-build-get-meta.outputs.branch }} + ref: ${{ needs.unstable-build-get-meta.outputs.branch }} + environment: unstable + unstable: ${{ needs.unstable-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + unstable-build-macos-package: + needs: + - unstable-build-get-meta + uses: ./.github/workflows/call-build-macos.yaml + with: + version: ${{ needs.unstable-build-get-meta.outputs.branch }} + ref: ${{ needs.unstable-build-get-meta.outputs.branch }} + environment: unstable + unstable: ${{ needs.unstable-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + # We already detain all artefacts as build output so just capture for an unstable release. + # We make all of these on a separate repo to prevent notification spam. + unstable-release: + runs-on: ubuntu-latest + needs: + - unstable-build-get-meta + - unstable-build-images + - unstable-build-packages + - unstable-build-windows-package + - unstable-build-macos-package + environment: unstable + permissions: + contents: read + steps: + # Required to make a release later + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download all artefacts + continue-on-error: true + uses: actions/download-artifact@v3 + with: + path: artifacts/ + + - name: Single packages tar + run: | + mkdir -p release-upload + # Optional JSON schema so ignore failure + mv -f artifacts/*.json release-upload/ || true + tar -czvf release-upload/packages-unstable-${{ needs.unstable-build-get-meta.outputs.branch }}.tar.gz -C artifacts/ . + shell: bash + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull containers as well (single arch only) + # May not be any/valid so ignore errors + continue-on-error: true + run: | + docker pull $IMAGE + docker save --output container-${{ needs.unstable-build-get-meta.outputs.branch }}.tar $IMAGE + docker pull $IMAGE-debug + docker save --output container-${{ needs.unstable-build-get-meta.outputs.branch }}-debug.tar $IMAGE-debug + shell: bash + working-directory: release-upload + env: + IMAGE: ghcr.io/${{ github.repository }}/unstable:${{ needs.unstable-build-get-meta.outputs.branch }} + + - name: Display structure of files to upload + run: ls -R + working-directory: release-upload + shell: bash + + - name: Remove any existing release + continue-on-error: true + run: gh release delete unstable-${{ needs.unstable-build-get-meta.outputs.branch }} --yes --repo ${{ secrets.RELEASE_REPO }} + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + shell: bash + + - name: Create Release + # Do not fail the job here + continue-on-error: true + run: gh release create unstable-${{ needs.unstable-build-get-meta.outputs.branch }} release-upload/*.* --repo ${{ secrets.RELEASE_REPO }} --generate-notes --prerelease --target ${{ needs.unstable-build-get-meta.outputs.branch }} --title "Nightly unstable ${{ needs.unstable-build-get-meta.outputs.branch }} build" + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + shell: bash diff --git a/fluent-bit/.github/workflows/master-integration-test.yaml b/fluent-bit/.github/workflows/master-integration-test.yaml new file mode 100644 index 00000000..85c321b8 --- /dev/null +++ b/fluent-bit/.github/workflows/master-integration-test.yaml @@ -0,0 +1,33 @@ +name: Build master container images and run integration tests +on: + push: + branches: + - master + +jobs: + master-integration-test-build: + name: Master - integration build + uses: ./.github/workflows/call-integration-image-build.yaml + with: + ref: ${{ github.sha }} + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/master + image-tag: x86_64 + environment: integration + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + master-integration-test-run-integration: + name: Master - integration test + needs: master-integration-test-build + uses: ./.github/workflows/call-run-integration-test.yaml + with: + image_name: ghcr.io/${{ github.repository }}/master + image_tag: x86_64 + secrets: + opensearch_aws_access_id: ${{ secrets.OPENSEARCH_AWS_ACCESS_ID }} + opensearch_aws_secret_key: ${{ secrets.OPENSEARCH_AWS_SECRET_KEY }} + opensearch_admin_password: ${{ secrets.OPENSEARCH_ADMIN_PASSWORD }} + terraform_api_token: ${{ secrets.TF_API_TOKEN }} + gcp-service-account-key: ${{ secrets.GCP_SA_KEY }} diff --git a/fluent-bit/.github/workflows/pr-closed-docker.yaml b/fluent-bit/.github/workflows/pr-closed-docker.yaml new file mode 100644 index 00000000..628f5e92 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-closed-docker.yaml @@ -0,0 +1,21 @@ +name: Remove docker images for stale/closed PR(s). +on: + pull_request: + branches: + - master + types: [closed] +jobs: + cleanup: + name: PR - cleanup pr-${{ github.event.number }} images + runs-on: ubuntu-latest + permissions: + # We may need a specific token here with `packages:admin` privileges which is not available to GITHUB_TOKEN + packages: write + steps: + - uses: vlaurin/action-ghcr-prune@v0.5.0 + with: + organization: fluent + container: fluent-bit/pr-${{ github.event.number }} + token: ${{ secrets.GITHUB_TOKEN }} + prune-untagged: true + keep-last: 0 diff --git a/fluent-bit/.github/workflows/pr-commit-message.yaml b/fluent-bit/.github/workflows/pr-commit-message.yaml new file mode 100644 index 00000000..596d9cdc --- /dev/null +++ b/fluent-bit/.github/workflows/pr-commit-message.yaml @@ -0,0 +1,21 @@ +name: 'Pull requests commit messages' +on: + pull_request: + types: + - opened + - edited + - reopened + - synchronize +jobs: + check-commit-message: + name: Check Commit Message + runs-on: ubuntu-latest + steps: + - name: Check commit subject complies with https://github.com/fluent/fluent-bit/blob/master/CONTRIBUTING.md#commit-changes + uses: gsactions/commit-message-checker@v2 + with: + pattern: '^[a-z0-9A-Z\-_\s\,\.\/]+\:[ ]{0,1}[a-zA-Z]+[a-zA-Z0-9 \-\.\:_\#\(\)=\/''\"\,><\+\[\]\!\*\\]+$' + error: 'Invalid commit subject. Please refer to: https://github.com/fluent/fluent-bit/blob/master/CONTRIBUTING.md#commit-changes' + checkAllCommitMessages: 'false' + excludeDescription: 'true' + accessToken: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file diff --git a/fluent-bit/.github/workflows/pr-compile-check.yaml b/fluent-bit/.github/workflows/pr-compile-check.yaml new file mode 100644 index 00000000..d9f3dc92 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-compile-check.yaml @@ -0,0 +1,32 @@ +name: 'Pull requests compile checks' +on: + pull_request: + # Only trigger if there is a code change or a CMake change that (could) affect code + paths: + - '**.c' + - '**.h' + - 'CMakeLists.txt' + - 'cmake/*' + workflow_dispatch: + +jobs: + # Sanity check for compilation using older compiler on CentOS 7 + pr-compile-centos-7: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout Fluent Bit code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Attempt to build current source for CentOS 7 + uses: docker/build-push-action@v5 + with: + context: . + file: ./dockerfiles/Dockerfile.centos7 + # No need to use after this so discard completely + push: false + load: false + provenance: false diff --git a/fluent-bit/.github/workflows/pr-fuzz.yaml b/fluent-bit/.github/workflows/pr-fuzz.yaml new file mode 100644 index 00000000..79371ed5 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-fuzz.yaml @@ -0,0 +1,41 @@ +name: CIFuzz +on: + pull_request: + # Only fuzz when C source files change + paths: + - '**.c' + - '**.h' +jobs: + fuzzing: + name: PR - fuzzing test + runs-on: ubuntu-latest + steps: + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'fluent-bit' + dry-run: false + language: c + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'fluent-bit' + fuzz-seconds: 600 + dry-run: false + language: c + output-sarif: true + - name: Upload Crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts + - name: Upload Sarif + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v2 + with: + # Path to SARIF file relative to the root of the repository + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif + category: CIFuzz diff --git a/fluent-bit/.github/workflows/pr-image-tests.yaml b/fluent-bit/.github/workflows/pr-image-tests.yaml new file mode 100644 index 00000000..2cdc3622 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-image-tests.yaml @@ -0,0 +1,121 @@ +name: Build images for all architectures +on: + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - 'dockerfiles/Dockerfile' + - 'dockerfiles/Dockerfile.windows' + - 'conf/**' + + workflow_dispatch: + +jobs: + pr-image-tests-build-images: + name: PR - Buildkit docker build test + runs-on: ubuntu-latest + permissions: + contents: read + # We do not push and this allows simpler workflow running for forks too + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata from Github + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }}/pr-${{ github.event.pull_request.number }} + tags: | + type=sha + + - name: Build the multi-arch images + id: build + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/Dockerfile + context: . + platforms: linux/amd64 + target: production + provenance: false + push: false + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Sanity check it runs + # We do this for a simple check of dependencies + run: | + docker run --rm -t ${{ steps.meta.outputs.tags }} --help + shell: bash + + - name: Build the debug multi-arch images + uses: docker/build-push-action@v5 + with: + file: ./dockerfiles/Dockerfile + context: . + platforms: linux/amd64 + target: debug + provenance: false + push: false + load: false + + pr-image-tests-classic-docker-build: + name: PR - Classic docker build test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build the classic test image + # We only want to confirm it builds with classic mode, nothing else + run: | + docker build -f ./dockerfiles/Dockerfile . + env: + # Ensure we disable buildkit + DOCKER_BUILDKIT: 0 + shell: bash + pr-image-tests-build-windows-images: + name: PR - Docker windows build test, windows 2019 and 2022 + runs-on: windows-${{ matrix.windows-base-version }} + strategy: + fail-fast: true + matrix: + windows-base-version: + # https://github.com/fluent/fluent-bit/blob/1d366594a889624ec3003819fe18588aac3f17cd/dockerfiles/Dockerfile.windows#L3 + - '2019' + - '2022' + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Extract metadata from Github + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }}/pr-${{ github.event.pull_request.number }} + tags: | + type=sha + flavor: | + suffix=-windows-${{ matrix.windows-base-version }} + + - name: Build the windows images + id: build + run: | + docker build -t ${{ steps.meta.outputs.tags }} --build-arg WINDOWS_VERSION=ltsc${{ matrix.windows-base-version }} -f ./dockerfiles/Dockerfile.windows . + + - name: Sanity check it runs + # We do this for a simple check of dependencies + run: | + docker run --rm -t ${{ steps.meta.outputs.tags }} --help + shell: bash + diff --git a/fluent-bit/.github/workflows/pr-install-script.yaml b/fluent-bit/.github/workflows/pr-install-script.yaml new file mode 100644 index 00000000..0eeeed65 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-install-script.yaml @@ -0,0 +1,29 @@ +name: Test install script for all targets +on: + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - 'packaging/test-release-packages.sh' + - 'install.sh' + + workflow_dispatch: +jobs: + test-install-script: + name: Run install tests + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 30 + steps: + - name: Checkout Fluent Bit code + uses: actions/checkout@v4 + + - name: Run install tests + run: | + ./packaging/test-release-packages.sh + shell: bash + env: + INSTALL_SCRIPT: ./install.sh diff --git a/fluent-bit/.github/workflows/pr-integration-test.yaml b/fluent-bit/.github/workflows/pr-integration-test.yaml new file mode 100644 index 00000000..2b18b1c6 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-integration-test.yaml @@ -0,0 +1,69 @@ +name: Build and run integration tests for PR +on: + # We need write token for upload to GHCR and we are protecting with labels too. + pull_request_target: + branches: + - master + types: + - labeled + - opened + - reopened + - synchronize + +jobs: + pr-integration-test-build: + name: PR - integration build + # We only need to test this once as the rest are chained from it. + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + uses: ./.github/workflows/call-integration-image-build.yaml + with: + ref: ${{ github.event.pull_request.head.sha }} + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/pr-${{ github.event.number }} + image-tag: ${{ github.sha }} + environment: integration + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + pr-integration-test-build-complete: + name: PR - integration build complete + runs-on: ubuntu-latest + needs: + - pr-integration-test-build + steps: + - uses: actions-ecosystem/action-add-labels@v1 + name: Label the PR + with: + labels: ci/integration-docker-ok + github_token: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} + + pr-integration-test-run-integration: + name: PR - K8S integration test + needs: + - pr-integration-test-build + uses: ./.github/workflows/call-run-integration-test.yaml + with: + image_name: ghcr.io/${{ github.repository }}/pr-${{ github.event.pull_request.number }} + image_tag: ${{ github.sha }} + secrets: + opensearch_aws_access_id: ${{ secrets.OPENSEARCH_AWS_ACCESS_ID }} + opensearch_aws_secret_key: ${{ secrets.OPENSEARCH_AWS_SECRET_KEY }} + opensearch_admin_password: ${{ secrets.OPENSEARCH_ADMIN_PASSWORD }} + terraform_api_token: ${{ secrets.TF_API_TOKEN }} + gcp-service-account-key: ${{ secrets.GCP_SA_KEY }} + + pr-integration-test-run-integration-post-label: + name: PR - integration test complete + runs-on: ubuntu-latest + needs: + - pr-integration-test-run-integration + steps: + - uses: actions-ecosystem/action-add-labels@v1 + name: Label the PR + with: + labels: ci/integration-test-ok + github_token: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} + repo: fluent/fluent-bit diff --git a/fluent-bit/.github/workflows/pr-labels.yaml b/fluent-bit/.github/workflows/pr-labels.yaml new file mode 100644 index 00000000..f8139304 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-labels.yaml @@ -0,0 +1,17 @@ +name: 'Pull requests labels' +on: + pull_request_target: + types: + - opened +jobs: + apply-default-labels: + name: PR - apply default labels + runs-on: ubuntu-latest + steps: + - uses: actions-ecosystem/action-add-labels@v1 + name: Label the PR with 'docs-required' by default. + with: + labels: docs-required + github_token: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} + repo: fluent/fluent-bit diff --git a/fluent-bit/.github/workflows/pr-lint.yaml b/fluent-bit/.github/workflows/pr-lint.yaml new file mode 100644 index 00000000..f8af79bb --- /dev/null +++ b/fluent-bit/.github/workflows/pr-lint.yaml @@ -0,0 +1,36 @@ +name: Lint PRs +on: + pull_request: + workflow_dispatch: + +jobs: + hadolint-pr: + runs-on: ubuntu-latest + name: PR - Hadolint + steps: + - uses: actions/checkout@v4 + # Ignores do not work: https://github.com/reviewdog/action-hadolint/issues/35 is resolved + - uses: reviewdog/action-hadolint@v1 + with: + exclude: | + packaging/testing/smoke/packages/Dockerfile.* + + shellcheck-pr: + runs-on: ubuntu-latest + name: PR - Shellcheck + steps: + - uses: actions/checkout@v4 + - uses: ludeeus/action-shellcheck@master + with: + ignore_paths: cmake/sanitizers-cmake lib plugins tests + + actionlint-pr: + runs-on: ubuntu-latest + name: PR - Actionlint + steps: + - uses: actions/checkout@v4 + - run: | + echo "::add-matcher::.github/actionlint-matcher.json" + bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + ./actionlint -color -shellcheck= + shell: bash diff --git a/fluent-bit/.github/workflows/pr-package-tests.yaml b/fluent-bit/.github/workflows/pr-package-tests.yaml new file mode 100644 index 00000000..b461ac4d --- /dev/null +++ b/fluent-bit/.github/workflows/pr-package-tests.yaml @@ -0,0 +1,92 @@ +name: PR - packaging tests run on-demand via label +on: + pull_request: + types: + - labeled + - opened + - reopened + - synchronize + branches: + - master + +# Cancel any running on push +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + # This job provides this metadata for the other jobs to use. + pr-package-test-build-get-meta: + # This is a long test to run so only on-demand for certain PRs + if: contains(github.event.pull_request.labels.*.name, 'ok-package-test') + name: Get metadata to add to build + runs-on: ubuntu-latest + outputs: + date: ${{ steps.date.outputs.date }} + permissions: + contents: none + steps: + # For cron builds, i.e. nightly, we provide date and time as extra parameter to distinguish them. + - name: Get current date + id: date + run: echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT + + - name: Debug event output + uses: hmarr/debug-action@v2 + + pr-package-test-build-generate-matrix: + name: PR - packages build matrix + needs: + - pr-package-test-build-get-meta + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix.outputs.build-matrix }} + permissions: + contents: read + steps: + - name: Checkout repository, always latest for action + uses: actions/checkout@v4 + + # Set up the list of target to build so we can pass the JSON to the reusable job + - uses: ./.github/actions/generate-package-build-matrix + id: set-matrix + with: + ref: master + + pr-package-test-build-packages: + name: PR - packages build Linux + needs: + - pr-package-test-build-get-meta + - pr-package-test-build-generate-matrix + uses: ./.github/workflows/call-build-linux-packages.yaml + with: + version: pr-${{ github.event.number }} + ref: ${{ github.ref }} + build_matrix: ${{ needs.pr-package-test-build-generate-matrix.outputs.build-matrix }} + unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + pr-package-test-build-windows-package: + name: PR - packages build Windows + needs: + - pr-package-test-build-get-meta + uses: ./.github/workflows/call-build-windows.yaml + with: + version: pr-${{ github.event.number }} + ref: ${{ github.ref }} + unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + + pr-package-test-build-macos-package: + name: PR - packages build MacOS + needs: + - pr-package-test-build-get-meta + uses: ./.github/workflows/call-build-macos.yaml + with: + version: pr-${{ github.event.number }} + ref: ${{ github.ref }} + unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/fluent-bit/.github/workflows/pr-perf-test.yaml b/fluent-bit/.github/workflows/pr-perf-test.yaml new file mode 100644 index 00000000..52b73799 --- /dev/null +++ b/fluent-bit/.github/workflows/pr-perf-test.yaml @@ -0,0 +1,97 @@ +name: Build and run performance tests for PR +on: + # We want to run on forks as we protect with a label. + # Without this we have no secrets to pass. + pull_request_target: + branches: + - master + types: + - labeled + +jobs: + + pr-perf-test-run: + # We only need to test this once as the rest are chained from it. + if: contains(github.event.pull_request.labels.*.name, 'ok-to-performance-test') + uses: fluent/fluent-bit-ci/.github/workflows/call-run-performance-test.yaml@main + with: + vm-name: fb-perf-test-pr-${{ github.event.number }} + git-branch: ${{ github.head_ref }} + test-directory: examples/perf_test + duration: 30 + service: fb-delta + secrets: + service-account: ${{ secrets.GCP_SA_KEY }} + + pr-perf-test-upload: + name: PR - performance test upload + # Label check from previous + runs-on: ubuntu-latest + needs: + - pr-perf-test-run + permissions: + pull-requests: write + steps: + - uses: actions/download-artifact@v3 + + - name: Upload plots to CML + run: | + docker pull -q dvcorg/cml-py3:latest + for FILE in *.png; do + echo "Handling $FILE" + docker run --rm -t \ + -v $PWD:/output:rw \ + dvcorg/cml-py3:latest \ + /bin/sh -c "cml-publish /output/$FILE --driver=github --repo=${{ github.repository }} --md --title=$FILE >> /output/report.md" + done + shell: bash + working-directory: output + + - name: Dump markdown to variable + id: report + run: | + file_test_newlines="" + while read line + do + file_test_newlines+=" $line" + done < report.md + echo "report=$file_test_newlines" >> $GITHUB_OUTPUT + shell: bash + working-directory: output + + - uses: actions/github-script@v6 + if: github.event_name == 'pull_request' + env: + REPORT: "Plots\n${{ steps.report.outputs.report }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const output = `#### Performance Test + Run URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + <details><summary>Show Performance Results</summary> + ${process.env.REPORT} + </details> + *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + + pr-perf-test-complete: + # Only runs on success + name: PR - performance test complete + # Label check from previous + runs-on: ubuntu-latest + needs: + - pr-perf-test-run + permissions: + pull-requests: write + steps: + - uses: actions-ecosystem/action-add-labels@v1 + name: Label the PR + with: + labels: ci/performance-test-ok + github_token: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} diff --git a/fluent-bit/.github/workflows/pr-windows-build.yaml b/fluent-bit/.github/workflows/pr-windows-build.yaml new file mode 100644 index 00000000..2831122f --- /dev/null +++ b/fluent-bit/.github/workflows/pr-windows-build.yaml @@ -0,0 +1,27 @@ +name: PR - Windows checks +on: + workflow_dispatch: + + pull_request: + # Limit to just those changes that 'might' affect Windows for automated builds + # We can always do a manual build for a branch + paths: + - '**.h' + - '**.c' + - '**.windows' + - './conf/**' + - './cmake/**' + types: + - opened + - reopened + - synchronize + +jobs: + pr-windows-build: + uses: ./.github/workflows/call-build-windows.yaml + with: + version: ${{ github.sha }} + ref: ${{ github.sha }} + environment: pr + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/fluent-bit/.github/workflows/resources/auto-build-test-workflow.png b/fluent-bit/.github/workflows/resources/auto-build-test-workflow.png Binary files differnew file mode 100644 index 00000000..40d3dfce --- /dev/null +++ b/fluent-bit/.github/workflows/resources/auto-build-test-workflow.png diff --git a/fluent-bit/.github/workflows/resources/release-approval-per-job.png b/fluent-bit/.github/workflows/resources/release-approval-per-job.png Binary files differnew file mode 100644 index 00000000..0f30245c --- /dev/null +++ b/fluent-bit/.github/workflows/resources/release-approval-per-job.png diff --git a/fluent-bit/.github/workflows/resources/release-approval.png b/fluent-bit/.github/workflows/resources/release-approval.png Binary files differnew file mode 100644 index 00000000..16a5570c --- /dev/null +++ b/fluent-bit/.github/workflows/resources/release-approval.png diff --git a/fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.png b/fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.png Binary files differnew file mode 100644 index 00000000..3708885b --- /dev/null +++ b/fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.png diff --git a/fluent-bit/.github/workflows/resources/release-from-staging-workflow.png b/fluent-bit/.github/workflows/resources/release-from-staging-workflow.png Binary files differnew file mode 100644 index 00000000..c6ea45ae --- /dev/null +++ b/fluent-bit/.github/workflows/resources/release-from-staging-workflow.png diff --git a/fluent-bit/.github/workflows/resources/release-version-failure.png b/fluent-bit/.github/workflows/resources/release-version-failure.png Binary files differnew file mode 100644 index 00000000..a96698cc --- /dev/null +++ b/fluent-bit/.github/workflows/resources/release-version-failure.png diff --git a/fluent-bit/.github/workflows/skipped-unit-tests.yaml b/fluent-bit/.github/workflows/skipped-unit-tests.yaml new file mode 100644 index 00000000..ab5adc5f --- /dev/null +++ b/fluent-bit/.github/workflows/skipped-unit-tests.yaml @@ -0,0 +1,21 @@ +# For skipped checks that are required to merge we trigger a fake job that succeeds. +# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks +name: Run unit tests +on: + pull_request: + paths: + - '.github/**' + - 'dockerfiles/**' + - 'docker_compose/**' + - 'packaging/**' + - '.gitignore' + - 'appveyor.yml' + - '**.sh' + - 'examples/**' + +jobs: + run-all-unit-tests: + runs-on: ubuntu-latest + name: Unit tests (matrix) + steps: + - run: echo "No unit tests required"
\ No newline at end of file diff --git a/fluent-bit/.github/workflows/staging-build.yaml b/fluent-bit/.github/workflows/staging-build.yaml new file mode 100644 index 00000000..2bf6524f --- /dev/null +++ b/fluent-bit/.github/workflows/staging-build.yaml @@ -0,0 +1,179 @@ +--- +name: Deploy to staging + +on: + push: + tags: + - '*' + + workflow_dispatch: + inputs: + version: + description: Version of Fluent Bit to build + required: true + default: master + target: + description: Only build a specific Linux target, intended for debug/test/quick builds only. + required: false + default: "" + ignore_failing_targets: + description: Optionally ignore any failing builds in the matrix and continue. + type: boolean + required: false + default: false + +# We do not want a new staging build to run whilst we are releasing the current staging build. +# We also do not want multiples to run for the same version. +concurrency: staging-build-release + +jobs: + + # This job strips off the `v` at the start of any tag provided. + # It then provides this metadata for the other jobs to use. + staging-build-get-meta: + name: Get metadata to build + runs-on: ubuntu-latest + outputs: + version: ${{ steps.formatted_version.outputs.replaced }} + steps: + + - run: | + echo "Version: ${{ inputs.version || github.ref_name }}" + shell: bash + + # This step is to consolidate the three different triggers into a single "version" + # 1. If manual dispatch - use the version provided. + # 2. If cron/regular build - use master. + # 3. If tag trigger, use that tag. + - name: Get the version + id: get_version + run: | + VERSION="${INPUT_VERSION}" + if [ -z "${VERSION}" ]; then + echo "Defaulting to master" + VERSION=master + fi + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + shell: bash + env: + # Use the dispatch variable in preference, if empty use the context ref_name which should + # only ever be a tag or the master branch for cron builds. + INPUT_VERSION: ${{ inputs.version || github.ref_name }} + + # String the 'v' prefix for tags. + - uses: frabert/replace-string-action@v2.4 + id: formatted_version + with: + pattern: '[v]*(.*)$' + string: "${{ steps.get_version.outputs.VERSION }}" + replace-with: '$1' + flags: 'g' + + staging-build-images: + needs: staging-build-get-meta + uses: ./.github/workflows/call-build-images.yaml + with: + version: ${{ needs.staging-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/staging + environment: staging + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }} + cosign_private_key_password: ${{ secrets.COSIGN_PASSWORD }} + + staging-build-upload-schema-s3: + needs: + - staging-build-get-meta + - staging-build-images + runs-on: ubuntu-latest + environment: staging + steps: + - name: Download the schema generated by call-build-images + # We may have no schema so ignore that failure + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: fluent-bit-schema-${{ needs.staging-build-get-meta.outputs.version }} + path: artifacts/ + + - name: Display structure of downloaded files + run: | + ls -R artifacts/ + shell: bash + + - name: Push schema to S3 bucket + # We may have no schema so ignore that failure + continue-on-error: true + run: | + aws s3 sync "artifacts/" "s3://${AWS_S3_BUCKET}/${DEST_DIR}" --no-progress + env: + DEST_DIR: "${{ needs.staging-build-get-meta.outputs.version }}/" + AWS_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }} + + staging-build-generate-matrix: + name: Staging build matrix + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix.outputs.build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Set up the list of target to build so we can pass the JSON to the reusable job + - uses: ./.github/actions/generate-package-build-matrix + id: set-matrix + with: + ref: ${{ inputs.version || github.ref_name }} + target: ${{ inputs.target || '' }} + + staging-build-packages: + needs: + - staging-build-get-meta + - staging-build-generate-matrix + uses: ./.github/workflows/call-build-linux-packages.yaml + with: + version: ${{ needs.staging-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + build_matrix: ${{ needs.staging-build-generate-matrix.outputs.build-matrix }} + environment: staging + ignore_failing_targets: ${{ inputs.ignore_failing_targets || false }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }} + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg_private_key_passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + staging-build-windows-packages: + needs: + - staging-build-get-meta + uses: ./.github/workflows/call-build-windows.yaml + with: + version: ${{ needs.staging-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + environment: staging + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }} + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + staging-build-macos-packages: + needs: + - staging-build-get-meta + uses: ./.github/workflows/call-build-macos.yaml + with: + version: ${{ needs.staging-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + environment: staging + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }} + access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/fluent-bit/.github/workflows/staging-release.yaml b/fluent-bit/.github/workflows/staging-release.yaml new file mode 100644 index 00000000..8ae352bd --- /dev/null +++ b/fluent-bit/.github/workflows/staging-release.yaml @@ -0,0 +1,955 @@ +--- +name: Release from staging + +# This is only expected to be invoked on-demand by a specific user. +on: + workflow_dispatch: + inputs: + version: + type: string + description: The version we want to release from staging, ensure this is numeric without the v prefix for the tag. + required: true + docker-image: + type: string + description: Optionally override the image name to push to on Docker Hub. + default: fluent/fluent-bit + required: false + github-image: + type: string + description: Optionally override the image name to push to on Github Container Registry. + default: fluent/fluent-bit + required: false + +# We do not want a new staging build to run whilst we are releasing the current staging build. +# We also do not want multiples to run for the same version. +concurrency: staging-build-release + +env: + STAGING_IMAGE_NAME: ghcr.io/${{ github.repository }}/staging + +jobs: + + staging-release-version-check: + name: Check staging release matches + environment: release # required to get bucket name + runs-on: ubuntu-latest + outputs: + major-version: ${{ steps.get_major_version.outputs.value }} + permissions: + contents: read + steps: + - name: Get the version on staging + run: | + curl --fail -LO "$AWS_URL/latest-version.txt" + cat latest-version.txt + STAGING_VERSION=$(cat latest-version.txt) + [[ "$STAGING_VERSION" != "$RELEASE_VERSION" ]] && echo "Latest version mismatch: $STAGING_VERSION != $RELEASE_VERSION" && exit 1 + # Must end in something that exits 0 + echo "Successfully confirmed version is as expected: $STAGING_VERSION" + shell: bash + env: + AWS_URL: https://${{ secrets.AWS_S3_BUCKET_STAGING }}.s3.amazonaws.com + RELEASE_VERSION: ${{ github.event.inputs.version }} + + # Get the major version, i.e. 1.9.3 --> 1.9, or just return the passed in version. + - name: Convert to major version format + id: get_major_version + run: | + MAJOR_VERSION="$RELEASE_VERSION" + if [[ $RELEASE_VERSION =~ ^[0-9]+\.[0-9]+ ]]; then + MAJOR_VERSION="${BASH_REMATCH[0]}" + fi + echo "value=$MAJOR_VERSION" >> $GITHUB_OUTPUT + shell: bash + env: + RELEASE_VERSION: ${{ github.event.inputs.version }} + + - name: Checkout repository + uses: actions/checkout@v4 + + # Check we can download the AppVeyor build which confirms it matches the version to release as well as being a successful build + - name: Get Appveyor binaries + run: | + ./packaging/appveyor-download.sh + shell: bash + env: + TAG: v${{ github.event.inputs.version }} + + staging-release-generate-package-matrix: + name: Get package matrix + runs-on: ubuntu-latest + outputs: + deb-build-matrix: ${{ steps.get-matrix.outputs.deb-build-matrix }} + rpm-build-matrix: ${{ steps.get-matrix.outputs.rpm-build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y jq + shell: bash + + # Cope with 1.9 as well as 2.0 + - uses: ./.github/actions/generate-package-build-matrix + id: get-matrix + with: + ref: v${{ inputs.version }} + + # Now annotate with whether it is Yum or Apt based + + # 1. Take packages from the staging bucket + # 2. Sign them with the release GPG key + # 3. Also take existing release packages from the release bucket. + # 4. Create a full repo configuration using the existing releases as well. + # 5. Upload to release bucket. + # Note we could resign all packages as well potentially if we wanted to update the key. + staging-release-yum-packages: + name: S3 - update YUM packages bucket + runs-on: ubuntu-22.04 # no createrepo on Ubuntu 20.04 + environment: release + needs: + - staging-release-version-check + - staging-release-generate-package-matrix + permissions: + contents: read + strategy: + matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.rpm-build-matrix) }} + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y createrepo-c rpm + shell: bash + + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + # Download the current release bucket + # Add everything from staging + # Sign and set up metadata for it all + # Upload to release bucket + + - name: Sync packages from buckets on S3 + run: | + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + - name: GPG set up keys for signing + run: | + gpg --export -a "${{ steps.import_gpg.outputs.name }}" > /tmp/fluentbit.key + rpm --import /tmp/fluentbit.key + shell: bash + + - name: Update repo info and remove any staging details + run: | + packaging/update-yum-repo.sh + env: + GPG_KEY: ${{ steps.import_gpg.outputs.name }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} + VERSION: ${{ github.event.inputs.version }} + BASE_PATH: "packaging/releases" + RPM_REPO: ${{ matrix.distro }} + shell: bash + + - name: Sync to release bucket on S3 + run: | + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + staging-release-apt-packages: + name: S3 - update APT packages bucket + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-version-check + - staging-release-generate-package-matrix + permissions: + contents: read + strategy: + matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.deb-build-matrix) }} + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup runner + run: | + sudo apt-get update + sudo apt-get install -y aptly debsigs distro-info rsync + shell: bash + + - name: Convert version to codename + id: get_codename + run: | + CODENAME="$DISTRO" + if [[ "$DISTRO" == ubuntu* ]]; then + echo "Converting Ubuntu version to codename" + UBUNTU_NAME=$(grep "${DISTRO##*/} LTS" /usr/share/distro-info/ubuntu.csv|cut -d ',' -f3) + echo "Got Ubuntu codename: $UBUNTU_NAME" + CODENAME="ubuntu/$UBUNTU_NAME" + fi + echo "Using codename: $CODENAME" + echo "CODENAME=$CODENAME" >> $GITHUB_OUTPUT + shell: bash + env: + DISTRO: ${{ matrix.distro }} + + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + - name: Sync packages from buckets on S3 + run: | + mkdir -p "packaging/releases/$CODENAME" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + CODENAME: ${{ steps.get_codename.outputs.CODENAME }} + shell: bash + + - name: Update repo info and remove any staging details + run: | + packaging/update-apt-repo.sh + env: + GPG_KEY: ${{ steps.import_gpg.outputs.name }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} + VERSION: ${{ github.event.inputs.version }} + BASE_PATH: "packaging/releases" + DEB_REPO: ${{ steps.get_codename.outputs.CODENAME }} + shell: bash + + - name: Sync to release bucket on S3 + run: | + aws s3 sync "packaging/releases/$CODENAME" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" --delete --follow-symlinks --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + CODENAME: ${{ steps.get_codename.outputs.CODENAME }} + shell: bash + + staging-release-update-non-linux-s3: + name: Update Windows and macOS packages + runs-on: ubuntu-22.04 + environment: release + needs: + - staging-release-version-check + permissions: + contents: none + strategy: + matrix: + distro: + - macos + - windows + fail-fast: false + steps: + - name: Sync packages from buckets on S3 + run: | + mkdir -p "packaging/releases/$DISTRO" + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + - name: Sync to release bucket on S3 + run: | + aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + DISTRO: ${{ matrix.distro }} + shell: bash + + staging-release-update-base-s3: + name: Update top-level bucket info + runs-on: ubuntu-22.04 + environment: release + needs: + - staging-release-apt-packages + - staging-release-yum-packages + permissions: + contents: none + steps: + - name: Import GPG key for signing + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} + + - name: GPG public key + run: | + gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./fluentbit.key + aws s3 cp ./fluentbit.key s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/fluentbit.key --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + shell: bash + + - name: JSON schema + continue-on-error: true + run: | + aws s3 sync "s3://${AWS_STAGING_S3_BUCKET}/${VERSION}" "s3://${AWS_RELEASE_S3_BUCKET}/${VERSION}" --no-progress + env: + VERSION: ${{ github.event.inputs.version }} + AWS_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_STAGING_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }} + AWS_RELEASE_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} + shell: bash + + staging-release-source-s3: + name: S3 - update source bucket + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-version-check + permissions: + contents: read + outputs: + windows-exe32-hash: ${{ steps.windows.outputs.WIN_32_EXE_HASH }} + windows-zip32-hash: ${{ steps.windows.outputs.WIN_32_ZIP_HASH }} + windows-exe64-hash: ${{ steps.windows.outputs.WIN_64_EXE_HASH }} + windows-zip64-hash: ${{ steps.windows.outputs.WIN_64_ZIP_HASH }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Sync packages from buckets on S3 + run: | + mkdir -p release staging + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" release/ --no-progress + aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/source/" staging/ --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + shell: bash + + - name: Get Appveyor binaries + run: | + ./packaging/appveyor-download.sh + shell: bash + env: + TAG: v${{ github.event.inputs.version }} + OUTPUT_DIR: appveyor + + - name: Move components from staging and setup + run: | + ./packaging/update-source-packages.sh + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + SOURCE_DIR: staging + WINDOWS_SOURCE_DIR: appveyor + TARGET_DIR: release + VERSION: ${{ github.event.inputs.version }} + MAJOR_VERSION: ${{ needs.staging-release-version-check.outputs.major-version }} + shell: bash + + - name: Sync to bucket on S3 + run: | + aws s3 sync release/ "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" --delete --follow-symlinks --no-progress + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + shell: bash + + - name: Provide output for documentation PR + id: windows + # do not fail the build for this + continue-on-error: true + run: | + ls -l $BASE_DIR/ + export WIN_32_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}') + export WIN_32_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}') + export WIN_64_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}') + export WIN_64_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}') + set | grep WIN_ + echo WIN_32_EXE_HASH="$WIN_32_EXE_HASH" >> $GITHUB_OUTPUT + echo WIN_32_ZIP_HASH="$WIN_32_ZIP_HASH" >> $GITHUB_OUTPUT + echo WIN_64_EXE_HASH="$WIN_64_EXE_HASH" >> $GITHUB_OUTPUT + echo WIN_64_ZIP_HASH="$WIN_64_ZIP_HASH" >> $GITHUB_OUTPUT + shell: bash + env: + BASE_DIR: release/${{ needs.staging-release-version-check.outputs.major-version }} + + # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing. + # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533 + staging-release-images: + name: Release ${{ matrix.tag }} Linux container images + runs-on: ubuntu-latest + needs: + - staging-release-version-check + environment: release + strategy: + fail-fast: false + matrix: + # All the explicit tags we want to release + tag: [ + "${{ github.event.inputs.version }}", + "${{ needs.staging-release-version-check.outputs.major-version }}", + "${{ github.event.inputs.version }}-debug", + "${{ needs.staging-release-version-check.outputs.major-version }}-debug", + ] + permissions: + packages: write + steps: + # Primarily because the skopeo errors are hard to parse and non-obvious + - name: Check the image exists + run: | + docker pull "$STAGING_IMAGE_NAME:$TAG" + env: + TAG: ${{ matrix.tag }} + shell: bash + + # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it. + - name: Promote container images from staging to Dockerhub + run: | + docker run --rm \ + quay.io/skopeo/stable:latest \ + copy \ + --all \ + --retry-times 10 \ + --src-no-creds \ + --dest-creds "$RELEASE_CREDS" \ + "docker://$STAGING_IMAGE_NAME:$TAG" \ + "docker://$RELEASE_IMAGE_NAME:$TAG" + env: + RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} + RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }} + TAG: ${{ matrix.tag }} + shell: bash + + - name: Promote container images from staging to GHCR.io + if: ${{ startsWith(github.event.inputs.version, '2.0') || ! startsWith(matrix.tag, 'latest') }} + run: | + docker run --rm \ + quay.io/skopeo/stable:latest \ + copy \ + --all \ + --retry-times 10 \ + --src-no-creds \ + --dest-creds "$RELEASE_CREDS" \ + "docker://$STAGING_IMAGE_NAME:$TAG" \ + "docker://$RELEASE_IMAGE_NAME:$TAG" + env: + RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} + RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} + TAG: ${{ matrix.tag }} + shell: bash + + # Part of resolution for: https://github.com/fluent/fluent-bit/issues/7748 + # More recent build-push-actions may mean legacy format is not preserved so we provide arch-specific tags just in case + staging-release-images-arch-specific-legacy-tags: + # TODO: remove next release once we are happy this all works, for now though do not block a release + continue-on-error: true + # + name: Release ${{ matrix.arch }} legacy format Linux container images + runs-on: ubuntu-latest + needs: + - staging-release-images + environment: release + strategy: + fail-fast: false + matrix: + arch: + - amd64 + - arm64 + - arm/v7 + permissions: + packages: write + env: + RELEASE_IMAGE_NAME: ${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} + RELEASE_TAG: ${{ github.event.inputs.version }} + steps: + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Convert arch to tag + id: get-tag + run: | + TAG="${RELEASE_TAG}-${{ matrix.arch }}" + echo "Input value: $TAG" + TAG=${TAG/\//-} + echo "Using tag: $TAG" + echo "tag=$TAG" >> $GITHUB_OUTPUT + shell: bash + + - name: Pull release image + run: docker pull --platform='linux/${{ matrix.arch }}' "$RELEASE_IMAGE_NAME:$RELEASE_TAG" + shell: bash + + - name: Tag and push legacy format image to DockerHub + run: | + docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" docker.io/"$RELEASE_IMAGE_NAME:$TAG" + docker push docker.io/"$RELEASE_IMAGE_NAME:$TAG" + shell: bash + env: + TAG: ${{ steps.get-tag.outputs.tag }} + + - name: Tag and push legacy format image to Github Container Registry + run: | + docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" ghcr.io/"$RELEASE_IMAGE_NAME:$TAG" + docker push ghcr.io/"$RELEASE_IMAGE_NAME:$TAG" + shell: bash + env: + TAG: ${{ steps.get-tag.outputs.tag }} + + staging-release-images-latest-tags: + # Only update latest tags for 2.1 releases + if: startsWith(github.event.inputs.version, '2.1') + name: Release latest Linux container images + runs-on: ubuntu-latest + needs: + - staging-release-images + environment: release + strategy: + fail-fast: false + matrix: + tag: [ + "latest", + "latest-debug" + ] + permissions: + packages: write + steps: + # Primarily because the skopeo errors are hard to parse and non-obvious + - name: Check the image exists + run: | + docker pull "$STAGING_IMAGE_NAME:$TAG" + env: + TAG: ${{ matrix.tag }} + shell: bash + + # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it. + - name: Promote container images from staging to Dockerhub + run: | + docker run --rm \ + quay.io/skopeo/stable:latest \ + copy \ + --all \ + --retry-times 10 \ + --src-no-creds \ + --dest-creds "$RELEASE_CREDS" \ + "docker://$STAGING_IMAGE_NAME:$TAG" \ + "docker://$RELEASE_IMAGE_NAME:$TAG" + env: + RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} + RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }} + TAG: ${{ matrix.tag }} + shell: bash + + - name: Promote container images from staging to GHCR.io + run: | + docker run --rm \ + quay.io/skopeo/stable:latest \ + copy \ + --all \ + --retry-times 10 \ + --src-no-creds \ + --dest-creds "$RELEASE_CREDS" \ + "docker://$STAGING_IMAGE_NAME:$TAG" \ + "docker://$RELEASE_IMAGE_NAME:$TAG" + env: + RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} + RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} + TAG: ${{ matrix.tag }} + shell: bash + + staging-release-images-windows: + name: Release Windows images + # Cannot be done by Skopeo on a Linux runner unfortunately + runs-on: windows-latest + needs: + - staging-release-version-check + environment: release + permissions: + packages: write + strategy: + fail-fast: false + matrix: + tag: [ + "windows-2019-${{ github.event.inputs.version }}", + "windows-2022-${{ github.event.inputs.version }}" + ] + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check the image exists + run: | + docker pull "$STAGING_IMAGE_NAME:$TAG" + env: + TAG: ${{ matrix.tag }} + shell: bash + + - name: Promote container images from staging to GHCR.io + run: | + docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG" + docker push "$RELEASE_IMAGE_NAME:$TAG" + env: + RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} + RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} + TAG: ${{ matrix.tag }} + shell: bash + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Promote container images from staging to Dockerhub + run: | + docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG" + docker push "$RELEASE_IMAGE_NAME:$TAG" + env: + RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} + RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }} + TAG: ${{ matrix.tag }} + shell: bash + + staging-release-images-sign: + name: Sign container image manifests + permissions: write-all + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-images + env: + DH_RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} + GHCR_RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} + steps: + - name: Install cosign + uses: sigstore/cosign-installer@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Cosign with a key + # Only run if we have a key defined + if: ${{ env.COSIGN_PRIVATE_KEY }} + # The key needs to cope with newlines + run: | + echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key + cosign sign --key /tmp/my_cosign.key --recursive \ + -a "repo=${{ github.repository }}" \ + -a "workflow=${{ github.workflow }}" \ + -a "release=${{ github.event.inputs.version }}" \ + "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ + "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \ + "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ + "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" + rm -f /tmp/my_cosign.key + shell: bash + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional + + - name: Cosign keyless signing using Rektor public transparency log + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance, and records it to the + # sigstore community Rekor transparency log. + # + # We use recursive signing on the manifest to cover all the images. + run: | + cosign sign --yes --recursive \ + -a "repo=${{ github.repository }}" \ + -a "workflow=${{ github.workflow }}" \ + -a "release=${{ github.event.inputs.version }}" \ + "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ + "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \ + "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ + "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" + shell: bash + env: + COSIGN_EXPERIMENTAL: true + + staging-release-upload-cosign-key: + name: Upload Cosign public key for verification + needs: + - staging-release-images-sign + permissions: + contents: none + runs-on: ubuntu-latest + steps: + - name: Install cosign + uses: sigstore/cosign-installer@v2 + + - name: Get public key and add to S3 bucket + # Only run if we have a key defined + if: ${{ env.COSIGN_PRIVATE_KEY }} + # The key needs to cope with newlines + run: | + echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key + cosign public-key --key /tmp/my_cosign.key > ./cosign.pub + rm -f /tmp/my_cosign.key + cat ./cosign.pub + aws s3 cp ./cosign.pub "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/cosign.pub" --no-progress + shell: bash + env: + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + + staging-release-smoke-test-packages: + name: Run package smoke tests + permissions: + contents: read + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-apt-packages + - staging-release-yum-packages + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Test release packages + run: | + ./packaging/test-release-packages.sh + shell: bash + env: + VERSION_TO_CHECK_FOR: ${{ github.event.inputs.version }} + FLUENT_BIT_PACKAGES_URL: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com + FLUENT_BIT_PACKAGES_KEY: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com/fluentbit.key + + staging-release-smoke-test-containers: + name: Run container smoke tests + permissions: + contents: read + packages: read + runs-on: ubuntu-latest + environment: release + needs: + - staging-release-images + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Test containers + run: | + ./packaging/testing/smoke/container/container-smoke-test.sh + shell: bash + env: + IMAGE_TAG: ${{ github.event.inputs.version }} + + staging-release-create-release: + name: Create the Github Release once packages and containers are up + needs: + - staging-release-images + - staging-release-apt-packages + - staging-release-yum-packages + permissions: + contents: write + environment: release + runs-on: ubuntu-latest + steps: + - name: Release 2.0 - not latest + uses: softprops/action-gh-release@v1 + if: startsWith(inputs.version, '2.0') + with: + body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" + draft: false + generate_release_notes: true + name: "Fluent Bit ${{ inputs.version }}" + tag_name: v${{ inputs.version }} + target_commitish: '2.0' + + - name: Release 2.1 and latest + uses: softprops/action-gh-release@v1 + if: startsWith(inputs.version, '2.1') + with: + body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" + draft: false + generate_release_notes: true + name: "Fluent Bit ${{ inputs.version }}" + tag_name: v${{ inputs.version }} + + staging-release-create-docs-pr: + name: Create docs updates for new release + needs: + - staging-release-images + - staging-release-source-s3 + permissions: + contents: none + environment: release + runs-on: ubuntu-latest + steps: + - name: Release 2.0 - not latest + if: startsWith(inputs.version, '2.0') + uses: actions/checkout@v4 + with: + repository: fluent/fluent-bit-docs + ref: 2.0 + token: ${{ secrets.GH_PA_TOKEN }} + + - name: Release 2.1 and latest + if: startsWith(inputs.version, '2.1') + uses: actions/checkout@v4 + with: + repository: fluent/fluent-bit-docs + token: ${{ secrets.GH_PA_TOKEN }} + + - name: Ensure we have the script we need + run: | + if [[ ! -f update-release-version-docs.sh ]] ; then + git checkout update-release-version-docs.sh -- master + fi + shell: bash + + - name: Update versions + # Uses https://github.com/fluent/fluent-bit-docs/blob/master/update-release-version-docs.sh + run: | + ./update-release-version-docs.sh + shell: bash + env: + NEW_VERSION: ${{ inputs.version }} + WIN_32_EXE_HASH: ${{ needs.staging-release-source-s3.outputs.windows-exe32-hash }} + WIN_32_ZIP_HASH: ${{ needs.staging-release-source-s3.outputs.windows-zip32-hash }} + WIN_64_EXE_HASH: ${{ needs.staging-release-source-s3.outputs.windows-exe64-hash }} + WIN_64_ZIP_HASH: ${{ needs.staging-release-source-s3.outputs.windows-zip64-hash }} + + - name: Raise docs PR + id: cpr + uses: peter-evans/create-pull-request@v5 + with: + commit-message: 'release: update to v${{ inputs.version }}' + signoff: true + delete-branch: true + title: 'release: update to v${{ inputs.version }}' + # We need workflows permission so have to use the GH_PA_TOKEN + token: ${{ secrets.GH_PA_TOKEN }} + labels: ci,automerge + body: | + Update release ${{ inputs.version }} version. + - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request + draft: false + + - name: Check outputs + if: ${{ steps.cpr.outputs.pull-request-number }} + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" + + staging-release-create-version-update-pr: + name: Create version update PR for new release + needs: + - staging-release-create-release + permissions: + contents: write + pull-requests: write + environment: release + runs-on: ubuntu-latest + steps: + - name: Release 2.0 + if: startsWith(inputs.version, '2.0') + uses: actions/checkout@v4 + with: + ref: 2.0 + + - name: Release 2.1 and latest + if: startsWith(inputs.version, '2.1') + uses: actions/checkout@v4 + + # Get the new version to use + - name: 'Get next minor version' + id: semvers + uses: "WyriHaximus/github-action-next-semvers@v1" + with: + version: ${{ inputs.version }} + strict: true + + - run: ./update_version.sh + shell: bash + env: + NEW_VERSION: ${{ steps.semvers.outputs.patch }} + # Ensure we use the PR action to do the work + DISABLE_COMMIT: 'yes' + + - name: Raise FB PR to update version + id: cpr + uses: peter-evans/create-pull-request@v5 + with: + commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}' + signoff: true + delete-branch: true + title: 'release: update to ${{ steps.semvers.outputs.patch }}' + labels: ci,automerge + body: | + Update next release to ${{ steps.semvers.outputs.patch }} version. + - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request + draft: false + + - name: Check outputs + if: ${{ steps.cpr.outputs.pull-request-number }} + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/fluent-bit/.github/workflows/staging-test.yaml b/fluent-bit/.github/workflows/staging-test.yaml new file mode 100644 index 00000000..e7a2d3ca --- /dev/null +++ b/fluent-bit/.github/workflows/staging-test.yaml @@ -0,0 +1,57 @@ +--- +name: Test staging +# The intention is this workflow is triggered either manually or +# after build has completed. +on: + workflow_run: + workflows: ['Deploy to staging'] + types: + - completed + workflow_dispatch: + +concurrency: integration-test + +jobs: + staging-test-images: + name: Container images staging tests + # Workflow run always triggers on completion regardless of status + # This prevents us from running if build fails. + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + uses: ./.github/workflows/call-test-images.yaml + with: + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/staging + image-tag: latest + environment: staging + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + cosign_key: ${{ secrets.COSIGN_PUBLIC_KEY }} + + # Called workflows cannot be nested + staging-test-images-integration: + name: run integration tests on GCP + # Wait for other tests to succeed + needs: staging-test-images + uses: ./.github/workflows/call-run-integration-test.yaml + with: + image_name: ghcr.io/${{ github.repository }}/staging + image_tag: latest + secrets: + opensearch_aws_access_id: ${{ secrets.OPENSEARCH_AWS_ACCESS_ID }} + opensearch_aws_secret_key: ${{ secrets.OPENSEARCH_AWS_SECRET_KEY }} + opensearch_admin_password: ${{ secrets.OPENSEARCH_ADMIN_PASSWORD }} + terraform_api_token: ${{ secrets.TF_API_TOKEN }} + gcp-service-account-key: ${{ secrets.GCP_SA_KEY }} + + staging-test-packages: + name: Binary packages staging test + # Workflow run always triggers on completion regardless of status + # This prevents us from running if build fails. + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + uses: ./.github/workflows/call-test-packages.yaml + with: + environment: staging + secrets: + bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/fluent-bit/.github/workflows/unit-tests.yaml b/fluent-bit/.github/workflows/unit-tests.yaml new file mode 100644 index 00000000..282879c5 --- /dev/null +++ b/fluent-bit/.github/workflows/unit-tests.yaml @@ -0,0 +1,125 @@ +name: Run unit tests +on: + push: + branches: + - master + - 2.0 + - 1.9 + - 1.8 + pull_request: + paths-ignore: + - '.github/**' + - 'dockerfiles/**' + - 'docker_compose/**' + - 'packaging/**' + - '.gitignore' + - 'appveyor.yml' + - '**.sh' + - 'examples/**' + branches: + - master + - 2.0 + - 1.9 + - 1.8 + types: [opened, reopened, synchronize] + workflow_dispatch: + +jobs: + run-ubuntu-unit-tests: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + flb_option: + - "-DFLB_JEMALLOC=On" + - "-DFLB_JEMALLOC=Off" + - "-DFLB_SMALL=On" + - "-DSANITIZE_ADDRESS=On" + - "-DSANITIZE_UNDEFINED=On" + - "-DFLB_COVERAGE=On" + - "-DFLB_SANITIZE_MEMORY=On" + - "-DFLB_SANITIZE_THREAD=On" + compiler: + - gcc + - clang + exclude: + - flb_option: "-DFLB_COVERAGE=On" + compiler: clang + permissions: + contents: read + steps: + - name: Setup environment + run: | + sudo apt-get update + sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev + sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true + + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + with: + repository: calyptia/fluent-bit-ci + path: ci + + - name: ${{ matrix.compiler }} - ${{ matrix.flb_option }} + run: | + echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT" + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90 + sudo usermod -a -G systemd-journal $(id -un) + sudo -E su -p $(id -un) -c "PATH=$PATH ci/scripts/run-unit-tests.sh" + env: + CC: ${{ matrix.compiler }} + CXX: ${{ matrix.compiler }} + FLB_OPT: ${{ matrix.flb_option }} + + run-macos-unit-tests: + # We chain this after Linux one as there are costs and restrictions associated + needs: + - run-ubuntu-unit-tests + runs-on: macos-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + flb_option: + - "-DFLB_JEMALLOC=Off" + - "-DFLB_SANITIZE_MEMORY=On" + - "-DFLB_SANITIZE_THREAD=On" + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: calyptia/fluent-bit-ci + path: ci + + - name: ${{ matrix.flb_option }} + run: | + echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT" + brew update + brew install bison flex openssl || true + ci/scripts/run-unit-tests.sh + env: + CC: gcc + CXX: g++ + FLB_OPT: ${{ matrix.flb_option }} + + # Required check looks at this so do not remove + run-all-unit-tests: + if: always() + runs-on: ubuntu-latest + name: Unit tests (matrix) + permissions: + contents: none + needs: + - run-macos-unit-tests + - run-ubuntu-unit-tests + steps: + - name: Check build matrix status + # Ignore MacOS failures + if: ${{ needs.run-ubuntu-unit-tests.result != 'success' }} + run: exit 1 diff --git a/fluent-bit/.github/workflows/update-dockerhub.yaml b/fluent-bit/.github/workflows/update-dockerhub.yaml new file mode 100644 index 00000000..cdd5dbe1 --- /dev/null +++ b/fluent-bit/.github/workflows/update-dockerhub.yaml @@ -0,0 +1,23 @@ +--- +name: Update Dockerhub description + +on: + workflow_dispatch: + +jobs: + update-dockerhub: + name: Update Dockerhub description + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ github.repository }} + readme-filepath: ./dockerfiles/dockerhub-description.md + short-description: 'Fluent Bit, lightweight logs and metrics collector and forwarder' |
