summaryrefslogtreecommitdiffstats
path: root/fluent-bit/.github
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 02:57:58 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 02:57:58 +0000
commitbe1c7e50e1e8809ea56f2c9d472eccd8ffd73a97 (patch)
tree9754ff1ca740f6346cf8483ec915d4054bc5da2d /fluent-bit/.github
parentInitial commit. (diff)
downloadnetdata-be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97.tar.xz
netdata-be1c7e50e1e8809ea56f2c9d472eccd8ffd73a97.zip
Adding upstream version 1.44.3.upstream/1.44.3upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fluent-bit/.github')
-rw-r--r--fluent-bit/.github/ISSUE_TEMPLATE.md9
-rw-r--r--fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md40
-rw-r--r--fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md22
-rw-r--r--fluent-bit/.github/PULL_REQUEST_TEMPLATE.md39
-rw-r--r--fluent-bit/.github/actionlint-matcher.json17
-rw-r--r--fluent-bit/.github/actionlint.yml4
-rw-r--r--fluent-bit/.github/actions/generate-package-build-matrix/action.yaml119
-rw-r--r--fluent-bit/.github/actions/sync-to-bucket/action.yaml88
-rw-r--r--fluent-bit/.github/dependabot.yml9
-rw-r--r--fluent-bit/.github/workflows/README.md227
-rw-r--r--fluent-bit/.github/workflows/build-branch-containers.yaml20
-rw-r--r--fluent-bit/.github/workflows/build-legacy-branch.yaml156
-rw-r--r--fluent-bit/.github/workflows/build-master-packages.yaml51
-rw-r--r--fluent-bit/.github/workflows/call-build-images.yaml329
-rw-r--r--fluent-bit/.github/workflows/call-build-linux-packages.yaml263
-rw-r--r--fluent-bit/.github/workflows/call-build-macos.yaml132
-rw-r--r--fluent-bit/.github/workflows/call-build-windows.yaml221
-rw-r--r--fluent-bit/.github/workflows/call-integration-image-build.yaml144
-rw-r--r--fluent-bit/.github/workflows/call-run-integration-test.yaml294
-rw-r--r--fluent-bit/.github/workflows/call-test-images.yaml205
-rw-r--r--fluent-bit/.github/workflows/call-test-packages.yaml55
-rw-r--r--fluent-bit/.github/workflows/cron-scorecards-analysis.yaml59
-rw-r--r--fluent-bit/.github/workflows/cron-stale.yaml26
-rw-r--r--fluent-bit/.github/workflows/cron-trivy.yaml87
-rw-r--r--fluent-bit/.github/workflows/cron-unstable-build.yaml213
-rw-r--r--fluent-bit/.github/workflows/master-integration-test.yaml33
-rw-r--r--fluent-bit/.github/workflows/pr-closed-docker.yaml21
-rw-r--r--fluent-bit/.github/workflows/pr-commit-message.yaml21
-rw-r--r--fluent-bit/.github/workflows/pr-compile-check.yaml32
-rw-r--r--fluent-bit/.github/workflows/pr-fuzz.yaml41
-rw-r--r--fluent-bit/.github/workflows/pr-image-tests.yaml121
-rw-r--r--fluent-bit/.github/workflows/pr-install-script.yaml29
-rw-r--r--fluent-bit/.github/workflows/pr-integration-test.yaml69
-rw-r--r--fluent-bit/.github/workflows/pr-labels.yaml17
-rw-r--r--fluent-bit/.github/workflows/pr-lint.yaml36
-rw-r--r--fluent-bit/.github/workflows/pr-package-tests.yaml92
-rw-r--r--fluent-bit/.github/workflows/pr-perf-test.yaml97
-rw-r--r--fluent-bit/.github/workflows/pr-windows-build.yaml27
-rw-r--r--fluent-bit/.github/workflows/resources/auto-build-test-workflow.pngbin0 -> 307383 bytes
-rw-r--r--fluent-bit/.github/workflows/resources/release-approval-per-job.pngbin0 -> 83634 bytes
-rw-r--r--fluent-bit/.github/workflows/resources/release-approval.pngbin0 -> 105348 bytes
-rw-r--r--fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.pngbin0 -> 231459 bytes
-rw-r--r--fluent-bit/.github/workflows/resources/release-from-staging-workflow.pngbin0 -> 140273 bytes
-rw-r--r--fluent-bit/.github/workflows/resources/release-version-failure.pngbin0 -> 72310 bytes
-rw-r--r--fluent-bit/.github/workflows/skipped-unit-tests.yaml21
-rw-r--r--fluent-bit/.github/workflows/staging-build.yaml179
-rw-r--r--fluent-bit/.github/workflows/staging-release.yaml955
-rw-r--r--fluent-bit/.github/workflows/staging-test.yaml57
-rw-r--r--fluent-bit/.github/workflows/unit-tests.yaml125
-rw-r--r--fluent-bit/.github/workflows/update-dockerhub.yaml23
50 files changed, 4825 insertions, 0 deletions
diff --git a/fluent-bit/.github/ISSUE_TEMPLATE.md b/fluent-bit/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 00000000..26e1d9fc
--- /dev/null
+++ b/fluent-bit/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,9 @@
+<!---
+Thanks for filing an issue 😄 ! Before you submit, please read the following:
+
+Please post all questions and issues on https://slack.fluentd.org/ on the fluent-bit before opening a Github Issue. Your questions will reach a wider audience there,
+and if we confirm that there is a bug, then you can open a new issue.
+
+Check the other issue templates if you are trying to submit a bug report, feature request, or question
+Search open/closed issues before submitting since someone might have asked the same thing before!
+-->
diff --git a/fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md b/fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 00000000..11535469
--- /dev/null
+++ b/fluent-bit/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,40 @@
+---
+name: "\U0001F41B Bug Report"
+about: "If something isn't working as expected \U0001F914."
+title: ''
+labels: 'status: waiting-for-triage'
+assignees: ''
+
+---
+
+## Bug Report
+
+**Describe the bug**
+<!--- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+- Rubular link if applicable:
+- Example log message if applicable:
+```
+{"log":"YOUR LOG MESSAGE HERE","stream":"stdout","time":"2018-06-11T14:37:30.681701731Z"}
+```
+- Steps to reproduce the problem:
+
+**Expected behavior**
+<!--- A clear and concise description of what you expected to happen. -->
+
+**Screenshots**
+<!--- If applicable, add screenshots to help explain your problem. -->
+
+**Your Environment**
+<!--- Include as many relevant details about the environment you experienced the bug in -->
+* Version used:
+* Configuration:
+* Environment name and version (e.g. Kubernetes? What version?):
+* Server type and version:
+* Operating System and version:
+* Filters and plugins:
+
+**Additional context**
+<!--- How has this issue affected you? What are you trying to accomplish? -->
+<!--- Providing context helps us come up with a solution that is most useful in the real world -->
diff --git a/fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md b/fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 00000000..db207e41
--- /dev/null
+++ b/fluent-bit/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+<!--- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+<!--- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+<!--- How has this issue affected you? What are you trying to accomplish? -->
+<!--- Providing context helps us come up with a solution that is most useful in the real world -->
+<!--- Add any other context or screenshots about the feature request here. -->
diff --git a/fluent-bit/.github/PULL_REQUEST_TEMPLATE.md b/fluent-bit/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 00000000..fc910708
--- /dev/null
+++ b/fluent-bit/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,39 @@
+<!-- Provide summary of changes -->
+
+<!-- Issue number, if available. E.g. "Fixes #31", "Addresses #42, #77" -->
+
+----
+Enter `[N/A]` in the box, if an item is not applicable to your change.
+
+**Testing**
+Before we can approve your change; please submit the following in a comment:
+- [ ] Example configuration file for the change
+- [ ] Debug log output from testing the change
+<!--
+Please refer to the Developer Guide for instructions on building Fluent Bit with Valgrind support:
+https://github.com/fluent/fluent-bit/blob/master/DEVELOPER_GUIDE.md#valgrind
+Invoke Fluent Bit and Valgrind as: $ valgrind --leak-check=full ./bin/fluent-bit <args>
+-->
+- [ ] Attached [Valgrind](https://valgrind.org/docs/manual/quick-start.html) output that shows no leaks or memory corruption was found
+
+If this is a change to packaging of containers or native binaries then please confirm it works for all targets.
+- [ ] Run [local packaging test](./packaging/local-build-all.sh) showing all targets (including any new ones) build.
+- [ ] Set `ok-package-test` label to test for all targets (requires maintainer to do).
+
+**Documentation**
+<!-- Docs can be edited at https://github.com/fluent/fluent-bit-docs -->
+- [ ] Documentation required for this feature
+
+<!-- Doc PR (not required but highly recommended) -->
+
+**Backporting**
+<!--
+PRs targeting the default master branch will go into the next major release usually.
+If this PR should be backported to the current or earlier releases then please submit a PR for that particular branch.
+-->
+- [ ] Backport to latest stable release.
+
+<!-- Other release PR (not required but highly recommended for quick turnaround) -->
+----
+
+Fluent Bit is licensed under Apache 2.0, by submitting this pull request I understand that this code will be released under the terms of that license.
diff --git a/fluent-bit/.github/actionlint-matcher.json b/fluent-bit/.github/actionlint-matcher.json
new file mode 100644
index 00000000..7b1ba251
--- /dev/null
+++ b/fluent-bit/.github/actionlint-matcher.json
@@ -0,0 +1,17 @@
+{
+ "problemMatcher": [
+ {
+ "owner": "actionlint",
+ "pattern": [
+ {
+ "regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$",
+ "file": 1,
+ "line": 2,
+ "column": 3,
+ "message": 4,
+ "code": 5
+ }
+ ]
+ }
+ ]
+ } \ No newline at end of file
diff --git a/fluent-bit/.github/actionlint.yml b/fluent-bit/.github/actionlint.yml
new file mode 100644
index 00000000..692d0852
--- /dev/null
+++ b/fluent-bit/.github/actionlint.yml
@@ -0,0 +1,4 @@
+self-hosted-runner:
+ labels:
+ - actuated
+ - actuated-aarch64
diff --git a/fluent-bit/.github/actions/generate-package-build-matrix/action.yaml b/fluent-bit/.github/actions/generate-package-build-matrix/action.yaml
new file mode 100644
index 00000000..f8355e4b
--- /dev/null
+++ b/fluent-bit/.github/actions/generate-package-build-matrix/action.yaml
@@ -0,0 +1,119 @@
+name: Composite action to generate the matrix of targets to build packages for.
+description: Remove any duplication of this information so we only have to update in one place.
+
+# Remember to add any new checks and target creation required.
+# For example, when 2.0 comes out we should detect it and add any target changes.
+
+inputs:
+ target:
+ description: Override to build a single target for debug/test only.
+ required: false
+ default: ""
+ ref:
+ description: The commit, tag or branch of Fluent Bit to checkout for building we then use to determine version for.
+ required: true
+outputs:
+ build-matrix:
+ description: The total build matrix we have created.
+ value: ${{ steps.set-matrix.outputs.matrix }}
+ deb-build-matrix:
+ description: The targets that provide DEB artefacts.
+ value: ${{ steps.set-matrix.outputs.debmatrix }}
+ rpm-build-matrix:
+ description: The targets that provide RPN artefacts.
+ value: ${{ steps.set-matrix.outputs.rpmmatrix }}
+runs:
+ using: "composite"
+ steps:
+ - name: Checkout code for version check
+ uses: actions/checkout@v3
+ with:
+ ref: ${{ inputs.ref }}
+ path: version-check
+
+ - name: Determine target type
+ id: determine-build-type
+ run: |
+ BUILD_TYPE="legacy"
+ if [[ -f "packaging/build-config.json" ]]; then
+ BUILD_TYPE="modern"
+ fi
+ echo "Detected type: $BUILD_TYPE"
+ echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_OUTPUT
+ shell: bash
+ working-directory: version-check
+
+ - name: 2.0+ targets
+ if: steps.determine-build-type.outputs.BUILD_TYPE == 'modern'
+ run: |
+ matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets|map(.target)' packaging/build-config.json)'}'|jq -c .)
+ echo "MATRIX=$matrix" >> $GITHUB_ENV
+
+ # The following are only used by release so exclude architecture as well
+
+ debtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="deb") | .target ]' packaging/build-config.json)
+ debmatrix=$(echo "{ \"distro\" : $debtargets }"|jq -c .)
+ echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV
+
+ rpmtargets=$(jq -cr '[.linux_targets[] | select(.target|contains("arm64v8")|not) | select(.type=="rpm") | .target ]' packaging/build-config.json)
+ rpmmatrix=$(echo "{ \"distro\" : $rpmtargets}"|jq -c .)
+ echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV
+ shell: bash
+
+ - name: 1.9 targets
+ if: steps.determine-build-type.outputs.BUILD_TYPE == 'legacy'
+ run: |
+ matrix=$((
+ echo '{ "distro" : ['
+ echo '"amazonlinux/2", "amazonlinux/2.arm64v8",'
+ echo '"centos/7", "centos/7.arm64v8", "centos/8", "centos/8.arm64v8",'
+ echo '"debian/buster", "debian/buster.arm64v8", "debian/bullseye", "debian/bullseye.arm64v8",'
+ echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",'
+ echo '"ubuntu/18.04.arm64v8", "ubuntu/20.04.arm64v8", "ubuntu/22.04.arm64v8",'
+ echo '"raspbian/buster", "raspbian/bullseye"'
+ echo ']}'
+ ) | jq -c .)
+ echo "MATRIX=$matrix" >> $GITHUB_ENV
+ debmatrix=$((
+ echo '{ "distro" : ['
+ echo '"debian/buster", "debian/bullseye",'
+ echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/22.04",'
+ echo '"raspbian/buster", "raspbian/bullseye"'
+ echo ']}'
+ ) | jq -c .)
+ echo "DEB_MATRIX=$debmatrix" >> $GITHUB_ENV
+ rpmmatrix=$((
+ echo '{ "distro" : ['
+ echo '"amazonlinux/2",'
+ echo '"centos/7", "centos/8"'
+ echo ']}'
+ ) | jq -c .)
+ echo "RPM_MATRIX=$rpmmatrix" >> $GITHUB_ENV
+ shell: bash
+
+ - name: Manual override of target
+ if: inputs.target != ''
+ run: |
+ if [ -n "${{ inputs.target || '' }}" ]; then
+ echo "Overriding matrix to build: ${{ inputs.target }}"
+ matrix=$((
+ echo '{ "distro" : ['
+ echo '"${{ inputs.target }}"'
+ echo ']}'
+ ) | jq -c .)
+ fi
+ echo "MATRIX=$matrix" >> $GITHUB_ENV
+ shell: bash
+
+ - id: set-matrix
+ run: |
+ echo $MATRIX
+ echo $MATRIX| jq .
+ echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+ echo $DEB_MATRIX
+ echo $DEB_MATRIX| jq .
+ echo "debmatrix=$DEB_MATRIX" >> $GITHUB_OUTPUT
+ echo $RPM_MATRIX
+ echo $RPM_MATRIX| jq .
+ echo "rpmmatrix=$RPM_MATRIX" >> $GITHUB_OUTPUT
+ shell: bash
diff --git a/fluent-bit/.github/actions/sync-to-bucket/action.yaml b/fluent-bit/.github/actions/sync-to-bucket/action.yaml
new file mode 100644
index 00000000..e79c0b3d
--- /dev/null
+++ b/fluent-bit/.github/actions/sync-to-bucket/action.yaml
@@ -0,0 +1,88 @@
+name: Composite action to sync S3 buckets
+description: Carry out all the tasks to sync to a bucket and make reusable.
+
+inputs:
+ bucket:
+ description: The name of the S3 (US-East) bucket to sync packages from.
+ required: true
+ access_key_id:
+ description: The S3 access key id for the bucket.
+ required: true
+ secret_access_key:
+ description: The S3 secret access key for the bucket.
+ required: true
+ bucket-directory:
+ description: The directory in the bucket to sync to.
+ required: true
+ source-directory:
+ description: The source directory to sync from.
+ required: true
+ aws-region:
+ description: The default region to use.
+ required: false
+ default: "us-east-1"
+ aws-custom-endpoint:
+ # To use with Minio locally (or update to whatever endpoint you want)
+ # '--endpoint http://localhost:9000'
+ description: A custom endpoint for S3 commands, e.g. for Minio.
+ required: false
+
+runs:
+ using: "composite"
+ steps:
+ - name: Local - Check and sync
+ run: |
+ if [[ ! -d "$SOURCE_DIR" ]]; then
+ echo "No source directory: $SOURCE_DIR"
+ ls -lR
+ exit 1
+ fi
+ echo "Valid source directory: $SOURCE_DIR"
+ ls -lR "$SOURCE_DIR"
+ env:
+ SOURCE_DIR: ${{ inputs.source-directory }}
+ shell: bash
+
+ - name: AWS - Check and sync
+ run: |
+ # For Minio, etc.
+ if [ -n "${AWS_S3_ENDPOINT}" ]; then
+ ENDPOINT="--endpoint-url ${AWS_S3_ENDPOINT}"
+ fi
+
+ # Check for non-empty values
+ if [ -z "$AWS_S3_BUCKET" ]; then
+ echo "Invalid (empty) bucket defined, check running on right environment to allow access to any secrets"
+ exit 1
+ fi
+ echo "$AWS_S3_BUCKET bucket is defined"
+
+ # Verify bucket access
+ bucketstatus=$(aws --region "$AWS_REGION" s3api head-bucket --bucket "${AWS_S3_BUCKET}" ${ENDPOINT} 2>&1)
+ echo "Response: $bucketstatus"
+ if echo "${bucketstatus}" | grep 'Not Found'; then
+ echo "$AWS_S3_BUCKET: bucket does not exist";
+ exit 1
+ elif echo "${bucketstatus}" | grep 'Forbidden'; then
+ echo "$AWS_S3_BUCKET: bucket exists but not owned"
+ exit 1
+ elif echo "${bucketstatus}" | grep 'Bad Request'; then
+ echo "$AWS_S3_BUCKET: bucket name specified is less than 3 or greater than 63 characters"
+ exit 1
+ else
+ echo "$AWS_S3_BUCKET: bucket owned and exists";
+ fi
+
+ # Sync to bucket
+ aws --region "$AWS_REGION" s3 sync "${SOURCE_DIR}" "s3://${AWS_S3_BUCKET}/${DEST_DIR}" --follow-symlinks --no-progress ${ENDPOINT}
+ env:
+ SOURCE_DIR: ${{ inputs.source-directory }}
+ DEST_DIR: ${{ inputs.bucket-directory }}
+ # Make sure to run in an environment with access to any secrets that are passed in.
+ # Otherwise they will be empty.
+ AWS_S3_BUCKET: ${{ inputs.bucket }}
+ AWS_REGION: ${{ inputs.aws-region }}
+ AWS_ACCESS_KEY_ID: ${{ inputs.access_key_id }}
+ AWS_SECRET_ACCESS_KEY: ${{ inputs.secret_access_key }}
+ AWS_S3_ENDPOINT: ${{ inputs.aws-custom-endpoint }}
+ shell: bash
diff --git a/fluent-bit/.github/dependabot.yml b/fluent-bit/.github/dependabot.yml
new file mode 100644
index 00000000..dc6a2c38
--- /dev/null
+++ b/fluent-bit/.github/dependabot.yml
@@ -0,0 +1,9 @@
+version: 2
+updates:
+ # Maintain dependencies for GitHub Actions
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "daily"
+ commit-message:
+ prefix: "workflows: " \ No newline at end of file
diff --git a/fluent-bit/.github/workflows/README.md b/fluent-bit/.github/workflows/README.md
new file mode 100644
index 00000000..aa52f593
--- /dev/null
+++ b/fluent-bit/.github/workflows/README.md
@@ -0,0 +1,227 @@
+# Available workflows
+
+| Workflow file | Description | Run event |
+| :---------------------------------------------------- | ------------------------ | ------------------------------------------------- |
+| [build-master-packages](./build-master-packages.yaml) | Builds packages using `master` for certain targets | on new commit/push on master / manual |
+| [cron-unstable-build](./cron-unstable-build.yaml) | Automated nightly builds of each supported branch | Scheduled/manual trigger |
+| [master-integration-test](./master-integration-test.yaml) | Runs the integration testing suite on master | on new commit/push on master |
+| [staging-build](./staging-build.yaml) | Builds the distro packages and docker images from a tagged release into staging (S3 and GHCR) | on new release/tag |
+| [staging-test](./staging-test.yaml) | Test the staging distro packages and docker images| manually or when `staging-build` completes successfully |
+| [staging-release](./staging-release.yaml) | Publishes the docker images/manifest on hub.docker.io/fluent/ and the distro packages | manual approval |
+| [pr-closed-docker](./pr-closed-docker.yaml) | Removes docker images for PR on hub.docker.io/fluentbitdev/| on pr closed|
+| [pr-compile-check](./pr-compile-check.yaml) | Runs some compilation sanity checks on a PR |
+| [pr-integration-test](./pr-integration-test.yaml) | Runs the integration testing suite on a PR branch | pr opened / label created 'ok-to-test' / on new commit/push on PR(s) |
+| [pr-package-tests](./pr-package-tests.yaml) | Runs the package build for all targets on a PR branch | pr opened / label created 'ok-package-test' / on new commit/push on PR(s) |
+| [pr-perf-test](./pr-integration-test.yaml) | Runs the performance testing suite on a PR branch | pr opened / label created 'ok-to-performance-test' / on new commit/push on PR(s) |
+| [pr-stale](./pr-stale.yaml) | Closes stale PR(s) with no activity in 30 days | scheduled daily 01:30 AM UTC|
+| [unit-tests](./unit-tests.yaml) | Runs the unit tests suite on master push or new PR | PR opened, merge in master branch |
+
+## Available labels
+
+| Label name | Description |
+| :----------|-------------|
+| docs-required| default tag used to request documentation, has to be removed before merge |
+| ok-package-test | Build for all possible targets |
+| ok-to-test | run all integration tests |
+| ok-to-merge | run mergebot and merge (rebase) current PR |
+| ci/integration-docker-ok | integration test is able to build docker image |
+| ci/integration-gcp-ok | integration test is able to run on GCP |
+| long-term | long running pull request, don't close |
+| exempt-stale | prevent stale checks running |
+
+## Required secrets
+
+* AWS_ACCESS_KEY_ID
+* AWS_SECRET_ACCESS_KEY
+* AWS_S3_BUCKET_STAGING
+* AWS_S3_BUCKET_RELEASE
+* GPG_PRIVATE_KEY
+* GPG_PRIVATE_KEY_PASSPHRASE
+
+These are only required for Cosign of the container images, will be skipped if not present:
+
+* COSIGN_PRIVATE_KEY
+* COSIGN_PRIVATE_KEY_PASSWORD - if set otherwise not required
+
+## Environments
+
+These environments are used:
+
+* `unstable` for all nightly builds
+* `staging` for all staging builds
+* `release` for running the promotion of staging to release, this can have additional approvals added
+
+If an environment is not present then it will be created but this may not have the appropriate permissions then.
+
+## Pushing to Github Container Registry
+
+Github actions require specific permissions to push to packages, see: <https://github.community/t/403-error-on-container-registry-push-from-github-action/173071/39>
+For some reason this is not automatically done via permission inheritance or similar.
+
+1. Verify you can push with a simple test, e.g. `docker pull alpine && docker tag alpine:latest ghcr.io/<repo>/fluent-bit:latest && docker push ghcr.io/<repo>/fluent-bit:latest`
+2. Once this is working locally, you should then be able to set up action permissions for the repository. If you already have a package no need to push a test one.
+3. Go to `https://github.com/users/USER/packages/container/fluent-bit/settings` and ensure the repository has access to `Write`.
+
+## Version-specific targets
+
+Each major version (e.g. 1.8 & 1.9) supports different targets to build for, e.g. 1.9 includes a CentOS 8 target and 1.8 has some other legacy targets.
+
+This is all handled by the [build matrix generation composite action](../actions/generate-package-build-matrix/action.yaml).
+This uses a [JSON file](../../packaging/build-config.json) to specify the targets so ensure this is updated.
+The build matrix is then fed into the [reusable job](./call-build-linux-packages.yaml) that builds packages which will then fire for the appropriate targets.
+The reusable job is used for all package builds including unstable/nightly and the PR `ok-package-test` triggered ones.
+
+## Releases
+
+The process at a high level is as follows:
+
+1. Tag created with `v` prefix.
+2. [Deploy to staging](https://github.com/fluent/fluent-bit/actions/workflows/staging-build.yaml) workflow runs.
+3. [Test staging](https://github.com/fluent/fluent-bit/actions/workflows/staging-test.yaml) workflow runs.
+4. Manually initiate [release from staging](https://github.com/fluent/fluent-bit/actions/workflows/staging-release.yaml) workflow.
+5. A PR is auto-created to increment the minor version now for Fluent Bit using the [`update_version.sh`](../../update_version.sh) script.
+6. Create PRs for doc updates - Windows & container versions. (WIP to automate).
+
+Breaking the steps down.
+
+### Deploy to staging and test
+
+This should run automatically when a tag is created matching the `v*` regex.
+It currently copes with 1.8+ builds although automation is only exercised for 1.9+ releases.
+
+Once this is completed successfully the staging tests should also run automatically.
+
+![Workflows for staging and test example](./resources/auto-build-test-workflow.png "Example of workflows for build and test")
+
+If both complete successfully then we are good to go.
+
+Occasional failures are seen with package builds not downloading dependencies (CentOS 7 in particular seems bad for this).
+A re-run of failed jobs should resolve this.
+
+The workflow builds all Linux, macOS and Windows targets to a staging S3 bucket plus the container images to ghcr.io.
+
+### Release from staging workflow
+
+This is a manually initiated workflow, the intention is multiple staging builds can happen but we only release one.
+Note that currently we do not support parallel staging builds of different versions, e.g. master and 1.9 branches.
+**We can only release the previous staging build and there is a check to confirm version.**
+
+Ensure AppVeyor build for the tag has completed successfully as well.
+
+To trigger: <https://github.com/fluent/fluent-bit/actions/workflows/staging-release.yaml>
+
+All this job does is copy the various artefacts from staging locations to release ones, it does not rebuild them.
+
+![Workflow for release example](./resources/release-from-staging-workflow-incorrect-version.png "Example of workflow for release")
+
+With this example you can see we used the wrong `version` as it requires it without the `v` prefix (it is used for container tag, etc.) and so it fails.
+
+![Workflow for release failure example](./resources/release-version-failure.png "Example of failing workflow for release")
+
+Make sure to provide without the `v` prefix.
+
+![Workflow for release example](./resources/release-from-staging-workflow.png "Example of successful workflow for release")
+
+Once this workflow is initiated you then also need to have it approved by the designated "release team" otherwise it will not progress.
+
+![Release approval example](./resources/release-approval.png "Release approval example")
+
+They will be notified for approval by Github.
+Unfortunately it has to be approved for each job in the sequence rather than a global approval for the whole workflow although that can be useful to check between jobs.
+
+![Release approval per-job required](./resources/release-approval-per-job.png "Release approval per-job required")
+
+This is quite useful to delay the final smoke test of packages until after the manual steps are done as it will then verify them all for you.
+
+#### Packages server sync
+
+The workflow above ensures all release artefacts are pushed to the appropriate container registry and S3 bucket for official releases.
+The packages server then periodically syncs from this bucket to pull down and serve the new packages so there may be a delay (up to 1 hour) before it serves the new versions.
+The syncs happen hourly.
+See <https://github.com/fluent/fluent-bit-infra/blob/main/terraform/provision/package-server-provision.sh.tftpl> for details of the dedicated packages server.
+
+The main reason for a separate server is to accurately track download statistics.
+Container images are handled by ghcr.io and Docker Hub, not this server.
+
+#### Transient container publishing failures
+
+The parallel publishing of multiple container tags for the same image seems to fail occasionally with network errors, particularly more for ghcr.io than DockerHub.
+This can be resolved by just re-running the failed jobs.
+
+#### Windows builds from AppVeyor
+
+This is automated, however confirm that the actual build is successful for the tag: <https://ci.appveyor.com/project/fluent/fluent-bit-2e87g/history>
+If not then ask a maintainer to retrigger.
+
+It can take a while to find the one for the specific tag...
+
+#### ARM builds
+
+All builds are carried out in containers and intended to be run on a valid Ubuntu host to match a standard Github Actions runner.
+This can take some time for ARM as we have to emulate the architecture via QEMU.
+
+<https://github.com/fluent/fluent-bit/pull/7527> introduces support to run ARM builds on a dedicated [actuated.dev](https://docs.actuated.dev/) ephemeral VM runner.
+A self-hosted ARM runner is sponsored by [Equinix Metal](https://deploy.equinix.com/metal/) and provisioned for this per the [documentation](https://docs.actuated.dev/provision-server/).
+For fork workflows, this should all be skipped and run on a normal Ubuntu Github hosted runner but be aware this may take some time.
+
+### Manual release
+
+As long as it is built to staging we can manually publish packages as well via the script here: <https://github.com/fluent/fluent-bit/blob/master/packaging/update-repos.sh>
+
+Containers can be promoted manually too, ensure to include all architectures and signatures.
+
+### Create PRs
+
+Once releases are published we need to provide PRs for the following documentation updates:
+
+1. Windows checksums: <https://docs.fluentbit.io/manual/installation/windows#installation-packages>
+2. Container versions: <https://docs.fluentbit.io/manual/installation/docker#tags-and-versions>
+
+<https://github.com/fluent/fluent-bit-docs> is the repo for updates to docs.
+
+Take the checksums from the release process above, the AppVeyor stage provides them all and we attempt to auto-create the PR with it.
+
+## Unstable/nightly builds
+
+These happen every 24 hours and [reuse the same workflow](./cron-unstable-build.yaml) as the staging build so are identical except they skip the upload to S3 step.
+This means all targets are built nightly for `master` and `2.0` branches including container images and Linux, macOS and Windows packages.
+
+The container images are available here (the tag refers to the branch):
+
+* [ghcr.io/fluent/fluent-bit/unstable:2.0](ghcr.io/fluent/fluent-bit/unstable:2.0)
+* [ghcr.io/fluent/fluent-bit/unstable:master](ghcr.io/fluent/fluent-bit/unstable:master)
+* [ghcr.io/fluent/fluent-bit/unstable:windows-2019-2.0](ghcr.io/fluent/fluent-bit/unstable:windows-2019-2.0)
+* [ghcr.io/fluent/fluent-bit/unstable:windows-2019-master](ghcr.io/fluent/fluent-bit/unstable:windows-2019-master)
+
+The Linux, macOS and Windows packages are available to download from the specific workflow run.
+
+## Integration tests
+
+On every commit to `master` we rebuild the [packages](./build-master-packages.yaml) and [container images](./master-integration-test.yaml).
+The container images are then used to [run the integration tests](./master-integration-test.yaml) from the <https://github.com/fluent/fluent-bit-ci> repository.
+The container images are available as:
+
+* [ghcr.io/fluent/fluent-bit/master:x86_64](ghcr.io/fluent/fluent-bit/master:x86_64)
+
+## PR checks
+
+Various workflows are run for PRs automatically:
+
+* [Unit tests](./unit-tests.yaml)
+* [Compile checks on CentOS 7 compilers](./pr-compile-check.yaml)
+* [Linting](./pr-lint.yaml)
+* [Windows builds](./pr-windows-build.yaml)
+* [Fuzzing](./pr-fuzz.yaml)
+* [Container image builds](./pr-image-tests.yaml)
+* [Install script checks](./pr-install-script.yaml)
+
+We try to guard these to only trigger when relevant files are changed to reduce any delays or resources used.
+**All should be able to be triggered manually for explicit branches as well.**
+
+The following workflows can be triggered manually for specific PRs too:
+
+* [Integration tests](./pr-integration-test.yaml): Build a container image and run the integration tests as per commits to `master`.
+* [Performance tests](./pr-perf-test.yaml): WIP to trigger a performance test on a dedicated VM and collect the results as a PR comment.
+* [Full package build](./pr-package-tests.yaml): builds all Linux, macOs and Windows packages as well as container images.
+
+To trigger these, apply the relevant label.
diff --git a/fluent-bit/.github/workflows/build-branch-containers.yaml b/fluent-bit/.github/workflows/build-branch-containers.yaml
new file mode 100644
index 00000000..ea5d3c6c
--- /dev/null
+++ b/fluent-bit/.github/workflows/build-branch-containers.yaml
@@ -0,0 +1,20 @@
+name: Build containers for a specific branch of 1.9+
+on:
+ workflow_dispatch:
+ inputs:
+ version:
+ description: Version of Fluent Bit to build, commit, branch, etc. The container image will be ghcr.io/fluent/fluent-bit/test/<this value>.
+ required: true
+ default: master
+jobs:
+ build-branch-containers:
+ uses: ./.github/workflows/call-build-images.yaml
+ with:
+ version: ${{ github.event.inputs.version }}
+ ref: ${{ github.event.inputs.version }}
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ image: ${{ github.repository }}/test/${{ github.event.inputs.version }}
+ unstable: ${{ github.event.inputs.version }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/fluent-bit/.github/workflows/build-legacy-branch.yaml b/fluent-bit/.github/workflows/build-legacy-branch.yaml
new file mode 100644
index 00000000..2bca2a71
--- /dev/null
+++ b/fluent-bit/.github/workflows/build-legacy-branch.yaml
@@ -0,0 +1,156 @@
+name: Build containers for a specific branch of 1.8
+on:
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: The code to build so a commit, branch, etc. The container image will be ghcr.io/fluent/fluent-bit/test/<this value>.
+ required: true
+ default: "1.8"
+
+env:
+ IMAGE_NAME: ghcr.io/${{ github.repository }}/test/${{ github.event.inputs.ref }}
+
+jobs:
+ build-legacy-branch-meta:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Check this is a 1.8 type build
+ run: |
+ if [[ -f "dockerfiles/Dockerfile" ]]; then
+ echo "Invalid branch as contains Dockerfile: ${{ inputs.ref }}"
+ exit 1
+ fi
+ shell: bash
+
+ # For 1.8 builds it is a little more complex so we have this build matrix to handle it.
+ # This creates separate images for each architecture.
+ # The later step then creates a multi-arch manifest for all of these.
+ build-legacy-images-matrix:
+ name: Build single arch legacy images
+ runs-on: ubuntu-latest
+ needs:
+ - build-legacy-branch-meta
+ strategy:
+ fail-fast: false
+ matrix:
+ arch: [amd64, arm64, arm/v7]
+ include:
+ - arch: amd64
+ suffix: x86_64
+ - arch: arm/v7
+ suffix: arm32v7
+ - arch: arm64
+ suffix: arm64v8
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - name: Checkout the docker build repo for legacy builds
+ uses: actions/checkout@v4
+ with:
+ repository: fluent/fluent-bit-docker-image
+ ref: "1.8" # Fixed to this branch
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - id: debug-meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.IMAGE_NAME }}
+ tags: |
+ raw,${{ inputs.ref }}-debug
+
+ - name: Build the legacy x86_64 debug image
+ if: matrix.arch == 'amd64'
+ uses: docker/build-push-action@v5
+ with:
+ file: ./Dockerfile.x86_64.debug
+ context: .
+ tags: ${{ steps.debug-meta.outputs.tags }}
+ labels: ${{ steps.debug-meta.outputs.labels }}
+ provenance: false
+ platforms: linux/amd64
+ push: true
+ load: false
+ build-args: |
+ FLB_TARBALL=https://github.com/fluent/fluent-bit/tarball/${{ inputs.ref }}
+
+ - name: Extract metadata from Github
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.IMAGE_NAME }}
+ tags: |
+ raw,${{ matrix.suffix }}-${{ inputs.ref }}
+
+ - name: Build the legacy ${{ matrix.arch }} image
+ uses: docker/build-push-action@v5
+ with:
+ file: ./Dockerfile.${{ matrix.suffix }}
+ context: .
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: linux/${{ matrix.arch }}
+ provenance: false
+ push: true
+ load: false
+ build-args: |
+ FLB_TARBALL=https://github.com/fluent/fluent-bit/tarball/${{ inputs.ref }}
+
+ # Create a multi-arch manifest for the separate 1.8 images.
+ build-legacy-image-manifests:
+ name: Deploy multi-arch container image manifests
+ permissions:
+ contents: read
+ packages: write
+ runs-on: ubuntu-latest
+ needs:
+ - build-legacy-branch-meta
+ - build-legacy-images-matrix
+ steps:
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Pull all the images
+ # Use platform to trigger warnings on invalid image metadata
+ run: |
+ docker pull --platform=linux/amd64 ${{ env.IMAGE_NAME }}:x86_64-${{ inputs.ref }}
+ docker pull --platform=linux/arm64 ${{ env.IMAGE_NAME }}:arm64v8-${{ inputs.ref }}
+ docker pull --platform=linux/arm/v7 ${{ env.IMAGE_NAME }}:arm32v7-${{ inputs.ref }}
+ shell: bash
+
+ - name: Create manifests for images
+ run: |
+ docker manifest create ${{ env.IMAGE_NAME }}:${{ inputs.ref }} \
+ --amend ${{ env.IMAGE_NAME }}:x86_64-${{ inputs.ref }} \
+ --amend ${{ env.IMAGE_NAME }}:arm64v8-${{ inputs.ref }} \
+ --amend ${{ env.IMAGE_NAME }}:arm32v7-${{ inputs.ref }}
+ docker manifest push --purge ${{ env.IMAGE_NAME }}:${{ inputs.ref }}
+ env:
+ DOCKER_CLI_EXPERIMENTAL: enabled
+ shell: bash
diff --git a/fluent-bit/.github/workflows/build-master-packages.yaml b/fluent-bit/.github/workflows/build-master-packages.yaml
new file mode 100644
index 00000000..5698abfe
--- /dev/null
+++ b/fluent-bit/.github/workflows/build-master-packages.yaml
@@ -0,0 +1,51 @@
+on:
+ push:
+ branches:
+ - master
+ workflow_dispatch:
+ inputs:
+ version:
+ description: Version of Fluent Bit to build
+ required: false
+ default: master
+ target:
+ description: Only build a specific target, intended for debug/test builds only.
+ required: false
+ default: ""
+
+name: Build packages for master
+jobs:
+ master-build-generate-matrix:
+ name: Staging build matrix
+ runs-on: ubuntu-latest
+ outputs:
+ build-matrix: ${{ steps.set-matrix.outputs.matrix }}
+ steps:
+ # Set up the list of target to build so we can pass the JSON to the reusable job
+ - id: set-matrix
+ run: |
+ matrix=$((
+ echo '{ "distro" : [ "debian/bullseye", "ubuntu/20.04", "ubuntu/22.04", "centos/7" ]}'
+ ) | jq -c .)
+ if [ -n "${{ github.event.inputs.target || '' }}" ]; then
+ echo "Overriding matrix to build: ${{ github.event.inputs.target }}"
+ matrix=$((
+ echo '{ "distro" : ['
+ echo '"${{ github.event.inputs.target }}"'
+ echo ']}'
+ ) | jq -c .)
+ fi
+ echo $matrix
+ echo $matrix| jq .
+ echo "matrix=$matrix" >> $GITHUB_OUTPUT
+ shell: bash
+
+ master-build-packages:
+ needs: master-build-generate-matrix
+ uses: ./.github/workflows/call-build-linux-packages.yaml
+ with:
+ version: master
+ ref: master
+ build_matrix: ${{ needs.master-build-generate-matrix.outputs.build-matrix }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/fluent-bit/.github/workflows/call-build-images.yaml b/fluent-bit/.github/workflows/call-build-images.yaml
new file mode 100644
index 00000000..fd2cf5b1
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-build-images.yaml
@@ -0,0 +1,329 @@
+---
+name: Reusable workflow to build container images
+
+on:
+ workflow_call:
+ inputs:
+ version:
+ description: The version of Fluent Bit to create.
+ type: string
+ required: true
+ ref:
+ description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+ type: string
+ required: true
+ registry:
+ description: The registry to push container images to.
+ type: string
+ required: true
+ username:
+ description: The username for the registry.
+ type: string
+ required: true
+ image:
+ description: The name of the container image to push to the registry.
+ type: string
+ required: true
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ unstable:
+ description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add.
+ type: string
+ required: false
+ default: ""
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with for the registry.
+ required: true
+ cosign_private_key:
+ description: The optional Cosign key to use for signing the images.
+ required: false
+ cosign_private_key_password:
+ description: If the Cosign key requires a password then specify here, otherwise not required.
+ required: false
+jobs:
+ call-build-images-meta:
+ name: Extract any supporting metadata
+ outputs:
+ major-version: ${{ steps.determine-major-version.outputs.replaced }}
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ # For main branch/releases we want to tag with the major version.
+ # E.g. if we build version 1.9.2 we want to tag with 1.9.2 and 1.9.
+ - name: Determine major version tag
+ id: determine-major-version
+ uses: frabert/replace-string-action@v2.4
+ with:
+ pattern: '^(\d+\.\d+).*$'
+ string: ${{ inputs.version }}
+ replace-with: "$1"
+ flags: "g"
+
+ # This is the intended approach to multi-arch image and all the other checks scanning,
+ # signing, etc only trigger from this.
+ call-build-images:
+ needs:
+ - call-build-images-meta
+ name: Multiarch container images to GHCR
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ packages: write
+ outputs:
+ production-digest: ${{ steps.build_push.outputs.digest }}
+ debug-digest: ${{ steps.debug_build_push.outputs.digest }}
+ steps:
+ - name: Checkout code for modern style builds
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Extract metadata from Github
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ inputs.registry }}/${{ inputs.image }}
+ tags: |
+ raw,${{ inputs.version }}
+ raw,${{ needs.call-build-images-meta.outputs.major-version }}
+ raw,latest
+
+ - name: Build the production images
+ id: build_push
+ uses: docker/build-push-action@v5
+ with:
+ file: ./dockerfiles/Dockerfile
+ context: .
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: linux/amd64, linux/arm64, linux/arm/v7
+ target: production
+ # Must be disabled to provide legacy format images from the registry
+ provenance: false
+ push: true
+ load: false
+ build-args: |
+ FLB_NIGHTLY_BUILD=${{ inputs.unstable }}
+ RELEASE_VERSION=${{ inputs.version }}
+
+ - id: debug-meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ inputs.registry }}/${{ inputs.image }}
+ tags: |
+ raw,${{ inputs.version }}-debug
+ raw,${{ needs.call-build-images-meta.outputs.major-version }}-debug
+ raw,latest-debug
+
+ - name: Build the debug multi-arch images
+ id: debug_build_push
+ uses: docker/build-push-action@v5
+ with:
+ file: ./dockerfiles/Dockerfile
+ context: .
+ tags: ${{ steps.debug-meta.outputs.tags }}
+ labels: ${{ steps.debug-meta.outputs.labels }}
+ platforms: linux/amd64, linux/arm64, linux/arm/v7
+ # Must be disabled to provide legacy format images from the registry
+ provenance: false
+ target: debug
+ push: true
+ load: false
+ build-args: |
+ FLB_NIGHTLY_BUILD=${{ inputs.unstable }}
+ RELEASE_VERSION=${{ inputs.version }}
+
+ call-build-images-generate-schema:
+ needs:
+ - call-build-images-meta
+ - call-build-images
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Generate schema
+ run: |
+ docker run --rm -t ${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.version }} -J > fluent-bit-schema-${{ inputs.version }}.json
+ cat fluent-bit-schema-${{ inputs.version }}.json | jq -M > fluent-bit-schema-pretty-${{ inputs.version }}.json
+ shell: bash
+
+ - name: Upload the schema
+ uses: actions/upload-artifact@v3
+ with:
+ path: ./fluent-bit-schema*.json
+ name: fluent-bit-schema-${{ inputs.version }}
+ if-no-files-found: error
+
+ call-build-images-scan:
+ needs:
+ - call-build-images-meta
+ - call-build-images
+ name: Trivy + Dockle image scan
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Trivy - multi-arch
+ uses: aquasecurity/trivy-action@master
+ with:
+ image-ref: "${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.version }}"
+ format: "table"
+ exit-code: "1"
+ ignore-unfixed: true
+ vuln-type: "os,library"
+ severity: "CRITICAL,HIGH"
+
+ - name: Dockle - multi-arch
+ uses: hands-lab/dockle-action@v1
+ with:
+ image: "${{ inputs.registry }}/${{ inputs.image }}:${{ inputs.version }}"
+ exit-code: "1"
+ exit-level: WARN
+
+ call-build-images-sign:
+ needs:
+ - call-build-images-meta
+ - call-build-images
+ name: Deploy and sign multi-arch container image manifests
+ permissions:
+ contents: read
+ packages: write
+ # This is used to complete the identity challenge
+ # with sigstore/fulcio when running outside of PRs.
+ id-token: write
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ steps:
+ - name: Install cosign
+ uses: sigstore/cosign-installer@v2
+
+ - name: Cosign keyless signing using Rektor public transparency log
+ # This step uses the identity token to provision an ephemeral certificate
+ # against the sigstore community Fulcio instance, and records it to the
+ # sigstore community Rekor transparency log.
+ #
+ # We use recursive signing on the manifest to cover all the images.
+ run: |
+ cosign sign --recursive \
+ -a "repo=${{ github.repository }}" \
+ -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+ -a "ref=${{ github.sha }}" \
+ -a "release=${{ inputs.version }}" \
+ "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \
+ "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}"
+ shell: bash
+ # Ensure we move on to key-based signing as well
+ continue-on-error: true
+ env:
+ COSIGN_EXPERIMENTAL: true
+
+ - name: Cosign with a key
+ # Only run if we have a key defined
+ if: ${{ env.COSIGN_PRIVATE_KEY }}
+ # The key needs to cope with newlines
+ run: |
+ echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key
+ cosign sign --key /tmp/my_cosign.key --recursive \
+ -a "repo=${{ github.repository }}" \
+ -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+ -a "ref=${{ github.sha }}" \
+ -a "release=${{ inputs.version }}" \
+ "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \
+ "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}"
+ rm -f /tmp/my_cosign.key
+ shell: bash
+ continue-on-error: true
+ env:
+ COSIGN_PRIVATE_KEY: ${{ secrets.cosign_private_key }}
+ COSIGN_PASSWORD: ${{ secrets.cosign_private_key_password }} # optional
+
+ # This takes a long time...
+ call-build-windows-container:
+ name: Windows container images
+ runs-on: windows-${{ matrix.windows-base-version }}
+ environment: ${{ inputs.environment }}
+ needs:
+ - call-build-images-meta
+ strategy:
+ fail-fast: true
+ matrix:
+ windows-base-version:
+ - '2019'
+ - '2022'
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Build the production images
+ run: |
+ docker build -t ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }} --build-arg FLB_NIGHTLY_BUILD=${{ inputs.unstable }} --build-arg WINDOWS_VERSION=ltsc${{ matrix.windows-base-version }} -f ./dockerfiles/Dockerfile.windows .
+ docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }}
+
+ # We cannot use this action as it requires privileged mode
+ # uses: docker/build-push-action@v5
+ # with:
+ # file: ./dockerfiles/Dockerfile.windows
+ # context: .
+ # tags: ${{ steps.meta.outputs.tags }}
+ # labels: ${{ steps.meta.outputs.labels }}
+ # platforms: windows/amd64
+ # target: runtime
+ # push: true
+ # load: false
+ # build-args: |
+ # FLB_NIGHTLY_BUILD=${{ inputs.unstable }}
+ # WINDOWS_VERSION=ltsc2019
diff --git a/fluent-bit/.github/workflows/call-build-linux-packages.yaml b/fluent-bit/.github/workflows/call-build-linux-packages.yaml
new file mode 100644
index 00000000..b8000225
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-build-linux-packages.yaml
@@ -0,0 +1,263 @@
+---
+name: Reusable workflow to build binary packages into S3 bucket
+
+on:
+ workflow_call:
+ inputs:
+ version:
+ description: The version of Fluent Bit to create.
+ type: string
+ required: true
+ ref:
+ description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+ type: string
+ required: true
+ build_matrix:
+ description: The build targets to produce as a JSON matrix.
+ type: string
+ required: true
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ unstable:
+ description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add.
+ type: string
+ required: false
+ default: ""
+ ignore_failing_targets:
+ description: Optionally ignore any failing builds in the matrix and continue.
+ type: boolean
+ required: false
+ default: false
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with.
+ required: true
+ bucket:
+ description: The name of the S3 (US-East) bucket to push packages into.
+ required: false
+ access_key_id:
+ description: The S3 access key id for the bucket.
+ required: false
+ secret_access_key:
+ description: The S3 secret access key for the bucket.
+ required: false
+ gpg_private_key:
+ description: The GPG key to use for signing the packages.
+ required: false
+ gpg_private_key_passphrase:
+ description: The GPG key passphrase to use for signing the packages.
+ required: false
+
+jobs:
+ call-build-capture-source:
+ # Capture source tarball and generate checksum for it
+ name: Extract any supporting metadata
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+ path: source
+
+ - name: Create tarball and checksums
+ run: |
+ tar -czvf $SOURCE_FILENAME_PREFIX.tar.gz -C source --exclude-vcs .
+ md5sum $SOURCE_FILENAME_PREFIX.tar.gz > $SOURCE_FILENAME_PREFIX.tar.gz.md5
+ sha256sum $SOURCE_FILENAME_PREFIX.tar.gz > $SOURCE_FILENAME_PREFIX.tar.gz.sha256
+ # Move to a directory to simplify upload/sync
+ mkdir -p source-packages
+ cp -fv $SOURCE_FILENAME_PREFIX* source-packages/
+ shell: bash
+ env:
+ SOURCE_FILENAME_PREFIX: source-${{ inputs.version }}
+
+ - name: Upload the source artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: source-${{ inputs.version }}
+ path: source-packages/*
+ if-no-files-found: error
+
+ # Pick up latest master version
+ - name: Checkout code for action
+ if: inputs.environment == 'staging'
+ uses: actions/checkout@v4
+ with:
+ path: action-support
+
+ - name: Push tarballs to S3
+ # Only upload for staging
+ if: inputs.environment == 'staging'
+ uses: ./action-support/.github/actions/sync-to-bucket
+ with:
+ bucket: ${{ secrets.bucket }}
+ access_key_id: ${{ secrets.access_key_id }}
+ secret_access_key: ${{ secrets.secret_access_key }}
+ bucket-directory: "${{ inputs.version }}/source"
+ source-directory: "source-packages/"
+
+ call-build-linux-packages:
+ name: ${{ matrix.distro }} package build and stage to S3
+ environment: ${{ inputs.environment }}
+ # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage.
+ runs-on: ${{ (contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit') && 'actuated-aarch64') || 'ubuntu-latest' }}
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJSON(inputs.build_matrix) }}
+ # Potentially we support continuing with all successful targets
+ continue-on-error: ${{ inputs.ignore_failing_targets || false }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Set up Actuated mirror
+ if: contains(matrix.distro, 'arm' ) && (github.repository == 'fluent/fluent-bit')
+ uses: self-actuated/hub-mirror@master
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Replace all special characters with dashes
+ id: formatted_distro
+ run:
+ output=${INPUT//[\/]/-}
+ echo "$INPUT --> $output"
+ echo "replaced=$output" >> "$GITHUB_OUTPUT"
+ shell: bash
+ env:
+ INPUT: ${{ matrix.distro }}
+
+ - name: fluent-bit - ${{ matrix.distro }} artifacts
+ run: |
+ ./build.sh
+ env:
+ FLB_DISTRO: ${{ matrix.distro }}
+ FLB_OUT_DIR: ${{ inputs.version }}/staging
+ FLB_NIGHTLY_BUILD: ${{ inputs.unstable }}
+ CMAKE_INSTALL_PREFIX: /opt/fluent-bit/
+ working-directory: packaging
+
+ - name: Upload the ${{ steps.formatted_distro.outputs.replaced }} artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: packages-${{ inputs.version }}-${{ steps.formatted_distro.outputs.replaced }}
+ path: packaging/packages/
+ if-no-files-found: error
+
+ - name: Retrieve target info for repo creation
+ id: get-target-info
+ # Remove any .arm648 suffix
+ # For ubuntu map to codename using the disto-info list (CSV)
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y distro-info awscli
+ TARGET=${DISTRO%*.arm64v8}
+ if [[ "$TARGET" == "ubuntu/"* ]]; then
+ UBUNTU_CODENAME=$(cut -d ',' -f 1,3 < "/usr/share/distro-info/ubuntu.csv"|grep "${TARGET##*/}"|cut -d ',' -f 2)
+ if [[ -n "$UBUNTU_CODENAME" ]]; then
+ TARGET="ubuntu/$UBUNTU_CODENAME"
+ else
+ echo "Unable to extract codename for $DISTRO"
+ exit 1
+ fi
+ fi
+ echo "$TARGET"
+ echo "target=$TARGET" >> $GITHUB_OUTPUT
+ env:
+ DISTRO: ${{ matrix.distro }}
+ shell: bash
+
+ - name: Verify output target
+ # Only upload for staging
+ # Make sure not to do a --delete on sync as it will remove the other architecture
+ run: |
+ if [ -z "${{ steps.get-target-info.outputs.target }}" ]; then
+ echo "Invalid (empty) target defined"
+ exit 1
+ fi
+ shell: bash
+
+ # Pick up latest master version
+ - name: Checkout code for action
+ if: inputs.environment == 'staging'
+ uses: actions/checkout@v4
+ with:
+ path: action-support
+
+ - name: Push packages to S3
+ # Only upload for staging
+ if: inputs.environment == 'staging'
+ uses: ./action-support/.github/actions/sync-to-bucket
+ with:
+ bucket: ${{ secrets.bucket }}
+ access_key_id: ${{ secrets.access_key_id }}
+ secret_access_key: ${{ secrets.secret_access_key }}
+ bucket-directory: "${{ inputs.version }}/${{ steps.get-target-info.outputs.target }}/"
+ source-directory: "packaging/packages/${{ matrix.distro }}/${{ inputs.version }}/staging/"
+
+ call-build-linux-packages-repo:
+ name: Create repo metadata in S3
+ # Only upload for staging
+ if: inputs.environment == 'staging'
+ # Need to use 18.04 as 20.04 has no createrepo available
+ runs-on: ubuntu-22.04
+ environment: ${{ inputs.environment }}
+ needs:
+ - call-build-linux-packages
+ steps:
+ - name: Install dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y createrepo-c aptly awscli
+
+ - name: Checkout code for repo metadata construction - always latest
+ uses: actions/checkout@v4
+
+ - name: Import GPG key for signing
+ id: import_gpg
+ uses: crazy-max/ghaction-import-gpg@v6
+ with:
+ gpg_private_key: ${{ secrets.gpg_private_key }}
+ passphrase: ${{ secrets.gpg_private_key_passphrase }}
+
+ - name: Create repositories on staging now
+ # We sync down what we have for the release directories.
+ # Create the repo metadata then upload to the root of the bucket.
+ # This will wipe out any versioned directories in the process.
+ run: |
+ rm -rf ./latest/
+ mkdir -p ./latest/
+ if [ -n "${AWS_S3_ENDPOINT}" ]; then
+ ENDPOINT="--endpoint-url ${AWS_S3_ENDPOINT}"
+ fi
+ aws s3 sync "s3://$AWS_S3_BUCKET/${{ inputs.version }}" ./latest/ --no-progress ${ENDPOINT}
+
+ gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./latest/fluentbit.key
+ rpm --import ./latest/fluentbit.key
+
+ ./update-repos.sh "./latest/"
+ echo "${{ inputs.version }}" > "./latest/latest-version.txt"
+ aws s3 sync "./latest/" "s3://$AWS_S3_BUCKET" --delete --follow-symlinks --no-progress ${ENDPOINT}
+ env:
+ GPG_KEY: ${{ steps.import_gpg.outputs.name }}
+ AWS_REGION: "us-east-1"
+ AWS_ACCESS_KEY_ID: ${{ secrets.access_key_id }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.secret_access_key }}
+ AWS_S3_BUCKET: ${{ secrets.bucket }}
+ # To use with Minio locally (or update to whatever endpoint you want)
+ # AWS_S3_ENDPOINT: http://localhost:9000
+ shell: bash
+ working-directory: packaging
diff --git a/fluent-bit/.github/workflows/call-build-macos.yaml b/fluent-bit/.github/workflows/call-build-macos.yaml
new file mode 100644
index 00000000..c97ee838
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-build-macos.yaml
@@ -0,0 +1,132 @@
+---
+name: Reusable workflow to build MacOS packages optionally into S3 bucket
+
+on:
+ workflow_call:
+ inputs:
+ version:
+ description: The version of Fluent Bit to create.
+ type: string
+ required: true
+ ref:
+ description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+ type: string
+ required: true
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ unstable:
+ description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add.
+ type: string
+ required: false
+ default: ''
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with.
+ required: true
+ bucket:
+ description: The name of the S3 (US-East) bucket to push packages into.
+ required: false
+ access_key_id:
+ description: The S3 access key id for the bucket.
+ required: false
+ secret_access_key:
+ description: The S3 secret access key for the bucket.
+ required: false
+
+jobs:
+ call-build-macos-legacy-check:
+ # Requires https://github.com/fluent/fluent-bit/pull/5247 so will not build for previous branches
+ name: Extract any supporting metadata
+ outputs:
+ build-type: ${{ steps.determine-build-type.outputs.BUILD_TYPE }}
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Determine build type
+ id: determine-build-type
+ run: |
+ BUILD_TYPE="legacy"
+ if [[ -f "conf/fluent-bit-macos.conf" ]]; then
+ BUILD_TYPE="modern"
+ fi
+ echo "Detected type: $BUILD_TYPE"
+ echo "BUILD_TYPE=$BUILD_TYPE" >> $GITHUB_OUTPUT
+ shell: bash
+
+ call-build-macos-package:
+ if: needs.call-build-macos-legacy-check.outputs.build-type == 'modern'
+ runs-on: macos-latest
+ environment: ${{ inputs.environment }}
+ needs:
+ - call-build-macos-legacy-check
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Install dependencies
+ run: |
+ brew update
+ brew install bison flex libyaml openssl pkgconfig || true
+
+ - name: Build Fluent Bit packages
+ run: |
+ export LIBRARY_PATH=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib:$LIBRARY_PATH
+ cmake -DCPACK_GENERATOR=productbuild -DFLB_NIGHTLY_BUILD=${{ inputs.unstable }} ../ -DOPENSSL_ROOT_DIR=$(brew --prefix openssl)
+ cmake --build .
+ cpack -G productbuild
+ working-directory: build
+
+ - name: Upload build packages
+ uses: actions/upload-artifact@v3
+ with:
+ name: macos-packages
+ path: |
+ build/fluent-bit-*-apple*
+ build/fluent-bit-*-intel*
+ if-no-files-found: error
+
+ call-build-macos-s3-upload:
+ name: Handle upload to S3
+ # The environment must be used that has access to any secrets required, even if passed in.
+ # If passed in but not in the environment here you end up with an empty secret.
+ environment: ${{ inputs.environment }}
+ runs-on: ubuntu-latest
+ needs:
+ - call-build-macos-package
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Download all artefacts
+ continue-on-error: true
+ uses: actions/download-artifact@v3
+ with:
+ name: macos-packages
+ path: artifacts/
+
+ - name: Push MacOS packages to S3
+ if: inputs.environment == 'staging'
+ uses: ./.github/actions/sync-to-bucket
+ with:
+ bucket: ${{ secrets.bucket }}
+ access_key_id: ${{ secrets.access_key_id }}
+ secret_access_key: ${{ secrets.secret_access_key }}
+ bucket-directory: "${{ inputs.version }}/macos/"
+ source-directory: "artifacts/"
diff --git a/fluent-bit/.github/workflows/call-build-windows.yaml b/fluent-bit/.github/workflows/call-build-windows.yaml
new file mode 100644
index 00000000..7adc229a
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-build-windows.yaml
@@ -0,0 +1,221 @@
+---
+name: Reusable workflow to build Windows packages optionally into S3 bucket
+
+on:
+ workflow_call:
+ inputs:
+ version:
+ description: The version of Fluent Bit to create.
+ type: string
+ required: true
+ ref:
+ description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+ type: string
+ required: true
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ unstable:
+ description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add.
+ type: string
+ required: false
+ default: ''
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with.
+ required: true
+ bucket:
+ description: The name of the S3 (US-East) bucket to push packages into.
+ required: false
+ access_key_id:
+ description: The S3 access key id for the bucket.
+ required: false
+ secret_access_key:
+ description: The S3 secret access key for the bucket.
+ required: false
+
+jobs:
+
+ call-build-windows-get-meta:
+ name: Determine build info
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ outputs:
+ armSupported: ${{ steps.armcheck.outputs.armSupported }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Determine if we are doing a build with ARM support
+ id: armcheck
+ # Check for new contents from https://github.com/fluent/fluent-bit/pull/6621
+ run: |
+ if grep -q "winarm64" CMakeLists.txt ; then
+ echo "armSupported=true" >> $GITHUB_OUTPUT
+ else
+ echo "armSupported=false" >> $GITHUB_OUTPUT
+ fi
+ shell: bash
+
+ call-build-windows-package:
+ runs-on: windows-latest
+ environment: ${{ inputs.environment }}
+ needs:
+ - call-build-windows-get-meta
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - name: "Windows 32bit"
+ arch: x86
+ openssl_dir: C:\vcpkg\packages\openssl_x86-windows-static
+ cmake_additional_opt: ""
+ vcpkg_triplet: x86-windows-static
+ - name: "Windows 64bit"
+ arch: x64
+ openssl_dir: C:\vcpkg\packages\openssl_x64-windows-static
+ cmake_additional_opt: ""
+ vcpkg_triplet: x64-windows-static
+ - name: "Windows 64bit (Arm64)"
+ arch: amd64_arm64
+ openssl_dir: C:\vcpkg\packages\openssl_arm64-windows-static
+ cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64"
+ vcpkg_triplet: arm64-windows-static
+ permissions:
+ contents: read
+ # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures.
+ env:
+ PATH: C:\ProgramData\Chocolatey\bin;c:/Program Files/Git/cmd;c:/Windows/system32;C:/Windows/System32/WindowsPowerShell/v1.0;$ENV:WIX/bin;C:/Program Files/CMake/bin;C:\vcpkg;
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Get dependencies
+ run: |
+ Invoke-WebRequest -O winflexbison.zip $env:WINFLEXBISON
+ Expand-Archive winflexbison.zip -Destination C:\WinFlexBison
+ Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe
+ Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe
+ echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append
+ env:
+ WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip
+ shell: pwsh
+
+ - name: Set up with Developer Command Prompt for Microsoft Visual C++
+ uses: ilammy/msvc-dev-cmd@v1
+ with:
+ arch: ${{ matrix.config.arch }}
+
+ - name: Get gzip command w/ chocolatey
+ uses: crazy-max/ghaction-chocolatey@v3
+ with:
+ args: install gzip -y
+
+ # http://man7.org/linux/man-pages/man1/date.1.html
+ - name: Get Date
+ id: get-date
+ run: |
+ echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+ shell: bash
+
+ - name: Restore cached packages of vcpkg
+ id: cache-vcpkg-sources
+ uses: actions/cache/restore@v3
+ with:
+ path: |
+ C:\vcpkg\packages
+ key: ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-${{ steps.get-date.outputs.date }}
+ restore-keys: |
+ ${{ runner.os }}-${{ matrix.config.arch }}-vcpkg-
+ enableCrossOsArchive: false
+
+ - name: Build openssl with vcpkg
+ run: |
+ C:\vcpkg\vcpkg install --recurse openssl --triplet ${{ matrix.config.vcpkg_triplet }}
+ shell: cmd
+
+ - name: Build libyaml with vcpkg
+ run: |
+ C:\vcpkg\vcpkg install --recurse libyaml --triplet ${{ matrix.config.vcpkg_triplet }}
+ shell: cmd
+
+ - name: Save packages of vcpkg
+ id: save-vcpkg-sources
+ uses: actions/cache/save@v3
+ with:
+ path: |
+ C:\vcpkg\packages
+ key: ${{ steps.cache-vcpkg-sources.outputs.cache-primary-key }}
+ enableCrossOsArchive: false
+
+ - name: Build Fluent Bit packages
+ # If we are using 2.0.* or earlier we need to exclude the ARM64 build as the dependencies fail to compile.
+ # Trying to do via an exclude for the job triggers linting errors.
+ # This is only supposed to be a workaround for now so can be easily removed later.
+ if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }}
+ run: |
+ cmake -G "NMake Makefiles" -DFLB_NIGHTLY_BUILD='${{ inputs.unstable }}' -DOPENSSL_ROOT_DIR='${{ matrix.config.openssl_dir }}' ${{ matrix.config.cmake_additional_opt }} -DFLB_LIBYAML_DIR=C:\vcpkg\packages\libyaml_${{ matrix.config.vcpkg_triplet }} ../
+ cmake --build .
+ cpack
+ working-directory: build
+
+ - name: Upload build packages
+ # Skip upload if we skipped build.
+ if: ${{ matrix.config.arch != 'amd64_arm64' || needs.call-build-windows-get-meta.outputs.armSupported == 'true' }}
+ uses: actions/upload-artifact@v3
+ with:
+ name: windows-packages
+ path: |
+ build/*-bit-*.exe
+ build/*-bit-*.msi
+ build/*-bit-*.zip
+ if-no-files-found: error
+
+ call-build-windows-s3-upload:
+ name: Handle upload to S3
+ # The environment must be used that has access to any secrets required, even if passed in.
+ # If passed in but not in the environment here you end up with an empty secret.
+ environment: ${{ inputs.environment }}
+ runs-on: ubuntu-latest
+ needs:
+ - call-build-windows-package
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ # Need latest for checksum packaging script
+ ref: master
+
+ - name: Download all artefacts
+ continue-on-error: true
+ uses: actions/download-artifact@v3
+ with:
+ name: windows-packages
+ path: artifacts/
+
+ - name: Set up Windows checksums
+ run: |
+ packaging/windows-checksums.sh
+ ls -lR artifacts/
+ shell: bash
+ env:
+ SOURCE_DIR: artifacts
+
+ - name: Push Windows packages to S3
+ # Only upload for staging
+ if: inputs.environment == 'staging'
+ uses: ./.github/actions/sync-to-bucket
+ with:
+ bucket: ${{ secrets.bucket }}
+ access_key_id: ${{ secrets.access_key_id }}
+ secret_access_key: ${{ secrets.secret_access_key }}
+ bucket-directory: "${{ inputs.version }}/windows/"
+ source-directory: "artifacts/"
diff --git a/fluent-bit/.github/workflows/call-integration-image-build.yaml b/fluent-bit/.github/workflows/call-integration-image-build.yaml
new file mode 100644
index 00000000..1644883c
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-integration-image-build.yaml
@@ -0,0 +1,144 @@
+name: Reusable workflow for integration testing
+on:
+ workflow_call:
+ inputs:
+ ref:
+ description: The SHA, commit or branch to checkout and build.
+ required: true
+ type: string
+ registry:
+ description: The registry to push container images to.
+ type: string
+ required: true
+ username:
+ description: The username for the registry.
+ type: string
+ required: true
+ image:
+ description: The name of the container image to push to the registry.
+ type: string
+ required: true
+ image-tag:
+ description: The tag of the image to for testing.
+ type: string
+ required: true
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with for the registry.
+ required: true
+jobs:
+ call-integration-image-build-container:
+ name: Integration test container image build
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Extract metadata from Github
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ inputs.registry }}/${{ inputs.image }}
+ tags: |
+ raw,${{ inputs.image-tag }}
+
+ - name: Build the AMD64 image
+ uses: docker/build-push-action@v5
+ with:
+ file: ./dockerfiles/Dockerfile
+ context: .
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: linux/amd64
+ target: production
+ provenance: false
+ push: true
+ load: false
+
+ - name: Upload the image just in case as an artefact
+ run: |
+ docker pull $IMAGE
+ docker save --output /tmp/pr-image.tar $IMAGE
+ env:
+ IMAGE: ${{ steps.meta.outputs.tags }}
+ shell: bash
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v3
+ with:
+ name: pr-${{ github.event.pull_request.number }}-image
+ path: /tmp/pr-image.tar
+ if-no-files-found: error
+
+ - name: Extract metadata from Github
+ id: meta-debug
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ inputs.registry }}/${{ inputs.image }}
+ tags: |
+ raw,${{ inputs.image-tag }}-debug
+
+ - name: Build the AMD64 debug image
+ uses: docker/build-push-action@v5
+ with:
+ file: ./dockerfiles/Dockerfile
+ context: .
+ tags: ${{ steps.meta-debug.outputs.tags }}
+ labels: ${{ steps.meta-debug.outputs.labels }}
+ provenance: false
+ target: debug
+ platforms: linux/amd64
+ push: true
+ load: false
+
+ call-integration-image-build-smoke-test:
+ name: Integration test image is valid
+ needs: call-integration-image-build-container
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Test the HTTP server is responding
+ timeout-minutes: 5
+ run: |
+ packaging/testing/smoke/container/container-smoke-test.sh
+ shell: bash
+ env:
+ CONTAINER_NAME: local-smoke-${{ inputs.image-tag }}
+ CONTAINER_ARCH: linux/amd64
+ REGISTRY: ${{ inputs.registry }}
+ IMAGE_NAME: ${{ inputs.image }}
+ IMAGE_TAG: ${{ inputs.image-tag }} \ No newline at end of file
diff --git a/fluent-bit/.github/workflows/call-run-integration-test.yaml b/fluent-bit/.github/workflows/call-run-integration-test.yaml
new file mode 100644
index 00000000..47f44a9b
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-run-integration-test.yaml
@@ -0,0 +1,294 @@
+---
+name: Reusable workflow to run integration tests with specific images
+on:
+ workflow_call:
+ secrets:
+ opensearch_aws_access_id:
+ description: AWS access ID to use within the opensearch integration tests.
+ required: true
+ opensearch_aws_secret_key:
+ description: AWS secret key to use within the opensearch integration tests.
+ required: true
+ opensearch_admin_password:
+ description: Default admin password use within the opensearch integration tests.
+ required: true
+ terraform_api_token:
+ description: Default terraform API token to use when running integration tests.
+ required: true
+ gcp-service-account-key:
+ description: The GCP service account key to use.
+ required: true
+ inputs:
+ image_name:
+ description: The image repository and name to use.
+ required: false
+ default: ghcr.io/fluent/fluent-bit/master
+ type: string
+ image_tag:
+ description: The image tag to use.
+ required: false
+ default: x86_64
+ type: string
+ ref:
+ description: The commit, tag or branch of the repository to checkout
+ type: string
+ required: false
+ default: main
+jobs:
+ call-run-terraform-setup:
+ name: Run Terraform set up
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ outputs:
+ aws-opensearch-endpoint: ${{ steps.aws-opensearch-endpoint.outputs.stdout }}
+ gke-cluster-name: ${{ steps.gke-cluster-name.outputs.stdout }}
+ gke-cluster-region: ${{ steps.gke-cluster-region.outputs.stdout }}
+ gke-cluster-zone: ${{ steps.gke-cluster-zone.outputs.stdout }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+ repository: fluent/fluent-bit-ci
+
+ - uses: hashicorp/setup-terraform@v2
+ with:
+ cli_config_credentials_hostname: 'app.terraform.io'
+ cli_config_credentials_token: ${{ secrets.terraform_api_token }}
+
+ - id: 'auth'
+ uses: 'google-github-actions/auth@v1'
+ with:
+ credentials_json: ${{ secrets.gcp-service-account-key }}
+
+ - name: 'Set up Cloud SDK'
+ uses: 'google-github-actions/setup-gcloud@v1'
+
+ - name: Replace terraform variables.
+ run: |
+ sed -i -e "s|\$OPENSEARCH_AWS_ACCESS_ID|${{ secrets.opensearch_aws_access_id }}|g" default.auto.tfvars
+ sed -i -e "s|\$OPENSEARCH_AWS_SECRET_KEY|${{ secrets.opensearch_aws_secret_key }}|g" default.auto.tfvars
+ sed -i -e "s|\$OPENSEARCH_ADMIN_PASSWORD|${{ secrets.opensearch_admin_password }}|g" default.auto.tfvars
+
+ cat <<EOT >> default.auto.tfvars
+ gcp_sa_key = <<-EOF
+ ${{ secrets.gcp-service-account-key }}
+ EOF
+ EOT
+ working-directory: terraform/
+ shell: bash
+
+ - name: Terraform init
+ id: init
+ run: terraform init
+ working-directory: terraform/
+
+ - name: Terraform fmt
+ id: fmt
+ run: |
+ find . -name "*.tf" -exec terraform fmt -check {} \;
+ working-directory: terraform
+
+ - name: Terraform validate
+ id: validate
+ run: terraform validate -no-color
+ working-directory: terraform/
+
+ - name: Terraform Plan
+ if: github.event_name == 'pull_request'
+ id: plan
+ run: terraform plan -no-color
+ working-directory: terraform
+ continue-on-error: true
+
+ - uses: actions/github-script@v6
+ if: github.event_name == 'pull_request'
+ env:
+ PLAN: "terraform\n${{ steps.plan.outputs.stdout }}"
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+ #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+ #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+ #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+ <details><summary>Show Plan</summary>
+ \`\`\`\n
+ ${process.env.PLAN}
+ \`\`\`
+ </details>
+ *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
+ github.rest.issues.createComment({
+ issue_number: context.issue.number,
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: output
+ })
+
+ - name: Terraform Plan Status
+ if: steps.plan.outcome == 'failure'
+ run: exit 1
+
+ - name: Terraform Apply
+ if: github.event_name != 'pull_request'
+ id: apply
+ run: terraform apply -input=false -auto-approve
+ working-directory: terraform
+ env:
+ TF_LOG: TRACE
+
+ # We're using the terraform wrapper here so separate steps for each output variable
+ - id: aws-opensearch-endpoint
+ run: terraform output -no-color -raw aws-opensearch-endpoint
+ working-directory: terraform
+ shell: bash
+
+ - id: gke-cluster-name
+ run: terraform output -no-color -raw gke_kubernetes_cluster_name
+ working-directory: terraform
+ shell: bash
+
+ - id: gke-cluster-region
+ run: terraform output -no-color -raw gke_region
+ working-directory: terraform
+ shell: bash
+
+ - id: gke-cluster-zone
+ run: terraform output -no-color -raw gke_zone
+ working-directory: terraform
+ shell: bash
+
+ call-run-integration-kind:
+ name: Run integration tests on KIND
+ needs:
+ - call-run-terraform-setup
+ # Can test for multiple K8S versions with KIND
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-release: [ 'v1.23.5', 'v1.22.7', 'v1.21.10' ]
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Test image exists and cache locally
+ run: docker pull ${{ inputs.image_name }}:${{ inputs.image_tag }}
+
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+ repository: fluent/fluent-bit-ci
+
+ - name: Configure system for Opensearch
+ run: |
+ sudo sysctl -w vm.max_map_count=262144
+ sysctl -p
+ shell: bash
+
+ - name: Setup BATS
+ uses: mig4/setup-bats@v1
+ with:
+ bats-version: 1.9.0
+
+ - name: Create k8s Kind Cluster
+ uses: helm/kind-action@v1.8.0
+ with:
+ node_image: kindest/node:${{ matrix.k8s-release }}
+ cluster_name: kind
+
+ - name: Set up Helm
+ uses: azure/setup-helm@v3.5
+ with:
+ version: v3.8.1
+
+ - name: Set up Kubectl
+ uses: azure/setup-kubectl@v3.2
+
+ - name: Run tests
+ timeout-minutes: 60
+ run: |
+ kind load docker-image ${{ inputs.image_name }}:${{ inputs.image_tag }}
+ ./run-tests.sh
+ shell: bash
+ env:
+ FLUENTBIT_IMAGE_REPOSITORY: ${{ inputs.image_name }}
+ FLUENTBIT_IMAGE_TAG: ${{ inputs.image_tag }}
+ HOSTED_OPENSEARCH_HOST: ${{ needs.call-run-terraform-setup.outputs.aws-opensearch-endpoint }}
+ HOSTED_OPENSEARCH_PORT: 443
+ HOSTED_OPENSEARCH_USERNAME: admin
+ HOSTED_OPENSEARCH_PASSWORD: ${{ secrets.opensearch_admin_password }}
+
+ call-run-integration-cloud:
+ name: Run integration tests on cloud providers
+ needs:
+ - call-run-terraform-setup
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ cloud:
+ - gke
+ env:
+ USE_GKE_GCLOUD_AUTH_PLUGIN: true
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+ repository: fluent/fluent-bit-ci
+
+ - if: matrix.cloud == 'gke'
+ uses: 'google-github-actions/auth@v1'
+ with:
+ credentials_json: ${{ secrets.gcp-service-account-key }}
+
+ - if: matrix.cloud == 'gke'
+ uses: 'google-github-actions/setup-gcloud@v1'
+ with:
+ install_components: 'gke-gcloud-auth-plugin'
+
+ - name: Setup BATS
+ uses: mig4/setup-bats@v1
+ with:
+ bats-version: 1.9.0
+
+ - name: Set up Helm
+ uses: azure/setup-helm@v3.5
+ with:
+ version: v3.8.1
+
+ - name: Set up Kubectl
+ uses: azure/setup-kubectl@v3.2
+
+ - name: Get the GKE Kubeconfig
+ if: matrix.cloud == 'gke'
+ uses: 'google-github-actions/get-gke-credentials@v1'
+ with:
+ cluster_name: ${{ needs.call-run-terraform-setup.outputs.gke-cluster-name }}
+ location: ${{ needs.call-run-terraform-setup.outputs.gke-cluster-zone }}
+
+ - name: Check Kubeconfig set up
+ run: |
+ kubectl cluster-info
+ kubectl get nodes --show-labels
+ kubectl get pods --all-namespaces --show-labels
+ kubectl get ns
+ shell: bash
+
+ - name: Run tests
+ timeout-minutes: 60
+ run: |
+ ./run-tests.sh
+ shell: bash
+ env:
+ # Namespace per test run to hopefully isolate a bit
+ TEST_NAMESPACE: test-${{ github.run_id }}
+ FLUENTBIT_IMAGE_REPOSITORY: ${{ inputs.image_name }}
+ FLUENTBIT_IMAGE_TAG: ${{ inputs.image_tag }}
+ HOSTED_OPENSEARCH_HOST: ${{ needs.call-run-terraform-setup.outputs.aws-opensearch-endpoint }}
+ HOSTED_OPENSEARCH_PORT: 443
+ HOSTED_OPENSEARCH_USERNAME: admin
+ USE_GKE_GCLOUD_AUTH_PLUGIN: true
+ HOSTED_OPENSEARCH_PASSWORD: ${{ secrets.opensearch_admin_password }}
diff --git a/fluent-bit/.github/workflows/call-test-images.yaml b/fluent-bit/.github/workflows/call-test-images.yaml
new file mode 100644
index 00000000..29bc0cd3
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-test-images.yaml
@@ -0,0 +1,205 @@
+---
+name: Reusable workflow to test container images
+on:
+ workflow_call:
+ inputs:
+ registry:
+ description: The registry to pull the images to test from.
+ type: string
+ required: true
+ username:
+ description: The username for authentication with the registry.
+ type: string
+ required: true
+ image:
+ description: The name of the image to pull from the registry for testing.
+ type: string
+ required: true
+ image-tag:
+ description: The tag of the image to pull from the registry for testing.
+ type: string
+ required: true
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ ref:
+ description: The commit, tag or branch to checkout for testing scripts.
+ type: string
+ default: master
+ required: false
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with for the registry.
+ required: true
+ cosign_key:
+ description: The optional Cosign key to use for verifying the images.
+ required: false
+jobs:
+ call-test-images-cosign-verify:
+ name: Cosign verification of container image
+ environment: ${{ inputs.environment }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Install cosign
+ uses: sigstore/cosign-installer@v2
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ # There is currently no way to verify a local image, e.g. for a particular architecture
+ # https://github.com/sigstore/cosign/issues/60
+ - name: Verify image with a key
+ # Only key-based verification currently
+ if: ${{ env.COSIGN_PUBLIC_KEY }}
+ run: |
+ echo -e "${COSIGN_PUBLIC_KEY}" > /tmp/my_cosign.pub
+ cosign verify --key /tmp/my_cosign.pub "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG"
+ rm -f /tmp/my_cosign.key
+ shell: bash
+ env:
+ COSIGN_PUBLIC_KEY: ${{ secrets.cosign_key }}
+ REGISTRY: ${{ inputs.registry }}
+ IMAGE_NAME: ${{ inputs.image }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
+
+ call-test-images-container-architecture:
+ name: ${{ matrix.arch }} image architecture verification
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ # Test as much as we can
+ continue-on-error: true
+ strategy:
+ fail-fast: false
+ matrix:
+ arch: [ linux/amd64, linux/arm64, linux/arm/v7 ]
+ include:
+ # Rather than extract the specific central arch we just provide it
+ - arch: linux/amd64
+ expected: amd64
+ - arch: linux/arm64
+ expected: arm64
+ - arch: linux/arm/v7
+ expected: arm
+ steps:
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Pull and extract architecture of image
+ id: extract_arch
+ run: |
+ docker pull --platform=${{ matrix.arch }} "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG"
+ ACTUAL_ARCH=$(docker image inspect --format '{{.Architecture}}' "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG")
+ echo "ACTUAL_ARCH=$ACTUAL_ARCH" >> $GITHUB_OUTPUT
+ docker image inspect "$REGISTRY/$IMAGE_NAME:$IMAGE_TAG"
+ shell: bash
+ env:
+ REGISTRY: ${{ inputs.registry }}
+ IMAGE_NAME: ${{ inputs.image }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
+
+ - name: Validate architecture of image
+ run: |
+ if [[ "$ACTUAL_ARCH" != "$EXPECTED_ARCH" ]]; then
+ echo "Invalid architecture for $REGISTRY/$IMAGE_NAME: $ACTUAL_ARCH != $EXPECTED_ARCH"
+ exit 1
+ fi
+ env:
+ EXPECTED_ARCH: ${{ matrix.expected }}
+ ACTUAL_ARCH: ${{ steps.extract_arch.outputs.ACTUAL_ARCH }}
+ shell: bash
+
+ call-test-images-container-smoke:
+ # Ensure each architecture container runs up correctly with default configuration.
+ name: ${{ matrix.arch }} smoke test for local container images
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ # No point running if the architecture is incorrect
+ needs: [ call-test-images-container-architecture ]
+ continue-on-error: true
+ strategy:
+ fail-fast: false # verify all
+ matrix:
+ arch: [ linux/amd64, linux/arm64, linux/arm/v7 ]
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ inputs.registry }}
+ username: ${{ inputs.username }}
+ password: ${{ secrets.token }}
+
+ - name: Set up QEMU using standard action
+ if: ${{ matrix.arch != 'linux/arm64' }}
+ uses: docker/setup-qemu-action@v3
+
+ # Without this QEMU fails for ARM64
+ - name: Set up binary emulation for QEMU
+ if: ${{ matrix.arch == 'linux/arm64' }}
+ run: |
+ docker run --privileged --rm tonistiigi/binfmt --install all
+
+ - name: Verify platform is supported with Alpine container
+ # We make sure there is not an inherent issue with this architecture on this runner
+ run: |
+ docker run --rm --platform=${{ matrix.arch }} alpine uname -a
+
+ - name: Test the HTTP server is responding
+ timeout-minutes: 10
+ run: |
+ packaging/testing/smoke/container/container-smoke-test.sh
+ shell: bash
+ env:
+ CONTAINER_NAME: local-smoke-${{ matrix.arch }}
+ CONTAINER_ARCH: ${{ matrix.arch }}
+ REGISTRY: ${{ inputs.registry }}
+ IMAGE_NAME: ${{ inputs.image }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
+
+ call-test-images-k8s-smoke:
+ # No need to test every architecture here, that is covered by local container tests.
+ # Testing helm chart deployment on KIND here.
+ name: Helm chart test on KIND
+ environment: ${{ inputs.environment }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+
+ - name: Create k8s Kind Cluster
+ uses: helm/kind-action@v1.8.0
+
+ - name: Set up Helm
+ uses: azure/setup-helm@v3.5
+ with:
+ version: v3.6.3
+
+ - name: Set up Kubectl
+ uses: azure/setup-kubectl@v3.2
+
+ - name: Test the HTTP server is responding
+ timeout-minutes: 5
+ run: |
+ packaging/testing/smoke/k8s/k8s-smoke-test.sh
+ shell: bash
+ env:
+ NAMESPACE: default
+ REGISTRY: ${{ inputs.registry }}
+ IMAGE_NAME: ${{ inputs.image }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
+
diff --git a/fluent-bit/.github/workflows/call-test-packages.yaml b/fluent-bit/.github/workflows/call-test-packages.yaml
new file mode 100644
index 00000000..90b8414c
--- /dev/null
+++ b/fluent-bit/.github/workflows/call-test-packages.yaml
@@ -0,0 +1,55 @@
+---
+name: Reusable workflow to test packages in S3 bucket
+on:
+ workflow_call:
+ inputs:
+ environment:
+ description: The Github environment to run this workflow on.
+ type: string
+ required: false
+ ref:
+ description: The commit, tag or branch to checkout for testing scripts.
+ type: string
+ default: master
+ required: false
+ secrets:
+ token:
+ description: The Github token or similar to authenticate with.
+ required: true
+ bucket:
+ description: The name of the S3 (US-East) bucket to pull packages from.
+ required: true
+
+jobs:
+ call-test-packaging:
+ # We use Dokken to run a series of test suites locally on containers representing
+ # each OS we want to install on. This creates custom images with the package
+ # installed and configured as per our documentation then verifies that the agent
+ # is running at startup.
+ name: ${{ matrix.distro }} package tests
+ runs-on: ubuntu-latest
+ environment: ${{ inputs.environment }}
+ env:
+ AWS_URL: https://${{ secrets.bucket }}.s3.amazonaws.com
+ strategy:
+ fail-fast: false
+ matrix:
+ distro: [ amazonlinux2022, amazonlinux2, centos7, centos8, debian10, debian11, ubuntu1804, ubuntu2004, ubuntu2204 ]
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Get the version
+ id: get_version
+ run: |
+ curl --fail -LO "$AWS_URL/latest-version.txt"
+ VERSION=$(cat latest-version.txt)
+ echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+ shell: bash
+
+ - name: Run package installation tests
+ run: |
+ packaging/testing/smoke/packages/run-package-tests.sh
+ env:
+ PACKAGE_TEST: ${{ matrix.distro }}
+ RELEASE_URL: https://packages.fluentbit.io
diff --git a/fluent-bit/.github/workflows/cron-scorecards-analysis.yaml b/fluent-bit/.github/workflows/cron-scorecards-analysis.yaml
new file mode 100644
index 00000000..065afdfd
--- /dev/null
+++ b/fluent-bit/.github/workflows/cron-scorecards-analysis.yaml
@@ -0,0 +1,59 @@
+
+---
+# https://openssf.org/blog/2022/01/19/reducing-security-risks-in-open-source-software-at-scale-scorecards-launches-v4/
+name: Scorecards supply-chain security
+on:
+ # Only the default branch is supported.
+ branch_protection_rule:
+ schedule:
+ # Weekly on Saturdays.
+ - cron: '30 1 * * 6'
+ workflow_dispatch:
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+ scorecard-analysis:
+ name: Scorecards analysis
+ runs-on: ubuntu-latest
+ permissions:
+ # Needed to upload the results to code-scanning dashboard.
+ security-events: write
+ # Needed for GitHub OIDC token if publish_results is true
+ id-token: write
+ steps:
+ - name: "Checkout code"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+
+ - name: "Run analysis"
+ uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031
+ with:
+ results_file: scorecard-results.sarif
+ results_format: sarif
+ # (Optional) fine-grained personal access token. Uncomment the `repo_token` line below if:
+ # - you want to enable the Branch-Protection check on a *public* repository, or
+ # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-fine-grained-pat-optional.
+ repo_token: ${{ secrets.SCORECARD_TOKEN }}
+ #
+ # Publish the results for public repositories to enable scorecard badges. For more details, see
+ # https://github.com/ossf/scorecard-action#publishing-results.
+ # For private repositories, `publish_results` will automatically be set to `false`, regardless
+ # of the value entered here.
+ publish_results: true
+
+ - name: "Upload artifact"
+ uses: actions/upload-artifact@v3
+ with:
+ name: SARIF file
+ path: scorecard-results.sarif
+ retention-days: 7
+
+ # Upload the results to GitHub's code scanning dashboard.
+ - name: "Upload to code-scanning"
+ uses: github/codeql-action/upload-sarif@v2
+ with:
+ sarif_file: scorecard-results.sarif
+ category: ossf-scorecard
diff --git a/fluent-bit/.github/workflows/cron-stale.yaml b/fluent-bit/.github/workflows/cron-stale.yaml
new file mode 100644
index 00000000..523685c5
--- /dev/null
+++ b/fluent-bit/.github/workflows/cron-stale.yaml
@@ -0,0 +1,26 @@
+name: 'Close stale issues and PR(s)'
+on:
+ schedule:
+ - cron: '30 1 * * *'
+
+jobs:
+ stale:
+ name: Mark stale
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/stale@v8
+ with:
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days. Maintainers can add the `exempt-stale` label.'
+ stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
+ close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.'
+ days-before-stale: 90
+ days-before-close: 5
+ days-before-pr-close: -1
+ exempt-all-pr-assignees: true
+ exempt-all-pr-milestones: true
+ exempt-issue-labels: 'long-term,enhancement,exempt-stale'
+ # start with the oldest
+ ascending: true
+ # keep an eye on this
+ operations-per-run: 250
diff --git a/fluent-bit/.github/workflows/cron-trivy.yaml b/fluent-bit/.github/workflows/cron-trivy.yaml
new file mode 100644
index 00000000..a1266819
--- /dev/null
+++ b/fluent-bit/.github/workflows/cron-trivy.yaml
@@ -0,0 +1,87 @@
+---
+# Separate action to allow us to initiate manually and run regularly
+name: Trivy security analysis of latest containers
+
+# Run on every push to master, or weekly.
+# Allow users to trigger an asynchronous run anytime too.
+on:
+ push:
+ branches: [master]
+ schedule:
+ # 13:44 on Thursday
+ - cron: 44 13 * * 4
+ workflow_dispatch:
+
+jobs:
+ # Run Trivy on the latest container and update the security code scanning results tab.
+ trivy-latest:
+ # Matrix job that pulls the latest image for each supported architecture via the multi-arch latest manifest.
+ # We then re-tag it locally to ensure that when Trivy runs it does not pull the latest for the wrong architecture.
+ name: ${{ matrix.arch }} container scan
+ runs-on: [ ubuntu-latest ]
+ continue-on-error: true
+ strategy:
+ fail-fast: false
+ # Matrix of architectures to test along with their local tags for special character substitution
+ matrix:
+ # The architecture for the container runtime to pull.
+ arch: [ linux/amd64, linux/arm64, linux/arm/v7 ]
+ # In a few cases we need the arch without slashes so provide a descriptive extra field for that.
+ # We could also extract or modify this via a regex but this seemed simpler and easier to follow.
+ include:
+ - arch: linux/amd64
+ local_tag: x86_64
+ - arch: linux/arm64
+ local_tag: arm64
+ - arch: linux/arm/v7
+ local_tag: arm32
+ steps:
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Pull the image for the architecture we're testing
+ run: |
+ docker pull --platform ${{ matrix.arch }} fluent/fluent-bit:latest
+
+ - name: Tag locally to ensure we do not pull wrong architecture
+ run: |
+ docker tag fluent/fluent-bit:latest local/fluent-bit:${{ matrix.local_tag }}
+
+ # Deliberately chosen master here to keep up-to-date.
+ - name: Run Trivy vulnerability scanner for any major issues
+ uses: aquasecurity/trivy-action@master
+ with:
+ image-ref: local/fluent-bit:${{ matrix.local_tag }}
+ # Filter out any that have no current fix.
+ ignore-unfixed: true
+ # Only include major issues.
+ severity: CRITICAL,HIGH
+ format: template
+ template: '@/contrib/sarif.tpl'
+ output: trivy-results-${{ matrix.local_tag }}.sarif
+
+ # Show all detected issues.
+ # Note this will show a lot more, including major un-fixed ones.
+ - name: Run Trivy vulnerability scanner for local output
+ uses: aquasecurity/trivy-action@master
+ with:
+ image-ref: local/fluent-bit:${{ matrix.local_tag }}
+ format: table
+
+ - name: Upload Trivy scan results to GitHub Security tab
+ uses: github/codeql-action/upload-sarif@v2
+ with:
+ sarif_file: trivy-results-${{ matrix.local_tag }}.sarif
+ category: ${{ matrix.arch }} container
+ wait-for-processing: true
+
+ # In case we need to analyse the uploaded files for some reason.
+ - name: Detain results for debug if needed
+ uses: actions/upload-artifact@v3
+ with:
+ name: trivy-results-${{ matrix.local_tag }}.sarif
+ path: trivy-results-${{ matrix.local_tag }}.sarif
+ if-no-files-found: error
diff --git a/fluent-bit/.github/workflows/cron-unstable-build.yaml b/fluent-bit/.github/workflows/cron-unstable-build.yaml
new file mode 100644
index 00000000..82235a41
--- /dev/null
+++ b/fluent-bit/.github/workflows/cron-unstable-build.yaml
@@ -0,0 +1,213 @@
+---
+name: Unstable build
+
+on:
+ workflow_dispatch:
+ inputs:
+ branch:
+ description: The branch to create an unstable release for/from.
+ type: string
+ default: master
+ required: true
+
+ # Run nightly build at this time, bit of trial and error but this seems good.
+ schedule:
+ - cron: "0 6 * * *" # master build
+ - cron: "0 12 * * *" # 2.0 build
+
+# We do not want a new unstable build to run whilst we are releasing the current unstable build.
+concurrency: unstable-build-release
+
+jobs:
+ # This job provides this metadata for the other jobs to use.
+ unstable-build-get-meta:
+ name: Get metadata to add to build
+ runs-on: ubuntu-latest
+ environment: unstable
+ outputs:
+ date: ${{ steps.date.outputs.date }}
+ branch: ${{ steps.branch.outputs.branch }}
+ permissions:
+ contents: none
+ steps:
+ # For cron builds, i.e. nightly, we provide date and time as extra parameter to distinguish them.
+ - name: Get current date
+ id: date
+ run: echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT
+
+ - name: Debug event output
+ uses: hmarr/debug-action@v2
+
+ # Now we need to determine which branch to build
+ - name: Manual run - get branch
+ if: github.event_name == 'workflow_dispatch'
+ run: |
+ echo "cron_branch=${{ github.event.inputs.branch }}" >> $GITHUB_ENV
+ shell: bash
+
+ - name: master run
+ if: github.event_name == 'schedule' && github.event.schedule=='0 6 * * *'
+ run: |
+ echo "cron_branch=master" >> $GITHUB_ENV
+ shell: bash
+
+ - name: 2.0 run
+ if: github.event_name == 'schedule' && github.event.schedule=='0 12 * * *'
+ run: |
+ echo "cron_branch=2.0" >> $GITHUB_ENV
+ shell: bash
+
+ - name: Output the branch to use
+ id: branch
+ run: |
+ echo "$cron_branch"
+ if [[ -z "$cron_branch" ]]; then
+ echo "Unable to determine branch to use"
+ exit 1
+ fi
+ echo "branch=$cron_branch" >> $GITHUB_OUTPUT
+ shell: bash
+
+ unstable-build-images:
+ needs: unstable-build-get-meta
+ uses: ./.github/workflows/call-build-images.yaml
+ with:
+ version: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ ref: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ image: ${{ github.repository }}/unstable
+ environment: unstable
+ unstable: ${{ needs.unstable-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ unstable-build-generate-matrix:
+ name: unstable build matrix
+ needs:
+ - unstable-build-get-meta
+ runs-on: ubuntu-latest
+ outputs:
+ build-matrix: ${{ steps.set-matrix.outputs.build-matrix }}
+ environment: unstable
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout repository, always latest for action
+ uses: actions/checkout@v4
+
+ # Set up the list of target to build so we can pass the JSON to the reusable job
+ - uses: ./.github/actions/generate-package-build-matrix
+ id: set-matrix
+ with:
+ ref: ${{ needs.unstable-build-get-meta.outputs.branch }}
+
+ unstable-build-packages:
+ needs:
+ - unstable-build-get-meta
+ - unstable-build-generate-matrix
+ uses: ./.github/workflows/call-build-linux-packages.yaml
+ with:
+ version: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ ref: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ build_matrix: ${{ needs.unstable-build-generate-matrix.outputs.build-matrix }}
+ environment: unstable
+ unstable: ${{ needs.unstable-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ unstable-build-windows-package:
+ needs:
+ - unstable-build-get-meta
+ uses: ./.github/workflows/call-build-windows.yaml
+ with:
+ version: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ ref: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ environment: unstable
+ unstable: ${{ needs.unstable-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ unstable-build-macos-package:
+ needs:
+ - unstable-build-get-meta
+ uses: ./.github/workflows/call-build-macos.yaml
+ with:
+ version: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ ref: ${{ needs.unstable-build-get-meta.outputs.branch }}
+ environment: unstable
+ unstable: ${{ needs.unstable-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ # We already detain all artefacts as build output so just capture for an unstable release.
+ # We make all of these on a separate repo to prevent notification spam.
+ unstable-release:
+ runs-on: ubuntu-latest
+ needs:
+ - unstable-build-get-meta
+ - unstable-build-images
+ - unstable-build-packages
+ - unstable-build-windows-package
+ - unstable-build-macos-package
+ environment: unstable
+ permissions:
+ contents: read
+ steps:
+ # Required to make a release later
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Download all artefacts
+ continue-on-error: true
+ uses: actions/download-artifact@v3
+ with:
+ path: artifacts/
+
+ - name: Single packages tar
+ run: |
+ mkdir -p release-upload
+ # Optional JSON schema so ignore failure
+ mv -f artifacts/*.json release-upload/ || true
+ tar -czvf release-upload/packages-unstable-${{ needs.unstable-build-get-meta.outputs.branch }}.tar.gz -C artifacts/ .
+ shell: bash
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Pull containers as well (single arch only)
+ # May not be any/valid so ignore errors
+ continue-on-error: true
+ run: |
+ docker pull $IMAGE
+ docker save --output container-${{ needs.unstable-build-get-meta.outputs.branch }}.tar $IMAGE
+ docker pull $IMAGE-debug
+ docker save --output container-${{ needs.unstable-build-get-meta.outputs.branch }}-debug.tar $IMAGE-debug
+ shell: bash
+ working-directory: release-upload
+ env:
+ IMAGE: ghcr.io/${{ github.repository }}/unstable:${{ needs.unstable-build-get-meta.outputs.branch }}
+
+ - name: Display structure of files to upload
+ run: ls -R
+ working-directory: release-upload
+ shell: bash
+
+ - name: Remove any existing release
+ continue-on-error: true
+ run: gh release delete unstable-${{ needs.unstable-build-get-meta.outputs.branch }} --yes --repo ${{ secrets.RELEASE_REPO }}
+ env:
+ GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+ shell: bash
+
+ - name: Create Release
+ # Do not fail the job here
+ continue-on-error: true
+ run: gh release create unstable-${{ needs.unstable-build-get-meta.outputs.branch }} release-upload/*.* --repo ${{ secrets.RELEASE_REPO }} --generate-notes --prerelease --target ${{ needs.unstable-build-get-meta.outputs.branch }} --title "Nightly unstable ${{ needs.unstable-build-get-meta.outputs.branch }} build"
+ env:
+ GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+ shell: bash
diff --git a/fluent-bit/.github/workflows/master-integration-test.yaml b/fluent-bit/.github/workflows/master-integration-test.yaml
new file mode 100644
index 00000000..85c321b8
--- /dev/null
+++ b/fluent-bit/.github/workflows/master-integration-test.yaml
@@ -0,0 +1,33 @@
+name: Build master container images and run integration tests
+on:
+ push:
+ branches:
+ - master
+
+jobs:
+ master-integration-test-build:
+ name: Master - integration build
+ uses: ./.github/workflows/call-integration-image-build.yaml
+ with:
+ ref: ${{ github.sha }}
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ image: ${{ github.repository }}/master
+ image-tag: x86_64
+ environment: integration
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ master-integration-test-run-integration:
+ name: Master - integration test
+ needs: master-integration-test-build
+ uses: ./.github/workflows/call-run-integration-test.yaml
+ with:
+ image_name: ghcr.io/${{ github.repository }}/master
+ image_tag: x86_64
+ secrets:
+ opensearch_aws_access_id: ${{ secrets.OPENSEARCH_AWS_ACCESS_ID }}
+ opensearch_aws_secret_key: ${{ secrets.OPENSEARCH_AWS_SECRET_KEY }}
+ opensearch_admin_password: ${{ secrets.OPENSEARCH_ADMIN_PASSWORD }}
+ terraform_api_token: ${{ secrets.TF_API_TOKEN }}
+ gcp-service-account-key: ${{ secrets.GCP_SA_KEY }}
diff --git a/fluent-bit/.github/workflows/pr-closed-docker.yaml b/fluent-bit/.github/workflows/pr-closed-docker.yaml
new file mode 100644
index 00000000..628f5e92
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-closed-docker.yaml
@@ -0,0 +1,21 @@
+name: Remove docker images for stale/closed PR(s).
+on:
+ pull_request:
+ branches:
+ - master
+ types: [closed]
+jobs:
+ cleanup:
+ name: PR - cleanup pr-${{ github.event.number }} images
+ runs-on: ubuntu-latest
+ permissions:
+ # We may need a specific token here with `packages:admin` privileges which is not available to GITHUB_TOKEN
+ packages: write
+ steps:
+ - uses: vlaurin/action-ghcr-prune@v0.5.0
+ with:
+ organization: fluent
+ container: fluent-bit/pr-${{ github.event.number }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+ prune-untagged: true
+ keep-last: 0
diff --git a/fluent-bit/.github/workflows/pr-commit-message.yaml b/fluent-bit/.github/workflows/pr-commit-message.yaml
new file mode 100644
index 00000000..596d9cdc
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-commit-message.yaml
@@ -0,0 +1,21 @@
+name: 'Pull requests commit messages'
+on:
+ pull_request:
+ types:
+ - opened
+ - edited
+ - reopened
+ - synchronize
+jobs:
+ check-commit-message:
+ name: Check Commit Message
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check commit subject complies with https://github.com/fluent/fluent-bit/blob/master/CONTRIBUTING.md#commit-changes
+ uses: gsactions/commit-message-checker@v2
+ with:
+ pattern: '^[a-z0-9A-Z\-_\s\,\.\/]+\:[ ]{0,1}[a-zA-Z]+[a-zA-Z0-9 \-\.\:_\#\(\)=\/''\"\,><\+\[\]\!\*\\]+$'
+ error: 'Invalid commit subject. Please refer to: https://github.com/fluent/fluent-bit/blob/master/CONTRIBUTING.md#commit-changes'
+ checkAllCommitMessages: 'false'
+ excludeDescription: 'true'
+ accessToken: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file
diff --git a/fluent-bit/.github/workflows/pr-compile-check.yaml b/fluent-bit/.github/workflows/pr-compile-check.yaml
new file mode 100644
index 00000000..d9f3dc92
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-compile-check.yaml
@@ -0,0 +1,32 @@
+name: 'Pull requests compile checks'
+on:
+ pull_request:
+ # Only trigger if there is a code change or a CMake change that (could) affect code
+ paths:
+ - '**.c'
+ - '**.h'
+ - 'CMakeLists.txt'
+ - 'cmake/*'
+ workflow_dispatch:
+
+jobs:
+ # Sanity check for compilation using older compiler on CentOS 7
+ pr-compile-centos-7:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - name: Checkout Fluent Bit code
+ uses: actions/checkout@v4
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Attempt to build current source for CentOS 7
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ file: ./dockerfiles/Dockerfile.centos7
+ # No need to use after this so discard completely
+ push: false
+ load: false
+ provenance: false
diff --git a/fluent-bit/.github/workflows/pr-fuzz.yaml b/fluent-bit/.github/workflows/pr-fuzz.yaml
new file mode 100644
index 00000000..79371ed5
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-fuzz.yaml
@@ -0,0 +1,41 @@
+name: CIFuzz
+on:
+ pull_request:
+ # Only fuzz when C source files change
+ paths:
+ - '**.c'
+ - '**.h'
+jobs:
+ fuzzing:
+ name: PR - fuzzing test
+ runs-on: ubuntu-latest
+ steps:
+ - name: Build Fuzzers
+ id: build
+ uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+ with:
+ oss-fuzz-project-name: 'fluent-bit'
+ dry-run: false
+ language: c
+ - name: Run Fuzzers
+ uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+ with:
+ oss-fuzz-project-name: 'fluent-bit'
+ fuzz-seconds: 600
+ dry-run: false
+ language: c
+ output-sarif: true
+ - name: Upload Crash
+ uses: actions/upload-artifact@v3
+ if: failure() && steps.build.outcome == 'success'
+ with:
+ name: artifacts
+ path: ./out/artifacts
+ - name: Upload Sarif
+ if: always() && steps.build.outcome == 'success'
+ uses: github/codeql-action/upload-sarif@v2
+ with:
+ # Path to SARIF file relative to the root of the repository
+ sarif_file: cifuzz-sarif/results.sarif
+ checkout_path: cifuzz-sarif
+ category: CIFuzz
diff --git a/fluent-bit/.github/workflows/pr-image-tests.yaml b/fluent-bit/.github/workflows/pr-image-tests.yaml
new file mode 100644
index 00000000..2cdc3622
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-image-tests.yaml
@@ -0,0 +1,121 @@
+name: Build images for all architectures
+on:
+ pull_request:
+ types:
+ - opened
+ - reopened
+ - synchronize
+ paths:
+ - 'dockerfiles/Dockerfile'
+ - 'dockerfiles/Dockerfile.windows'
+ - 'conf/**'
+
+ workflow_dispatch:
+
+jobs:
+ pr-image-tests-build-images:
+ name: PR - Buildkit docker build test
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ # We do not push and this allows simpler workflow running for forks too
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Extract metadata from Github
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ github.repository }}/pr-${{ github.event.pull_request.number }}
+ tags: |
+ type=sha
+
+ - name: Build the multi-arch images
+ id: build
+ uses: docker/build-push-action@v5
+ with:
+ file: ./dockerfiles/Dockerfile
+ context: .
+ platforms: linux/amd64
+ target: production
+ provenance: false
+ push: false
+ load: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+
+ - name: Sanity check it runs
+ # We do this for a simple check of dependencies
+ run: |
+ docker run --rm -t ${{ steps.meta.outputs.tags }} --help
+ shell: bash
+
+ - name: Build the debug multi-arch images
+ uses: docker/build-push-action@v5
+ with:
+ file: ./dockerfiles/Dockerfile
+ context: .
+ platforms: linux/amd64
+ target: debug
+ provenance: false
+ push: false
+ load: false
+
+ pr-image-tests-classic-docker-build:
+ name: PR - Classic docker build test
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Build the classic test image
+ # We only want to confirm it builds with classic mode, nothing else
+ run: |
+ docker build -f ./dockerfiles/Dockerfile .
+ env:
+ # Ensure we disable buildkit
+ DOCKER_BUILDKIT: 0
+ shell: bash
+ pr-image-tests-build-windows-images:
+ name: PR - Docker windows build test, windows 2019 and 2022
+ runs-on: windows-${{ matrix.windows-base-version }}
+ strategy:
+ fail-fast: true
+ matrix:
+ windows-base-version:
+ # https://github.com/fluent/fluent-bit/blob/1d366594a889624ec3003819fe18588aac3f17cd/dockerfiles/Dockerfile.windows#L3
+ - '2019'
+ - '2022'
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Extract metadata from Github
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ github.repository }}/pr-${{ github.event.pull_request.number }}
+ tags: |
+ type=sha
+ flavor: |
+ suffix=-windows-${{ matrix.windows-base-version }}
+
+ - name: Build the windows images
+ id: build
+ run: |
+ docker build -t ${{ steps.meta.outputs.tags }} --build-arg WINDOWS_VERSION=ltsc${{ matrix.windows-base-version }} -f ./dockerfiles/Dockerfile.windows .
+
+ - name: Sanity check it runs
+ # We do this for a simple check of dependencies
+ run: |
+ docker run --rm -t ${{ steps.meta.outputs.tags }} --help
+ shell: bash
+
diff --git a/fluent-bit/.github/workflows/pr-install-script.yaml b/fluent-bit/.github/workflows/pr-install-script.yaml
new file mode 100644
index 00000000..0eeeed65
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-install-script.yaml
@@ -0,0 +1,29 @@
+name: Test install script for all targets
+on:
+ pull_request:
+ types:
+ - opened
+ - reopened
+ - synchronize
+ paths:
+ - 'packaging/test-release-packages.sh'
+ - 'install.sh'
+
+ workflow_dispatch:
+jobs:
+ test-install-script:
+ name: Run install tests
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ timeout-minutes: 30
+ steps:
+ - name: Checkout Fluent Bit code
+ uses: actions/checkout@v4
+
+ - name: Run install tests
+ run: |
+ ./packaging/test-release-packages.sh
+ shell: bash
+ env:
+ INSTALL_SCRIPT: ./install.sh
diff --git a/fluent-bit/.github/workflows/pr-integration-test.yaml b/fluent-bit/.github/workflows/pr-integration-test.yaml
new file mode 100644
index 00000000..2b18b1c6
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-integration-test.yaml
@@ -0,0 +1,69 @@
+name: Build and run integration tests for PR
+on:
+ # We need write token for upload to GHCR and we are protecting with labels too.
+ pull_request_target:
+ branches:
+ - master
+ types:
+ - labeled
+ - opened
+ - reopened
+ - synchronize
+
+jobs:
+ pr-integration-test-build:
+ name: PR - integration build
+ # We only need to test this once as the rest are chained from it.
+ if: contains(github.event.pull_request.labels.*.name, 'ok-to-test')
+ uses: ./.github/workflows/call-integration-image-build.yaml
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ image: ${{ github.repository }}/pr-${{ github.event.number }}
+ image-tag: ${{ github.sha }}
+ environment: integration
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ pr-integration-test-build-complete:
+ name: PR - integration build complete
+ runs-on: ubuntu-latest
+ needs:
+ - pr-integration-test-build
+ steps:
+ - uses: actions-ecosystem/action-add-labels@v1
+ name: Label the PR
+ with:
+ labels: ci/integration-docker-ok
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ number: ${{ github.event.pull_request.number }}
+
+ pr-integration-test-run-integration:
+ name: PR - K8S integration test
+ needs:
+ - pr-integration-test-build
+ uses: ./.github/workflows/call-run-integration-test.yaml
+ with:
+ image_name: ghcr.io/${{ github.repository }}/pr-${{ github.event.pull_request.number }}
+ image_tag: ${{ github.sha }}
+ secrets:
+ opensearch_aws_access_id: ${{ secrets.OPENSEARCH_AWS_ACCESS_ID }}
+ opensearch_aws_secret_key: ${{ secrets.OPENSEARCH_AWS_SECRET_KEY }}
+ opensearch_admin_password: ${{ secrets.OPENSEARCH_ADMIN_PASSWORD }}
+ terraform_api_token: ${{ secrets.TF_API_TOKEN }}
+ gcp-service-account-key: ${{ secrets.GCP_SA_KEY }}
+
+ pr-integration-test-run-integration-post-label:
+ name: PR - integration test complete
+ runs-on: ubuntu-latest
+ needs:
+ - pr-integration-test-run-integration
+ steps:
+ - uses: actions-ecosystem/action-add-labels@v1
+ name: Label the PR
+ with:
+ labels: ci/integration-test-ok
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ number: ${{ github.event.pull_request.number }}
+ repo: fluent/fluent-bit
diff --git a/fluent-bit/.github/workflows/pr-labels.yaml b/fluent-bit/.github/workflows/pr-labels.yaml
new file mode 100644
index 00000000..f8139304
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-labels.yaml
@@ -0,0 +1,17 @@
+name: 'Pull requests labels'
+on:
+ pull_request_target:
+ types:
+ - opened
+jobs:
+ apply-default-labels:
+ name: PR - apply default labels
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions-ecosystem/action-add-labels@v1
+ name: Label the PR with 'docs-required' by default.
+ with:
+ labels: docs-required
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ number: ${{ github.event.pull_request.number }}
+ repo: fluent/fluent-bit
diff --git a/fluent-bit/.github/workflows/pr-lint.yaml b/fluent-bit/.github/workflows/pr-lint.yaml
new file mode 100644
index 00000000..f8af79bb
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-lint.yaml
@@ -0,0 +1,36 @@
+name: Lint PRs
+on:
+ pull_request:
+ workflow_dispatch:
+
+jobs:
+ hadolint-pr:
+ runs-on: ubuntu-latest
+ name: PR - Hadolint
+ steps:
+ - uses: actions/checkout@v4
+ # Ignores do not work: https://github.com/reviewdog/action-hadolint/issues/35 is resolved
+ - uses: reviewdog/action-hadolint@v1
+ with:
+ exclude: |
+ packaging/testing/smoke/packages/Dockerfile.*
+
+ shellcheck-pr:
+ runs-on: ubuntu-latest
+ name: PR - Shellcheck
+ steps:
+ - uses: actions/checkout@v4
+ - uses: ludeeus/action-shellcheck@master
+ with:
+ ignore_paths: cmake/sanitizers-cmake lib plugins tests
+
+ actionlint-pr:
+ runs-on: ubuntu-latest
+ name: PR - Actionlint
+ steps:
+ - uses: actions/checkout@v4
+ - run: |
+ echo "::add-matcher::.github/actionlint-matcher.json"
+ bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
+ ./actionlint -color -shellcheck=
+ shell: bash
diff --git a/fluent-bit/.github/workflows/pr-package-tests.yaml b/fluent-bit/.github/workflows/pr-package-tests.yaml
new file mode 100644
index 00000000..b461ac4d
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-package-tests.yaml
@@ -0,0 +1,92 @@
+name: PR - packaging tests run on-demand via label
+on:
+ pull_request:
+ types:
+ - labeled
+ - opened
+ - reopened
+ - synchronize
+ branches:
+ - master
+
+# Cancel any running on push
+concurrency:
+ group: ${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ # This job provides this metadata for the other jobs to use.
+ pr-package-test-build-get-meta:
+ # This is a long test to run so only on-demand for certain PRs
+ if: contains(github.event.pull_request.labels.*.name, 'ok-package-test')
+ name: Get metadata to add to build
+ runs-on: ubuntu-latest
+ outputs:
+ date: ${{ steps.date.outputs.date }}
+ permissions:
+ contents: none
+ steps:
+ # For cron builds, i.e. nightly, we provide date and time as extra parameter to distinguish them.
+ - name: Get current date
+ id: date
+ run: echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT
+
+ - name: Debug event output
+ uses: hmarr/debug-action@v2
+
+ pr-package-test-build-generate-matrix:
+ name: PR - packages build matrix
+ needs:
+ - pr-package-test-build-get-meta
+ runs-on: ubuntu-latest
+ outputs:
+ build-matrix: ${{ steps.set-matrix.outputs.build-matrix }}
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout repository, always latest for action
+ uses: actions/checkout@v4
+
+ # Set up the list of target to build so we can pass the JSON to the reusable job
+ - uses: ./.github/actions/generate-package-build-matrix
+ id: set-matrix
+ with:
+ ref: master
+
+ pr-package-test-build-packages:
+ name: PR - packages build Linux
+ needs:
+ - pr-package-test-build-get-meta
+ - pr-package-test-build-generate-matrix
+ uses: ./.github/workflows/call-build-linux-packages.yaml
+ with:
+ version: pr-${{ github.event.number }}
+ ref: ${{ github.ref }}
+ build_matrix: ${{ needs.pr-package-test-build-generate-matrix.outputs.build-matrix }}
+ unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ pr-package-test-build-windows-package:
+ name: PR - packages build Windows
+ needs:
+ - pr-package-test-build-get-meta
+ uses: ./.github/workflows/call-build-windows.yaml
+ with:
+ version: pr-${{ github.event.number }}
+ ref: ${{ github.ref }}
+ unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ pr-package-test-build-macos-package:
+ name: PR - packages build MacOS
+ needs:
+ - pr-package-test-build-get-meta
+ uses: ./.github/workflows/call-build-macos.yaml
+ with:
+ version: pr-${{ github.event.number }}
+ ref: ${{ github.ref }}
+ unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/fluent-bit/.github/workflows/pr-perf-test.yaml b/fluent-bit/.github/workflows/pr-perf-test.yaml
new file mode 100644
index 00000000..52b73799
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-perf-test.yaml
@@ -0,0 +1,97 @@
+name: Build and run performance tests for PR
+on:
+ # We want to run on forks as we protect with a label.
+ # Without this we have no secrets to pass.
+ pull_request_target:
+ branches:
+ - master
+ types:
+ - labeled
+
+jobs:
+
+ pr-perf-test-run:
+ # We only need to test this once as the rest are chained from it.
+ if: contains(github.event.pull_request.labels.*.name, 'ok-to-performance-test')
+ uses: fluent/fluent-bit-ci/.github/workflows/call-run-performance-test.yaml@main
+ with:
+ vm-name: fb-perf-test-pr-${{ github.event.number }}
+ git-branch: ${{ github.head_ref }}
+ test-directory: examples/perf_test
+ duration: 30
+ service: fb-delta
+ secrets:
+ service-account: ${{ secrets.GCP_SA_KEY }}
+
+ pr-perf-test-upload:
+ name: PR - performance test upload
+ # Label check from previous
+ runs-on: ubuntu-latest
+ needs:
+ - pr-perf-test-run
+ permissions:
+ pull-requests: write
+ steps:
+ - uses: actions/download-artifact@v3
+
+ - name: Upload plots to CML
+ run: |
+ docker pull -q dvcorg/cml-py3:latest
+ for FILE in *.png; do
+ echo "Handling $FILE"
+ docker run --rm -t \
+ -v $PWD:/output:rw \
+ dvcorg/cml-py3:latest \
+ /bin/sh -c "cml-publish /output/$FILE --driver=github --repo=${{ github.repository }} --md --title=$FILE >> /output/report.md"
+ done
+ shell: bash
+ working-directory: output
+
+ - name: Dump markdown to variable
+ id: report
+ run: |
+ file_test_newlines=""
+ while read line
+ do
+ file_test_newlines+=" $line"
+ done < report.md
+ echo "report=$file_test_newlines" >> $GITHUB_OUTPUT
+ shell: bash
+ working-directory: output
+
+ - uses: actions/github-script@v6
+ if: github.event_name == 'pull_request'
+ env:
+ REPORT: "Plots\n${{ steps.report.outputs.report }}"
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const output = `#### Performance Test
+ Run URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ <details><summary>Show Performance Results</summary>
+ ${process.env.REPORT}
+ </details>
+ *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
+ github.rest.issues.createComment({
+ issue_number: context.issue.number,
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ body: output
+ })
+
+ pr-perf-test-complete:
+ # Only runs on success
+ name: PR - performance test complete
+ # Label check from previous
+ runs-on: ubuntu-latest
+ needs:
+ - pr-perf-test-run
+ permissions:
+ pull-requests: write
+ steps:
+ - uses: actions-ecosystem/action-add-labels@v1
+ name: Label the PR
+ with:
+ labels: ci/performance-test-ok
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ number: ${{ github.event.pull_request.number }}
diff --git a/fluent-bit/.github/workflows/pr-windows-build.yaml b/fluent-bit/.github/workflows/pr-windows-build.yaml
new file mode 100644
index 00000000..2831122f
--- /dev/null
+++ b/fluent-bit/.github/workflows/pr-windows-build.yaml
@@ -0,0 +1,27 @@
+name: PR - Windows checks
+on:
+ workflow_dispatch:
+
+ pull_request:
+ # Limit to just those changes that 'might' affect Windows for automated builds
+ # We can always do a manual build for a branch
+ paths:
+ - '**.h'
+ - '**.c'
+ - '**.windows'
+ - './conf/**'
+ - './cmake/**'
+ types:
+ - opened
+ - reopened
+ - synchronize
+
+jobs:
+ pr-windows-build:
+ uses: ./.github/workflows/call-build-windows.yaml
+ with:
+ version: ${{ github.sha }}
+ ref: ${{ github.sha }}
+ environment: pr
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/fluent-bit/.github/workflows/resources/auto-build-test-workflow.png b/fluent-bit/.github/workflows/resources/auto-build-test-workflow.png
new file mode 100644
index 00000000..40d3dfce
--- /dev/null
+++ b/fluent-bit/.github/workflows/resources/auto-build-test-workflow.png
Binary files differ
diff --git a/fluent-bit/.github/workflows/resources/release-approval-per-job.png b/fluent-bit/.github/workflows/resources/release-approval-per-job.png
new file mode 100644
index 00000000..0f30245c
--- /dev/null
+++ b/fluent-bit/.github/workflows/resources/release-approval-per-job.png
Binary files differ
diff --git a/fluent-bit/.github/workflows/resources/release-approval.png b/fluent-bit/.github/workflows/resources/release-approval.png
new file mode 100644
index 00000000..16a5570c
--- /dev/null
+++ b/fluent-bit/.github/workflows/resources/release-approval.png
Binary files differ
diff --git a/fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.png b/fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.png
new file mode 100644
index 00000000..3708885b
--- /dev/null
+++ b/fluent-bit/.github/workflows/resources/release-from-staging-workflow-incorrect-version.png
Binary files differ
diff --git a/fluent-bit/.github/workflows/resources/release-from-staging-workflow.png b/fluent-bit/.github/workflows/resources/release-from-staging-workflow.png
new file mode 100644
index 00000000..c6ea45ae
--- /dev/null
+++ b/fluent-bit/.github/workflows/resources/release-from-staging-workflow.png
Binary files differ
diff --git a/fluent-bit/.github/workflows/resources/release-version-failure.png b/fluent-bit/.github/workflows/resources/release-version-failure.png
new file mode 100644
index 00000000..a96698cc
--- /dev/null
+++ b/fluent-bit/.github/workflows/resources/release-version-failure.png
Binary files differ
diff --git a/fluent-bit/.github/workflows/skipped-unit-tests.yaml b/fluent-bit/.github/workflows/skipped-unit-tests.yaml
new file mode 100644
index 00000000..ab5adc5f
--- /dev/null
+++ b/fluent-bit/.github/workflows/skipped-unit-tests.yaml
@@ -0,0 +1,21 @@
+# For skipped checks that are required to merge we trigger a fake job that succeeds.
+# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
+name: Run unit tests
+on:
+ pull_request:
+ paths:
+ - '.github/**'
+ - 'dockerfiles/**'
+ - 'docker_compose/**'
+ - 'packaging/**'
+ - '.gitignore'
+ - 'appveyor.yml'
+ - '**.sh'
+ - 'examples/**'
+
+jobs:
+ run-all-unit-tests:
+ runs-on: ubuntu-latest
+ name: Unit tests (matrix)
+ steps:
+ - run: echo "No unit tests required" \ No newline at end of file
diff --git a/fluent-bit/.github/workflows/staging-build.yaml b/fluent-bit/.github/workflows/staging-build.yaml
new file mode 100644
index 00000000..2bf6524f
--- /dev/null
+++ b/fluent-bit/.github/workflows/staging-build.yaml
@@ -0,0 +1,179 @@
+---
+name: Deploy to staging
+
+on:
+ push:
+ tags:
+ - '*'
+
+ workflow_dispatch:
+ inputs:
+ version:
+ description: Version of Fluent Bit to build
+ required: true
+ default: master
+ target:
+ description: Only build a specific Linux target, intended for debug/test/quick builds only.
+ required: false
+ default: ""
+ ignore_failing_targets:
+ description: Optionally ignore any failing builds in the matrix and continue.
+ type: boolean
+ required: false
+ default: false
+
+# We do not want a new staging build to run whilst we are releasing the current staging build.
+# We also do not want multiples to run for the same version.
+concurrency: staging-build-release
+
+jobs:
+
+ # This job strips off the `v` at the start of any tag provided.
+ # It then provides this metadata for the other jobs to use.
+ staging-build-get-meta:
+ name: Get metadata to build
+ runs-on: ubuntu-latest
+ outputs:
+ version: ${{ steps.formatted_version.outputs.replaced }}
+ steps:
+
+ - run: |
+ echo "Version: ${{ inputs.version || github.ref_name }}"
+ shell: bash
+
+ # This step is to consolidate the three different triggers into a single "version"
+ # 1. If manual dispatch - use the version provided.
+ # 2. If cron/regular build - use master.
+ # 3. If tag trigger, use that tag.
+ - name: Get the version
+ id: get_version
+ run: |
+ VERSION="${INPUT_VERSION}"
+ if [ -z "${VERSION}" ]; then
+ echo "Defaulting to master"
+ VERSION=master
+ fi
+ echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+ shell: bash
+ env:
+ # Use the dispatch variable in preference, if empty use the context ref_name which should
+ # only ever be a tag or the master branch for cron builds.
+ INPUT_VERSION: ${{ inputs.version || github.ref_name }}
+
+ # String the 'v' prefix for tags.
+ - uses: frabert/replace-string-action@v2.4
+ id: formatted_version
+ with:
+ pattern: '[v]*(.*)$'
+ string: "${{ steps.get_version.outputs.VERSION }}"
+ replace-with: '$1'
+ flags: 'g'
+
+ staging-build-images:
+ needs: staging-build-get-meta
+ uses: ./.github/workflows/call-build-images.yaml
+ with:
+ version: ${{ needs.staging-build-get-meta.outputs.version }}
+ ref: ${{ inputs.version || github.ref_name }}
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ image: ${{ github.repository }}/staging
+ environment: staging
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }}
+ cosign_private_key_password: ${{ secrets.COSIGN_PASSWORD }}
+
+ staging-build-upload-schema-s3:
+ needs:
+ - staging-build-get-meta
+ - staging-build-images
+ runs-on: ubuntu-latest
+ environment: staging
+ steps:
+ - name: Download the schema generated by call-build-images
+ # We may have no schema so ignore that failure
+ continue-on-error: true
+ uses: actions/download-artifact@v3
+ with:
+ name: fluent-bit-schema-${{ needs.staging-build-get-meta.outputs.version }}
+ path: artifacts/
+
+ - name: Display structure of downloaded files
+ run: |
+ ls -R artifacts/
+ shell: bash
+
+ - name: Push schema to S3 bucket
+ # We may have no schema so ignore that failure
+ continue-on-error: true
+ run: |
+ aws s3 sync "artifacts/" "s3://${AWS_S3_BUCKET}/${DEST_DIR}" --no-progress
+ env:
+ DEST_DIR: "${{ needs.staging-build-get-meta.outputs.version }}/"
+ AWS_REGION: "us-east-1"
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }}
+
+ staging-build-generate-matrix:
+ name: Staging build matrix
+ runs-on: ubuntu-latest
+ outputs:
+ build-matrix: ${{ steps.set-matrix.outputs.build-matrix }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ # Set up the list of target to build so we can pass the JSON to the reusable job
+ - uses: ./.github/actions/generate-package-build-matrix
+ id: set-matrix
+ with:
+ ref: ${{ inputs.version || github.ref_name }}
+ target: ${{ inputs.target || '' }}
+
+ staging-build-packages:
+ needs:
+ - staging-build-get-meta
+ - staging-build-generate-matrix
+ uses: ./.github/workflows/call-build-linux-packages.yaml
+ with:
+ version: ${{ needs.staging-build-get-meta.outputs.version }}
+ ref: ${{ inputs.version || github.ref_name }}
+ build_matrix: ${{ needs.staging-build-generate-matrix.outputs.build-matrix }}
+ environment: staging
+ ignore_failing_targets: ${{ inputs.ignore_failing_targets || false }}
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }}
+ access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+ gpg_private_key_passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}
+
+ staging-build-windows-packages:
+ needs:
+ - staging-build-get-meta
+ uses: ./.github/workflows/call-build-windows.yaml
+ with:
+ version: ${{ needs.staging-build-get-meta.outputs.version }}
+ ref: ${{ inputs.version || github.ref_name }}
+ environment: staging
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }}
+ access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+ staging-build-macos-packages:
+ needs:
+ - staging-build-get-meta
+ uses: ./.github/workflows/call-build-macos.yaml
+ with:
+ version: ${{ needs.staging-build-get-meta.outputs.version }}
+ ref: ${{ inputs.version || github.ref_name }}
+ environment: staging
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }}
+ access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/fluent-bit/.github/workflows/staging-release.yaml b/fluent-bit/.github/workflows/staging-release.yaml
new file mode 100644
index 00000000..8ae352bd
--- /dev/null
+++ b/fluent-bit/.github/workflows/staging-release.yaml
@@ -0,0 +1,955 @@
+---
+name: Release from staging
+
+# This is only expected to be invoked on-demand by a specific user.
+on:
+ workflow_dispatch:
+ inputs:
+ version:
+ type: string
+ description: The version we want to release from staging, ensure this is numeric without the v prefix for the tag.
+ required: true
+ docker-image:
+ type: string
+ description: Optionally override the image name to push to on Docker Hub.
+ default: fluent/fluent-bit
+ required: false
+ github-image:
+ type: string
+ description: Optionally override the image name to push to on Github Container Registry.
+ default: fluent/fluent-bit
+ required: false
+
+# We do not want a new staging build to run whilst we are releasing the current staging build.
+# We also do not want multiples to run for the same version.
+concurrency: staging-build-release
+
+env:
+ STAGING_IMAGE_NAME: ghcr.io/${{ github.repository }}/staging
+
+jobs:
+
+ staging-release-version-check:
+ name: Check staging release matches
+ environment: release # required to get bucket name
+ runs-on: ubuntu-latest
+ outputs:
+ major-version: ${{ steps.get_major_version.outputs.value }}
+ permissions:
+ contents: read
+ steps:
+ - name: Get the version on staging
+ run: |
+ curl --fail -LO "$AWS_URL/latest-version.txt"
+ cat latest-version.txt
+ STAGING_VERSION=$(cat latest-version.txt)
+ [[ "$STAGING_VERSION" != "$RELEASE_VERSION" ]] && echo "Latest version mismatch: $STAGING_VERSION != $RELEASE_VERSION" && exit 1
+ # Must end in something that exits 0
+ echo "Successfully confirmed version is as expected: $STAGING_VERSION"
+ shell: bash
+ env:
+ AWS_URL: https://${{ secrets.AWS_S3_BUCKET_STAGING }}.s3.amazonaws.com
+ RELEASE_VERSION: ${{ github.event.inputs.version }}
+
+ # Get the major version, i.e. 1.9.3 --> 1.9, or just return the passed in version.
+ - name: Convert to major version format
+ id: get_major_version
+ run: |
+ MAJOR_VERSION="$RELEASE_VERSION"
+ if [[ $RELEASE_VERSION =~ ^[0-9]+\.[0-9]+ ]]; then
+ MAJOR_VERSION="${BASH_REMATCH[0]}"
+ fi
+ echo "value=$MAJOR_VERSION" >> $GITHUB_OUTPUT
+ shell: bash
+ env:
+ RELEASE_VERSION: ${{ github.event.inputs.version }}
+
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ # Check we can download the AppVeyor build which confirms it matches the version to release as well as being a successful build
+ - name: Get Appveyor binaries
+ run: |
+ ./packaging/appveyor-download.sh
+ shell: bash
+ env:
+ TAG: v${{ github.event.inputs.version }}
+
+ staging-release-generate-package-matrix:
+ name: Get package matrix
+ runs-on: ubuntu-latest
+ outputs:
+ deb-build-matrix: ${{ steps.get-matrix.outputs.deb-build-matrix }}
+ rpm-build-matrix: ${{ steps.get-matrix.outputs.rpm-build-matrix }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup runner
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y jq
+ shell: bash
+
+ # Cope with 1.9 as well as 2.0
+ - uses: ./.github/actions/generate-package-build-matrix
+ id: get-matrix
+ with:
+ ref: v${{ inputs.version }}
+
+ # Now annotate with whether it is Yum or Apt based
+
+ # 1. Take packages from the staging bucket
+ # 2. Sign them with the release GPG key
+ # 3. Also take existing release packages from the release bucket.
+ # 4. Create a full repo configuration using the existing releases as well.
+ # 5. Upload to release bucket.
+ # Note we could resign all packages as well potentially if we wanted to update the key.
+ staging-release-yum-packages:
+ name: S3 - update YUM packages bucket
+ runs-on: ubuntu-22.04 # no createrepo on Ubuntu 20.04
+ environment: release
+ needs:
+ - staging-release-version-check
+ - staging-release-generate-package-matrix
+ permissions:
+ contents: read
+ strategy:
+ matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.rpm-build-matrix) }}
+ fail-fast: false
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Setup runner
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y createrepo-c rpm
+ shell: bash
+
+ - name: Import GPG key for signing
+ id: import_gpg
+ uses: crazy-max/ghaction-import-gpg@v6
+ with:
+ gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+ passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}
+
+ # Download the current release bucket
+ # Add everything from staging
+ # Sign and set up metadata for it all
+ # Upload to release bucket
+
+ - name: Sync packages from buckets on S3
+ run: |
+ mkdir -p "packaging/releases/$DISTRO"
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ DISTRO: ${{ matrix.distro }}
+ shell: bash
+
+ - name: GPG set up keys for signing
+ run: |
+ gpg --export -a "${{ steps.import_gpg.outputs.name }}" > /tmp/fluentbit.key
+ rpm --import /tmp/fluentbit.key
+ shell: bash
+
+ - name: Update repo info and remove any staging details
+ run: |
+ packaging/update-yum-repo.sh
+ env:
+ GPG_KEY: ${{ steps.import_gpg.outputs.name }}
+ AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }}
+ VERSION: ${{ github.event.inputs.version }}
+ BASE_PATH: "packaging/releases"
+ RPM_REPO: ${{ matrix.distro }}
+ shell: bash
+
+ - name: Sync to release bucket on S3
+ run: |
+ aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ DISTRO: ${{ matrix.distro }}
+ shell: bash
+
+ staging-release-apt-packages:
+ name: S3 - update APT packages bucket
+ runs-on: ubuntu-latest
+ environment: release
+ needs:
+ - staging-release-version-check
+ - staging-release-generate-package-matrix
+ permissions:
+ contents: read
+ strategy:
+ matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.deb-build-matrix) }}
+ fail-fast: false
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Setup runner
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y aptly debsigs distro-info rsync
+ shell: bash
+
+ - name: Convert version to codename
+ id: get_codename
+ run: |
+ CODENAME="$DISTRO"
+ if [[ "$DISTRO" == ubuntu* ]]; then
+ echo "Converting Ubuntu version to codename"
+ UBUNTU_NAME=$(grep "${DISTRO##*/} LTS" /usr/share/distro-info/ubuntu.csv|cut -d ',' -f3)
+ echo "Got Ubuntu codename: $UBUNTU_NAME"
+ CODENAME="ubuntu/$UBUNTU_NAME"
+ fi
+ echo "Using codename: $CODENAME"
+ echo "CODENAME=$CODENAME" >> $GITHUB_OUTPUT
+ shell: bash
+ env:
+ DISTRO: ${{ matrix.distro }}
+
+ - name: Import GPG key for signing
+ id: import_gpg
+ uses: crazy-max/ghaction-import-gpg@v6
+ with:
+ gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+ passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}
+
+ - name: Sync packages from buckets on S3
+ run: |
+ mkdir -p "packaging/releases/$CODENAME"
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ CODENAME: ${{ steps.get_codename.outputs.CODENAME }}
+ shell: bash
+
+ - name: Update repo info and remove any staging details
+ run: |
+ packaging/update-apt-repo.sh
+ env:
+ GPG_KEY: ${{ steps.import_gpg.outputs.name }}
+ AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }}
+ VERSION: ${{ github.event.inputs.version }}
+ BASE_PATH: "packaging/releases"
+ DEB_REPO: ${{ steps.get_codename.outputs.CODENAME }}
+ shell: bash
+
+ - name: Sync to release bucket on S3
+ run: |
+ aws s3 sync "packaging/releases/$CODENAME" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" --delete --follow-symlinks --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ CODENAME: ${{ steps.get_codename.outputs.CODENAME }}
+ shell: bash
+
+ staging-release-update-non-linux-s3:
+ name: Update Windows and macOS packages
+ runs-on: ubuntu-22.04
+ environment: release
+ needs:
+ - staging-release-version-check
+ permissions:
+ contents: none
+ strategy:
+ matrix:
+ distro:
+ - macos
+ - windows
+ fail-fast: false
+ steps:
+ - name: Sync packages from buckets on S3
+ run: |
+ mkdir -p "packaging/releases/$DISTRO"
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ DISTRO: ${{ matrix.distro }}
+ shell: bash
+
+ - name: Sync to release bucket on S3
+ run: |
+ aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ DISTRO: ${{ matrix.distro }}
+ shell: bash
+
+ staging-release-update-base-s3:
+ name: Update top-level bucket info
+ runs-on: ubuntu-22.04
+ environment: release
+ needs:
+ - staging-release-apt-packages
+ - staging-release-yum-packages
+ permissions:
+ contents: none
+ steps:
+ - name: Import GPG key for signing
+ id: import_gpg
+ uses: crazy-max/ghaction-import-gpg@v6
+ with:
+ gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+ passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }}
+
+ - name: GPG public key
+ run: |
+ gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./fluentbit.key
+ aws s3 cp ./fluentbit.key s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/fluentbit.key --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ shell: bash
+
+ - name: JSON schema
+ continue-on-error: true
+ run: |
+ aws s3 sync "s3://${AWS_STAGING_S3_BUCKET}/${VERSION}" "s3://${AWS_RELEASE_S3_BUCKET}/${VERSION}" --no-progress
+ env:
+ VERSION: ${{ github.event.inputs.version }}
+ AWS_REGION: "us-east-1"
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_STAGING_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }}
+ AWS_RELEASE_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }}
+ shell: bash
+
+ staging-release-source-s3:
+ name: S3 - update source bucket
+ runs-on: ubuntu-latest
+ environment: release
+ needs:
+ - staging-release-version-check
+ permissions:
+ contents: read
+ outputs:
+ windows-exe32-hash: ${{ steps.windows.outputs.WIN_32_EXE_HASH }}
+ windows-zip32-hash: ${{ steps.windows.outputs.WIN_32_ZIP_HASH }}
+ windows-exe64-hash: ${{ steps.windows.outputs.WIN_64_EXE_HASH }}
+ windows-zip64-hash: ${{ steps.windows.outputs.WIN_64_ZIP_HASH }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Sync packages from buckets on S3
+ run: |
+ mkdir -p release staging
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" release/ --no-progress
+ aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/source/" staging/ --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ shell: bash
+
+ - name: Get Appveyor binaries
+ run: |
+ ./packaging/appveyor-download.sh
+ shell: bash
+ env:
+ TAG: v${{ github.event.inputs.version }}
+ OUTPUT_DIR: appveyor
+
+ - name: Move components from staging and setup
+ run: |
+ ./packaging/update-source-packages.sh
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ SOURCE_DIR: staging
+ WINDOWS_SOURCE_DIR: appveyor
+ TARGET_DIR: release
+ VERSION: ${{ github.event.inputs.version }}
+ MAJOR_VERSION: ${{ needs.staging-release-version-check.outputs.major-version }}
+ shell: bash
+
+ - name: Sync to bucket on S3
+ run: |
+ aws s3 sync release/ "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" --delete --follow-symlinks --no-progress
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+ shell: bash
+
+ - name: Provide output for documentation PR
+ id: windows
+ # do not fail the build for this
+ continue-on-error: true
+ run: |
+ ls -l $BASE_DIR/
+ export WIN_32_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}')
+ export WIN_32_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}')
+ export WIN_64_EXE_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}')
+ export WIN_64_ZIP_HASH=$(cat "$BASE_DIR/fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}')
+ set | grep WIN_
+ echo WIN_32_EXE_HASH="$WIN_32_EXE_HASH" >> $GITHUB_OUTPUT
+ echo WIN_32_ZIP_HASH="$WIN_32_ZIP_HASH" >> $GITHUB_OUTPUT
+ echo WIN_64_EXE_HASH="$WIN_64_EXE_HASH" >> $GITHUB_OUTPUT
+ echo WIN_64_ZIP_HASH="$WIN_64_ZIP_HASH" >> $GITHUB_OUTPUT
+ shell: bash
+ env:
+ BASE_DIR: release/${{ needs.staging-release-version-check.outputs.major-version }}
+
+ # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing.
+ # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533
+ staging-release-images:
+ name: Release ${{ matrix.tag }} Linux container images
+ runs-on: ubuntu-latest
+ needs:
+ - staging-release-version-check
+ environment: release
+ strategy:
+ fail-fast: false
+ matrix:
+ # All the explicit tags we want to release
+ tag: [
+ "${{ github.event.inputs.version }}",
+ "${{ needs.staging-release-version-check.outputs.major-version }}",
+ "${{ github.event.inputs.version }}-debug",
+ "${{ needs.staging-release-version-check.outputs.major-version }}-debug",
+ ]
+ permissions:
+ packages: write
+ steps:
+ # Primarily because the skopeo errors are hard to parse and non-obvious
+ - name: Check the image exists
+ run: |
+ docker pull "$STAGING_IMAGE_NAME:$TAG"
+ env:
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it.
+ - name: Promote container images from staging to Dockerhub
+ run: |
+ docker run --rm \
+ quay.io/skopeo/stable:latest \
+ copy \
+ --all \
+ --retry-times 10 \
+ --src-no-creds \
+ --dest-creds "$RELEASE_CREDS" \
+ "docker://$STAGING_IMAGE_NAME:$TAG" \
+ "docker://$RELEASE_IMAGE_NAME:$TAG"
+ env:
+ RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
+ RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ - name: Promote container images from staging to GHCR.io
+ if: ${{ startsWith(github.event.inputs.version, '2.0') || ! startsWith(matrix.tag, 'latest') }}
+ run: |
+ docker run --rm \
+ quay.io/skopeo/stable:latest \
+ copy \
+ --all \
+ --retry-times 10 \
+ --src-no-creds \
+ --dest-creds "$RELEASE_CREDS" \
+ "docker://$STAGING_IMAGE_NAME:$TAG" \
+ "docker://$RELEASE_IMAGE_NAME:$TAG"
+ env:
+ RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
+ RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ # Part of resolution for: https://github.com/fluent/fluent-bit/issues/7748
+ # More recent build-push-actions may mean legacy format is not preserved so we provide arch-specific tags just in case
+ staging-release-images-arch-specific-legacy-tags:
+ # TODO: remove next release once we are happy this all works, for now though do not block a release
+ continue-on-error: true
+ #
+ name: Release ${{ matrix.arch }} legacy format Linux container images
+ runs-on: ubuntu-latest
+ needs:
+ - staging-release-images
+ environment: release
+ strategy:
+ fail-fast: false
+ matrix:
+ arch:
+ - amd64
+ - arm64
+ - arm/v7
+ permissions:
+ packages: write
+ env:
+ RELEASE_IMAGE_NAME: ${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
+ RELEASE_TAG: ${{ github.event.inputs.version }}
+ steps:
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Convert arch to tag
+ id: get-tag
+ run: |
+ TAG="${RELEASE_TAG}-${{ matrix.arch }}"
+ echo "Input value: $TAG"
+ TAG=${TAG/\//-}
+ echo "Using tag: $TAG"
+ echo "tag=$TAG" >> $GITHUB_OUTPUT
+ shell: bash
+
+ - name: Pull release image
+ run: docker pull --platform='linux/${{ matrix.arch }}' "$RELEASE_IMAGE_NAME:$RELEASE_TAG"
+ shell: bash
+
+ - name: Tag and push legacy format image to DockerHub
+ run: |
+ docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" docker.io/"$RELEASE_IMAGE_NAME:$TAG"
+ docker push docker.io/"$RELEASE_IMAGE_NAME:$TAG"
+ shell: bash
+ env:
+ TAG: ${{ steps.get-tag.outputs.tag }}
+
+ - name: Tag and push legacy format image to Github Container Registry
+ run: |
+ docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" ghcr.io/"$RELEASE_IMAGE_NAME:$TAG"
+ docker push ghcr.io/"$RELEASE_IMAGE_NAME:$TAG"
+ shell: bash
+ env:
+ TAG: ${{ steps.get-tag.outputs.tag }}
+
+ staging-release-images-latest-tags:
+ # Only update latest tags for 2.1 releases
+ if: startsWith(github.event.inputs.version, '2.1')
+ name: Release latest Linux container images
+ runs-on: ubuntu-latest
+ needs:
+ - staging-release-images
+ environment: release
+ strategy:
+ fail-fast: false
+ matrix:
+ tag: [
+ "latest",
+ "latest-debug"
+ ]
+ permissions:
+ packages: write
+ steps:
+ # Primarily because the skopeo errors are hard to parse and non-obvious
+ - name: Check the image exists
+ run: |
+ docker pull "$STAGING_IMAGE_NAME:$TAG"
+ env:
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it.
+ - name: Promote container images from staging to Dockerhub
+ run: |
+ docker run --rm \
+ quay.io/skopeo/stable:latest \
+ copy \
+ --all \
+ --retry-times 10 \
+ --src-no-creds \
+ --dest-creds "$RELEASE_CREDS" \
+ "docker://$STAGING_IMAGE_NAME:$TAG" \
+ "docker://$RELEASE_IMAGE_NAME:$TAG"
+ env:
+ RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
+ RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ - name: Promote container images from staging to GHCR.io
+ run: |
+ docker run --rm \
+ quay.io/skopeo/stable:latest \
+ copy \
+ --all \
+ --retry-times 10 \
+ --src-no-creds \
+ --dest-creds "$RELEASE_CREDS" \
+ "docker://$STAGING_IMAGE_NAME:$TAG" \
+ "docker://$RELEASE_IMAGE_NAME:$TAG"
+ env:
+ RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
+ RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ staging-release-images-windows:
+ name: Release Windows images
+ # Cannot be done by Skopeo on a Linux runner unfortunately
+ runs-on: windows-latest
+ needs:
+ - staging-release-version-check
+ environment: release
+ permissions:
+ packages: write
+ strategy:
+ fail-fast: false
+ matrix:
+ tag: [
+ "windows-2019-${{ github.event.inputs.version }}",
+ "windows-2022-${{ github.event.inputs.version }}"
+ ]
+ steps:
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Check the image exists
+ run: |
+ docker pull "$STAGING_IMAGE_NAME:$TAG"
+ env:
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ - name: Promote container images from staging to GHCR.io
+ run: |
+ docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG"
+ docker push "$RELEASE_IMAGE_NAME:$TAG"
+ env:
+ RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
+ RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Promote container images from staging to Dockerhub
+ run: |
+ docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG"
+ docker push "$RELEASE_IMAGE_NAME:$TAG"
+ env:
+ RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
+ RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}
+ TAG: ${{ matrix.tag }}
+ shell: bash
+
+ staging-release-images-sign:
+ name: Sign container image manifests
+ permissions: write-all
+ runs-on: ubuntu-latest
+ environment: release
+ needs:
+ - staging-release-images
+ env:
+ DH_RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }}
+ GHCR_RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }}
+ steps:
+ - name: Install cosign
+ uses: sigstore/cosign-installer@v2
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Cosign with a key
+ # Only run if we have a key defined
+ if: ${{ env.COSIGN_PRIVATE_KEY }}
+ # The key needs to cope with newlines
+ run: |
+ echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key
+ cosign sign --key /tmp/my_cosign.key --recursive \
+ -a "repo=${{ github.repository }}" \
+ -a "workflow=${{ github.workflow }}" \
+ -a "release=${{ github.event.inputs.version }}" \
+ "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
+ "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \
+ "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
+ "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug"
+ rm -f /tmp/my_cosign.key
+ shell: bash
+ env:
+ COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+ COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional
+
+ - name: Cosign keyless signing using Rektor public transparency log
+ # This step uses the identity token to provision an ephemeral certificate
+ # against the sigstore community Fulcio instance, and records it to the
+ # sigstore community Rekor transparency log.
+ #
+ # We use recursive signing on the manifest to cover all the images.
+ run: |
+ cosign sign --yes --recursive \
+ -a "repo=${{ github.repository }}" \
+ -a "workflow=${{ github.workflow }}" \
+ -a "release=${{ github.event.inputs.version }}" \
+ "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
+ "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \
+ "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \
+ "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug"
+ shell: bash
+ env:
+ COSIGN_EXPERIMENTAL: true
+
+ staging-release-upload-cosign-key:
+ name: Upload Cosign public key for verification
+ needs:
+ - staging-release-images-sign
+ permissions:
+ contents: none
+ runs-on: ubuntu-latest
+ steps:
+ - name: Install cosign
+ uses: sigstore/cosign-installer@v2
+
+ - name: Get public key and add to S3 bucket
+ # Only run if we have a key defined
+ if: ${{ env.COSIGN_PRIVATE_KEY }}
+ # The key needs to cope with newlines
+ run: |
+ echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key
+ cosign public-key --key /tmp/my_cosign.key > ./cosign.pub
+ rm -f /tmp/my_cosign.key
+ cat ./cosign.pub
+ aws s3 cp ./cosign.pub "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/cosign.pub" --no-progress
+ shell: bash
+ env:
+ COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+ COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: "us-east-1"
+
+ staging-release-smoke-test-packages:
+ name: Run package smoke tests
+ permissions:
+ contents: read
+ runs-on: ubuntu-latest
+ environment: release
+ needs:
+ - staging-release-apt-packages
+ - staging-release-yum-packages
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Test release packages
+ run: |
+ ./packaging/test-release-packages.sh
+ shell: bash
+ env:
+ VERSION_TO_CHECK_FOR: ${{ github.event.inputs.version }}
+ FLUENT_BIT_PACKAGES_URL: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com
+ FLUENT_BIT_PACKAGES_KEY: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com/fluentbit.key
+
+ staging-release-smoke-test-containers:
+ name: Run container smoke tests
+ permissions:
+ contents: read
+ packages: read
+ runs-on: ubuntu-latest
+ environment: release
+ needs:
+ - staging-release-images
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Test containers
+ run: |
+ ./packaging/testing/smoke/container/container-smoke-test.sh
+ shell: bash
+ env:
+ IMAGE_TAG: ${{ github.event.inputs.version }}
+
+ staging-release-create-release:
+ name: Create the Github Release once packages and containers are up
+ needs:
+ - staging-release-images
+ - staging-release-apt-packages
+ - staging-release-yum-packages
+ permissions:
+ contents: write
+ environment: release
+ runs-on: ubuntu-latest
+ steps:
+ - name: Release 2.0 - not latest
+ uses: softprops/action-gh-release@v1
+ if: startsWith(inputs.version, '2.0')
+ with:
+ body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
+ draft: false
+ generate_release_notes: true
+ name: "Fluent Bit ${{ inputs.version }}"
+ tag_name: v${{ inputs.version }}
+ target_commitish: '2.0'
+
+ - name: Release 2.1 and latest
+ uses: softprops/action-gh-release@v1
+ if: startsWith(inputs.version, '2.1')
+ with:
+ body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
+ draft: false
+ generate_release_notes: true
+ name: "Fluent Bit ${{ inputs.version }}"
+ tag_name: v${{ inputs.version }}
+
+ staging-release-create-docs-pr:
+ name: Create docs updates for new release
+ needs:
+ - staging-release-images
+ - staging-release-source-s3
+ permissions:
+ contents: none
+ environment: release
+ runs-on: ubuntu-latest
+ steps:
+ - name: Release 2.0 - not latest
+ if: startsWith(inputs.version, '2.0')
+ uses: actions/checkout@v4
+ with:
+ repository: fluent/fluent-bit-docs
+ ref: 2.0
+ token: ${{ secrets.GH_PA_TOKEN }}
+
+ - name: Release 2.1 and latest
+ if: startsWith(inputs.version, '2.1')
+ uses: actions/checkout@v4
+ with:
+ repository: fluent/fluent-bit-docs
+ token: ${{ secrets.GH_PA_TOKEN }}
+
+ - name: Ensure we have the script we need
+ run: |
+ if [[ ! -f update-release-version-docs.sh ]] ; then
+ git checkout update-release-version-docs.sh -- master
+ fi
+ shell: bash
+
+ - name: Update versions
+ # Uses https://github.com/fluent/fluent-bit-docs/blob/master/update-release-version-docs.sh
+ run: |
+ ./update-release-version-docs.sh
+ shell: bash
+ env:
+ NEW_VERSION: ${{ inputs.version }}
+ WIN_32_EXE_HASH: ${{ needs.staging-release-source-s3.outputs.windows-exe32-hash }}
+ WIN_32_ZIP_HASH: ${{ needs.staging-release-source-s3.outputs.windows-zip32-hash }}
+ WIN_64_EXE_HASH: ${{ needs.staging-release-source-s3.outputs.windows-exe64-hash }}
+ WIN_64_ZIP_HASH: ${{ needs.staging-release-source-s3.outputs.windows-zip64-hash }}
+
+ - name: Raise docs PR
+ id: cpr
+ uses: peter-evans/create-pull-request@v5
+ with:
+ commit-message: 'release: update to v${{ inputs.version }}'
+ signoff: true
+ delete-branch: true
+ title: 'release: update to v${{ inputs.version }}'
+ # We need workflows permission so have to use the GH_PA_TOKEN
+ token: ${{ secrets.GH_PA_TOKEN }}
+ labels: ci,automerge
+ body: |
+ Update release ${{ inputs.version }} version.
+ - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request
+ draft: false
+
+ - name: Check outputs
+ if: ${{ steps.cpr.outputs.pull-request-number }}
+ run: |
+ echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+ echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
+
+ staging-release-create-version-update-pr:
+ name: Create version update PR for new release
+ needs:
+ - staging-release-create-release
+ permissions:
+ contents: write
+ pull-requests: write
+ environment: release
+ runs-on: ubuntu-latest
+ steps:
+ - name: Release 2.0
+ if: startsWith(inputs.version, '2.0')
+ uses: actions/checkout@v4
+ with:
+ ref: 2.0
+
+ - name: Release 2.1 and latest
+ if: startsWith(inputs.version, '2.1')
+ uses: actions/checkout@v4
+
+ # Get the new version to use
+ - name: 'Get next minor version'
+ id: semvers
+ uses: "WyriHaximus/github-action-next-semvers@v1"
+ with:
+ version: ${{ inputs.version }}
+ strict: true
+
+ - run: ./update_version.sh
+ shell: bash
+ env:
+ NEW_VERSION: ${{ steps.semvers.outputs.patch }}
+ # Ensure we use the PR action to do the work
+ DISABLE_COMMIT: 'yes'
+
+ - name: Raise FB PR to update version
+ id: cpr
+ uses: peter-evans/create-pull-request@v5
+ with:
+ commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}'
+ signoff: true
+ delete-branch: true
+ title: 'release: update to ${{ steps.semvers.outputs.patch }}'
+ labels: ci,automerge
+ body: |
+ Update next release to ${{ steps.semvers.outputs.patch }} version.
+ - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request
+ draft: false
+
+ - name: Check outputs
+ if: ${{ steps.cpr.outputs.pull-request-number }}
+ run: |
+ echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+ echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
diff --git a/fluent-bit/.github/workflows/staging-test.yaml b/fluent-bit/.github/workflows/staging-test.yaml
new file mode 100644
index 00000000..e7a2d3ca
--- /dev/null
+++ b/fluent-bit/.github/workflows/staging-test.yaml
@@ -0,0 +1,57 @@
+---
+name: Test staging
+# The intention is this workflow is triggered either manually or
+# after build has completed.
+on:
+ workflow_run:
+ workflows: ['Deploy to staging']
+ types:
+ - completed
+ workflow_dispatch:
+
+concurrency: integration-test
+
+jobs:
+ staging-test-images:
+ name: Container images staging tests
+ # Workflow run always triggers on completion regardless of status
+ # This prevents us from running if build fails.
+ if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
+ uses: ./.github/workflows/call-test-images.yaml
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ image: ${{ github.repository }}/staging
+ image-tag: latest
+ environment: staging
+ secrets:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ cosign_key: ${{ secrets.COSIGN_PUBLIC_KEY }}
+
+ # Called workflows cannot be nested
+ staging-test-images-integration:
+ name: run integration tests on GCP
+ # Wait for other tests to succeed
+ needs: staging-test-images
+ uses: ./.github/workflows/call-run-integration-test.yaml
+ with:
+ image_name: ghcr.io/${{ github.repository }}/staging
+ image_tag: latest
+ secrets:
+ opensearch_aws_access_id: ${{ secrets.OPENSEARCH_AWS_ACCESS_ID }}
+ opensearch_aws_secret_key: ${{ secrets.OPENSEARCH_AWS_SECRET_KEY }}
+ opensearch_admin_password: ${{ secrets.OPENSEARCH_ADMIN_PASSWORD }}
+ terraform_api_token: ${{ secrets.TF_API_TOKEN }}
+ gcp-service-account-key: ${{ secrets.GCP_SA_KEY }}
+
+ staging-test-packages:
+ name: Binary packages staging test
+ # Workflow run always triggers on completion regardless of status
+ # This prevents us from running if build fails.
+ if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
+ uses: ./.github/workflows/call-test-packages.yaml
+ with:
+ environment: staging
+ secrets:
+ bucket: ${{ secrets.AWS_S3_BUCKET_STAGING }}
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/fluent-bit/.github/workflows/unit-tests.yaml b/fluent-bit/.github/workflows/unit-tests.yaml
new file mode 100644
index 00000000..282879c5
--- /dev/null
+++ b/fluent-bit/.github/workflows/unit-tests.yaml
@@ -0,0 +1,125 @@
+name: Run unit tests
+on:
+ push:
+ branches:
+ - master
+ - 2.0
+ - 1.9
+ - 1.8
+ pull_request:
+ paths-ignore:
+ - '.github/**'
+ - 'dockerfiles/**'
+ - 'docker_compose/**'
+ - 'packaging/**'
+ - '.gitignore'
+ - 'appveyor.yml'
+ - '**.sh'
+ - 'examples/**'
+ branches:
+ - master
+ - 2.0
+ - 1.9
+ - 1.8
+ types: [opened, reopened, synchronize]
+ workflow_dispatch:
+
+jobs:
+ run-ubuntu-unit-tests:
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ flb_option:
+ - "-DFLB_JEMALLOC=On"
+ - "-DFLB_JEMALLOC=Off"
+ - "-DFLB_SMALL=On"
+ - "-DSANITIZE_ADDRESS=On"
+ - "-DSANITIZE_UNDEFINED=On"
+ - "-DFLB_COVERAGE=On"
+ - "-DFLB_SANITIZE_MEMORY=On"
+ - "-DFLB_SANITIZE_THREAD=On"
+ compiler:
+ - gcc
+ - clang
+ exclude:
+ - flb_option: "-DFLB_COVERAGE=On"
+ compiler: clang
+ permissions:
+ contents: read
+ steps:
+ - name: Setup environment
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev
+ sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
+
+ - uses: actions/checkout@v4
+
+ - uses: actions/checkout@v4
+ with:
+ repository: calyptia/fluent-bit-ci
+ path: ci
+
+ - name: ${{ matrix.compiler }} - ${{ matrix.flb_option }}
+ run: |
+ echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
+ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
+ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
+ sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 90
+ sudo usermod -a -G systemd-journal $(id -un)
+ sudo -E su -p $(id -un) -c "PATH=$PATH ci/scripts/run-unit-tests.sh"
+ env:
+ CC: ${{ matrix.compiler }}
+ CXX: ${{ matrix.compiler }}
+ FLB_OPT: ${{ matrix.flb_option }}
+
+ run-macos-unit-tests:
+ # We chain this after Linux one as there are costs and restrictions associated
+ needs:
+ - run-ubuntu-unit-tests
+ runs-on: macos-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ flb_option:
+ - "-DFLB_JEMALLOC=Off"
+ - "-DFLB_SANITIZE_MEMORY=On"
+ - "-DFLB_SANITIZE_THREAD=On"
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/checkout@v4
+ with:
+ repository: calyptia/fluent-bit-ci
+ path: ci
+
+ - name: ${{ matrix.flb_option }}
+ run: |
+ echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
+ brew update
+ brew install bison flex openssl || true
+ ci/scripts/run-unit-tests.sh
+ env:
+ CC: gcc
+ CXX: g++
+ FLB_OPT: ${{ matrix.flb_option }}
+
+ # Required check looks at this so do not remove
+ run-all-unit-tests:
+ if: always()
+ runs-on: ubuntu-latest
+ name: Unit tests (matrix)
+ permissions:
+ contents: none
+ needs:
+ - run-macos-unit-tests
+ - run-ubuntu-unit-tests
+ steps:
+ - name: Check build matrix status
+ # Ignore MacOS failures
+ if: ${{ needs.run-ubuntu-unit-tests.result != 'success' }}
+ run: exit 1
diff --git a/fluent-bit/.github/workflows/update-dockerhub.yaml b/fluent-bit/.github/workflows/update-dockerhub.yaml
new file mode 100644
index 00000000..cdd5dbe1
--- /dev/null
+++ b/fluent-bit/.github/workflows/update-dockerhub.yaml
@@ -0,0 +1,23 @@
+---
+name: Update Dockerhub description
+
+on:
+ workflow_dispatch:
+
+jobs:
+ update-dockerhub:
+ name: Update Dockerhub description
+ permissions:
+ contents: read
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Docker Hub Description
+ uses: peter-evans/dockerhub-description@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ repository: ${{ github.repository }}
+ readme-filepath: ./dockerfiles/dockerhub-description.md
+ short-description: 'Fluent Bit, lightweight logs and metrics collector and forwarder'