###################################################### # WARNING! Action needed when changing this file # ###################################################### # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the # CI configuration stored on the repository. To work around that this file is # expanded by a tool in the repository, and the expansion is committed as well. # # After you make any change to the file you'll need to run this command: # # ./x.py run src/tools/expand-yaml-anchors # # ...and commit the file it updated in addition to this one. If you forget this # step CI will fail. --- ############################### # YAML Anchors Definition # ############################### # This key contains most of the YAML anchors that will be used later in the # document. YAML anchors allows us to greatly reduce duplication inside the CI # configuration by reusing parts of the configuration. # # YAML anchors work by defining an anchor with `&anchor-name` and reusing its # content in another place with `*anchor-name`. The special `<<` map key merges # the content of the map with the content of the anchor (or list of anchors). # # The expand-yaml-anchors tool will automatically remove this block from the # output YAML file. x--expand-yaml-anchors--remove: - &shared-ci-variables CI_JOB_NAME: ${{ matrix.name }} CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - &public-variables SCCACHE_BUCKET: rust-lang-ci-sccache2 TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate CACHE_DOMAIN: ci-caches.rust-lang.org - &prod-variables SCCACHE_BUCKET: rust-lang-ci-sccache2 DEPLOY_BUCKET: rust-lang-ci2 TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues TOOLSTATE_PUBLISH: 1 # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to # rotate them in a single branch while keeping the old key in another # branch, which wouldn't be possible if the key was named with the kind # (caches, artifacts...). CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 AWS_REGION: us-west-1 CACHE_DOMAIN: ci-caches.rust-lang.org - &dummy-variables SCCACHE_BUCKET: rust-lang-gha-caches DEPLOY_BUCKET: rust-lang-gha TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues TOOLSTATE_PUBLISH: 1 # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to # rotate them in a single branch while keeping the old key in another # branch, which wouldn't be possible if the key was named with the kind # (caches, artifacts...). CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF AWS_REGION: us-west-1 CACHE_DOMAIN: ci-caches-gha.rust-lang.org - &base-job env: {} - &job-linux-4c os: ubuntu-20.04-4core-16gb <<: *base-job - &job-linux-8c os: ubuntu-20.04-8core-32gb <<: *base-job - &job-linux-16c os: ubuntu-20.04-16core-64gb <<: *base-job - &job-macos-xl os: macos-13 # We use the standard runner for now <<: *base-job - &job-windows-8c os: windows-2019-8core-32gb <<: *base-job - &job-windows-16c os: windows-2019-16core-64gb <<: *base-job - &job-aarch64-linux os: [self-hosted, ARM64, linux] - &step if: success() && !env.SKIP_JOB - &base-ci-job timeout-minutes: 600 runs-on: "${{ matrix.os }}" env: *shared-ci-variables steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - name: checkout the source code uses: actions/checkout@v4 with: fetch-depth: 2 # Rust Log Analyzer can't currently detect the PR number of a GitHub # Actions build on its own, so a hint in the log message is needed to # point it in the right direction. - name: configure the PR in which the error message will be posted run: echo "[CI_PR_NUMBER=$num]" env: num: ${{ github.event.number }} if: success() && !env.SKIP_JOB && github.event_name == 'pull_request' - name: add extra environment variables run: src/ci/scripts/setup-environment.sh env: # Since it's not possible to merge `${{ matrix.env }}` with the other # variables in `job..env`, the variables defined in the matrix # are passed to the `setup-environment.sh` script encoded in JSON, # which then uses log commands to actually set them. EXTRA_VARIABLES: ${{ toJson(matrix.env) }} <<: *step - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh <<: *step - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh <<: *step - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh <<: *step - name: show the current environment run: src/ci/scripts/dump-environment.sh <<: *step - name: install sccache run: src/ci/scripts/install-sccache.sh <<: *step - name: select Xcode run: src/ci/scripts/select-xcode.sh <<: *step - name: install clang run: src/ci/scripts/install-clang.sh <<: *step - name: install WIX run: src/ci/scripts/install-wix.sh <<: *step - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh <<: *step - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh <<: *step - name: install MSYS2 run: src/ci/scripts/install-msys2.sh <<: *step - name: install MinGW run: src/ci/scripts/install-mingw.sh <<: *step - name: install ninja run: src/ci/scripts/install-ninja.sh <<: *step - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh <<: *step # Disable automatic line ending conversion (again). On Windows, when we're # installing dependencies, something switches the git configuration directory or # re-enables autocrlf. We've not tracked down the exact cause -- and there may # be multiple -- but this should ensure submodules are checked out with the # appropriate line endings. - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh <<: *step - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh <<: *step - name: ensure backported commits are in upstream branches run: src/ci/scripts/verify-backported-commits.sh <<: *step - name: ensure the stable version number is correct run: src/ci/scripts/verify-stable-version-number.sh <<: *step - name: run the build run: src/ci/scripts/run-build-from-ci.sh env: AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} <<: *step - name: create github artifacts run: src/ci/scripts/create-doc-artifacts.sh <<: *step - name: upload artifacts to github uses: actions/upload-artifact@v3 with: # name is set in previous step name: ${{ env.DOC_ARTIFACT_NAME }} path: obj/artifacts/doc if-no-files-found: ignore retention-days: 5 <<: *step - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy # builders *should* have the AWS credentials available. Still, explicitly # adding the condition is helpful as this way CI will not silently skip # deploying artifacts from a dist builder if the variables are misconfigured, # erroring about invalid credentials instead. if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') <<: *step # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs. # Check out their documentation for more information on why they're needed. - &base-outcome-job name: bors build finished runs-on: ubuntu-latest - &base-success-job steps: - name: mark the job as a success run: exit 0 shell: bash <<: *base-outcome-job - &base-failure-job steps: - name: mark the job as a failure run: exit 1 shell: bash <<: *base-outcome-job ########################### # Builders definition # ########################### name: CI on: push: branches: - auto - try - try-perf - master pull_request: branches: - "**" permissions: contents: read defaults: run: # On Linux, macOS, and Windows, use the system-provided bash as the default # shell. (This should only make a difference on Windows, where the default # shell is PowerShell.) shell: bash concurrency: # For a given workflow, if we push to the same branch, cancel all previous builds on that branch. # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which # are all triggered on the same branch, but which should be able to run concurrently. group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }} cancel-in-progress: true jobs: pr: <<: *base-ci-job name: PR - ${{ matrix.name }} env: <<: [*shared-ci-variables, *public-variables] PR_CI_JOB: 1 if: github.event_name == 'pull_request' continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} strategy: matrix: include: - name: mingw-check <<: *job-linux-4c - name: mingw-check-tidy <<: *job-linux-4c - name: x86_64-gnu-llvm-15 <<: *job-linux-16c - name: x86_64-gnu-tools <<: *job-linux-16c auto: <<: *base-ci-job name: auto - ${{ matrix.name }} env: <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' strategy: matrix: include: ############################# # Linux/Docker builders # ############################# - name: aarch64-gnu <<: *job-aarch64-linux - name: arm-android <<: *job-linux-8c - name: armhf-gnu <<: *job-linux-8c - name: dist-aarch64-linux <<: *job-linux-8c - name: dist-android <<: *job-linux-8c - name: dist-arm-linux <<: *job-linux-16c - name: dist-armhf-linux <<: *job-linux-8c - name: dist-armv7-linux <<: *job-linux-8c - name: dist-i586-gnu-i586-i686-musl <<: *job-linux-8c - name: dist-i686-linux <<: *job-linux-8c - name: dist-loongarch64-linux <<: *job-linux-8c - name: dist-powerpc-linux <<: *job-linux-8c - name: dist-powerpc64-linux <<: *job-linux-8c - name: dist-powerpc64le-linux <<: *job-linux-8c - name: dist-riscv64-linux <<: *job-linux-8c - name: dist-s390x-linux <<: *job-linux-8c - name: dist-various-1 <<: *job-linux-8c - name: dist-various-2 <<: *job-linux-8c - name: dist-x86_64-freebsd <<: *job-linux-8c - name: dist-x86_64-illumos <<: *job-linux-8c - &dist-x86_64-linux name: dist-x86_64-linux <<: *job-linux-16c - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux <<: *job-linux-16c - name: dist-x86_64-musl <<: *job-linux-8c - name: dist-x86_64-netbsd <<: *job-linux-8c - name: i686-gnu <<: *job-linux-8c - name: i686-gnu-nopt <<: *job-linux-8c - name: mingw-check <<: *job-linux-4c - name: test-various <<: *job-linux-8c - name: wasm32 env: # Running emscripten tests currently requires that we are # building a nightly toolchain. Otherwise, we cannot pass # -Zunstable-options to libtest. Normally we workaround this by # setting RUSTC_BOOTSTRAP in the environment, but that doesn't # work for emscripten as environment variables are not threaded # into the compiled code. # # For more details see: # https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html#environment-variables RUST_CI_OVERRIDE_RELEASE_CHANNEL: nightly <<: *job-linux-8c - name: x86_64-gnu <<: *job-linux-4c # This job ensures commits landing on nightly still pass the full # test suite on the stable channel. There are some UI tests that # depend on the channel being built (for example if they include the # channel name on the output), and this builder prevents landing # changes that would result in broken builds after a promotion. - name: x86_64-gnu-stable env: IMAGE: x86_64-gnu RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable # Only run this job on the nightly channel. Running this on beta # could cause failures when `dev: 1` in `stage0.txt`, and running # this on stable is useless. CI_ONLY_WHEN_CHANNEL: nightly <<: *job-linux-4c - name: x86_64-gnu-aux <<: *job-linux-4c - name: x86_64-gnu-debug <<: *job-linux-8c - name: x86_64-gnu-distcheck <<: *job-linux-8c - name: x86_64-gnu-llvm-16 env: RUST_BACKTRACE: 1 <<: *job-linux-8c - name: x86_64-gnu-llvm-15 env: RUST_BACKTRACE: 1 <<: *job-linux-8c - name: x86_64-gnu-nopt <<: *job-linux-4c - name: x86_64-gnu-tools env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json <<: *job-linux-8c #################### # macOS Builders # #################### - name: dist-x86_64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.12 SELECT_XCODE: /Applications/Xcode_13.4.1.app NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-macos-xl - name: dist-apple-various env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.12 SELECT_XCODE: /Applications/Xcode_13.4.1.app NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 <<: *job-macos-xl - name: x86_64-apple-1 env: &env-x86_64-apple-tests SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.12 MACOSX_STD_DEPLOYMENT_TARGET: 10.12 NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 <<: *job-macos-xl - name: x86_64-apple-2 env: SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps <<: *env-x86_64-apple-tests <<: *job-macos-xl # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 RUST_CONFIGURE_ARGS: >- --build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 SELECT_XCODE: /Applications/Xcode_13.4.1.app USE_XCODE_CLANG: 1 MACOSX_DEPLOYMENT_TARGET: 11.0 MACOSX_STD_DEPLOYMENT_TARGET: 11.0 NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 DIST_REQUIRE_ALL_TOOLS: 1 # Corresponds to 16K page size # # Shouldn't be needed if jemalloc-sys is updated to # handle this platform like iOS or if we build on # aarch64-apple-darwin itself. # # https://github.com/gnzlbg/jemallocator/blob/c27a859e98e3cb790dc269773d9da71a1e918458/jemalloc-sys/build.rs#L237 JEMALLOC_SYS_WITH_LG_PAGE: 14 <<: *job-macos-xl ###################### # Windows Builders # ###################### - name: x86_64-msvc env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-msvc <<: *job-windows-8c - name: i686-msvc env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-msvc <<: *job-windows-8c - name: x86_64-msvc-ext env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json DEPLOY_TOOLSTATES_JSON: toolstates-windows.json <<: *job-windows-8c # 32/64-bit MinGW builds. # # We are using MinGW with POSIX threads since LLVM requires # C++'s std::thread which is disabled in libstdc++ with win32 threads. # FIXME: Libc++ doesn't have this limitation so we can avoid # winpthreads if we switch to it. # # Instead of relying on the MinGW version installed on CI we download # and install one ourselves so we won't be surprised by changes to CI's # build image. # # Finally, note that the downloads below are all in the `rust-lang-ci` S3 # bucket, but they clearly didn't originate there! The downloads originally # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - name: i686-mingw env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-8c - name: x86_64-mingw env: SCRIPT: make ci-mingw RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-profiler # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-8c - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-i686-mingw env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 SCRIPT: python x.py dist bootstrap --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-x86_64-mingw env: SCRIPT: python x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths <<: *job-windows-8c try: <<: *base-ci-job name: try - ${{ matrix.name }} env: DIST_TRY_BUILD: 1 <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust' strategy: matrix: include: - &dist-x86_64-linux name: dist-x86_64-linux <<: *job-linux-16c master: name: master runs-on: ubuntu-latest env: <<: [*prod-variables] if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust' steps: - name: checkout the source code uses: actions/checkout@v4 with: fetch-depth: 2 - name: publish toolstate run: src/ci/publish_toolstate.sh shell: bash env: TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} <<: *step # These jobs don't actually test anything, but they're used to tell bors the # build completed, as there is no practical way to detect when a workflow is # successful listening to webhooks only. try-success: needs: [try] if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" <<: *base-success-job try-failure: needs: [try] if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job auto-success: needs: [auto] if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" <<: *base-success-job auto-failure: needs: [auto] if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job