diff options
Diffstat (limited to '.github/workflows')
| -rw-r--r-- | .github/workflows/build.yml | 487 | ||||
| -rw-r--r-- | .github/workflows/codeql.yml | 65 | ||||
| -rw-r--r-- | .github/workflows/core.yml | 61 | ||||
| -rw-r--r-- | .github/workflows/download.yml | 48 | ||||
| -rw-r--r-- | .github/workflows/quick-test.yml | 35 | ||||
| -rw-r--r-- | .github/workflows/release-master.yml | 29 | ||||
| -rw-r--r-- | .github/workflows/release-nightly.yml | 42 | ||||
| -rw-r--r-- | .github/workflows/release.yml | 387 |
8 files changed, 1154 insertions, 0 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..4bed5af --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,487 @@ +name: Build Artifacts +on: + workflow_call: + inputs: + version: + required: true + type: string + channel: + required: false + default: stable + type: string + unix: + default: true + type: boolean + linux_arm: + default: true + type: boolean + macos: + default: true + type: boolean + macos_legacy: + default: true + type: boolean + windows: + default: true + type: boolean + windows32: + default: true + type: boolean + meta_files: + default: true + type: boolean + origin: + required: false + default: '' + type: string + secrets: + GPG_SIGNING_KEY: + required: false + + workflow_dispatch: + inputs: + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + required: true + type: string + channel: + description: | + SOURCE of this build's updates: stable/nightly/master/<repo> + required: true + default: stable + type: string + unix: + description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip + default: true + type: boolean + linux_arm: + description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l + default: true + type: boolean + macos: + description: yt-dlp_macos, yt-dlp_macos.zip + default: true + type: boolean + macos_legacy: + description: yt-dlp_macos_legacy + default: true + type: boolean + windows: + description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip + default: true + type: boolean + windows32: + description: yt-dlp_x86.exe + default: true + type: boolean + meta_files: + description: SHA2-256SUMS, SHA2-512SUMS, _update_spec + default: true + type: boolean + origin: + description: Origin + required: false + default: 'current repo' + type: choice + options: + - 'current repo' + +permissions: + contents: read + +jobs: + process: + runs-on: ubuntu-latest + outputs: + origin: ${{ steps.process_origin.outputs.origin }} + steps: + - name: Process origin + id: process_origin + run: | + echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT" + + unix: + needs: process + if: inputs.unix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Mambaforge + use-mamba: true + channels: conda-forge + auto-update-conda: true + activate-environment: "" + auto-activate-base: false + - name: Install Requirements + run: | + sudo apt -y install zip pandoc man sed + cat > ./requirements.txt << EOF + python=3.10.* + brotli-python + EOF + python devscripts/install_deps.py --print \ + --exclude brotli --exclude brotlicffi \ + --include secretstorage --include pyinstaller >> ./requirements.txt + mamba create -n build --file ./requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py + - name: Build Unix platform-independent binary + run: | + make all tar + - name: Build Unix standalone binary + shell: bash -l {0} + run: | + unset LD_LIBRARY_PATH # Harmful; set by setup-python + conda activate build + python -m bundle.pyinstaller --onedir + (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) + python -m bundle.pyinstaller + mv ./dist/yt-dlp_linux ./yt-dlp_linux + mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + binaries=("yt-dlp" "yt-dlp_linux") + for binary in "${binaries[@]}"; do + chmod +x ./${binary} + cp ./${binary} ./${binary}_downgraded + version="$(./${binary} --version)" + ./${binary}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./${binary}_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + done + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + yt-dlp + yt-dlp.tar.gz + yt-dlp_linux + yt-dlp_linux.zip + compression-level: 0 + + linux_arm: + needs: process + if: inputs.linux_arm + permissions: + contents: read + packages: write # for creating cache + runs-on: ubuntu-latest + strategy: + matrix: + architecture: + - armv7 + - aarch64 + + steps: + - uses: actions/checkout@v4 + with: + path: ./repo + - name: Virtualized Install, Prepare & Build + uses: yt-dlp/run-on-arch-action@v2 + with: + # Ref: https://github.com/uraimo/run-on-arch-action/issues/55 + env: | + GITHUB_WORKFLOW: build + githubToken: ${{ github.token }} # To cache image + arch: ${{ matrix.architecture }} + distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS + dockerRunArgs: --volume "${PWD}/repo:/repo" + install: | # Installing Python 3.10 from the Deadsnakes repo raises errors + apt update + apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip + python3.8 -m pip install -U pip setuptools wheel + # Cannot access any files from the repo directory at this stage + python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi + + run: | + cd repo + python3.8 devscripts/install_deps.py -o --include build + python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage # Cached version may be out of date + python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3.8 devscripts/make_lazy_extractors.py + python3.8 -m bundle.pyinstaller + + if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then + arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}" + chmod +x ./dist/yt-dlp_linux_${arch} + cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded + version="$(./dist/yt-dlp_linux_${arch} --version)" + ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + fi + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-linux_${{ matrix.architecture }} + path: | # run-on-arch-action designates armv7l as armv7 + repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} + compression-level: 0 + + macos: + needs: process + if: inputs.macos + runs-on: macos-11 + + steps: + - uses: actions/checkout@v4 + # NB: Building universal2 does not work with python from actions/setup-python + - name: Install Requirements + run: | + brew install coreutils + python3 devscripts/install_deps.py --user -o --include build + python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt + # We need to ignore wheels otherwise we break universal2 builds + python3 -m pip install -U --user --no-binary :all: -r requirements.txt + + - name: Prepare + run: | + python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + python3 -m bundle.pyinstaller --target-architecture universal2 --onedir + (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .) + python3 -m bundle.pyinstaller --target-architecture universal2 + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + chmod +x ./dist/yt-dlp_macos + cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded + version="$(./dist/yt-dlp_macos --version)" + ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + dist/yt-dlp_macos + dist/yt-dlp_macos.zip + compression-level: 0 + + macos_legacy: + needs: process + if: inputs.macos_legacy + runs-on: macos-latest + + steps: + - uses: actions/checkout@v4 + - name: Install Python + # We need the official Python, because the GA ones only support newer macOS versions + env: + PYTHON_VERSION: 3.10.5 + MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools + run: | + # Hack to get the latest patch version. Uncomment if needed + #brew install python@3.10 + #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 ) + curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg" + sudo installer -pkg python.pkg -target / + python3 --version + - name: Install Requirements + run: | + brew install coreutils + python3 devscripts/install_deps.py --user -o --include build + python3 devscripts/install_deps.py --user --include pyinstaller + + - name: Prepare + run: | + python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + python3 -m bundle.pyinstaller + mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + chmod +x ./dist/yt-dlp_macos_legacy + cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded + version="$(./dist/yt-dlp_macos_legacy --version)" + ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + dist/yt-dlp_macos_legacy + compression-level: 0 + + windows: + needs: process + if: inputs.windows + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: # 3.8 is used for Win7 support + python-version: "3.8" + - name: Install Requirements + run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds + python devscripts/install_deps.py -o --include build + python devscripts/install_deps.py --include py2exe + python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" + + - name: Prepare + run: | + python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python -m bundle.py2exe + Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe + python -m bundle.pyinstaller + python -m bundle.pyinstaller --onedir + Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + foreach ($name in @("yt-dlp","yt-dlp_min")) { + Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" + $version = & "./dist/${name}.exe" --version + & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04 + $downgraded_version = & "./dist/${name}_downgraded.exe" --version + if ($version -eq $downgraded_version) { + exit 1 + } + } + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + dist/yt-dlp.exe + dist/yt-dlp_min.exe + dist/yt-dlp_win.zip + compression-level: 0 + + windows32: + needs: process + if: inputs.windows32 + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.8" + architecture: "x86" + - name: Install Requirements + run: | + python devscripts/install_deps.py -o --include build + python devscripts/install_deps.py + python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" + + - name: Prepare + run: | + python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python -m bundle.pyinstaller + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + foreach ($name in @("yt-dlp_x86")) { + Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" + $version = & "./dist/${name}.exe" --version + & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04 + $downgraded_version = & "./dist/${name}_downgraded.exe" --version + if ($version -eq $downgraded_version) { + exit 1 + } + } + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + dist/yt-dlp_x86.exe + compression-level: 0 + + meta_files: + if: inputs.meta_files && always() && !cancelled() + needs: + - process + - unix + - linux_arm + - macos + - macos_legacy + - windows + - windows32 + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: artifact + pattern: build-bin-* + merge-multiple: true + + - name: Make SHA2-SUMS files + run: | + cd ./artifact/ + sha256sum * > ../SHA2-256SUMS + sha512sum * > ../SHA2-512SUMS + + - name: Make Update spec + run: | + cat >> _update_spec << EOF + # This file is used for regulating self-update + lock 2022.08.18.36 .+ Python 3\.6 + lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7 + lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) + lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6 + lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7 + lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) + lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7 + lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server) + lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7 + lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server) + EOF + + - name: Sign checksum files + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + if: env.GPG_SIGNING_KEY != '' + run: | + gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}" + for signfile in ./SHA*SUMS; do + gpg --batch --detach-sign "$signfile" + done + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-${{ github.job }} + path: | + _update_spec + SHA*SUMS* + compression-level: 0 + overwrite: true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..170a6ac --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,65 @@ +name: "CodeQL" + +on: + push: + branches: [ 'master', 'gh-pages', 'release' ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ 'master' ] + schedule: + - cron: '59 11 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # âšī¸ Command-line programs to run using the OS shell. + # đ See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml new file mode 100644 index 0000000..ba86306 --- /dev/null +++ b/.github/workflows/core.yml @@ -0,0 +1,61 @@ +name: Core Tests +on: + push: + paths: + - .github/** + - devscripts/** + - test/** + - yt_dlp/**.py + - '!yt_dlp/extractor/*.py' + - yt_dlp/extractor/__init__.py + - yt_dlp/extractor/common.py + - yt_dlp/extractor/extractors.py + pull_request: + paths: + - .github/** + - devscripts/** + - test/** + - yt_dlp/**.py + - '!yt_dlp/extractor/*.py' + - yt_dlp/extractor/__init__.py + - yt_dlp/extractor/common.py + - yt_dlp/extractor/extractors.py +permissions: + contents: read + +concurrency: + group: core-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + tests: + name: Core Tests + if: "!contains(github.event.head_commit.message, 'ci skip')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + # CPython 3.8 is in quick-test + python-version: ['3.9', '3.10', '3.11', '3.12', pypy-3.8, pypy-3.10] + include: + # atleast one of each CPython/PyPy tests must be in windows + - os: windows-latest + python-version: '3.8' + - os: windows-latest + python-version: '3.12' + - os: windows-latest + python-version: pypy-3.9 + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install test requirements + run: python3 ./devscripts/install_deps.py --include dev + - name: Run tests + continue-on-error: False + run: | + python3 -m yt_dlp -v || true # Print debug head + python3 ./devscripts/run_tests.py core diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml new file mode 100644 index 0000000..7256804 --- /dev/null +++ b/.github/workflows/download.yml @@ -0,0 +1,48 @@ +name: Download Tests +on: [push, pull_request] +permissions: + contents: read + +jobs: + quick: + name: Quick Download Tests + if: "contains(github.event.head_commit.message, 'ci run dl')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Install test requirements + run: python3 ./devscripts/install_deps.py --include dev + - name: Run tests + continue-on-error: true + run: python3 ./devscripts/run_tests.py download + + full: + name: Full Download Tests + if: "contains(github.event.head_commit.message, 'ci run dl all')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10] + include: + # atleast one of each CPython/PyPy tests must be in windows + - os: windows-latest + python-version: '3.8' + - os: windows-latest + python-version: pypy-3.9 + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install test requirements + run: python3 ./devscripts/install_deps.py --include dev + - name: Run tests + continue-on-error: true + run: python3 ./devscripts/run_tests.py download diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml new file mode 100644 index 0000000..3114e7b --- /dev/null +++ b/.github/workflows/quick-test.yml @@ -0,0 +1,35 @@ +name: Quick Test +on: [push, pull_request] +permissions: + contents: read + +jobs: + tests: + name: Core Test + if: "!contains(github.event.head_commit.message, 'ci skip all')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.8 + uses: actions/setup-python@v5 + with: + python-version: '3.8' + - name: Install test requirements + run: python3 ./devscripts/install_deps.py --include dev + - name: Run tests + run: | + python3 -m yt_dlp -v || true + python3 ./devscripts/run_tests.py core + flake8: + name: Linter + if: "!contains(github.event.head_commit.message, 'ci skip all')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - name: Install flake8 + run: python3 ./devscripts/install_deps.py -o --include dev + - name: Make lazy extractors + run: python3 ./devscripts/make_lazy_extractors.py + - name: Run flake8 + run: flake8 . diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml new file mode 100644 index 0000000..a845475 --- /dev/null +++ b/.github/workflows/release-master.yml @@ -0,0 +1,29 @@ +name: Release (master) +on: + push: + branches: + - master + paths: + - "yt_dlp/**.py" + - "!yt_dlp/version.py" + - "bundle/*.py" + - "pyproject.toml" + - "Makefile" + - ".github/workflows/build.yml" +concurrency: + group: release-master +permissions: + contents: read + +jobs: + release: + if: vars.BUILD_MASTER != '' + uses: ./.github/workflows/release.yml + with: + prerelease: true + source: master + permissions: + contents: write + packages: write + id-token: write # mandatory for trusted publishing + secrets: inherit diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml new file mode 100644 index 0000000..f459a3a --- /dev/null +++ b/.github/workflows/release-nightly.yml @@ -0,0 +1,42 @@ +name: Release (nightly) +on: + schedule: + - cron: '23 23 * * *' +permissions: + contents: read + +jobs: + check_nightly: + if: vars.BUILD_NIGHTLY != '' + runs-on: ubuntu-latest + outputs: + commit: ${{ steps.check_for_new_commits.outputs.commit }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check for new commits + id: check_for_new_commits + run: | + relevant_files=( + "yt_dlp/*.py" + ':!yt_dlp/version.py' + "bundle/*.py" + "pyproject.toml" + "Makefile" + ".github/workflows/build.yml" + ) + echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT" + + release: + needs: [check_nightly] + if: ${{ needs.check_nightly.outputs.commit }} + uses: ./.github/workflows/release.yml + with: + prerelease: true + source: nightly + permissions: + contents: write + packages: write + id-token: write # mandatory for trusted publishing + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..fd99cec --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,387 @@ +name: Release +on: + workflow_call: + inputs: + prerelease: + required: false + default: true + type: boolean + source: + required: false + default: '' + type: string + target: + required: false + default: '' + type: string + version: + required: false + default: '' + type: string + workflow_dispatch: + inputs: + source: + description: | + SOURCE of this release's updates: + channel, repo, tag, or channel/repo@tag + (default: <current_repo>) + required: false + default: '' + type: string + target: + description: | + TARGET to publish this release to: + channel, tag, or channel@tag + (default: <source> if writable else <current_repo>[@source_tag]) + required: false + default: '' + type: string + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + (default: auto-generated) + required: false + default: '' + type: string + prerelease: + description: Pre-release + default: false + type: boolean + +permissions: + contents: read + +jobs: + prepare: + permissions: + contents: write + runs-on: ubuntu-latest + outputs: + channel: ${{ steps.setup_variables.outputs.channel }} + version: ${{ steps.setup_variables.outputs.version }} + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + target_repo_token: ${{ steps.setup_variables.outputs.target_repo_token }} + target_tag: ${{ steps.setup_variables.outputs.target_tag }} + pypi_project: ${{ steps.setup_variables.outputs.pypi_project }} + pypi_suffix: ${{ steps.setup_variables.outputs.pypi_suffix }} + head_sha: ${{ steps.get_target.outputs.head_sha }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Process inputs + id: process_inputs + run: | + cat << EOF + ::group::Inputs + prerelease=${{ inputs.prerelease }} + source=${{ inputs.source }} + target=${{ inputs.target }} + version=${{ inputs.version }} + ::endgroup:: + EOF + IFS='@' read -r source_repo source_tag <<<"${{ inputs.source }}" + IFS='@' read -r target_repo target_tag <<<"${{ inputs.target }}" + cat << EOF >> "$GITHUB_OUTPUT" + source_repo=${source_repo} + source_tag=${source_tag} + target_repo=${target_repo} + target_tag=${target_tag} + EOF + + - name: Setup variables + id: setup_variables + env: + source_repo: ${{ steps.process_inputs.outputs.source_repo }} + source_tag: ${{ steps.process_inputs.outputs.source_tag }} + target_repo: ${{ steps.process_inputs.outputs.target_repo }} + target_tag: ${{ steps.process_inputs.outputs.target_tag }} + run: | + # unholy bash monstrosity (sincere apologies) + fallback_token () { + if ${{ !secrets.ARCHIVE_REPO_TOKEN }}; then + echo "::error::Repository access secret ${target_repo_token^^} not found" + exit 1 + fi + target_repo_token=ARCHIVE_REPO_TOKEN + return 0 + } + + source_is_channel=0 + [[ "${source_repo}" == 'stable' ]] && source_repo='yt-dlp/yt-dlp' + if [[ -z "${source_repo}" ]]; then + source_repo='${{ github.repository }}' + elif [[ '${{ vars[format('{0}_archive_repo', env.source_repo)] }}' ]]; then + source_is_channel=1 + source_channel='${{ vars[format('{0}_archive_repo', env.source_repo)] }}' + elif [[ -z "${source_tag}" && "${source_repo}" != */* ]]; then + source_tag="${source_repo}" + source_repo='${{ github.repository }}' + fi + resolved_source="${source_repo}" + if [[ "${source_tag}" ]]; then + resolved_source="${resolved_source}@${source_tag}" + elif [[ "${source_repo}" == 'yt-dlp/yt-dlp' ]]; then + resolved_source='stable' + fi + + revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}" + version="$( + python devscripts/update-version.py \ + -c "${resolved_source}" -r "${{ github.repository }}" ${{ inputs.version || '$revision' }} | \ + grep -Po "version=\K\d+\.\d+\.\d+(\.\d+)?")" + + if [[ "${target_repo}" ]]; then + if [[ -z "${target_tag}" ]]; then + if [[ '${{ vars[format('{0}_archive_repo', env.target_repo)] }}' ]]; then + target_tag="${source_tag:-${version}}" + else + target_tag="${target_repo}" + target_repo='${{ github.repository }}' + fi + fi + if [[ "${target_repo}" != '${{ github.repository}}' ]]; then + target_repo='${{ vars[format('{0}_archive_repo', env.target_repo)] }}' + target_repo_token='${{ env.target_repo }}_archive_repo_token' + ${{ !!secrets[format('{0}_archive_repo_token', env.target_repo)] }} || fallback_token + pypi_project='${{ vars[format('{0}_pypi_project', env.target_repo)] }}' + pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.target_repo)] }}' + fi + else + target_tag="${source_tag:-${version}}" + if ((source_is_channel)); then + target_repo="${source_channel}" + target_repo_token='${{ env.source_repo }}_archive_repo_token' + ${{ !!secrets[format('{0}_archive_repo_token', env.source_repo)] }} || fallback_token + pypi_project='${{ vars[format('{0}_pypi_project', env.source_repo)] }}' + pypi_suffix='${{ vars[format('{0}_pypi_suffix', env.source_repo)] }}' + else + target_repo='${{ github.repository }}' + fi + fi + + if [[ "${target_repo}" == '${{ github.repository }}' ]] && ${{ !inputs.prerelease }}; then + pypi_project='${{ vars.PYPI_PROJECT }}' + fi + + echo "::group::Output variables" + cat << EOF | tee -a "$GITHUB_OUTPUT" + channel=${resolved_source} + version=${version} + target_repo=${target_repo} + target_repo_token=${target_repo_token} + target_tag=${target_tag} + pypi_project=${pypi_project} + pypi_suffix=${pypi_suffix} + EOF + echo "::endgroup::" + + - name: Update documentation + env: + version: ${{ steps.setup_variables.outputs.version }} + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + !inputs.prerelease && env.target_repo == github.repository + run: | + make doc + sed '/### /Q' Changelog.md >> ./CHANGELOG + echo '### ${{ env.version }}' >> ./CHANGELOG + python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG + echo >> ./CHANGELOG + grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG + cat ./CHANGELOG > Changelog.md + + - name: Push to release + id: push_release + env: + version: ${{ steps.setup_variables.outputs.version }} + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + !inputs.prerelease && env.target_repo == github.repository + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -u + git commit -m "Release ${{ env.version }}" \ + -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl" + git push origin --force ${{ github.event.ref }}:release + + - name: Get target commitish + id: get_target + run: | + echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Update master + env: + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease && env.target_repo == github.repository + run: git push origin ${{ github.event.ref }} + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + channel: ${{ needs.prepare.outputs.channel }} + origin: ${{ needs.prepare.outputs.target_repo }} + permissions: + contents: read + packages: write # For package cache + secrets: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + + publish_pypi: + needs: [prepare, build] + if: ${{ needs.prepare.outputs.pypi_project }} + runs-on: ubuntu-latest + permissions: + id-token: write # mandatory for trusted publishing + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + sudo apt -y install pandoc man + python devscripts/install_deps.py -o --include build + + - name: Prepare + env: + version: ${{ needs.prepare.outputs.version }} + suffix: ${{ needs.prepare.outputs.pypi_suffix }} + channel: ${{ needs.prepare.outputs.channel }} + target_repo: ${{ needs.prepare.outputs.target_repo }} + pypi_project: ${{ needs.prepare.outputs.pypi_project }} + run: | + python devscripts/update-version.py -c "${{ env.channel }}" -r "${{ env.target_repo }}" -s "${{ env.suffix }}" "${{ env.version }}" + python devscripts/make_lazy_extractors.py + sed -i -E '0,/(name = ")[^"]+(")/s//\1${{ env.pypi_project }}\2/' pyproject.toml + + - name: Build + run: | + rm -rf dist/* + make pypi-files + printf '%s\n\n' \ + 'Official repository: <https://github.com/yt-dlp/yt-dlp>' \ + '**PS**: Some links in this document will not work since this is a copy of the README.md from Github' > ./README.md.new + cat ./README.md >> ./README.md.new && mv -f ./README.md.new ./README.md + python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update" + make clean-cache + python -m build --no-isolation . + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true + + publish: + needs: [prepare, build] + permissions: + contents: write + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v4 + with: + path: artifact + pattern: build-* + merge-multiple: true + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Generate release notes + env: + head_sha: ${{ needs.prepare.outputs.head_sha }} + target_repo: ${{ needs.prepare.outputs.target_repo }} + target_tag: ${{ needs.prepare.outputs.target_tag }} + run: | + printf '%s' \ + '[]' \ + '(https://github.com/${{ github.repository }}#installation "Installation instructions") ' \ + '[]' \ + '(https://discord.gg/H5MNcFW63r "Discord") ' \ + '[]' \ + '(https://github.com/yt-dlp/yt-dlp/blob/master/Collaborators.md#collaborators "Donate") ' \ + '[]' \ + '(https://github.com/${{ github.repository }}' \ + '${{ env.target_repo == github.repository && format('/tree/{0}', env.target_tag) || '' }}#readme "Documentation") ' \ + ${{ env.target_repo == 'yt-dlp/yt-dlp' && '\ + "[]" \ + "(https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest \"Nightly builds\") " \ + "[]" \ + "(https://github.com/yt-dlp/yt-dlp-master-builds/releases/latest \"Master builds\")"' || '' }} > ./RELEASE_NOTES + printf '\n\n' >> ./RELEASE_NOTES + cat >> ./RELEASE_NOTES << EOF + #### A description of the various files are in the [README](https://github.com/${{ github.repository }}#release-files) + --- + $(python ./devscripts/make_changelog.py -vv --collapsible) + EOF + printf '%s\n\n' '**This is a pre-release build**' >> ./PRERELEASE_NOTES + cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES + printf '%s\n\n' 'Generated from: https://github.com/${{ github.repository }}/commit/${{ env.head_sha }}' >> ./ARCHIVE_NOTES + cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES + + - name: Publish to archive repo + env: + GH_TOKEN: ${{ secrets[needs.prepare.outputs.target_repo_token] }} + GH_REPO: ${{ needs.prepare.outputs.target_repo }} + version: ${{ needs.prepare.outputs.version }} + channel: ${{ needs.prepare.outputs.channel }} + if: | + inputs.prerelease && env.GH_TOKEN != '' && env.GH_REPO != '' && env.GH_REPO != github.repository + run: | + title="${{ startswith(env.GH_REPO, 'yt-dlp/') && 'yt-dlp ' || '' }}${{ env.channel }}" + gh release create \ + --notes-file ARCHIVE_NOTES \ + --title "${title} ${{ env.version }}" \ + ${{ env.version }} \ + artifact/* + + - name: Prune old release + env: + GH_TOKEN: ${{ github.token }} + version: ${{ needs.prepare.outputs.version }} + target_repo: ${{ needs.prepare.outputs.target_repo }} + target_tag: ${{ needs.prepare.outputs.target_tag }} + if: | + env.target_repo == github.repository && env.target_tag != env.version + run: | + gh release delete --yes --cleanup-tag "${{ env.target_tag }}" || true + git tag --delete "${{ env.target_tag }}" || true + sleep 5 # Enough time to cover deletion race condition + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + version: ${{ needs.prepare.outputs.version }} + target_repo: ${{ needs.prepare.outputs.target_repo }} + target_tag: ${{ needs.prepare.outputs.target_tag }} + head_sha: ${{ needs.prepare.outputs.head_sha }} + if: | + env.target_repo == github.repository + run: | + title="${{ github.repository == 'yt-dlp/yt-dlp' && 'yt-dlp ' || '' }}" + title+="${{ env.target_tag != env.version && format('{0} ', env.target_tag) || '' }}" + gh release create \ + --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \ + --target ${{ env.head_sha }} \ + --title "${title}${{ env.version }}" \ + ${{ inputs.prerelease && '--prerelease' || '' }} \ + ${{ env.target_tag }} \ + artifact/* |