diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 17:37:42 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 17:37:42 +0000 |
commit | c7bab7c39fd51c0812f70020172766303191bc01 (patch) | |
tree | 56c05fbdd4fc47409d48ba318a4b621a7b0d299a /.github/workflows | |
parent | Initial commit. (diff) | |
download | yt-dlp-c7bab7c39fd51c0812f70020172766303191bc01.tar.xz yt-dlp-c7bab7c39fd51c0812f70020172766303191bc01.zip |
Adding upstream version 2023.03.04.upstream/2023.03.04upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '.github/workflows')
-rw-r--r-- | .github/workflows/build.yml | 356 | ||||
-rw-r--r-- | .github/workflows/core.yml | 38 | ||||
-rw-r--r-- | .github/workflows/download.yml | 51 | ||||
-rw-r--r-- | .github/workflows/publish.yml | 81 | ||||
-rw-r--r-- | .github/workflows/quick-test.yml | 35 | ||||
-rw-r--r-- | .github/workflows/release-nightly.yml | 51 | ||||
-rw-r--r-- | .github/workflows/release.yml | 129 |
7 files changed, 741 insertions, 0 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..93668a7 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,356 @@ +name: Build Artifacts +on: + workflow_call: + inputs: + version: + required: true + type: string + channel: + required: false + default: stable + type: string + unix: + default: true + type: boolean + linux_arm: + default: true + type: boolean + macos: + default: true + type: boolean + macos_legacy: + default: true + type: boolean + windows: + default: true + type: boolean + windows32: + default: true + type: boolean + meta_files: + default: true + type: boolean + secrets: + GPG_SIGNING_KEY: + required: false + + workflow_dispatch: + inputs: + version: + description: Version tag (YYYY.MM.DD[.REV]) + required: true + type: string + channel: + description: Update channel (stable/nightly) + required: true + default: stable + type: string + unix: + description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip + default: true + type: boolean + linux_arm: + description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l + default: true + type: boolean + macos: + description: yt-dlp_macos, yt-dlp_macos.zip + default: true + type: boolean + macos_legacy: + description: yt-dlp_macos_legacy + default: true + type: boolean + windows: + description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip + default: true + type: boolean + windows32: + description: yt-dlp_x86.exe + default: true + type: boolean + meta_files: + description: SHA2-256SUMS, SHA2-512SUMS, _update_spec + default: true + type: boolean + +permissions: + contents: read + +jobs: + unix: + if: inputs.unix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + use-mamba: true + channels: conda-forge + auto-update-conda: true + activate-environment: "" + auto-activate-base: false + - name: Install Requirements + run: | + sudo apt-get -y install zip pandoc man sed + python -m pip install -U pip setuptools wheel + python -m pip install -U Pyinstaller -r requirements.txt + reqs=$(mktemp) + cat > $reqs << EOF + python=3.10.* + pyinstaller + cffi + brotli-python + EOF + sed '/^brotli.*/d' requirements.txt >> $reqs + mamba create -n build --file $reqs + + - name: Prepare + run: | + python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + python devscripts/make_lazy_extractors.py + - name: Build Unix platform-independent binary + run: | + make all tar + - name: Build Unix standalone binary + shell: bash -l {0} + run: | + unset LD_LIBRARY_PATH # Harmful; set by setup-python + conda activate build + python pyinst.py --onedir + (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) + python pyinst.py + mv ./dist/yt-dlp_linux ./yt-dlp_linux + mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + yt-dlp + yt-dlp.tar.gz + yt-dlp_linux + yt-dlp_linux.zip + + linux_arm: + if: inputs.linux_arm + permissions: + contents: read + packages: write # for creating cache + runs-on: ubuntu-latest + strategy: + matrix: + architecture: + - armv7 + - aarch64 + + steps: + - uses: actions/checkout@v3 + with: + path: ./repo + - name: Virtualized Install, Prepare & Build + uses: yt-dlp/run-on-arch-action@v2 + with: + # Ref: https://github.com/uraimo/run-on-arch-action/issues/55 + env: | + GITHUB_WORKFLOW: build + githubToken: ${{ github.token }} # To cache image + arch: ${{ matrix.architecture }} + distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS + dockerRunArgs: --volume "${PWD}/repo:/repo" + install: | # Installing Python 3.10 from the Deadsnakes repo raises errors + apt update + apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip + python3.8 -m pip install -U pip setuptools wheel + # Cannot access requirements.txt from the repo directory at this stage + python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi + + run: | + cd repo + python3.8 -m pip install -U Pyinstaller -r requirements.txt # Cached version may be out of date + python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + python3.8 devscripts/make_lazy_extractors.py + python3.8 pyinst.py + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | # run-on-arch-action designates armv7l as armv7 + repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} + + macos: + if: inputs.macos + runs-on: macos-11 + + steps: + - uses: actions/checkout@v3 + # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used + - name: Install Requirements + run: | + brew install coreutils + /usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt + + - name: Prepare + run: | + /usr/bin/python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + /usr/bin/python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir + (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .) + /usr/bin/python3 pyinst.py --target-architecture universal2 + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_macos + dist/yt-dlp_macos.zip + + macos_legacy: + if: inputs.macos_legacy + runs-on: macos-latest + + steps: + - uses: actions/checkout@v3 + - name: Install Python + # We need the official Python, because the GA ones only support newer macOS versions + env: + PYTHON_VERSION: 3.10.5 + MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools + run: | + # Hack to get the latest patch version. Uncomment if needed + #brew install python@3.10 + #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 ) + curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg" + sudo installer -pkg python.pkg -target / + python3 --version + - name: Install Requirements + run: | + brew install coreutils + python3 -m pip install -U --user pip Pyinstaller -r requirements.txt + + - name: Prepare + run: | + python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + python3 pyinst.py + mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_macos_legacy + + windows: + if: inputs.windows + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: # 3.8 is used for Win7 support + python-version: "3.8" + - name: Install Requirements + run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds + python -m pip install -U pip setuptools wheel py2exe + pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python setup.py py2exe + Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe + python pyinst.py + python pyinst.py --onedir + Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp.exe + dist/yt-dlp_min.exe + dist/yt-dlp_win.zip + + windows32: + if: inputs.windows32 + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 + python-version: "3.7" + architecture: "x86" + - name: Install Requirements + run: | + python -m pip install -U pip setuptools wheel + pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }} + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python pyinst.py + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_x86.exe + + meta_files: + if: inputs.meta_files && always() + needs: + - unix + - linux_arm + - macos + - macos_legacy + - windows + - windows32 + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v3 + + - name: Make SHA2-SUMS files + run: | + cd ./artifact/ + sha256sum * > ../SHA2-256SUMS + sha512sum * > ../SHA2-512SUMS + + - name: Make Update spec + run: | + cat >> _update_spec << EOF + # This file is used for regulating self-update + lock 2022.08.18.36 .+ Python 3.6 + EOF + + - name: Sign checksum files + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + if: env.GPG_SIGNING_KEY != '' + run: | + gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}" + for signfile in ./SHA*SUMS; do + gpg --batch --detach-sign "$signfile" + done + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + SHA*SUMS* + _update_spec diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml new file mode 100644 index 0000000..dead444 --- /dev/null +++ b/.github/workflows/core.yml @@ -0,0 +1,38 @@ +name: Core Tests +on: [push, pull_request] +permissions: + contents: read + +jobs: + tests: + name: Core Tests + if: "!contains(github.event.head_commit.message, 'ci skip')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + # CPython 3.11 is in quick-test + python-version: ['3.8', '3.9', '3.10', pypy-3.7, pypy-3.8] + run-tests-ext: [sh] + include: + # atleast one of each CPython/PyPy tests must be in windows + - os: windows-latest + python-version: '3.7' + run-tests-ext: bat + - os: windows-latest + python-version: pypy-3.9 + run-tests-ext: bat + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install pytest + run: pip install pytest + - name: Run tests + continue-on-error: False + run: | + python3 -m yt_dlp -v || true # Print debug head + ./devscripts/run_tests.${{ matrix.run-tests-ext }} core diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml new file mode 100644 index 0000000..2b2387d --- /dev/null +++ b/.github/workflows/download.yml @@ -0,0 +1,51 @@ +name: Download Tests +on: [push, pull_request] +permissions: + contents: read + +jobs: + quick: + name: Quick Download Tests + if: "contains(github.event.head_commit.message, 'ci run dl')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install test requirements + run: pip install pytest + - name: Run tests + continue-on-error: true + run: ./devscripts/run_tests.sh download + + full: + name: Full Download Tests + if: "contains(github.event.head_commit.message, 'ci run dl all')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + python-version: ['3.7', '3.10', 3.11-dev, pypy-3.7, pypy-3.8] + run-tests-ext: [sh] + include: + # atleast one of each CPython/PyPy tests must be in windows + - os: windows-latest + python-version: '3.8' + run-tests-ext: bat + - os: windows-latest + python-version: pypy-3.9 + run-tests-ext: bat + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install pytest + run: pip install pytest + - name: Run tests + continue-on-error: true + run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..8a1bd9a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,81 @@ +name: Publish +on: + workflow_call: + inputs: + nightly: + default: false + required: false + type: boolean + version: + required: true + type: string + target_commitish: + required: true + type: string + secrets: + ARCHIVE_REPO_TOKEN: + required: false + +permissions: + contents: write + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Generate release notes + run: | + cat >> ./RELEASE_NOTES << EOF + #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files) + --- + <details><summary><h3>Changelog</h3></summary> + $(python ./devscripts/make_changelog.py -vv) + </details> + EOF + echo "**This is an automated nightly pre-release build**" >> ./PRERELEASE_NOTES + cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES + echo "Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}" >> ./ARCHIVE_NOTES + cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES + + - name: Archive nightly release + env: + GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} + GH_REPO: ${{ vars.ARCHIVE_REPO }} + if: | + inputs.nightly && env.GH_TOKEN != '' && env.GH_REPO != '' + run: | + gh release create \ + --notes-file ARCHIVE_NOTES \ + --title "yt-dlp nightly ${{ inputs.version }}" \ + ${{ inputs.version }} \ + artifact/* + + - name: Prune old nightly release + if: inputs.nightly && !vars.ARCHIVE_REPO + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release delete --yes --cleanup-tag "nightly" || true + git tag --delete "nightly" || true + sleep 5 # Enough time to cover deletion race condition + + - name: Publish release${{ inputs.nightly && ' (nightly)' || '' }} + env: + GH_TOKEN: ${{ github.token }} + if: (inputs.nightly && !vars.ARCHIVE_REPO) || !inputs.nightly + run: | + gh release create \ + --notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \ + --target ${{ inputs.target_commitish }} \ + --title "yt-dlp ${{ inputs.nightly && 'nightly ' || '' }}${{ inputs.version }}" \ + ${{ inputs.nightly && '--prerelease "nightly"' || inputs.version }} \ + artifact/* diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml new file mode 100644 index 0000000..930e581 --- /dev/null +++ b/.github/workflows/quick-test.yml @@ -0,0 +1,35 @@ +name: Quick Test +on: [push, pull_request] +permissions: + contents: read + +jobs: + tests: + name: Core Test + if: "!contains(github.event.head_commit.message, 'ci skip all')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install test requirements + run: pip install pytest pycryptodomex + - name: Run tests + run: | + python3 -m yt_dlp -v || true + ./devscripts/run_tests.sh core + flake8: + name: Linter + if: "!contains(github.event.head_commit.message, 'ci skip all')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Install flake8 + run: pip install flake8 + - name: Make lazy extractors + run: python devscripts/make_lazy_extractors.py + - name: Run flake8 + run: flake8 . diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml new file mode 100644 index 0000000..d4f01ab --- /dev/null +++ b/.github/workflows/release-nightly.yml @@ -0,0 +1,51 @@ +name: Release (nightly) +on: + push: + branches: + - master + paths: + - "yt_dlp/**.py" + - "!yt_dlp/version.py" +concurrency: + group: release-nightly + cancel-in-progress: true +permissions: + contents: read + +jobs: + prepare: + if: vars.BUILD_NIGHTLY != '' + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.version }} + + steps: + - uses: actions/checkout@v3 + - name: Get version + id: get_version + run: | + python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT" + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + channel: nightly + permissions: + contents: read + packages: write # For package cache + secrets: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + + publish: + needs: [prepare, build] + uses: ./.github/workflows/publish.yml + secrets: + ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} + permissions: + contents: write + with: + nightly: true + version: ${{ needs.prepare.outputs.version }} + target_commitish: ${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e07fc0c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,129 @@ +name: Release +on: workflow_dispatch +permissions: + contents: read + +jobs: + prepare: + permissions: + contents: write + runs-on: ubuntu-latest + outputs: + version: ${{ steps.update_version.outputs.version }} + head_sha: ${{ steps.push_release.outputs.head_sha }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Update version + id: update_version + run: | + python devscripts/update-version.py ${{ vars.PUSH_VERSION_COMMIT == '' && '"$(date -u +"%H%M%S")"' || '' }} | \ + grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT" + + - name: Update documentation + run: | + make doc + sed '/### /Q' Changelog.md >> ./CHANGELOG + echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG + python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG + echo >> ./CHANGELOG + grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG + cat ./CHANGELOG > Changelog.md + + - name: Push to release + id: push_release + run: | + git config --global user.name github-actions + git config --global user.email github-actions@example.com + git add -u + git commit -m "Release ${{ steps.update_version.outputs.version }}" \ + -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl" + git push origin --force ${{ github.event.ref }}:release + echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Update master + if: vars.PUSH_VERSION_COMMIT != '' + run: git push origin ${{ github.event.ref }} + + publish_pypi_homebrew: + needs: prepare + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Requirements + run: | + sudo apt-get -y install pandoc man + python -m pip install -U pip setuptools wheel twine + python -m pip install -U -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py ${{ needs.prepare.outputs.version }} + python devscripts/make_lazy_extractors.py + + - name: Build and publish on PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + if: env.TWINE_PASSWORD != '' + run: | + rm -rf dist/* + make pypi-files + python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update" + python setup.py sdist bdist_wheel + twine upload dist/* + + - name: Checkout Homebrew repository + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' + uses: actions/checkout@v3 + with: + repository: yt-dlp/homebrew-taps + path: taps + ssh-key: ${{ secrets.BREW_TOKEN }} + + - name: Update Homebrew Formulae + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != '' + run: | + python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}" + git -C taps/ config user.name github-actions + git -C taps/ config user.email github-actions@example.com + git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}' + git -C taps/ push + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + permissions: + contents: read + packages: write # For package cache + secrets: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + + publish: + needs: [prepare, build] + uses: ./.github/workflows/publish.yml + permissions: + contents: write + with: + version: ${{ needs.prepare.outputs.version }} + target_commitish: ${{ needs.prepare.outputs.head_sha }} |