diff options
Diffstat (limited to '')
| -rw-r--r-- | devscripts/changelog_override.json | 21 | ||||
| -rwxr-xr-x | devscripts/install_deps.py | 38 | ||||
| -rw-r--r-- | devscripts/make_changelog.py | 51 | ||||
| -rw-r--r-- | devscripts/prepare_manpage.py | 29 | ||||
| -rwxr-xr-x | devscripts/tomlparse.py | 10 | ||||
| -rw-r--r-- | devscripts/update-version.py | 4 | ||||
| -rwxr-xr-x | devscripts/update_changelog.py | 26 |
7 files changed, 128 insertions, 51 deletions
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index 2a34ad0..046060c 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -126,5 +126,26 @@ "when": "4ce57d3b873c2887814cbec03d029533e82f7db5", "short": "[ie] Support multi-period MPD streams (#6654)", "authors": ["alard", "pukkandan"] + }, + { + "action": "change", + "when": "aa7e9ae4f48276bd5d0173966c77db9484f65a0a", + "short": "[ie/xvideos] Support new URL format (#9502)", + "authors": ["sta1us"] + }, + { + "action": "remove", + "when": "22e4dfacb61f62dfbb3eb41b31c7b69ba1059b80" + }, + { + "action": "change", + "when": "e3a3ed8a981d9395c4859b6ef56cd02bc3148db2", + "short": "[cleanup:ie] No `from` stdlib imports in extractors", + "authors": ["pukkandan"] + }, + { + "action": "add", + "when": "9590cc6b4768e190183d7d071a6c78170889116a", + "short": "[priority] Security: [[CVE-2024-22423](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-22423)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-hjq6-52gw-2g7p)\n - The shell escape function now properly escapes `%`, `\\` and `\\n`.\n - `utils.Popen` has been patched accordingly." } ] diff --git a/devscripts/install_deps.py b/devscripts/install_deps.py index 889d9ab..d33fc63 100755 --- a/devscripts/install_deps.py +++ b/devscripts/install_deps.py @@ -10,6 +10,8 @@ import argparse import re import subprocess +from pathlib import Path + from devscripts.tomlparse import parse_toml from devscripts.utils import read_file @@ -17,17 +19,23 @@ from devscripts.utils import read_file def parse_args(): parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp') parser.add_argument( - 'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)') + 'input', nargs='?', metavar='TOMLFILE', default=Path(__file__).parent.parent / 'pyproject.toml', + help='input file (default: %(default)s)') parser.add_argument( - '-e', '--exclude', metavar='DEPENDENCY', action='append', help='Exclude a dependency') + '-e', '--exclude', metavar='DEPENDENCY', action='append', + help='exclude a dependency') parser.add_argument( - '-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group') + '-i', '--include', metavar='GROUP', action='append', + help='include an optional dependency group') parser.add_argument( - '-o', '--only-optional', action='store_true', help='Only install optional dependencies') + '-o', '--only-optional', action='store_true', + help='only install optional dependencies') parser.add_argument( - '-p', '--print', action='store_true', help='Only print a requirements.txt to stdout') + '-p', '--print', action='store_true', + help='only print requirements to stdout') parser.add_argument( - '-u', '--user', action='store_true', help='Install with pip as --user') + '-u', '--user', action='store_true', + help='install with pip as --user') return parser.parse_args() @@ -37,24 +45,16 @@ def main(): optional_groups = project_table['optional-dependencies'] excludes = args.exclude or [] - deps = [] + targets = [] if not args.only_optional: # `-o` should exclude 'dependencies' and the 'default' group - deps.extend(project_table['dependencies']) + targets.extend(project_table['dependencies']) if 'default' not in excludes: # `--exclude default` should exclude entire 'default' group - deps.extend(optional_groups['default']) - - def name(dependency): - return re.match(r'[\w-]+', dependency)[0].lower() - - target_map = {name(dep): dep for dep in deps} + targets.extend(optional_groups['default']) for include in filter(None, map(optional_groups.get, args.include or [])): - target_map.update(zip(map(name, include), include)) - - for exclude in map(name, excludes): - target_map.pop(exclude, None) + targets.extend(include) - targets = list(target_map.values()) + targets = [t for t in targets if re.match(r'[\w-]+', t).group(0).lower() not in excludes] if args.print: for target in targets: diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py index faab5fa..8e199e7 100644 --- a/devscripts/make_changelog.py +++ b/devscripts/make_changelog.py @@ -445,7 +445,32 @@ def get_new_contributors(contributors_path, commits): return sorted(new_contributors, key=str.casefold) -if __name__ == '__main__': +def create_changelog(args): + logging.basicConfig( + datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', + level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) + + commits = CommitRange(None, args.commitish, args.default_author) + + if not args.no_override: + if args.override_path.exists(): + overrides = json.loads(read_file(args.override_path)) + commits.apply_overrides(overrides) + else: + logger.warning(f'File {args.override_path.as_posix()} does not exist') + + logger.info(f'Loaded {len(commits)} commits') + + new_contributors = get_new_contributors(args.contributors_path, commits) + if new_contributors: + if args.contributors: + write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a') + logger.info(f'New contributors: {", ".join(new_contributors)}') + + return Changelog(commits.groups(), args.repo, args.collapsible) + + +def create_parser(): import argparse parser = argparse.ArgumentParser( @@ -477,27 +502,9 @@ if __name__ == '__main__': parser.add_argument( '--collapsible', action='store_true', help='make changelog collapsible (default: %(default)s)') - args = parser.parse_args() - - logging.basicConfig( - datefmt='%Y-%m-%d %H-%M-%S', format='{asctime} | {levelname:<8} | {message}', - level=logging.WARNING - 10 * args.verbosity, style='{', stream=sys.stderr) - - commits = CommitRange(None, args.commitish, args.default_author) - - if not args.no_override: - if args.override_path.exists(): - overrides = json.loads(read_file(args.override_path)) - commits.apply_overrides(overrides) - else: - logger.warning(f'File {args.override_path.as_posix()} does not exist') - logger.info(f'Loaded {len(commits)} commits') + return parser - new_contributors = get_new_contributors(args.contributors_path, commits) - if new_contributors: - if args.contributors: - write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a') - logger.info(f'New contributors: {", ".join(new_contributors)}') - print(Changelog(commits.groups(), args.repo, args.collapsible)) +if __name__ == '__main__': + print(create_changelog(create_parser().parse_args())) diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 9b12e71..47188e9 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -24,7 +24,7 @@ PREFIX = r'''%yt-dlp(1) # NAME -yt\-dlp \- A youtube-dl fork with additional features and patches +yt\-dlp \- A feature\-rich command\-line audio/video downloader # SYNOPSIS @@ -43,6 +43,27 @@ def filter_excluded_sections(readme): '', readme) +def _convert_code_blocks(readme): + current_code_block = None + + for line in readme.splitlines(True): + if current_code_block: + if line == current_code_block: + current_code_block = None + yield '\n' + else: + yield f' {line}' + elif line.startswith('```'): + current_code_block = line.count('`') * '`' + '\n' + yield '\n' + else: + yield line + + +def convert_code_blocks(readme): + return ''.join(_convert_code_blocks(readme)) + + def move_sections(readme): MOVE_TAG_TEMPLATE = '<!-- MANPAGE: MOVE "%s" SECTION HERE -->' sections = re.findall(r'(?m)^%s$' % ( @@ -65,8 +86,10 @@ def move_sections(readme): def filter_options(readme): section = re.search(r'(?sm)^# USAGE AND OPTIONS\n.+?(?=^# )', readme).group(0) + section_new = section.replace('*', R'\*') + options = '# OPTIONS\n' - for line in section.split('\n')[1:]: + for line in section_new.split('\n')[1:]: mobj = re.fullmatch(r'''(?x) \s{4}(?P<opt>-(?:,\s|[^\s])+) (?:\s(?P<meta>(?:[^\s]|\s(?!\s))+))? @@ -86,7 +109,7 @@ def filter_options(readme): return readme.replace(section, options, 1) -TRANSFORM = compose_functions(filter_excluded_sections, move_sections, filter_options) +TRANSFORM = compose_functions(filter_excluded_sections, convert_code_blocks, move_sections, filter_options) def main(): diff --git a/devscripts/tomlparse.py b/devscripts/tomlparse.py index 85ac4ee..ac9ea31 100755 --- a/devscripts/tomlparse.py +++ b/devscripts/tomlparse.py @@ -11,7 +11,7 @@ IMPORTANT: INVALID FILES OR MULTILINE STRINGS ARE NOT SUPPORTED! from __future__ import annotations -import datetime +import datetime as dt import json import re @@ -115,9 +115,9 @@ def parse_value(data: str, index: int): for func in [ int, float, - datetime.time.fromisoformat, - datetime.date.fromisoformat, - datetime.datetime.fromisoformat, + dt.time.fromisoformat, + dt.date.fromisoformat, + dt.datetime.fromisoformat, {'true': True, 'false': False}.get, ]: try: @@ -179,7 +179,7 @@ def main(): data = file.read() def default(obj): - if isinstance(obj, (datetime.date, datetime.time, datetime.datetime)): + if isinstance(obj, (dt.date, dt.time, dt.datetime)): return obj.isoformat() print(json.dumps(parse_toml(data), default=default)) diff --git a/devscripts/update-version.py b/devscripts/update-version.py index da54a6a..07a0717 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -9,15 +9,15 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import argparse import contextlib +import datetime as dt import sys -from datetime import datetime, timezone from devscripts.utils import read_version, run_process, write_file def get_new_version(version, revision): if not version: - version = datetime.now(timezone.utc).strftime('%Y.%m.%d') + version = dt.datetime.now(dt.timezone.utc).strftime('%Y.%m.%d') if revision: assert revision.isdecimal(), 'Revision must be a number' diff --git a/devscripts/update_changelog.py b/devscripts/update_changelog.py new file mode 100755 index 0000000..36b9a8e --- /dev/null +++ b/devscripts/update_changelog.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from pathlib import Path + +from devscripts.make_changelog import create_changelog, create_parser +from devscripts.utils import read_file, read_version, write_file + +# Always run after devscripts/update-version.py, and run before `make doc|pypi-files|tar|all` + +if __name__ == '__main__': + parser = create_parser() + parser.description = 'Update an existing changelog file with an entry for a new release' + parser.add_argument( + '--changelog-path', type=Path, default=Path(__file__).parent.parent / 'Changelog.md', + help='path to the Changelog file') + args = parser.parse_args() + new_entry = create_changelog(args) + + header, sep, changelog = read_file(args.changelog_path).partition('\n### ') + write_file(args.changelog_path, f'{header}{sep}{read_version()}\n{new_entry}\n{sep}{changelog}') |