From 24684db1d6b60add93efd6e78a1a6474f7e9e99a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 30 May 2024 05:10:54 +0200 Subject: Merging upstream version 2024.05.27. Signed-off-by: Daniel Baumann --- CONTRIBUTORS | 1 + Changelog.md | 11 +++++++++++ README.md | 3 +++ yt_dlp/extractor/common.py | 2 +- yt_dlp/extractor/peertube.py | 8 +++++++- yt_dlp/extractor/piksel.py | 19 +++++++++++-------- yt_dlp/extractor/tiktok.py | 8 ++++++-- yt_dlp/options.py | 3 ++- yt_dlp/version.py | 6 +++--- 9 files changed, 45 insertions(+), 16 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index b2a476b..e0d1668 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -630,3 +630,4 @@ TuxCoder voidful vtexier WyohKnott +trueauracoral diff --git a/Changelog.md b/Changelog.md index 0d27f1a..2673302 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,17 @@ # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master --> +### 2024.05.27 + +#### Extractor changes +- [Fix parsing of base URL in SMIL manifest](https://github.com/yt-dlp/yt-dlp/commit/26603d0b34898818992bee4598e0607c07059511) ([#9225](https://github.com/yt-dlp/yt-dlp/issues/9225)) by [seproDev](https://github.com/seproDev) +- **peertube**: [Support livestreams](https://github.com/yt-dlp/yt-dlp/commit/12b248ce60be1aa1362edd839d915bba70dbee4b) ([#10044](https://github.com/yt-dlp/yt-dlp/issues/10044)) by [bashonly](https://github.com/bashonly), [trueauracoral](https://github.com/trueauracoral) +- **piksel**: [Update domain](https://github.com/yt-dlp/yt-dlp/commit/ae2194e1dd4a99d32eb3cab7c48a0ff03101ef3b) ([#9223](https://github.com/yt-dlp/yt-dlp/issues/9223)) by [seproDev](https://github.com/seproDev) +- **tiktok**: user: [Fix extraction loop](https://github.com/yt-dlp/yt-dlp/commit/c53c2e40fde8f2e15c7c62f8ca1a5d9e90ddc079) ([#10035](https://github.com/yt-dlp/yt-dlp/issues/10035)) by [bashonly](https://github.com/bashonly) + +#### Misc. changes +- **cleanup**: Miscellaneous: [5e3e19c](https://github.com/yt-dlp/yt-dlp/commit/5e3e19c93c52830da98d9d1ed84ea7a559efefbd) by [bashonly](https://github.com/bashonly) + ### 2024.05.26 #### Core changes diff --git a/README.md b/README.md index e757567..e8cd6d3 100644 --- a/README.md +++ b/README.md @@ -401,6 +401,9 @@ If you fork the project on GitHub, you can run your fork's [build workflow](.git --impersonate CLIENT[:OS] Client to impersonate for requests. E.g. chrome, chrome-110, chrome:windows-10. Pass --impersonate="" to impersonate any client. + Note that forcing impersonation for all + requests may have a detrimental impact on + download speed and stability --list-impersonate-targets List available clients to impersonate. -4, --force-ipv4 Make all connections via IPv4 -6, --force-ipv6 Make all connections via IPv6 diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index b99b7e5..1d2c443 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -2451,7 +2451,7 @@ class InfoExtractor: }) continue - src_url = src if src.startswith('http') else urllib.parse.urljoin(base, src) + src_url = src if src.startswith('http') else urllib.parse.urljoin(f'{base}/', src) src_url = src_url.strip() if proto == 'm3u8' or src_ext == 'm3u8': diff --git a/yt_dlp/extractor/peertube.py b/yt_dlp/extractor/peertube.py index b7919c0..fb4d025 100644 --- a/yt_dlp/extractor/peertube.py +++ b/yt_dlp/extractor/peertube.py @@ -1470,11 +1470,15 @@ class PeerTubeIE(InfoExtractor): title = video['name'] - formats = [] + formats, is_live = [], False files = video.get('files') or [] for playlist in (video.get('streamingPlaylists') or []): if not isinstance(playlist, dict): continue + if playlist_url := url_or_none(playlist.get('playlistUrl')): + is_live = True + formats.extend(self._extract_m3u8_formats( + playlist_url, video_id, fatal=False, live=True)) playlist_files = playlist.get('files') if not (playlist_files and isinstance(playlist_files, list)): continue @@ -1498,6 +1502,7 @@ class PeerTubeIE(InfoExtractor): f['vcodec'] = 'none' else: f['fps'] = int_or_none(file_.get('fps')) + is_live = False formats.append(f) description = video.get('description') @@ -1555,6 +1560,7 @@ class PeerTubeIE(InfoExtractor): 'categories': categories, 'formats': formats, 'subtitles': subtitles, + 'is_live': is_live, 'webpage_url': webpage_url, } diff --git a/yt_dlp/extractor/piksel.py b/yt_dlp/extractor/piksel.py index 8870d7b..02ae2fe 100644 --- a/yt_dlp/extractor/piksel.py +++ b/yt_dlp/extractor/piksel.py @@ -25,29 +25,31 @@ class PikselIE(InfoExtractor): )| (?:api|player)\.multicastmedia| (?:api-ovp|player)\.piksel - )\.com| + )\.(?:com|tech)| (?: mz-edge\.stream\.co| movie-s\.nhk\.or )\.jp| vidego\.baltimorecity\.gov )/v/(?:refid/(?P[^/]+)/prefid/)?(?P[\w-]+)''' - _EMBED_REGEX = [r']+src=["\'](?P(?:https?:)?//player\.piksel\.com/v/[a-z0-9]+)'] + _EMBED_REGEX = [r']+src=["\'](?P(?:https?:)?//player\.piksel\.(?:com|tech)/v/[a-z0-9]+)'] _TESTS = [ { - 'url': 'http://player.piksel.com/v/ums2867l', + 'url': 'http://player.piksel.tech/v/ums2867l', 'md5': '34e34c8d89dc2559976a6079db531e85', 'info_dict': { 'id': 'ums2867l', 'ext': 'mp4', 'title': 'GX-005 with Caption', 'timestamp': 1481335659, - 'upload_date': '20161210' + 'upload_date': '20161210', + 'description': '', + 'thumbnail': 'https://thumbs.piksel.tech/thumbs/aid/t1488331553/3238987.jpg?w=640&h=480', } }, { # Original source: http://www.uscourts.gov/cameras-courts/state-washington-vs-donald-j-trump-et-al - 'url': 'https://player.piksel.com/v/v80kqp41', + 'url': 'https://player.piksel.tech/v/v80kqp41', 'md5': '753ddcd8cc8e4fa2dda4b7be0e77744d', 'info_dict': { 'id': 'v80kqp41', @@ -55,7 +57,8 @@ class PikselIE(InfoExtractor): 'title': 'WAW- State of Washington vs. Donald J. Trump, et al', 'description': 'State of Washington vs. Donald J. Trump, et al, Case Number 17-CV-00141-JLR, TRO Hearing, Civil Rights Case, 02/3/2017, 1:00 PM (PST), Seattle Federal Courthouse, Seattle, WA, Judge James L. Robart presiding.', 'timestamp': 1486171129, - 'upload_date': '20170204' + 'upload_date': '20170204', + 'thumbnail': 'https://thumbs.piksel.tech/thumbs/aid/t1495569155/3279887.jpg?w=640&h=360', } }, { @@ -65,7 +68,7 @@ class PikselIE(InfoExtractor): } ] - def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.com', fatal=True): + def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.tech', fatal=True): url = urljoin(host, f'/ws/ws_{resource}/api/{app_token}/mode/json/apiv/5') response = traverse_obj( self._download_json(url, display_id, query=query, fatal=fatal), ('response', {dict})) or {} @@ -146,7 +149,7 @@ class PikselIE(InfoExtractor): smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil']) if smil_url: - transform_source = None + transform_source = lambda x: x.replace('src="/', 'src="') if ref_id == 'nhkworld': # TODO: figure out if this is something to be fixed in urljoin, # _parse_smil_formats or keep it here diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py index ab8efc1..7bcfded 100644 --- a/yt_dlp/extractor/tiktok.py +++ b/yt_dlp/extractor/tiktok.py @@ -940,6 +940,7 @@ class TikTokUserIE(TikTokBaseIE): def _entries(self, sec_uid, user_name): display_id = user_name or sec_uid + seen_ids = set() cursor = int(time.time() * 1E3) for page in itertools.count(1): @@ -949,6 +950,9 @@ class TikTokUserIE(TikTokBaseIE): for video in traverse_obj(response, ('itemList', lambda _, v: v['id'])): video_id = video['id'] + if video_id in seen_ids: + continue + seen_ids.add(video_id) webpage_url = self._create_url(display_id, video_id) yield self.url_result( webpage_url, TikTokIE, @@ -956,8 +960,8 @@ class TikTokUserIE(TikTokBaseIE): old_cursor = cursor cursor = traverse_obj( - response, ('itemList', -1, 'createTime', {functools.partial(int_or_none, invscale=1E3)})) - if not cursor: + response, ('itemList', -1, 'createTime', {lambda x: int(x * 1E3)})) + if not cursor or old_cursor == cursor: # User may not have posted within this ~1 week lookback, so manually adjust cursor cursor = old_cursor - 7 * 86_400_000 # In case 'hasMorePrevious' is wrong, break if we have gone back before TikTok existed diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 997b575..9615bfb 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -520,7 +520,8 @@ def create_parser(): metavar='CLIENT[:OS]', dest='impersonate', default=None, help=( 'Client to impersonate for requests. E.g. chrome, chrome-110, chrome:windows-10. ' - 'Pass --impersonate="" to impersonate any client.'), + 'Pass --impersonate="" to impersonate any client. Note that forcing impersonation ' + 'for all requests may have a detrimental impact on download speed and stability'), ) network.add_option( '--list-impersonate-targets', diff --git a/yt_dlp/version.py b/yt_dlp/version.py index 415dc0e..a90b288 100644 --- a/yt_dlp/version.py +++ b/yt_dlp/version.py @@ -1,8 +1,8 @@ # Autogenerated by devscripts/update-version.py -__version__ = '2024.05.26' +__version__ = '2024.05.27' -RELEASE_GIT_HEAD = 'ae2af1104f80caf2f47544763a33db2c17a3e1de' +RELEASE_GIT_HEAD = '12b248ce60be1aa1362edd839d915bba70dbee4b' VARIANT = None @@ -12,4 +12,4 @@ CHANNEL = 'stable' ORIGIN = 'yt-dlp/yt-dlp' -_pkg_version = '2024.05.26' +_pkg_version = '2024.05.27' -- cgit v1.2.3