From 65de7d204ce88c0225df1321060304baab85dbd8 Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Tue, 26 Dec 2023 01:40:24 +0100 Subject: [PATCH] Update to ytdl-commit-be008e6 (#8836) - [utils] Make restricted filenames ignore some Unicode categories (by dirkf) - [ie/telewebion] Fix extraction (by Grub4K) - [ie/imgur] Overhaul extractor (by bashonly, Grub4K) - [ie/EpidemicSound] Add extractor (by Grub4K) Authored by: bashonly, dirkf, Grub4K Co-authored-by: bashonly --- yt_dlp/extractor/_extractors.py | 1 + yt_dlp/extractor/epidemicsound.py | 107 +++++++++ yt_dlp/extractor/imgur.py | 364 ++++++++++++++++++++++++------ yt_dlp/extractor/telewebion.py | 157 +++++++++---- yt_dlp/utils/_utils.py | 2 +- 5 files changed, 518 insertions(+), 113 deletions(-) create mode 100644 yt_dlp/extractor/epidemicsound.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 5c34bb7f4..62103f13c 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -548,6 +548,7 @@ EpiconIE, EpiconSeriesIE, ) +from .epidemicsound import EpidemicSoundIE from .eplus import EplusIbIE from .epoch import EpochIE from .eporner import EpornerIE diff --git a/yt_dlp/extractor/epidemicsound.py b/yt_dlp/extractor/epidemicsound.py new file mode 100644 index 000000000..0d81b11c8 --- /dev/null +++ b/yt_dlp/extractor/epidemicsound.py @@ -0,0 +1,107 @@ +from .common import InfoExtractor +from ..utils import ( + float_or_none, + int_or_none, + orderedSet, + parse_iso8601, + parse_qs, + parse_resolution, + str_or_none, + traverse_obj, + url_or_none, +) + + +class EpidemicSoundIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?epidemicsound\.com/track/(?P[0-9a-zA-Z]+)' + _TESTS = [{ + 'url': 'https://www.epidemicsound.com/track/yFfQVRpSPz/', + 'md5': 'd98ff2ddb49e8acab9716541cbc9dfac', + 'info_dict': { + 'id': '45014', + 'display_id': 'yFfQVRpSPz', + 'ext': 'mp3', + 'title': 'Door Knock Door 1', + 'alt_title': 'Door Knock Door 1', + 'tags': ['foley', 'door', 'knock', 'glass', 'window', 'glass door knock'], + 'categories': ['Misc. Door'], + 'duration': 1, + 'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/default-sfx/3000x3000.jpg', + 'timestamp': 1415320353, + 'upload_date': '20141107', + }, + }, { + 'url': 'https://www.epidemicsound.com/track/mj8GTTwsZd/', + 'md5': 'c82b745890f9baf18dc2f8d568ee3830', + 'info_dict': { + 'id': '148700', + 'display_id': 'mj8GTTwsZd', + 'ext': 'mp3', + 'title': 'Noplace', + 'tags': ['liquid drum n bass', 'energetic'], + 'categories': ['drum and bass'], + 'duration': 237, + 'timestamp': 1694426482, + 'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/11138/3000x3000.jpg', + 'upload_date': '20230911', + 'release_timestamp': 1700535606, + 'release_date': '20231121', + }, + }] + + @staticmethod + def _epidemic_parse_thumbnail(url: str): + if not url_or_none(url): + return None + + return { + 'url': url, + **(traverse_obj(url, ({parse_qs}, { + 'width': ('width', 0, {int_or_none}), + 'height': ('height', 0, {int_or_none}), + })) or parse_resolution(url)), + } + + @staticmethod + def _epidemic_fmt_or_none(f): + if not f.get('format'): + f['format'] = f.get('format_id') + elif not f.get('format_id'): + f['format_id'] = f['format'] + if not f['url'] or not f['format']: + return None + if f.get('format_note'): + f['format_note'] = f'track ID {f["format_note"]}' + if f['format'] != 'full': + f['preference'] = -2 + return f + + def _real_extract(self, url): + video_id = self._match_id(url) + json_data = self._download_json(f'https://www.epidemicsound.com/json/track/{video_id}', video_id) + + thumbnails = traverse_obj(json_data, [('imageUrl', 'cover')]) + thumb_base_url = traverse_obj(json_data, ('coverArt', 'baseUrl', {url_or_none})) + if thumb_base_url: + thumbnails.extend(traverse_obj(json_data, ( + 'coverArt', 'sizes', ..., {thumb_base_url.__add__}))) + + return traverse_obj(json_data, { + 'id': ('id', {str_or_none}), + 'display_id': ('publicSlug', {str}), + 'title': ('title', {str}), + 'alt_title': ('oldTitle', {str}), + 'duration': ('length', {float_or_none}), + 'timestamp': ('added', {parse_iso8601}), + 'release_timestamp': ('releaseDate', {parse_iso8601}), + 'categories': ('genres', ..., 'tag', {str}), + 'tags': ('metadataTags', ..., {str}), + 'age_limit': ('isExplicit', {lambda b: 18 if b else None}), + 'thumbnails': ({lambda _: thumbnails}, {orderedSet}, ..., {self._epidemic_parse_thumbnail}), + 'formats': ('stems', {dict.items}, ..., { + 'format': (0, {str_or_none}), + 'format_note': (1, 's3TrackId', {str_or_none}), + 'format_id': (1, 'stemType', {str}), + 'url': (1, 'lqMp3Url', {url_or_none}), + }, {self._epidemic_fmt_or_none}), + }) diff --git a/yt_dlp/extractor/imgur.py b/yt_dlp/extractor/imgur.py index bff6ed57f..1fa0a2a79 100644 --- a/yt_dlp/extractor/imgur.py +++ b/yt_dlp/extractor/imgur.py @@ -1,99 +1,243 @@ +import functools import re from .common import InfoExtractor from ..utils import ( + ExtractorError, + determine_ext, + float_or_none, int_or_none, js_to_json, mimetype2ext, - ExtractorError, + parse_iso8601, + str_or_none, + strip_or_none, + traverse_obj, + url_or_none, ) -class ImgurIE(InfoExtractor): - _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|(?:t(?:opic)?|r)/[^/]+)/)(?P[a-zA-Z0-9]+)' +class ImgurBaseIE(InfoExtractor): + _CLIENT_ID = '546c25a59c58ad7' + + @classmethod + def _imgur_result(cls, item_id): + return cls.url_result(f'https://imgur.com/{item_id}', ImgurIE, item_id) + + def _call_api(self, endpoint, video_id, **kwargs): + return self._download_json( + f'https://api.imgur.com/post/v1/{endpoint}/{video_id}?client_id={self._CLIENT_ID}&include=media,account', + video_id, **kwargs) + + @staticmethod + def get_description(s): + if 'Discover the magic of the internet at Imgur' in s: + return None + return s or None + + +class ImgurIE(ImgurBaseIE): + _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)(?P[a-zA-Z0-9]+)' _TESTS = [{ - 'url': 'https://i.imgur.com/A61SaA1.gifv', + 'url': 'https://imgur.com/A61SaA1', 'info_dict': { 'id': 'A61SaA1', 'ext': 'mp4', - 'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$', + 'title': 'MRW gifv is up and running without any bugs', + 'timestamp': 1416446068, + 'upload_date': '20141120', + 'dislike_count': int, + 'comment_count': int, + 'release_timestamp': 1416446068, + 'release_date': '20141120', + 'like_count': int, + 'thumbnail': 'https://i.imgur.com/A61SaA1h.jpg', }, }, { - 'url': 'https://imgur.com/A61SaA1', + 'url': 'https://i.imgur.com/A61SaA1.gifv', 'only_matching': True, }, { 'url': 'https://i.imgur.com/crGpqCV.mp4', 'only_matching': True, }, { - # no title 'url': 'https://i.imgur.com/jxBXAMC.gifv', - 'only_matching': True, + 'info_dict': { + 'id': 'jxBXAMC', + 'ext': 'mp4', + 'title': 'Fahaka puffer feeding', + 'timestamp': 1533835503, + 'upload_date': '20180809', + 'release_date': '20180809', + 'like_count': int, + 'duration': 30.0, + 'comment_count': int, + 'release_timestamp': 1533835503, + 'thumbnail': 'https://i.imgur.com/jxBXAMCh.jpg', + 'dislike_count': int, + }, }] def _real_extract(self, url): video_id = self._match_id(url) + data = self._call_api('media', video_id) + if not traverse_obj(data, ('media', 0, ( + ('type', {lambda t: t == 'video' or None}), + ('metadata', 'is_animated'))), get_all=False): + raise ExtractorError(f'{video_id} is not a video or animated image', expected=True) webpage = self._download_webpage( - 'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id) + f'https://i.imgur.com/{video_id}.gifv', video_id, fatal=False) or '' + formats = [] - width = int_or_none(self._og_search_property( - 'video:width', webpage, default=None)) - height = int_or_none(self._og_search_property( - 'video:height', webpage, default=None)) + media_fmt = traverse_obj(data, ('media', 0, { + 'url': ('url', {url_or_none}), + 'ext': ('ext', {str}), + 'width': ('width', {int_or_none}), + 'height': ('height', {int_or_none}), + 'filesize': ('size', {int_or_none}), + 'acodec': ('metadata', 'has_sound', {lambda b: None if b else 'none'}), + })) + media_url = media_fmt.get('url') + if media_url: + if not media_fmt.get('ext'): + media_fmt['ext'] = mimetype2ext(traverse_obj( + data, ('media', 0, 'mime_type'))) or determine_ext(media_url) + if traverse_obj(data, ('media', 0, 'type')) == 'image': + media_fmt['acodec'] = 'none' + media_fmt.setdefault('preference', -10) + formats.append(media_fmt) video_elements = self._search_regex( r'(?s)
(.*?)
', webpage, 'video elements', default=None) - if not video_elements: - raise ExtractorError( - 'No sources found for video %s. Maybe an image?' % video_id, - expected=True) - formats = [] - for m in re.finditer(r'[^"]+)"\s+type="(?P[^"]+)"', video_elements): - formats.append({ - 'format_id': m.group('type').partition('/')[2], - 'url': self._proto_relative_url(m.group('src')), - 'ext': mimetype2ext(m.group('type')), - 'width': width, - 'height': height, - 'http_headers': { - 'User-Agent': 'yt-dlp (like wget)', - }, - }) + if video_elements: + def og_get_size(media_type): + return { + p: int_or_none(self._og_search_property(f'{media_type}:{p}', webpage, default=None)) + for p in ('width', 'height') + } - gif_json = self._search_regex( - r'(?s)var\s+videoItem\s*=\s*(\{.*?\})', - webpage, 'GIF code', fatal=False) - if gif_json: - gifd = self._parse_json( - gif_json, video_id, transform_source=js_to_json) - formats.append({ - 'format_id': 'gif', - 'preference': -10, # gifs are worse than videos - 'width': width, - 'height': height, - 'ext': 'gif', - 'acodec': 'none', - 'vcodec': 'gif', - 'container': 'gif', - 'url': self._proto_relative_url(gifd['gifUrl']), - 'filesize': gifd.get('size'), - 'http_headers': { - 'User-Agent': 'yt-dlp (like wget)', - }, + size = og_get_size('video') + if not any(size.values()): + size = og_get_size('image') + + formats = traverse_obj( + re.finditer(r'[^"]+)"\s+type="(?P[^"]+)"', video_elements), + (..., { + 'format_id': ('type', {lambda s: s.partition('/')[2]}), + 'url': ('src', {self._proto_relative_url}), + 'ext': ('type', {mimetype2ext}), + })) + for f in formats: + f.update(size) + + # We can get the original gif format from the webpage as well + gif_json = traverse_obj(self._search_json( + r'var\s+videoItem\s*=', webpage, 'GIF info', video_id, + transform_source=js_to_json, fatal=False), { + 'url': ('gifUrl', {self._proto_relative_url}), + 'filesize': ('size', {int_or_none}), }) + if gif_json: + gif_json.update(size) + gif_json.update({ + 'format_id': 'gif', + 'preference': -10, # gifs < videos + 'ext': 'gif', + 'acodec': 'none', + 'vcodec': 'gif', + 'container': 'gif', + }) + formats.append(gif_json) + + search = functools.partial(self._html_search_meta, html=webpage, default=None) + + twitter_fmt = { + 'format_id': 'twitter', + 'url': url_or_none(search('twitter:player:stream')), + 'ext': mimetype2ext(search('twitter:player:stream:content_type')), + 'width': int_or_none(search('twitter:width')), + 'height': int_or_none(search('twitter:height')), + } + if twitter_fmt['url']: + formats.append(twitter_fmt) + + if not formats: + self.raise_no_formats( + f'No sources found for video {video_id}. Maybe a plain image?', expected=True) + self._remove_duplicate_formats(formats) return { + 'title': self._og_search_title(webpage, default=None), + 'description': self.get_description(self._og_search_description(webpage, default='')), + **traverse_obj(data, { + 'uploader_id': ('account_id', {lambda a: str(a) if int_or_none(a) else None}), + 'uploader': ('account', 'username', {lambda x: strip_or_none(x) or None}), + 'uploader_url': ('account', 'avatar_url', {url_or_none}), + 'like_count': ('upvote_count', {int_or_none}), + 'dislike_count': ('downvote_count', {int_or_none}), + 'comment_count': ('comment_count', {int_or_none}), + 'age_limit': ('is_mature', {lambda x: 18 if x else None}), + 'timestamp': (('updated_at', 'created_at'), {parse_iso8601}), + 'release_timestamp': ('created_at', {parse_iso8601}), + }, get_all=False), + **traverse_obj(data, ('media', 0, 'metadata', { + 'title': ('title', {lambda x: strip_or_none(x) or None}), + 'description': ('description', {self.get_description}), + 'duration': ('duration', {float_or_none}), + 'timestamp': (('updated_at', 'created_at'), {parse_iso8601}), + 'release_timestamp': ('created_at', {parse_iso8601}), + }), get_all=False), 'id': video_id, 'formats': formats, - 'title': self._og_search_title(webpage, default=video_id), + 'thumbnail': url_or_none(search('thumbnailUrl')), } -class ImgurGalleryIE(InfoExtractor): +class ImgurGalleryBaseIE(ImgurBaseIE): + _GALLERY = True + + def _real_extract(self, url): + gallery_id = self._match_id(url) + + data = self._call_api('albums', gallery_id, fatal=False, expected_status=404) + + info = traverse_obj(data, { + 'title': ('title', {lambda x: strip_or_none(x) or None}), + 'description': ('description', {self.get_description}), + }) + + if traverse_obj(data, 'is_album'): + + def yield_media_ids(): + for m_id in traverse_obj(data, ( + 'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'], + 'id', {lambda x: str_or_none(x) or None})): + yield m_id + + # if a gallery with exactly one video, apply album metadata to video + media_id = ( + self._GALLERY + and traverse_obj(data, ('image_count', {lambda c: c == 1})) + and next(yield_media_ids(), None)) + + if not media_id: + result = self.playlist_result( + map(self._imgur_result, yield_media_ids()), gallery_id) + result.update(info) + return result + gallery_id = media_id + + result = self._imgur_result(gallery_id) + info['_type'] = 'url_transparent' + result.update(info) + return result + + +class ImgurGalleryIE(ImgurGalleryBaseIE): IE_NAME = 'imgur:gallery' - _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P[a-zA-Z0-9]+)' + _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/?#]+)/(?P[a-zA-Z0-9]+)' _TESTS = [{ 'url': 'http://imgur.com/gallery/Q95ko', @@ -102,49 +246,121 @@ class ImgurGalleryIE(InfoExtractor): 'title': 'Adding faces make every GIF better', }, 'playlist_count': 25, + 'skip': 'Zoinks! You\'ve taken a wrong turn.', }, { + # TODO: static images - replace with animated/video gallery 'url': 'http://imgur.com/topic/Aww/ll5Vk', 'only_matching': True, }, { 'url': 'https://imgur.com/gallery/YcAQlkx', + 'add_ies': ['Imgur'], 'info_dict': { 'id': 'YcAQlkx', 'ext': 'mp4', 'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....', - } + 'timestamp': 1358554297, + 'upload_date': '20130119', + 'uploader_id': '1648642', + 'uploader': 'wittyusernamehere', + 'release_timestamp': 1358554297, + 'thumbnail': 'https://i.imgur.com/YcAQlkxh.jpg', + 'release_date': '20130119', + 'uploader_url': 'https://i.imgur.com/u3R4I2S_d.png?maxwidth=290&fidelity=grand', + 'comment_count': int, + 'dislike_count': int, + 'like_count': int, + }, }, { + # TODO: static image - replace with animated/video gallery 'url': 'http://imgur.com/topic/Funny/N8rOudd', 'only_matching': True, }, { 'url': 'http://imgur.com/r/aww/VQcQPhM', - 'only_matching': True, + 'add_ies': ['Imgur'], + 'info_dict': { + 'id': 'VQcQPhM', + 'ext': 'mp4', + 'title': 'The boss is here', + 'timestamp': 1476494751, + 'upload_date': '20161015', + 'uploader_id': '19138530', + 'uploader': 'thematrixcam', + 'comment_count': int, + 'dislike_count': int, + 'uploader_url': 'https://i.imgur.com/qCjr5Pi_d.png?maxwidth=290&fidelity=grand', + 'release_timestamp': 1476494751, + 'like_count': int, + 'release_date': '20161015', + 'thumbnail': 'https://i.imgur.com/VQcQPhMh.jpg', + }, + }, + # from https://github.com/ytdl-org/youtube-dl/pull/16674 + { + 'url': 'https://imgur.com/t/unmuted/6lAn9VQ', + 'info_dict': { + 'id': '6lAn9VQ', + 'title': 'Penguins !', + }, + 'playlist_count': 3, + }, { + 'url': 'https://imgur.com/t/unmuted/kx2uD3C', + 'add_ies': ['Imgur'], + 'info_dict': { + 'id': 'ZVMv45i', + 'ext': 'mp4', + 'title': 'Intruder', + 'timestamp': 1528129683, + 'upload_date': '20180604', + 'release_timestamp': 1528129683, + 'release_date': '20180604', + 'like_count': int, + 'dislike_count': int, + 'comment_count': int, + 'duration': 30.03, + 'thumbnail': 'https://i.imgur.com/ZVMv45ih.jpg', + }, + }, { + 'url': 'https://imgur.com/t/unmuted/wXSK0YH', + 'add_ies': ['Imgur'], + 'info_dict': { + 'id': 'JCAP4io', + 'ext': 'mp4', + 'title': 're:I got the blues$', + 'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.', + 'timestamp': 1527809525, + 'upload_date': '20180531', + 'like_count': int, + 'dislike_count': int, + 'duration': 30.03, + 'comment_count': int, + 'release_timestamp': 1527809525, + 'thumbnail': 'https://i.imgur.com/JCAP4ioh.jpg', + 'release_date': '20180531', + }, }] - def _real_extract(self, url): - gallery_id = self._match_id(url) - data = self._download_json( - 'https://imgur.com/gallery/%s.json' % gallery_id, - gallery_id)['data']['image'] - - if data.get('is_album'): - entries = [ - self.url_result('http://imgur.com/%s' % image['hash'], ImgurIE.ie_key(), image['hash']) - for image in data['album_images']['images'] if image.get('hash')] - return self.playlist_result(entries, gallery_id, data.get('title'), data.get('description')) - - return self.url_result('http://imgur.com/%s' % gallery_id, ImgurIE.ie_key(), gallery_id) - - -class ImgurAlbumIE(ImgurGalleryIE): # XXX: Do not subclass from concrete IE +class ImgurAlbumIE(ImgurGalleryBaseIE): IE_NAME = 'imgur:album' _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P[a-zA-Z0-9]+)' - + _GALLERY = False _TESTS = [{ + # TODO: only static images - replace with animated/video gallery 'url': 'http://imgur.com/a/j6Orj', + 'only_matching': True, + }, + # from https://github.com/ytdl-org/youtube-dl/pull/21693 + { + 'url': 'https://imgur.com/a/iX265HX', 'info_dict': { - 'id': 'j6Orj', - 'title': 'A Literary Analysis of "Star Wars: The Force Awakens"', + 'id': 'iX265HX', + 'title': 'enen-no-shouboutai' }, - 'playlist_count': 12, + 'playlist_count': 2, + }, { + 'url': 'https://imgur.com/a/8pih2Ed', + 'info_dict': { + 'id': '8pih2Ed' + }, + 'playlist_mincount': 1, }] diff --git a/yt_dlp/extractor/telewebion.py b/yt_dlp/extractor/telewebion.py index 550549f05..9378ed021 100644 --- a/yt_dlp/extractor/telewebion.py +++ b/yt_dlp/extractor/telewebion.py @@ -1,52 +1,133 @@ +from __future__ import annotations + +import json +from functools import partial +from textwrap import dedent + from .common import InfoExtractor +from ..utils import ExtractorError, format_field, int_or_none, parse_iso8601 +from ..utils.traversal import traverse_obj + + +def _fmt_url(url): + return partial(format_field, template=url, default=None) class TelewebionIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?telewebion\.com/#!/episode/(?P\d+)' - - _TEST = { - 'url': 'http://www.telewebion.com/#!/episode/1263668/', + _VALID_URL = r'https?://(?:www\.)?telewebion\.com/episode/(?P(?:0x[a-fA-F\d]+|\d+))' + _TESTS = [{ + 'url': 'http://www.telewebion.com/episode/0x1b3139c/', 'info_dict': { - 'id': '1263668', + 'id': '0x1b3139c', 'ext': 'mp4', - 'title': 'قرعه\u200cکشی لیگ قهرمانان اروپا', - 'thumbnail': r're:^https?://.*\.jpg', + 'title': 'قرعه‌کشی لیگ قهرمانان اروپا', + 'series': '+ فوتبال', + 'series_id': '0x1b2505c', + 'channel': 'شبکه 3', + 'channel_id': '0x1b1a761', + 'channel_url': 'https://telewebion.com/live/tv3', + 'timestamp': 1425522414, + 'upload_date': '20150305', + 'release_timestamp': 1425517020, + 'release_date': '20150305', + 'duration': 420, 'view_count': int, + 'tags': ['ورزشی', 'لیگ اروپا', 'اروپا'], + 'thumbnail': 'https://static.telewebion.com/episodeImages/YjFhM2MxMDBkMDNiZTU0MjE5YjQ3ZDY0Mjk1ZDE0ZmUwZWU3OTE3OWRmMDAyODNhNzNkNjdmMWMzMWIyM2NmMA/default', }, - 'params': { - # m3u8 download - 'skip_download': True, + 'skip_download': 'm3u8', + }, { + 'url': 'https://telewebion.com/episode/162175536', + 'info_dict': { + 'id': '0x9aa9a30', + 'ext': 'mp4', + 'title': 'کارما یعنی این !', + 'series': 'پاورقی', + 'series_id': '0x29a7426', + 'channel': 'شبکه 2', + 'channel_id': '0x1b1a719', + 'channel_url': 'https://telewebion.com/live/tv2', + 'timestamp': 1699979968, + 'upload_date': '20231114', + 'release_timestamp': 1699991638, + 'release_date': '20231114', + 'duration': 78, + 'view_count': int, + 'tags': ['کلیپ های منتخب', ' کلیپ طنز ', ' کلیپ سیاست ', 'پاورقی', 'ویژه فلسطین'], + 'thumbnail': 'https://static.telewebion.com/episodeImages/871e9455-7567-49a5-9648-34c22c197f5f/default', }, - } + 'skip_download': 'm3u8', + }] + + def _call_graphql_api( + self, operation, video_id, query, + variables: dict[str, tuple[str, str]] | None = None, + note='Downloading GraphQL JSON metadata', + ): + parameters = '' + if variables: + parameters = ', '.join(f'${name}: {type_}' for name, (type_, _) in variables.items()) + parameters = f'({parameters})' + + result = self._download_json('https://graph.telewebion.com/graphql', video_id, note, data=json.dumps({ + 'operationName': operation, + 'query': f'query {operation}{parameters} @cacheControl(maxAge: 60) {{{query}\n}}\n', + 'variables': {name: value for name, (_, value) in (variables or {}).items()} + }, separators=(',', ':')).encode(), headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }) + if not result or traverse_obj(result, 'errors'): + message = ', '.join(traverse_obj(result, ('errors', ..., 'message', {str}))) + raise ExtractorError(message or 'Unknown GraphQL API error') + + return result['data'] def _real_extract(self, url): video_id = self._match_id(url) + if not video_id.startswith('0x'): + video_id = hex(int(video_id)) - secure_token = self._download_webpage( - 'http://m.s2.telewebion.com/op/op?action=getSecurityToken', video_id) - episode_details = self._download_json( - 'http://m.s2.telewebion.com/op/op', video_id, - query={'action': 'getEpisodeDetails', 'episode_id': video_id}) + episode_data = self._call_graphql_api('getEpisodeDetail', video_id, dedent(''' + queryEpisode(filter: {EpisodeID: $EpisodeId}, first: 1) { + title + program { + ProgramID + title + } + image + view_count + duration + started_at + created_at + channel { + ChannelID + name + descriptor + } + tags { + name + } + } + '''), {'EpisodeId': ('[ID!]', video_id)}) - m3u8_url = 'http://m.s1.telewebion.com/smil/%s.m3u8?filepath=%s&m3u8=1&secure_token=%s' % ( - video_id, episode_details['file_path'], secure_token) - formats = self._extract_m3u8_formats( - m3u8_url, video_id, ext='mp4', m3u8_id='hls') - - picture_paths = [ - episode_details.get('picture_path'), - episode_details.get('large_picture_path'), - ] - - thumbnails = [{ - 'url': picture_path, - 'preference': idx, - } for idx, picture_path in enumerate(picture_paths) if picture_path is not None] - - return { - 'id': video_id, - 'title': episode_details['title'], - 'formats': formats, - 'thumbnails': thumbnails, - 'view_count': episode_details.get('view_count'), - } + info_dict = traverse_obj(episode_data, ('queryEpisode', 0, { + 'title': ('title', {str}), + 'view_count': ('view_count', {int_or_none}), + 'duration': ('duration', {int_or_none}), + 'tags': ('tags', ..., 'name', {str}), + 'release_timestamp': ('started_at', {parse_iso8601}), + 'timestamp': ('created_at', {parse_iso8601}), + 'series': ('program', 'title', {str}), + 'series_id': ('program', 'ProgramID', {str}), + 'channel': ('channel', 'name', {str}), + 'channel_id': ('channel', 'ChannelID', {str}), + 'channel_url': ('channel', 'descriptor', {_fmt_url('https://telewebion.com/live/%s')}), + 'thumbnail': ('image', {_fmt_url('https://static.telewebion.com/episodeImages/%s/default')}), + 'formats': ( + 'channel', 'descriptor', {str}, + {_fmt_url(f'https://cdna.telewebion.com/%s/episode/{video_id}/playlist.m3u8')}, + {partial(self._extract_m3u8_formats, video_id=video_id, ext='mp4', m3u8_id='hls')}), + })) + info_dict['id'] = video_id + return info_dict diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index f1d7cead6..361617c02 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -636,7 +636,7 @@ def replace_insane(char): elif char in '\\/|*<>': return '\0_' if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace() or ord(char) > 127): - return '\0_' + return '' if unicodedata.category(char)[0] in 'CM' else '\0_' return char # Replace look-alike Unicode glyphs