From b90dbe6c198f51921d137c3c5cd517d4222bca18 Mon Sep 17 00:00:00 2001 From: i6t <62123048+i6t@users.noreply.github.com> Date: Fri, 4 Mar 2022 22:53:43 +0900 Subject: [PATCH] [Gettr] Improve extractor (#2920) Authored by: i6t --- yt_dlp/extractor/gettr.py | 82 ++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/yt_dlp/extractor/gettr.py b/yt_dlp/extractor/gettr.py index 9842edd81..327a4d0b8 100644 --- a/yt_dlp/extractor/gettr.py +++ b/yt_dlp/extractor/gettr.py @@ -8,8 +8,8 @@ dict_get, float_or_none, int_or_none, - remove_end, str_or_none, + traverse_obj, try_get, url_or_none, urljoin, @@ -36,8 +36,9 @@ class GettrIE(GettrBaseIE): 'ext': 'mp4', 'uploader': 'EpochTV', 'uploader_id': 'epochtv', + 'upload_date': '20210927', 'thumbnail': r're:^https?://.+/out\.jpg', - 'timestamp': 1632782451058, + 'timestamp': 1632782451.058, 'duration': 58.5585, 'tags': ['hornofafrica', 'explorations'], } @@ -50,43 +51,69 @@ class GettrIE(GettrBaseIE): 'ext': 'mp4', 'uploader': 'Neues Forum Freiheit', 'uploader_id': 'nf_freiheit', + 'upload_date': '20210718', 'thumbnail': r're:^https?://.+/out\.jpg', - 'timestamp': 1626594455017, + 'timestamp': 1626594455.017, 'duration': 23, 'tags': 'count:12', } + }, { + # quote post + 'url': 'https://gettr.com/post/pxn5b743a9', + 'only_matching': True, + }, { + # quote with video + 'url': 'https://gettr.com/post/pxtiiz5ca2', + 'only_matching': True, + }, { + # streaming embed + 'url': 'https://gettr.com/post/pxlu8p3b13', + 'only_matching': True, + }, { + # youtube embed + 'url': 'https://gettr.com/post/pv6wp9e24c', + 'only_matching': True, + 'add_ie': ['Youtube'], }] def _real_extract(self, url): post_id = self._match_id(url) webpage = self._download_webpage(url, post_id) - api_data = self._call_api('post/%s?incl="poststats|userinfo"' % post_id, post_id) post_data = api_data.get('data') - user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']]) or {} + user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']], dict) or {} - if post_data.get('nfound'): - raise ExtractorError(post_data.get('txt'), expected=True) + vid = post_data.get('vid') + ovid = post_data.get('ovid') + + if post_data.get('p_type') == 'stream': + return self.url_result(f'https://gettr.com/streaming/{post_id}', ie='GettrStreaming', video_id=post_id) + + if not (ovid or vid): + embed_url = url_or_none(post_data.get('prevsrc')) + shared_post_id = traverse_obj(api_data, ('aux', 'shrdpst', '_id'), ('data', 'rpstIds', 0), expected_type=str) + + if embed_url: + return self.url_result(embed_url) + elif shared_post_id: + return self.url_result(f'https://gettr.com/post/{shared_post_id}', ie='Gettr', video_id=shared_post_id) + else: + raise ExtractorError('There\'s no video in this post.') title = description = str_or_none( post_data.get('txt') or self._og_search_description(webpage)) uploader = str_or_none( user_data.get('nickname') - or remove_end(self._og_search_title(webpage), ' on GETTR')) + or self._search_regex(r'^(.+?) on GETTR', self._og_search_title(webpage, default=''), 'uploader', fatal=False)) + if uploader: title = '%s - %s' % (uploader, title) - if not dict_get(post_data, ['vid', 'ovid']): - raise ExtractorError('There\'s no video in this post.') - - vid = post_data.get('vid') - ovid = post_data.get('ovid') - - formats = self._extract_m3u8_formats( + formats, subtitles = self._extract_m3u8_formats_and_subtitles( urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4', - entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else [] + entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else ([], {}) if ovid: formats.append({ @@ -103,15 +130,16 @@ def _real_extract(self, url): 'id': post_id, 'title': title, 'description': description, - 'thumbnail': url_or_none( - urljoin(self._MEDIA_BASE_URL, post_data.get('main')) - or self._og_search_thumbnail(webpage)), - 'timestamp': int_or_none(post_data.get('cdate')), + 'formats': formats, + 'subtitles': subtitles, + 'uploader': uploader, 'uploader_id': str_or_none( dict_get(user_data, ['_id', 'username']) or post_data.get('uid')), - 'uploader': uploader, - 'formats': formats, + 'thumbnail': url_or_none( + urljoin(self._MEDIA_BASE_URL, post_data.get('main')) + or self._html_search_meta(['og:image', 'image'], webpage, 'thumbnail', fatal=False)), + 'timestamp': float_or_none(dict_get(post_data, ['cdate', 'udate']), scale=1000), 'duration': float_or_none(post_data.get('vid_dur')), 'tags': post_data.get('htgs'), } @@ -165,19 +193,19 @@ def _real_extract(self, url): thumbnails = [{ 'url': urljoin(self._MEDIA_BASE_URL, thumbnail), - } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []] + } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs'], list) or []] self._sort_formats(formats) return { 'id': video_id, - 'title': try_get(video_info, lambda x: x['postData']['ttl']), - 'description': try_get(video_info, lambda x: x['postData']['dsc']), + 'title': try_get(video_info, lambda x: x['postData']['ttl'], str), + 'description': try_get(video_info, lambda x: x['postData']['dsc'], str), 'formats': formats, 'subtitles': subtitles, 'thumbnails': thumbnails, - 'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']), - 'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']), + 'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname'], str), + 'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id'], str), 'view_count': int_or_none(live_info.get('viewsCount')), 'timestamp': float_or_none(live_info.get('startAt'), scale=1000), 'duration': float_or_none(live_info.get('duration'), scale=1000),