From 1abfe53cda1399d97ecb67a217b01a36f25e0f0b Mon Sep 17 00:00:00 2001 From: SsSsS <54671367+u-spec-png@users.noreply.github.com> Date: Thu, 17 Jun 2021 12:03:06 +0000 Subject: [PATCH 1/5] [newgrounds] fix extarctor and improved extractor (fix #27397) - added thumbnail extractor - added upload date - added description - added view count - fix video extraction - fix playlist extraction --- youtube_dl/extractor/newgrounds.py | 94 ++++++++++++++++++------------ 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 82e7cf522..277a96c12 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -5,15 +5,18 @@ import re from .common import InfoExtractor from ..utils import ( extract_attributes, - int_or_none, parse_duration, parse_filesize, + parse_count, unified_timestamp, + unified_strdate, + RegexNotFoundError, + ExtractorError, ) class NewgroundsIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P[0-9]+)(?:/format/flash)?' _TESTS = [{ 'url': 'https://www.newgrounds.com/audio/listen/549479', 'md5': 'fe6033d297591288fa1c1f780386f07a', @@ -27,15 +30,14 @@ class NewgroundsIE(InfoExtractor): 'duration': 143, }, }, { - 'url': 'https://www.newgrounds.com/portal/view/673111', - 'md5': '3394735822aab2478c31b1004fe5e5bc', + 'url': 'https://www.newgrounds.com/portal/view/297383', 'info_dict': { - 'id': '673111', + 'id': '297383', 'ext': 'mp4', - 'title': 'Dancin', - 'uploader': 'Squirrelman82', - 'timestamp': 1460256780, - 'upload_date': '20160410', + 'title': 'Metal Gear Awesome', + 'uploader': 'Egoraptor', + 'timestamp': 1140663240, + 'upload_date': '20060223', }, }, { # source format unavailable, additional mp4 formats @@ -59,32 +61,37 @@ class NewgroundsIE(InfoExtractor): webpage = self._download_webpage(url, media_id) title = self._html_search_regex( - r'([^>]+)', webpage, 'title') + r'(.+?)', webpage, 'title') - media_url = self._parse_json(self._search_regex( - r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) + try: + media_url = self._parse_json(self._search_regex( + r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) + except RegexNotFoundError or ExtractorError: + media_url = None + formats = [] - formats = [{ - 'url': media_url, - 'format_id': 'source', - 'quality': 1, - }] + if media_url: + formats = [{ + 'url': media_url, + 'format_id': 'source', + 'quality': 1, + }] + else: + json_data = self._download_json('https://www.newgrounds.com/portal/video/' + media_id, media_id, headers={ + 'Accept': 'application/json, text/javascript, */*; q=0.01', + 'Accept-Encoding': 'gzip, deflate, br', + 'X-Requested-With': 'XMLHttpRequest', + 'Connection': 'keep-alive', + }) - max_resolution = int_or_none(self._search_regex( - r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution', - default=None)) - if max_resolution: - url_base = media_url.rpartition('.')[0] - for resolution in (360, 720, 1080): - if resolution > max_resolution: - break - formats.append({ - 'url': '%s.%dp.mp4' % (url_base, resolution), - 'format_id': '%dp' % resolution, - 'height': resolution, - }) + for resolution in ('360p', '720p', '1080p'): + if resolution in json_data['sources']: + formats.append({ + 'url': json_data['sources'][resolution][0]['src'], + 'format_id': resolution, + 'height': int(resolution[:-1]), + }) - self._check_formats(formats, media_id) self._sort_formats(formats) uploader = self._html_search_regex( @@ -92,14 +99,26 @@ class NewgroundsIE(InfoExtractor): r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', fatal=False) - timestamp = unified_timestamp(self._html_search_regex( + timestamp = self._html_search_regex( (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', - default=None)) + default=None) + + upload_date = unified_strdate(timestamp) + + timestamp = unified_timestamp(timestamp) + + thumbnail = self._og_search_thumbnail(webpage) + duration = parse_duration(self._search_regex( r'(?s)
\s*Song\s*
\s*
.+?
\s*
([^<]+)', webpage, 'duration', default=None)) + description = self._og_search_description(webpage) + + view_count = parse_count(self._html_search_regex(r'(?s)
\s*Views\s*
\s*
([\d\.,]+)
', webpage, + 'view_count', fatal=False, default=None)) + filesize_approx = parse_filesize(self._html_search_regex( r'(?s)
\s*Song\s*
\s*
(.+?)
', webpage, 'filesize', default=None)) @@ -108,7 +127,6 @@ class NewgroundsIE(InfoExtractor): if '
Song' in webpage: formats[0]['vcodec'] = 'none' - return { 'id': media_id, 'title': title, @@ -116,6 +134,10 @@ class NewgroundsIE(InfoExtractor): 'timestamp': timestamp, 'duration': duration, 'formats': formats, + 'description': description, + 'thumbnail': thumbnail, + 'view_count': view_count, + 'upload_date': upload_date, } @@ -155,14 +177,14 @@ class NewgroundsPlaylistIE(InfoExtractor): entries = [] for a, path, media_id in re.findall( - r'(]+\bhref=["\']/?((?:portal/view|audio/listen)/(\d+))[^>]+>)', + r'(]+href="https?://[^/]+/(audio/listen|portal/view)/([0-9]+)"[^>]+>)', webpage): a_class = extract_attributes(a).get('class') if a_class not in ('item-portalsubmission', 'item-audiosubmission'): continue entries.append( self.url_result( - 'https://www.newgrounds.com/%s' % path, + 'https://www.newgrounds.com/ + path + '/' + media_id, ie=NewgroundsIE.ie_key(), video_id=media_id)) return self.playlist_result(entries, playlist_id, title) From 417bfcf4131672423112a013108b5c532c157efd Mon Sep 17 00:00:00 2001 From: SsSsS <54671367+u-spec-png@users.noreply.github.com> Date: Thu, 17 Jun 2021 12:12:55 +0000 Subject: [PATCH 2/5] [newgrounds] whoops --- youtube_dl/extractor/newgrounds.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 277a96c12..105297005 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -149,14 +149,7 @@ class NewgroundsPlaylistIE(InfoExtractor): 'id': 'cats', 'title': 'Cats', }, - 'playlist_mincount': 46, - }, { - 'url': 'http://www.newgrounds.com/portal/search/author/ZONE-SAMA', - 'info_dict': { - 'id': 'ZONE-SAMA', - 'title': 'Portal Search: ZONE-SAMA', - }, - 'playlist_mincount': 47, + 'playlist_mincount': 45, }, { 'url': 'http://www.newgrounds.com/audio/search/title/cats', 'only_matching': True, @@ -184,7 +177,7 @@ class NewgroundsPlaylistIE(InfoExtractor): continue entries.append( self.url_result( - 'https://www.newgrounds.com/ + path + '/' + media_id, + 'https://www.newgrounds.com/' + path + '/' + media_id, ie=NewgroundsIE.ie_key(), video_id=media_id)) return self.playlist_result(entries, playlist_id, title) From 39681044bc73d80f19d95b23e0a4245b33f1f35c Mon Sep 17 00:00:00 2001 From: u-spec-png Date: Thu, 26 Aug 2021 16:41:17 +0200 Subject: [PATCH 3/5] [Newgrounds] Apply suggestion --- youtube_dl/extractor/newgrounds.py | 67 ++++++++++++++++-------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 105297005..0cf042c63 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -5,13 +5,11 @@ import re from .common import InfoExtractor from ..utils import ( extract_attributes, + int_or_none, parse_duration, parse_filesize, parse_count, unified_timestamp, - unified_strdate, - RegexNotFoundError, - ExtractorError, ) @@ -28,6 +26,7 @@ class NewgroundsIE(InfoExtractor): 'timestamp': 1378878540, 'upload_date': '20130911', 'duration': 143, + 'description': 'Sonic blahblahblah I\'m late again SEPTEMBER 11TH: Busmode-The story of ants on a log. ', }, }, { 'url': 'https://www.newgrounds.com/portal/view/297383', @@ -38,6 +37,7 @@ class NewgroundsIE(InfoExtractor): 'uploader': 'Egoraptor', 'timestamp': 1140663240, 'upload_date': '20060223', + 'description': 'Metal Gear is awesome is so is this movie.', }, }, { # source format unavailable, additional mp4 formats @@ -46,9 +46,10 @@ class NewgroundsIE(InfoExtractor): 'id': '689400', 'ext': 'mp4', 'title': 'ZTV News Episode 8', - 'uploader': 'BennettTheSage', + 'uploader': 'ZONE-SAMA', 'timestamp': 1487965140, 'upload_date': '20170224', + 'description': 'ZTV News Episode 8 (February 2017)', }, 'params': { 'skip_download': True, @@ -57,20 +58,18 @@ class NewgroundsIE(InfoExtractor): def _real_extract(self, url): media_id = self._match_id(url) + formats = [] + uploader = None webpage = self._download_webpage(url, media_id) title = self._html_search_regex( r'(.+?)', webpage, 'title') - try: - media_url = self._parse_json(self._search_regex( - r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) - except RegexNotFoundError or ExtractorError: - media_url = None - formats = [] + media_url_string = self._search_regex(r'"url"\s*:\s*("[^"]+"),', webpage, 'media url string', fatal=False, default=None) - if media_url: + if media_url_string: + media_url = self._parse_json(media_url_string, media_id) formats = [{ 'url': media_url, 'format_id': 'source', @@ -81,32 +80,29 @@ class NewgroundsIE(InfoExtractor): 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, br', 'X-Requested-With': 'XMLHttpRequest', - 'Connection': 'keep-alive', }) - for resolution in ('360p', '720p', '1080p'): - if resolution in json_data['sources']: + uploader = json_data.get('author') + media_formats = json_data.get('sources', []) + for media_format in media_formats: + media_sources = media_formats[media_format] + for source in media_sources: formats.append({ - 'url': json_data['sources'][resolution][0]['src'], - 'format_id': resolution, - 'height': int(resolution[:-1]), + 'format_id': media_format, + 'quality': int_or_none(media_format[:-1]), + 'url': source.get('src') }) - self._sort_formats(formats) + if not uploader: + uploader = self._html_search_regex( + (r'(?s)]*>(.+?).*?\s*Author\s*', + r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', + fatal=False) - uploader = self._html_search_regex( - (r'(?s)]*>(.+?).*?\s*Author\s*', - r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', - fatal=False) - - timestamp = self._html_search_regex( + timestamp = unified_timestamp(self._html_search_regex( (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', - default=None) - - upload_date = unified_strdate(timestamp) - - timestamp = unified_timestamp(timestamp) + default=None)) thumbnail = self._og_search_thumbnail(webpage) @@ -127,6 +123,9 @@ class NewgroundsIE(InfoExtractor): if '
Song' in webpage: formats[0]['vcodec'] = 'none' + + self._sort_formats(formats) + return { 'id': media_id, 'title': title, @@ -137,7 +136,6 @@ class NewgroundsIE(InfoExtractor): 'description': description, 'thumbnail': thumbnail, 'view_count': view_count, - 'upload_date': upload_date, } @@ -150,6 +148,13 @@ class NewgroundsPlaylistIE(InfoExtractor): 'title': 'Cats', }, 'playlist_mincount': 45, + }, { + 'url': 'https://www.newgrounds.com/collection/dogs', + 'info_dict': { + 'id': 'dogs', + 'title': 'Dogs', + }, + 'playlist_mincount': 25, }, { 'url': 'http://www.newgrounds.com/audio/search/title/cats', 'only_matching': True, @@ -177,7 +182,7 @@ class NewgroundsPlaylistIE(InfoExtractor): continue entries.append( self.url_result( - 'https://www.newgrounds.com/' + path + '/' + media_id, + 'https://www.newgrounds.com/%s/%s' % (path, media_id), ie=NewgroundsIE.ie_key(), video_id=media_id)) return self.playlist_result(entries, playlist_id, title) From f3da91008cf996cc7cfa18065f8e3b23d8068c37 Mon Sep 17 00:00:00 2001 From: u-spec-png Date: Thu, 26 Aug 2021 18:24:44 +0200 Subject: [PATCH 4/5] [Newgrounds] Apply suggestion --- youtube_dl/extractor/newgrounds.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 0cf042c63..65fc5c2ba 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -9,6 +9,7 @@ from ..utils import ( parse_duration, parse_filesize, parse_count, + parse_resolution, unified_timestamp, ) @@ -83,15 +84,17 @@ class NewgroundsIE(InfoExtractor): }) uploader = json_data.get('author') - media_formats = json_data.get('sources', []) + media_formats = json_data['sources'] for media_format in media_formats: media_sources = media_formats[media_format] for source in media_sources: - formats.append({ + format = { 'format_id': media_format, - 'quality': int_or_none(media_format[:-1]), - 'url': source.get('src') - }) + 'url': source.get('src'), + } + format.update(parse_resolution(media_format)) + format['quality'] = format.get('height') + formats.append(format) if not uploader: uploader = self._html_search_regex( From 8f7352509eddd2ce4e1a7238cd9daf238b618896 Mon Sep 17 00:00:00 2001 From: u-spec-png Date: Thu, 26 Aug 2021 18:29:13 +0200 Subject: [PATCH 5/5] [Newgrounds] Add _check_formats() --- youtube_dl/extractor/newgrounds.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 65fc5c2ba..2e95c6744 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -127,6 +127,7 @@ class NewgroundsIE(InfoExtractor): if '
Song' in webpage: formats[0]['vcodec'] = 'none' + self._check_formats(formats, media_id) self._sort_formats(formats) return {