From f877c6ae5a6e252d6904f90d597479451d2107aa Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 19 Aug 2015 23:11:25 +0800 Subject: [PATCH 1/5] [theplatform] Use InfoExtractor._parse_smil_formats() --- youtube_dl/extractor/common.py | 8 +++- youtube_dl/extractor/theplatform.py | 72 ++++++----------------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 65835d257..ac12be933 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1052,7 +1052,7 @@ def _parse_smil_namespace(self, smil): return self._search_regex( r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None) - def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None): + def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None): base = smil_url for meta in smil.findall(self._xpath_ns('./head/meta', namespace)): b = meta.get('base') or meta.get('httpBase') @@ -1091,6 +1091,12 @@ def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_para 'width': width, 'height': height, }) + if transform_rtmp_url: + streamer, src = transform_rtmp_url(streamer, src) + formats[-1].update({ + 'url': streamer, + 'play_path': src, + }) continue src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index 0643eccaf..29f938a76 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -9,9 +9,6 @@ from .common import InfoExtractor -from ..compat import ( - compat_str, -) from ..utils import ( determine_ext, ExtractorError, @@ -20,7 +17,8 @@ int_or_none, ) -_x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'}) +default_ns = 'http://www.w3.org/2005/SMIL21/Language' +_x = lambda p: xpath_with_ns(p, {'smil': default_ns}) class ThePlatformIE(InfoExtractor): @@ -145,63 +143,19 @@ def _real_extract(self, url): 'url': src, }] - head = meta.find(_x('smil:head')) - body = meta.find(_x('smil:body')) - - f4m_node = body.find(_x('smil:seq//smil:video')) - if f4m_node is None: - f4m_node = body.find(_x('smil:seq/smil:video')) - if f4m_node is not None and '.f4m' in f4m_node.attrib['src']: - f4m_url = f4m_node.attrib['src'] - if 'manifest.f4m?' not in f4m_url: - f4m_url += '?' + formats = self._parse_smil_formats( + meta, smil_url, video_id, namespace=default_ns, # the parameters are from syfy.com, other sites may use others, # they also work for nbc.com - f4m_url += '&g=UXWGVKRWHFSP&hdcore=3.0.3' - formats = self._extract_f4m_formats(f4m_url, video_id) - else: - formats = [] - switch = body.find(_x('smil:switch')) - if switch is None: - switch = body.find(_x('smil:par//smil:switch')) - if switch is None: - switch = body.find(_x('smil:par/smil:switch')) - if switch is None: - switch = body.find(_x('smil:par')) - if switch is not None: - base_url = head.find(_x('smil:meta')).attrib['base'] - for f in switch.findall(_x('smil:video')): - attr = f.attrib - width = int_or_none(attr.get('width')) - height = int_or_none(attr.get('height')) - vbr = int_or_none(attr.get('system-bitrate'), 1000) - format_id = '%dx%d_%dk' % (width, height, vbr) - formats.append({ - 'format_id': format_id, - 'url': base_url, - 'play_path': 'mp4:' + attr['src'], - 'ext': 'flv', - 'width': width, - 'height': height, - 'vbr': vbr, - }) - else: - switch = body.find(_x('smil:seq//smil:switch')) - if switch is None: - switch = body.find(_x('smil:seq/smil:switch')) - for f in switch.findall(_x('smil:video')): - attr = f.attrib - vbr = int_or_none(attr.get('system-bitrate'), 1000) - ext = determine_ext(attr['src']) - if ext == 'once': - ext = 'mp4' - formats.append({ - 'format_id': compat_str(vbr), - 'url': attr['src'], - 'vbr': vbr, - 'ext': ext, - }) - self._sort_formats(formats) + f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'}, + transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src)) + + for _format in formats: + ext = determine_ext(_format['url']) + if ext == 'once': + _format['ext'] = 'mp4' + + self._sort_formats(formats) return { 'id': video_id, From 26e1c3514f4af1ed60cd1114a653fe49e1fa8d11 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 20 Aug 2015 01:24:32 +0800 Subject: [PATCH 2/5] [theplatform] Add ThePlatformFeedIE --- youtube_dl/extractor/__init__.py | 5 +- youtube_dl/extractor/theplatform.py | 162 +++++++++++++++++++++------- 2 files changed, 126 insertions(+), 41 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index fa9acc923..b5f7ff9a9 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -612,7 +612,10 @@ from .testtube import TestTubeIE from .tf1 import TF1IE from .theonion import TheOnionIE -from .theplatform import ThePlatformIE +from .theplatform import ( + ThePlatformIE, + ThePlatformFeedIE, +) from .thesixtyone import TheSixtyOneIE from .thisamericanlife import ThisAmericanLifeIE from .thisav import ThisAVIE diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index 29f938a76..f02e0f58d 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -15,13 +15,68 @@ xpath_with_ns, unsmuggle_url, int_or_none, + url_basename, + float_or_none, ) default_ns = 'http://www.w3.org/2005/SMIL21/Language' _x = lambda p: xpath_with_ns(p, {'smil': default_ns}) -class ThePlatformIE(InfoExtractor): +class ThePlatformBaseIE(InfoExtractor): + def _extract_theplatform_smil_formats(self, smil_url, video_id, note='Downloading SMIL data'): + meta = self._download_xml(smil_url, video_id, note=note) + try: + error_msg = next( + n.attrib['abstract'] + for n in meta.findall(_x('.//smil:ref')) + if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired') + except StopIteration: + pass + else: + raise ExtractorError(error_msg, expected=True) + + formats = self._parse_smil_formats( + meta, smil_url, video_id, namespace=default_ns, + # the parameters are from syfy.com, other sites may use others, + # they also work for nbc.com + f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'}, + transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src)) + + for _format in formats: + ext = determine_ext(_format['url']) + if ext == 'once': + _format['ext'] = 'mp4' + + self._sort_formats(formats) + + return formats + + def get_metadata(self, path, video_id): + info_url = 'http://link.theplatform.com/s/%s?format=preview' % path + info_json = self._download_webpage(info_url, video_id) + info = json.loads(info_json) + + subtitles = {} + captions = info.get('captions') + if isinstance(captions, list): + for caption in captions: + lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type') + subtitles[lang] = [{ + 'ext': 'srt' if mime == 'text/srt' else 'ttml', + 'url': src, + }] + + return { + 'title': info['title'], + 'subtitles': subtitles, + 'description': info['description'], + 'thumbnail': info['defaultThumbnailUrl'], + 'duration': int_or_none(info.get('duration'), 1000), + } + + +class ThePlatformIE(ThePlatformBaseIE): _VALID_URL = r'''(?x) (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P[^/]+)/ (?:(?P(?:[^/]+/)+select/media/)|(?P(?:[^/\?]+/(?:swf|config)|onsite)/select/))? @@ -118,51 +173,78 @@ def _real_extract(self, url): if sig: smil_url = self._sign_url(smil_url, sig['key'], sig['secret']) - meta = self._download_xml(smil_url, video_id) - try: - error_msg = next( - n.attrib['abstract'] - for n in meta.findall(_x('.//smil:ref')) - if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired') - except StopIteration: - pass - else: - raise ExtractorError(error_msg, expected=True) + formats = self._extract_theplatform_smil_formats(smil_url, video_id) - info_url = 'http://link.theplatform.com/s/%s?format=preview' % path - info_json = self._download_webpage(info_url, video_id) - info = json.loads(info_json) + ret = self.get_metadata(path, video_id) + ret.update({ + 'id': video_id, + 'formats': formats, + }) - subtitles = {} - captions = info.get('captions') - if isinstance(captions, list): - for caption in captions: - lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type') - subtitles[lang] = [{ - 'ext': 'srt' if mime == 'text/srt' else 'ttml', - 'url': src, - }] + return ret - formats = self._parse_smil_formats( - meta, smil_url, video_id, namespace=default_ns, - # the parameters are from syfy.com, other sites may use others, - # they also work for nbc.com - f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'}, - transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src)) - for _format in formats: - ext = determine_ext(_format['url']) - if ext == 'once': - _format['ext'] = 'mp4' +class ThePlatformFeedIE(ThePlatformBaseIE): + _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&byGuid=%s' + _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P[^/]+)/(?P[^?/]+)\?(?:[^&]+&)*byGuid=(?P[a-zA-Z0-9_]+)' + _TEST = { + # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207 + 'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207', + 'md5': '22d2b84f058d3586efcd99e57d59d314', + 'info_dict': { + 'id': 'n_hardball_5biden_140207', + 'ext': 'mp4', + 'title': 'The Biden factor: will Joe run in 2016?', + 'description': 'Could Vice President Joe Biden be preparing a 2016 campaign? Mark Halperin and Sam Stein weigh in.', + 'thumbnail': 're:^https?://.*\.jpg$', + 'upload_date': '20140208', + 'timestamp': 1391824260, + 'duration': 467.0, + 'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'], + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group('id') + provider_id = mobj.group('provider_id') + feed_id = mobj.group('feed_id') + + real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, video_id) + feed = self._download_json(real_url, video_id) + entry = feed['entries'][0] + + formats = [] + first_video_id = None + duration = None + for item in entry['media$content']: + smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M' + cur_video_id = url_basename(smil_url) + if first_video_id is None: + first_video_id = cur_video_id + duration = float_or_none(item.get('plfile$duration')) + formats.extend(self._extract_theplatform_smil_formats(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id)) self._sort_formats(formats) - return { + thumbnails = [{ + 'url': thumbnail['plfile$url'], + 'width': int_or_none(thumbnail.get('plfile$width')), + 'height': int_or_none(thumbnail.get('plfile$height')), + } for thumbnail in entry.get('media$thumbnails', [])] + + timestamp = int_or_none(entry.get('media$availableDate'), scale=1000) + categories = [item['media$name'] for item in entry.get('media$categories', [])] + + ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id) + ret.update({ 'id': video_id, - 'title': info['title'], - 'subtitles': subtitles, 'formats': formats, - 'description': info['description'], - 'thumbnail': info['defaultThumbnailUrl'], - 'duration': int_or_none(info.get('duration'), 1000), - } + 'thumbnails': thumbnails, + 'duration': duration, + 'timestamp': timestamp, + 'categories': categories, + }) + + return ret From 05fe2594e4589b4e714a423550172eeec3949a70 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 20 Aug 2015 01:38:39 +0800 Subject: [PATCH 3/5] [theplatform] Support URLs with 'guid=' --- youtube_dl/extractor/theplatform.py | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index f02e0f58d..883bf491c 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -9,6 +9,10 @@ from .common import InfoExtractor +from ..compat import ( + compat_parse_qs, + compat_urllib_parse_urlparse, +) from ..utils import ( determine_ext, ExtractorError, @@ -120,6 +124,20 @@ class ThePlatformIE(ThePlatformBaseIE): }, { 'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7', 'only_matching': True, + }, { + 'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701', + 'md5': '734f3790fb5fc4903da391beeebc4836', + 'info_dict': { + 'id': 'tdy_or_siri_150701', + 'ext': 'mp4', + 'title': 'iPhone Siri’s sassy response to a math question has people talking', + 'description': 'md5:a565d1deadd5086f3331d57298ec6333', + 'duration': 83.0, + 'thumbnail': 're:^https?://.*\.jpg$', + 'timestamp': 1435752600, + 'upload_date': '20150701', + 'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"], + }, }] @staticmethod @@ -154,6 +172,24 @@ def _real_extract(self, url): path += '/media' path += '/' + video_id + qs_dict = compat_parse_qs(compat_urllib_parse_urlparse(url).query) + if 'guid' in qs_dict: + webpage = self._download_webpage(url, video_id) + scripts = re.findall(r']+src="([^"]+)"', webpage) + feed_id = None + # feed id usually locates in the last script. + # Seems there's no pattern for the interested script filename, so + # I try one by one + for script in reversed(scripts): + feed_script = self._download_webpage(script, video_id, 'Downloading feed script') + feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None) + if feed_id is not None: + break + if feed_id is None: + raise ExtractorError('Unable to find feed id') + return self.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % ( + provider_id, feed_id, qs_dict['guid'][0])) + if smuggled_data.get('force_smil_url', False): smil_url = url elif mobj.group('config'): From dac14bf311fd1b3c6af6c57b3b03878a11ef5aae Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 20 Aug 2015 01:41:18 +0800 Subject: [PATCH 4/5] [nbc] Add MSNBCIE --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/nbc.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index b5f7ff9a9..86ea0576a 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -358,6 +358,7 @@ NBCNewsIE, NBCSportsIE, NBCSportsVPlayerIE, + MSNBCIE, ) from .ndr import ( NDRIE, diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py index ccdbfb6c9..e683d24c4 100644 --- a/youtube_dl/extractor/nbc.py +++ b/youtube_dl/extractor/nbc.py @@ -236,3 +236,28 @@ def _real_extract(self, url): 'url': info['videoAssets'][-1]['publicUrl'], 'ie_key': 'ThePlatform', } + + +class MSNBCIE(InfoExtractor): + # https URLs redirect to corresponding http ones + _VALID_URL = r'http://www\.msnbc\.com/[^/]+/watch/(?P[^/]+)' + _TEST = { + 'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924', + 'md5': '6d236bf4f3dddc226633ce6e2c3f814d', + 'info_dict': { + 'id': 'n_hayes_Aimm_140801_272214', + 'ext': 'mp4', + 'title': 'The chaotic GOP immigration vote', + 'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.', + 'thumbnail': 're:^https?://.*\.jpg$', + 'timestamp': 1406937606, + 'upload_date': '20140802', + 'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'], + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + embed_url = self._html_search_meta('embedURL', webpage) + return self.url_result(embed_url) From aa6cd05ed82b14af0e3827b2ff43eed02087b574 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 20 Aug 2015 01:47:55 +0800 Subject: [PATCH 5/5] [theplatform] Fix Python 2: declare coding --- youtube_dl/extractor/theplatform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index 883bf491c..adaec3375 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals import re