From 0d97ef43bec006157870fd4a5cedfac1eaebf3a9 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Thu, 26 Feb 2015 23:45:54 +0200 Subject: [PATCH 01/17] [kaltura] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/kaltura.py | 134 +++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 youtube_dl/extractor/kaltura.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ddb9d6670e..e3b2cb54fe 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -227,6 +227,7 @@ from .jove import JoveIE from .jukebox import JukeboxIE from .jpopsukitv import JpopsukiIE +from .kaltura import KalturaIE from .kankan import KankanIE from .karaoketv import KaraoketvIE from .keezmovies import KeezMoviesIE diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py new file mode 100644 index 0000000000..2aff410c54 --- /dev/null +++ b/youtube_dl/extractor/kaltura.py @@ -0,0 +1,134 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_urllib_parse +from ..utils import ( + ExtractorError, + int_or_none, +) + + +class KalturaIE(InfoExtractor): + _VALID_URL = r'''(?x) + (?:kaltura:| + https?://(:?www\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_ + )(?P\d+) + (?::| + /(?:[^/]+/)*?entry_id/ + )(?P[0-9a-z_]+)''' + _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?' + _TESTS = [ + { + 'url': 'kaltura:269692:1_1jc2y3e4', + 'md5': '3adcbdb3dcc02d647539e53f284ba171', + 'info_dict': { + 'id': '1_1jc2y3e4', + 'ext': 'mp4', + 'title': 'Track 4', + 'upload_date': '20131219', + 'uploader_id': 'mlundberg@wolfgangsvault.com', + 'description': 'The Allman Brothers Band, 12/16/1981', + 'thumbnail': 're:^https?://.*/thumbnail/.*', + 'timestamp': int, + }, + }, + { + 'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4', + 'only_matching': True, + }, + ] + + def _kaltura_api_call(self, video_id, actions, *args, **kwargs): + params = actions[0] + if len(actions) > 1: + for i, a in enumerate(actions[1:], start=1): + for k, v in a.items(): + params['%d:%s' % (i, k)] = v + + query = compat_urllib_parse.urlencode(params) + url = self._API_BASE + query + data = self._download_json(url, video_id, *args, **kwargs) + + status = data if len(actions) == 1 else data[0] + if status.get('objectType') == 'KalturaAPIException': + raise ExtractorError( + '%s said: %s' % (self.IE_NAME, status['message'])) + + return data + + def _get_kaltura_signature(self, video_id, partner_id): + actions = [{ + 'apiVersion': '3.1', + 'expiry': 86400, + 'format': 1, + 'service': 'session', + 'action': 'startWidgetSession', + 'widgetId': '_%s' % partner_id, + }] + return self._kaltura_api_call( + video_id, actions, note='Downloading Kaltura signature')['ks'] + + def _get_video_info(self, video_id, partner_id): + signature = self._get_kaltura_signature(video_id, partner_id) + actions = [ + { + 'action': 'null', + 'apiVersion': '3.1.5', + 'clientTag': 'kdp:v3.8.5', + 'format': 1, # JSON, 2 = XML, 3 = PHP + 'service': 'multirequest', + 'ks': signature, + }, + { + 'action': 'get', + 'entryId': video_id, + 'service': 'baseentry', + 'version': '-1', + }, + { + 'action': 'getContextData', + 'contextDataParams:objectType': 'KalturaEntryContextDataParams', + 'contextDataParams:referrer': 'http://www.kaltura.com/', + 'contextDataParams:streamerType': 'http', + 'entryId': video_id, + 'service': 'baseentry', + }, + ] + return self._kaltura_api_call( + video_id, actions, note='Downloading video info JSON') + + def _real_extract(self, url): + video_id = self._match_id(url) + mobj = re.match(self._VALID_URL, url) + partner_id, entry_id = mobj.group('partner_id'), mobj.group('id') + + info, source_data = self._get_video_info(entry_id, partner_id) + + formats = [{ + 'format_id': '%(fileExt)s-%(bitrate)s' % f, + 'ext': f['fileExt'], + 'tbr': f['bitrate'], + 'fps': f.get('frameRate'), + 'filesize_approx': int_or_none(f.get('size'), invscale=1024), + 'container': f.get('containerFormat'), + 'vcodec': f.get('videoCodecId'), + 'height': f.get('height'), + 'width': f.get('width'), + 'url': '%s/flavorId/%s' % (info['dataUrl'], f['id']), + } for f in source_data['flavorAssets']] + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': info['name'], + 'formats': formats, + 'description': info.get('description'), + 'thumbnail': info.get('thumbnailUrl'), + 'duration': info.get('duration'), + 'timestamp': info.get('createdAt'), + 'uploader_id': info.get('userId'), + 'view_count': info.get('plays'), + } From da419e23320aef560c05f543c5ee375e8956c95a Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Thu, 26 Feb 2015 23:47:45 +0200 Subject: [PATCH 02/17] [musicvault] Use the Kaltura extractor --- youtube_dl/extractor/musicvault.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/musicvault.py b/youtube_dl/extractor/musicvault.py index ebb1eb8e95..0e46ac7c1f 100644 --- a/youtube_dl/extractor/musicvault.py +++ b/youtube_dl/extractor/musicvault.py @@ -3,17 +3,13 @@ import re from .common import InfoExtractor -from ..utils import ( - parse_duration, - unified_strdate, -) class MusicVaultIE(InfoExtractor): _VALID_URL = r'https?://www\.musicvault\.com/(?P[^/?#]*)/video/(?P[^/?#]*)_(?P[0-9]+)\.html' _TEST = { 'url': 'http://www.musicvault.com/the-allman-brothers-band/video/straight-from-the-heart_1010863.html', - 'md5': '2cdbb3ae75f7fb3519821507d2fb3c15', + 'md5': '3adcbdb3dcc02d647539e53f284ba171', 'info_dict': { 'id': '1010863', 'ext': 'mp4', @@ -22,9 +18,10 @@ class MusicVaultIE(InfoExtractor): 'duration': 244, 'uploader': 'The Allman Brothers Band', 'thumbnail': 're:^https?://.*/thumbnail/.*', - 'upload_date': '19811216', + 'upload_date': '20131219', 'location': 'Capitol Theatre (Passaic, NJ)', 'description': 'Listen to The Allman Brothers Band perform Straight from the Heart at Capitol Theatre (Passaic, NJ) on Dec 16, 1981', + 'timestamp': int, } } @@ -43,34 +40,24 @@ def _real_extract(self, url): r'(.*?)', data_div, 'uploader', fatal=False) title = self._html_search_regex( r'(.*?)', data_div, 'title') - upload_date = unified_strdate(self._html_search_regex( - r'(.*?)', data_div, 'uploader', fatal=False)) location = self._html_search_regex( r'(.*?)', data_div, 'location', fatal=False) - duration = parse_duration(self._html_search_meta('duration', webpage)) - - VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http' kaltura_id = self._search_regex( r'
Date: Fri, 27 Feb 2015 00:34:19 +0200 Subject: [PATCH 03/17] [generic] Support dynamic Kaltura embeds (#5016) (#5073) --- youtube_dl/extractor/generic.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 3aff57e303..27e2bc3001 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -557,6 +557,18 @@ class GenericIE(InfoExtractor): 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !', } }, + # Kaltura embed + { + 'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15', + 'info_dict': { + 'id': '1_eergr3h1', + 'ext': 'mp4', + 'upload_date': '20150226', + 'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com', + 'timestamp': int, + 'title': 'John Carlson Postgame 2/25/15', + }, + }, ] def report_following_redirect(self, new_url): @@ -1113,6 +1125,12 @@ def _playlist_from_matches(matches, getter=None, ie=None): if mobj is not None: return self.url_result(mobj.group('url'), 'Zapiks') + # Look for Kaltura embeds + mobj = re.search( + r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P[^']+)',.*?'entry_id'\s*:\s*'(?P[^']+)',", webpage) + if mobj is not None: + return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura') + def check_video(vurl): if YoutubeIE.suitable(vurl): return True From 0eba1e178230a88d1b316f54edbb671d216c1d02 Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 27 Feb 2015 00:51:22 +0200 Subject: [PATCH 04/17] [lynda] Fixed subtitles broken file --- youtube_dl/extractor/lynda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py index 109055e720..8bb21ee1d7 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dl/extractor/lynda.py @@ -152,7 +152,7 @@ def _fix_subtitles(self, subs): continue appear_time = m_current.group('timecode') disappear_time = m_next.group('timecode') - text = seq_current['Caption'] + text = seq_current['Caption'].strip() srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text) if srt: return srt From f3bff94cf96587462baa8da6d7e5f17801381753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 27 Feb 2015 12:24:51 +0100 Subject: [PATCH 05/17] [rtve] Extract duration --- youtube_dl/extractor/rtve.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py index c0fd23ff10..b42442d127 100644 --- a/youtube_dl/extractor/rtve.py +++ b/youtube_dl/extractor/rtve.py @@ -8,8 +8,9 @@ from .common import InfoExtractor from ..compat import compat_urlparse from ..utils import ( - struct_unpack, + float_or_none, remove_end, + struct_unpack, ) @@ -67,6 +68,7 @@ class RTVEALaCartaIE(InfoExtractor): 'id': '2491869', 'ext': 'mp4', 'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia', + 'duration': 5024.566, }, }, { 'note': 'Live stream', @@ -113,6 +115,7 @@ def _real_extract(self, url): 'thumbnail': info.get('image'), 'page_url': url, 'subtitles': subtitles, + 'duration': float_or_none(info.get('duration'), scale=1000), } def _get_subtitles(self, video_id, sub_file): From 7862ad88b725daae957ad27ae60993e360c01e13 Mon Sep 17 00:00:00 2001 From: HanYOLO Date: Fri, 27 Feb 2015 15:41:58 +0100 Subject: [PATCH 06/17] puls4 Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/puls4.py | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 youtube_dl/extractor/puls4.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index e3b2cb54fe..d137e11049 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -373,6 +373,7 @@ from .pornoxo import PornoXOIE from .promptfile import PromptFileIE from .prosiebensat1 import ProSiebenSat1IE +from .puls4 import Puls4IE from .pyvideo import PyvideoIE from .quickvid import QuickVidIE from .r7 import R7IE diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py new file mode 100644 index 0000000000..70dedbff3e --- /dev/null +++ b/youtube_dl/extractor/puls4.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +from .common import InfoExtractor + +import re + + +class Puls4IE(InfoExtractor): + + _VALID_URL = r'https?://www.puls4.com/video/.+?/play/(?P[0-9]+)' + _TESTS = [{ + 'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816', + 'md5': '49f6a6629747eeec43cef6a46b5df81d', + 'info_dict': { + 'id': '2716816', + 'ext': 'mp4', + 'title': 'Pro und Contra vom 23.02.2015'}}, + { + 'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106', + 'md5': '6a48316c8903ece8dab9b9a7bf7a59ec', + 'info_dict': { + 'id': '1298106', + 'ext': 'mp4', + 'title': 'Lucky Fritz'}} + ] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + # if fsk-button + real_url = self._html_search_regex(r'\"fsk-button\".+?href=\"([^"]+)', + webpage, 'fsk_button', default=None) + if real_url: + webpage = self._download_webpage(real_url, video_id) + + title = self._html_search_regex( + r'
.+?

(.+?)

', + webpage, 'title', flags=re.DOTALL) + + sd_url = self._html_search_regex( + r'{\"url\":\"([^"]+?)\",\"hd\":false', + webpage, 'sd_url').replace('\\', '') + + formats = [{'format_id': 'sd', 'url': sd_url, 'quality': -2}] + + hd_url = self._html_search_regex( + r'{\"url\":\"([^"]+?)\",\"hd\":true', + webpage, 'hd_url', default=None) + if hd_url: + hd_url = hd_url.replace('\\', '') + formats.append({'format_id': 'hd', 'url': hd_url, 'quality': -1}) + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'ext': 'mp4' + } From a0d646135aaf417e0aa000419974c676335d164a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 27 Feb 2015 20:56:06 +0600 Subject: [PATCH 07/17] [lynda] Extend _VALID_URL --- youtube_dl/extractor/lynda.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py index 109055e720..e7e9d80def 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dl/extractor/lynda.py @@ -18,7 +18,7 @@ class LyndaIE(InfoExtractor): IE_NAME = 'lynda' IE_DESC = 'lynda.com videos' - _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html' + _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(\d+)' _LOGIN_URL = 'https://www.lynda.com/login/login.aspx' _NETRC_MACHINE = 'lynda' @@ -27,7 +27,7 @@ class LyndaIE(InfoExtractor): ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.' - _TEST = { + _TESTS = [{ 'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html', 'md5': 'ecfc6862da89489161fb9cd5f5a6fac1', 'info_dict': { @@ -36,7 +36,10 @@ class LyndaIE(InfoExtractor): 'title': 'Using the exercise files', 'duration': 68 } - } + }, { + 'url': 'https://www.lynda.com/player/embed/133770?tr=foo=1;bar=g;fizz=rt&fs=0', + 'only_matching': True, + }] def _real_initialize(self): self._login() From 781a7ef60ab03cf1e646a80ecf4cbd70f1532db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 27 Feb 2015 16:18:18 +0100 Subject: [PATCH 08/17] [lynda] Use 'lstrip' for the subtitles The newlines at the end are important, they separate each piece of text. --- youtube_dl/extractor/lynda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py index bfd9b73d25..5dc22da22a 100644 --- a/youtube_dl/extractor/lynda.py +++ b/youtube_dl/extractor/lynda.py @@ -155,7 +155,7 @@ def _fix_subtitles(self, subs): continue appear_time = m_current.group('timecode') disappear_time = m_next.group('timecode') - text = seq_current['Caption'].strip() + text = seq_current['Caption'].lstrip() srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text) if srt: return srt From 4ffbf77886b1c6c3f8d767223ca088395d08ab14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Feb 2015 00:15:03 +0600 Subject: [PATCH 09/17] [odnoklassniki] Add extractor (Closes #5075) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/odnoklassniki.py | 85 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 youtube_dl/extractor/odnoklassniki.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index e3b2cb54fe..aecb67bf43 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -346,6 +346,7 @@ from .ntvru import NTVRuIE from .nytimes import NYTimesIE from .nuvid import NuvidIE +from .odnoklassniki import OdnoklassnikiIE from .oktoberfesttv import OktoberfestTVIE from .ooyala import OoyalaIE from .openfilm import OpenFilmIE diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py new file mode 100644 index 0000000000..155d0ee6a8 --- /dev/null +++ b/youtube_dl/extractor/odnoklassniki.py @@ -0,0 +1,85 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + unified_strdate, + int_or_none, + qualities, +) + + +class OdnoklassnikiIE(InfoExtractor): + _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P\d+)' + _TESTS = [{ + 'url': 'http://ok.ru/video/20079905452', + 'md5': '8e24ad2da6f387948e7a7d44eb8668fe', + 'info_dict': { + 'id': '20079905452', + 'ext': 'mp4', + 'title': 'Культура меняет нас (прекрасный ролик!))', + 'duration': 100, + 'upload_date': '20141207', + 'uploader_id': '330537914540', + 'uploader': 'Виталий Добровольский', + 'like_count': int, + 'age_limit': 0, + }, + }, { + 'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452', + 'only_matching': True, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage(url, video_id) + + player = self._parse_json( + self._search_regex( + r"OKVideo\.start\(({.+?})\s*,\s*'VideoAutoplay_player'", webpage, 'player'), + video_id) + + metadata = self._parse_json(player['flashvars']['metadata'], video_id) + + movie = metadata['movie'] + title = movie['title'] + thumbnail = movie.get('poster') + duration = int_or_none(movie.get('duration')) + + author = metadata.get('author', {}) + uploader_id = author.get('id') + uploader = author.get('name') + + upload_date = unified_strdate(self._html_search_meta( + 'ya:ovs:upload_date', webpage, 'upload date')) + + age_limit = None + adult = self._html_search_meta( + 'ya:ovs:adult', webpage, 'age limit') + if adult: + age_limit = 18 if adult == 'true' else 0 + + like_count = int_or_none(metadata.get('likeCount')) + + quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd')) + + formats = [{ + 'url': f['url'], + 'ext': 'mp4', + 'format_id': f['name'], + 'quality': quality(f['name']), + } for f in metadata['videos']] + + return { + 'id': video_id, + 'title': title, + 'thumbnail': thumbnail, + 'duration': duration, + 'upload_date': upload_date, + 'uploader': uploader, + 'uploader_id': uploader_id, + 'like_count': like_count, + 'age_limit': age_limit, + 'formats': formats, + } From bd3749ed693ae96becd3832f20e765e1efe01476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Feb 2015 00:19:31 +0600 Subject: [PATCH 10/17] [kaltura] Extend _VALID_URL (Closes #5081) --- youtube_dl/extractor/kaltura.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py index 2aff410c54..d287304920 100644 --- a/youtube_dl/extractor/kaltura.py +++ b/youtube_dl/extractor/kaltura.py @@ -14,7 +14,7 @@ class KalturaIE(InfoExtractor): _VALID_URL = r'''(?x) (?:kaltura:| - https?://(:?www\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_ + https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_ )(?P\d+) (?::| /(?:[^/]+/)*?entry_id/ @@ -39,6 +39,10 @@ class KalturaIE(InfoExtractor): 'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4', 'only_matching': True, }, + { + 'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3', + 'only_matching': True, + }, ] def _kaltura_api_call(self, video_id, actions, *args, **kwargs): From 40b077bc7ee2722463f23974ce488acb7f11815a Mon Sep 17 00:00:00 2001 From: thc202 Date: Fri, 27 Feb 2015 22:27:30 +0000 Subject: [PATCH 11/17] [oppetarkiv] Add new extractor Some, if not all, of the videos appear to be geo-blocked (Sweden). Test might fail (403 Forbidden) if not run through a Swedish connection. --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/oppetarkiv.py | 56 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 youtube_dl/extractor/oppetarkiv.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index aecb67bf43..1544f1059c 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -350,6 +350,7 @@ from .oktoberfesttv import OktoberfestTVIE from .ooyala import OoyalaIE from .openfilm import OpenFilmIE +from .oppetarkiv import OppetArkivIE from .orf import ( ORFTVthekIE, ORFOE1IE, diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py new file mode 100644 index 0000000000..6dd1fad3f0 --- /dev/null +++ b/youtube_dl/extractor/oppetarkiv.py @@ -0,0 +1,56 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import ( + determine_ext, +) + + +class OppetArkivIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?oppetarkiv.se/video/(?P[0-9]+)' + _TEST = { + 'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318', + 'md5': '7b95ca9bedeead63012b2d7c3992c28f', + 'info_dict': { + 'id': '1058509', + 'ext': 'mp4', + 'title': 'Farlig kryssning', + 'duration': 2566, + 'thumbnail': 're:^https?://.*[\.-]jpg$', + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + info = self._download_json( + 'http://www.oppetarkiv.se/video/%s?output=json' % video_id, video_id) + + title = info['context']['title'] + thumbnail = info['context'].get('thumbnailImage') + + video_info = info['video'] + formats = [] + for vr in video_info['videoReferences']: + vurl = vr['url'] + if determine_ext(vurl) == 'm3u8': + formats.extend(self._extract_m3u8_formats( + vurl, video_id, + ext='mp4', entry_protocol='m3u8_native', + m3u8_id=vr.get('playerType'))) + else: + formats.append({ + 'format_id': vr.get('playerType'), + 'url': vurl, + }) + self._sort_formats(formats) + + duration = video_info.get('materialLength') + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'thumbnail': thumbnail, + 'duration': duration, + } From 0f2c0d335b14ba1596e6608db7a6f29a0d9e1c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 28 Feb 2015 14:03:27 +0100 Subject: [PATCH 12/17] [YoutubeDL] Use the InfoExtractor._download_webpage method for getting the subtitles It handles encodings better, for example for 'http://www.npo.nl/nos-journaal/14-02-2015/POW_00942207' --- youtube_dl/YoutubeDL.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 76fc394bcf..74e4261680 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1300,17 +1300,18 @@ def process_info(self, info_dict): # subtitles download errors are already managed as troubles in relevant IE # that way it will silently go on when used with unsupporting IE subtitles = info_dict['requested_subtitles'] + ie = self.get_info_extractor(info_dict['extractor_key']) for sub_lang, sub_info in subtitles.items(): sub_format = sub_info['ext'] if sub_info.get('data') is not None: sub_data = sub_info['data'] else: try: - uf = self.urlopen(sub_info['url']) - sub_data = uf.read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + sub_data = ie._download_webpage( + sub_info['url'], info_dict['id'], note=False) + except ExtractorError as err: self.report_warning('Unable to download subtitle for "%s": %s' % - (sub_lang, compat_str(err))) + (sub_lang, compat_str(err.cause))) continue try: sub_filename = subtitles_filename(filename, sub_lang, sub_format) From e9fade72f347cbcce779ff176d516467a425b43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 28 Feb 2015 14:43:24 +0100 Subject: [PATCH 13/17] Add postprocessor for converting subtitles (closes #4954) --- youtube_dl/__init__.py | 8 ++++++ youtube_dl/options.py | 4 +++ youtube_dl/postprocessor/__init__.py | 2 ++ youtube_dl/postprocessor/ffmpeg.py | 38 ++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 5ce2018009..49f382695d 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -170,6 +170,9 @@ def _real_main(argv=None): if opts.recodevideo is not None: if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']: parser.error('invalid video recode format specified') + if opts.convertsubtitles is not None: + if opts.convertsubtitles not in ['srt', 'vtt', 'ass']: + parser.error('invalid subtitle format specified') if opts.date is not None: date = DateRange.day(opts.date) @@ -223,6 +226,11 @@ def _real_main(argv=None): 'key': 'FFmpegVideoConvertor', 'preferedformat': opts.recodevideo, }) + if opts.convertsubtitles: + postprocessors.append({ + 'key': 'FFmpegSubtitlesConvertor', + 'format': opts.convertsubtitles, + }) if opts.embedsubtitles: postprocessors.append({ 'key': 'FFmpegEmbedSubtitle', diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 886ce96132..58f811162e 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -751,6 +751,10 @@ def _hide_login_info(opts): '--exec', metavar='CMD', dest='exec_cmd', help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'') + postproc.add_option( + '--convert-subtitles', '--convert-subs', + metavar='FORMAT', dest='convertsubtitles', default=None, + help='Convert the subtitles to other format (currently supported: srt|ass|vtt)') parser.add_option_group(general) parser.add_option_group(network) diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py index 0ffbca2585..708df3dd49 100644 --- a/youtube_dl/postprocessor/__init__.py +++ b/youtube_dl/postprocessor/__init__.py @@ -11,6 +11,7 @@ FFmpegMergerPP, FFmpegMetadataPP, FFmpegVideoConvertorPP, + FFmpegSubtitlesConvertorPP, ) from .xattrpp import XAttrMetadataPP from .execafterdownload import ExecAfterDownloadPP @@ -31,6 +32,7 @@ def get_postprocessor(key): 'FFmpegMergerPP', 'FFmpegMetadataPP', 'FFmpegPostProcessor', + 'FFmpegSubtitlesConvertorPP', 'FFmpegVideoConvertorPP', 'XAttrMetadataPP', ] diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 398fe050ed..30094c2f37 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import io import os import subprocess import sys @@ -635,3 +636,40 @@ def run(self, info): os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return True, info + + +class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): + def __init__(self, downloader=None, format=None): + super(FFmpegSubtitlesConvertorPP, self).__init__(downloader) + self.format = format + + def run(self, info): + subs = info.get('requested_subtitles') + filename = info['filepath'] + new_ext = self.format + new_format = new_ext + if new_format == 'vtt': + new_format = 'webvtt' + if subs is None: + self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert') + return True, info + self._downloader.to_screen('[ffmpeg] Converting subtitles') + for lang, sub in subs.items(): + ext = sub['ext'] + if ext == new_ext: + self._downloader.to_screen( + '[ffmpeg] Subtitle file for %s is already in the requested' + 'format' % new_ext) + continue + new_file = subtitles_filename(filename, lang, new_ext) + self.run_ffmpeg( + subtitles_filename(filename, lang, ext), + new_file, ['-f', new_format]) + + with io.open(new_file, 'rt', encoding='utf-8') as f: + subs[lang] = { + 'ext': ext, + 'data': f.read(), + } + + return True, info From e143f5dae9c767529b8b522a9df63ac0ee8fc356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Feb 2015 21:12:06 +0600 Subject: [PATCH 14/17] [oppetarkiv] Extract f4m formats and age limit --- youtube_dl/extractor/oppetarkiv.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py index 6dd1fad3f0..ae6a283081 100644 --- a/youtube_dl/extractor/oppetarkiv.py +++ b/youtube_dl/extractor/oppetarkiv.py @@ -8,17 +8,19 @@ class OppetArkivIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?oppetarkiv.se/video/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?oppetarkiv\.se/video/(?P[0-9]+)' _TEST = { 'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318', - 'md5': '7b95ca9bedeead63012b2d7c3992c28f', + 'md5': '5c1eb616e59f733d4af77edc5177d2fe', 'info_dict': { 'id': '1058509', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'Farlig kryssning', 'duration': 2566, 'thumbnail': 're:^https?://.*[\.-]jpg$', + 'age_limit': 0, }, + 'skip': 'Only works from Sweden', } def _real_extract(self, url): @@ -33,11 +35,16 @@ def _real_extract(self, url): formats = [] for vr in video_info['videoReferences']: vurl = vr['url'] - if determine_ext(vurl) == 'm3u8': + ext = determine_ext(vurl) + if ext == 'm3u8': formats.extend(self._extract_m3u8_formats( vurl, video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id=vr.get('playerType'))) + elif ext == 'f4m': + formats.extend(self._extract_f4m_formats( + vurl + '?hdcore=3.3.0', video_id, + f4m_id=vr.get('playerType'))) else: formats.append({ 'format_id': vr.get('playerType'), @@ -47,10 +54,13 @@ def _real_extract(self, url): duration = video_info.get('materialLength') + age_limit = 18 if video_info.get('inappropriateForChildren') else 0 + return { 'id': video_id, 'title': title, 'formats': formats, 'thumbnail': thumbnail, 'duration': duration, + 'age_limit': age_limit, } From df5ae3eb16effec4dc2609ed3520ddc068305efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Feb 2015 21:25:04 +0600 Subject: [PATCH 15/17] [oppetarkiv] Merge with svtplay --- youtube_dl/extractor/__init__.py | 1 - youtube_dl/extractor/oppetarkiv.py | 66 ------------------------------ youtube_dl/extractor/svtplay.py | 42 +++++++++++++++---- 3 files changed, 34 insertions(+), 75 deletions(-) delete mode 100644 youtube_dl/extractor/oppetarkiv.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1544f1059c..aecb67bf43 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -350,7 +350,6 @@ from .oktoberfesttv import OktoberfestTVIE from .ooyala import OoyalaIE from .openfilm import OpenFilmIE -from .oppetarkiv import OppetArkivIE from .orf import ( ORFTVthekIE, ORFOE1IE, diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py deleted file mode 100644 index ae6a283081..0000000000 --- a/youtube_dl/extractor/oppetarkiv.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -from .common import InfoExtractor -from ..utils import ( - determine_ext, -) - - -class OppetArkivIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?oppetarkiv\.se/video/(?P[0-9]+)' - _TEST = { - 'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318', - 'md5': '5c1eb616e59f733d4af77edc5177d2fe', - 'info_dict': { - 'id': '1058509', - 'ext': 'flv', - 'title': 'Farlig kryssning', - 'duration': 2566, - 'thumbnail': 're:^https?://.*[\.-]jpg$', - 'age_limit': 0, - }, - 'skip': 'Only works from Sweden', - } - - def _real_extract(self, url): - video_id = self._match_id(url) - info = self._download_json( - 'http://www.oppetarkiv.se/video/%s?output=json' % video_id, video_id) - - title = info['context']['title'] - thumbnail = info['context'].get('thumbnailImage') - - video_info = info['video'] - formats = [] - for vr in video_info['videoReferences']: - vurl = vr['url'] - ext = determine_ext(vurl) - if ext == 'm3u8': - formats.extend(self._extract_m3u8_formats( - vurl, video_id, - ext='mp4', entry_protocol='m3u8_native', - m3u8_id=vr.get('playerType'))) - elif ext == 'f4m': - formats.extend(self._extract_f4m_formats( - vurl + '?hdcore=3.3.0', video_id, - f4m_id=vr.get('playerType'))) - else: - formats.append({ - 'format_id': vr.get('playerType'), - 'url': vurl, - }) - self._sort_formats(formats) - - duration = video_info.get('materialLength') - - age_limit = 18 if video_info.get('inappropriateForChildren') else 0 - - return { - 'id': video_id, - 'title': title, - 'formats': formats, - 'thumbnail': thumbnail, - 'duration': duration, - 'age_limit': age_limit, - } diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py index eadb9ccb44..433dfd1cb0 100644 --- a/youtube_dl/extractor/svtplay.py +++ b/youtube_dl/extractor/svtplay.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import ( determine_ext, @@ -8,23 +10,40 @@ class SVTPlayIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P[0-9]+)' - _TEST = { + IE_DESC = 'SVT Play and Öppet arkiv' + _VALID_URL = r'https?://(?:www\.)?(?Psvtplay|oppetarkiv)\.se/video/(?P[0-9]+)' + _TESTS = [{ 'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final', - 'md5': 'f4a184968bc9c802a9b41316657aaa80', + 'md5': 'ade3def0643fa1c40587a422f98edfd9', 'info_dict': { 'id': '2609989', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'SM veckan vinter, Örebro - Rally, final', 'duration': 4500, 'thumbnail': 're:^https?://.*[\.-]jpg$', + 'age_limit': 0, }, - } + }, { + 'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318', + 'md5': 'c3101a17ce9634f4c1f9800f0746c187', + 'info_dict': { + 'id': '1058509', + 'ext': 'flv', + 'title': 'Farlig kryssning', + 'duration': 2566, + 'thumbnail': 're:^https?://.*[\.-]jpg$', + 'age_limit': 0, + }, + 'skip': 'Only works from Sweden', + }] def _real_extract(self, url): - video_id = self._match_id(url) + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + host = mobj.group('host') + info = self._download_json( - 'http://www.svtplay.se/video/%s?output=json' % video_id, video_id) + 'http://www.%s.se/video/%s?output=json' % (host, video_id), video_id) title = info['context']['title'] thumbnail = info['context'].get('thumbnailImage') @@ -33,11 +52,16 @@ def _real_extract(self, url): formats = [] for vr in video_info['videoReferences']: vurl = vr['url'] - if determine_ext(vurl) == 'm3u8': + ext = determine_ext(vurl) + if ext == 'm3u8': formats.extend(self._extract_m3u8_formats( vurl, video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id=vr.get('playerType'))) + elif ext == 'f4m': + formats.extend(self._extract_f4m_formats( + vurl + '?hdcore=3.3.0', video_id, + f4m_id=vr.get('playerType'))) else: formats.append({ 'format_id': vr.get('playerType'), @@ -46,6 +70,7 @@ def _real_extract(self, url): self._sort_formats(formats) duration = video_info.get('materialLength') + age_limit = 18 if video_info.get('inappropriateForChildren') else 0 return { 'id': video_id, @@ -53,4 +78,5 @@ def _real_extract(self, url): 'formats': formats, 'thumbnail': thumbnail, 'duration': duration, + 'age_limit': age_limit, } From 6c87c2eea8b7d14c4178aaae3d74559347a772e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Feb 2015 22:25:57 +0600 Subject: [PATCH 16/17] [puls4] Improve and extract more metadata --- youtube_dl/extractor/puls4.py | 89 +++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py index 70dedbff3e..cce84b9e4d 100644 --- a/youtube_dl/extractor/puls4.py +++ b/youtube_dl/extractor/puls4.py @@ -1,61 +1,88 @@ # -*- coding: utf-8 -*- - from __future__ import unicode_literals from .common import InfoExtractor - -import re +from ..utils import ( + ExtractorError, + unified_strdate, + int_or_none, +) class Puls4IE(InfoExtractor): - - _VALID_URL = r'https?://www.puls4.com/video/.+?/play/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?puls4\.com/video/[^/]+/play/(?P[0-9]+)' _TESTS = [{ 'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816', 'md5': '49f6a6629747eeec43cef6a46b5df81d', 'info_dict': { 'id': '2716816', 'ext': 'mp4', - 'title': 'Pro und Contra vom 23.02.2015'}}, - { + 'title': 'Pro und Contra vom 23.02.2015', + 'description': 'md5:293e44634d9477a67122489994675db6', + 'duration': 2989, + 'upload_date': '20150224', + 'uploader': 'PULS_4', + }, + 'skip': 'Only works from Germany', + }, { 'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106', 'md5': '6a48316c8903ece8dab9b9a7bf7a59ec', 'info_dict': { 'id': '1298106', 'ext': 'mp4', - 'title': 'Lucky Fritz'}} - ] + 'title': 'Lucky Fritz', + }, + 'skip': 'Only works from Germany', + }] def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - # if fsk-button - real_url = self._html_search_regex(r'\"fsk-button\".+?href=\"([^"]+)', - webpage, 'fsk_button', default=None) + error_message = self._html_search_regex( + r'
(.+?)
', + webpage, 'error message', default=None) + if error_message: + raise ExtractorError( + '%s returned error: %s' % (self.IE_NAME, error_message), expected=True) + + real_url = self._html_search_regex( + r'\"fsk-button\".+?href=\"([^"]+)', + webpage, 'fsk_button', default=None) if real_url: webpage = self._download_webpage(real_url, video_id) - title = self._html_search_regex( - r'
.+?

(.+?)

', - webpage, 'title', flags=re.DOTALL) + player = self._search_regex( + r'p4_video_player(?:_iframe)?\("video_\d+_container"\s*,(.+?)\);\s*\}', + webpage, 'player') - sd_url = self._html_search_regex( - r'{\"url\":\"([^"]+?)\",\"hd\":false', - webpage, 'sd_url').replace('\\', '') + player_json = self._parse_json( + '[%s]' % player, video_id, + transform_source=lambda s: s.replace('undefined,', '')) - formats = [{'format_id': 'sd', 'url': sd_url, 'quality': -2}] + formats = None + result = None - hd_url = self._html_search_regex( - r'{\"url\":\"([^"]+?)\",\"hd\":true', - webpage, 'hd_url', default=None) - if hd_url: - hd_url = hd_url.replace('\\', '') - formats.append({'format_id': 'hd', 'url': hd_url, 'quality': -1}) + for v in player_json: + if isinstance(v, list) and not formats: + formats = [{ + 'url': f['url'], + 'format': 'hd' if f.get('hd') else 'sd', + 'width': int_or_none(f.get('size_x')), + 'height': int_or_none(f.get('size_y')), + 'tbr': int_or_none(f.get('bitrate')), + } for f in v] + self._sort_formats(formats) + elif isinstance(v, dict) and not result: + result = { + 'id': video_id, + 'title': v['videopartname'].strip(), + 'description': v.get('videotitle'), + 'duration': int_or_none(v.get('videoduration') or v.get('episodeduration')), + 'upload_date': unified_strdate(v.get('clipreleasetime')), + 'uploader': v.get('channel'), + } - return { - 'id': video_id, - 'title': title, - 'formats': formats, - 'ext': 'mp4' - } + result['formats'] = formats + + return result From 0d103de3b0b03c5027f0015327c2c44b9073513a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 28 Feb 2015 22:59:55 +0600 Subject: [PATCH 17/17] [twitch] Pass api_token along with every request (Closes #3986) --- youtube_dl/extractor/twitch.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index 4b0d8988d9..4b0ce54df4 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -34,7 +34,15 @@ def _handle_error(self, response): expected=True) def _download_json(self, url, video_id, note='Downloading JSON metadata'): - response = super(TwitchBaseIE, self)._download_json(url, video_id, note) + headers = { + 'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2', + 'X-Requested-With': 'XMLHttpRequest', + } + for cookie in self._downloader.cookiejar: + if cookie.name == 'api_token': + headers['Twitch-Api-Token'] = cookie.value + request = compat_urllib_request.Request(url, headers=headers) + response = super(TwitchBaseIE, self)._download_json(request, video_id, note) self._handle_error(response) return response