From 4698b14b768e099955c58e2dc2ee458e390dcbd7 Mon Sep 17 00:00:00 2001 From: Naglis Jonaitis Date: Sun, 23 Nov 2014 13:28:09 +0200 Subject: [PATCH 1/5] [rtlxl] Strip additional dot from video URL (#4115) --- youtube_dl/extractor/rtlnl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py index 4a188e5d46..d029b0ec52 100644 --- a/youtube_dl/extractor/rtlnl.py +++ b/youtube_dl/extractor/rtlnl.py @@ -44,7 +44,7 @@ def _real_extract(self, url): formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4') - video_urlpart = videopath.split('/flash/')[1][:-4] + video_urlpart = videopath.split('/flash/')[1][:-5] PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4' formats.extend([ From 598c218f7b5c3e78f98dad40f45646c0c9ec773e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sun, 23 Nov 2014 23:53:41 +0600 Subject: [PATCH 2/5] [smotri] Adapt to new API and modernize --- youtube_dl/extractor/smotri.py | 149 ++++++++++----------------------- 1 file changed, 43 insertions(+), 106 deletions(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index 9bd5defa7a..6c1d82dd7b 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -1,7 +1,6 @@ # encoding: utf-8 from __future__ import unicode_literals -import os.path import re import json import hashlib @@ -12,15 +11,15 @@ compat_urllib_parse, compat_urllib_request, ExtractorError, - url_basename, int_or_none, + unified_strdate, ) class SmotriIE(InfoExtractor): IE_DESC = 'Smotri.com' IE_NAME = 'smotri' - _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?Pv(?P[0-9]+)[a-z0-9]{4})' + _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?Pv(?P[0-9]+)[a-z0-9]{4})' _NETRC_MACHINE = 'smotri' _TESTS = [ @@ -35,7 +34,6 @@ class SmotriIE(InfoExtractor): 'uploader': 'rbc2008', 'uploader_id': 'rbc08', 'upload_date': '20131118', - 'description': 'катастрофа с камер видеонаблюдения, видео катастрофа с камер видеонаблюдения', 'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg', }, }, @@ -50,7 +48,6 @@ class SmotriIE(InfoExtractor): 'uploader': 'Support Photofile@photofile', 'uploader_id': 'support-photofile', 'upload_date': '20070704', - 'description': 'test, видео test', 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg', }, }, @@ -66,7 +63,6 @@ class SmotriIE(InfoExtractor): 'uploader_id': 'timoxa40', 'upload_date': '20100404', 'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg', - 'description': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1, видео TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1', }, 'params': { 'videopassword': 'qwerty', @@ -85,7 +81,6 @@ class SmotriIE(InfoExtractor): 'upload_date': '20101001', 'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg', 'age_limit': 18, - 'description': 'этот ролик не покажут по ТВ, видео этот ролик не покажут по ТВ', }, 'params': { 'videopassword': '333' @@ -102,17 +97,11 @@ class SmotriIE(InfoExtractor): 'uploader': 'HannahL', 'uploader_id': 'lisaha95', 'upload_date': '20090331', - 'description': 'Shakira - Don\'t Bother, видео Shakira - Don\'t Bother', 'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg', }, }, ] - _SUCCESS = 0 - _PASSWORD_NOT_VERIFIED = 1 - _PASSWORD_DETECTED = 2 - _VIDEO_NOT_FOUND = 3 - @classmethod def _extract_url(cls, webpage): mobj = re.search( @@ -137,44 +126,44 @@ def _search_meta(self, name, html, display_name=None): return self._html_search_meta(name, html, display_name) def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('videoid') - real_video_id = mobj.group('realvideoid') + video_id = self._match_id(url) - # Download video JSON data - video_json_url = 'http://smotri.com/vt.php?id=%s' % real_video_id - video_json_page = self._download_webpage(video_json_url, video_id, 'Downloading video JSON') - video_json = json.loads(video_json_page) + video_form = { + 'ticket': video_id, + 'video_url': '1', + 'frame_url': '1', + 'devid': 'LoadupFlashPlayer', + 'getvideoinfo': '1', + } - status = video_json['status'] - if status == self._VIDEO_NOT_FOUND: + request = compat_urllib_request.Request( + 'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form)) + request.add_header('Content-Type', 'application/x-www-form-urlencoded') + + video = self._download_json(request, video_id, 'Downloading video JSON') + + if video.get('_moderate_no') or not video.get('moderated'): + raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True) + + if video.get('error'): raise ExtractorError('Video %s does not exist' % video_id, expected=True) - elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with - # video-password set - video_password = self._downloader.params.get('videopassword', None) - if not video_password: - raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True) - video_json_url += '&md5pass=%s' % hashlib.md5(video_password.encode('utf-8')).hexdigest() - video_json_page = self._download_webpage(video_json_url, video_id, 'Downloading video JSON (video-password set)') - video_json = json.loads(video_json_page) - status = video_json['status'] - if status == self._PASSWORD_NOT_VERIFIED: - raise ExtractorError('Video password is invalid', expected=True) - if status != self._SUCCESS: - raise ExtractorError('Unexpected status value %s' % status) - - # Extract the URL of the video - video_url = video_json['file_data'] + video_url = video.get('_vidURL') or video.get('_vidURL_mp4') + title = video['title'] + thumbnail = video['_imgURL'] + upload_date = unified_strdate(video['added']) + uploader = video['userNick'] + uploader_id = video['userLogin'] + duration = int_or_none(video['duration']) # Video JSON does not provide enough meta data # We will extract some from the video web page instead - video_page_url = 'http://smotri.com/video/view/?id=%s' % video_id - video_page = self._download_webpage(video_page_url, video_id, 'Downloading video page') + webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id + webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page') # Warning if video is unavailable warning = self._html_search_regex( - r'
(.*?)
', video_page, + r'
(.*?)
', webpage, 'warning message', default=None) if warning is not None: self._downloader.report_warning( @@ -182,84 +171,32 @@ def _real_extract(self, url): (video_id, warning)) # Adult content - if re.search('EroConfirmText">', video_page) is not None: + if re.search('EroConfirmText">', webpage) is not None: self.report_age_confirmation() confirm_string = self._html_search_regex( r'' % video_id, - video_page, 'confirm string') - confirm_url = video_page_url + '&confirm=%s' % confirm_string - video_page = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)') + webpage, 'confirm string') + confirm_url = webpage_url + '&confirm=%s' % confirm_string + webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)') adult_content = True else: adult_content = False - # Extract the rest of meta data - video_title = self._search_meta('name', video_page, 'title') - if not video_title: - video_title = os.path.splitext(url_basename(video_url))[0] - - video_description = self._search_meta('description', video_page) - END_TEXT = ' на сайте Smotri.com' - if video_description and video_description.endswith(END_TEXT): - video_description = video_description[:-len(END_TEXT)] - START_TEXT = 'Смотреть онлайн ролик ' - if video_description and video_description.startswith(START_TEXT): - video_description = video_description[len(START_TEXT):] - video_thumbnail = self._search_meta('thumbnail', video_page) - - upload_date_str = self._search_meta('uploadDate', video_page, 'upload date') - if upload_date_str: - upload_date_m = re.search(r'(?P\d{4})\.(?P\d{2})\.(?P\d{2})T', upload_date_str) - video_upload_date = ( - ( - upload_date_m.group('year') + - upload_date_m.group('month') + - upload_date_m.group('day') - ) - if upload_date_m else None - ) - else: - video_upload_date = None - - duration_str = self._search_meta('duration', video_page) - if duration_str: - duration_m = re.search(r'T(?P[0-9]{2})H(?P[0-9]{2})M(?P[0-9]{2})S', duration_str) - video_duration = ( - ( - (int(duration_m.group('hours')) * 60 * 60) + - (int(duration_m.group('minutes')) * 60) + - int(duration_m.group('seconds')) - ) - if duration_m else None - ) - else: - video_duration = None - - video_uploader = self._html_search_regex( - '
Автор.*?onmouseover="popup_user_info[^"]+">(.*?)', - video_page, 'uploader', fatal=False, flags=re.MULTILINE|re.DOTALL) - - video_uploader_id = self._html_search_regex( - '
Автор.*?onmouseover="popup_user_info\\(.*?\'([^\']+)\'\\);">', - video_page, 'uploader id', fatal=False, flags=re.MULTILINE|re.DOTALL) - - video_view_count = self._html_search_regex( + view_count = self._html_search_regex( 'Общее количество просмотров.*?(\\d+)', - video_page, 'view count', fatal=False, flags=re.MULTILINE|re.DOTALL) + webpage, 'view count', fatal=False, flags=re.MULTILINE|re.DOTALL) return { 'id': video_id, 'url': video_url, - 'title': video_title, - 'thumbnail': video_thumbnail, - 'description': video_description, - 'uploader': video_uploader, - 'upload_date': video_upload_date, - 'uploader_id': video_uploader_id, - 'duration': video_duration, - 'view_count': int_or_none(video_view_count), + 'title': title, + 'thumbnail': thumbnail, + 'uploader': uploader, + 'upload_date': upload_date, + 'uploader_id': uploader_id, + 'duration': duration, + 'view_count': int_or_none(view_count), 'age_limit': 18 if adult_content else 0, - 'video_page_url': video_page_url } From 5f6a1245ffa9276c1af59b0835afeef67e2fb5b1 Mon Sep 17 00:00:00 2001 From: Jouke Waleson Date: Sun, 23 Nov 2014 20:41:03 +0100 Subject: [PATCH 3/5] PEP8 applied --- devscripts/bash-completion.py | 5 +- devscripts/buildserver.py | 3 +- devscripts/fish-completion.py | 1 + devscripts/gh-pages/sign-versions.py | 18 +-- devscripts/gh-pages/update-copyright.py | 2 +- devscripts/gh-pages/update-feed.py | 1 - devscripts/gh-pages/update-sites.py | 1 + devscripts/transition_helper.py | 8 +- devscripts/transition_helper_exe/setup.py | 2 +- .../transition_helper_exe/youtube-dl.py | 22 ++- test/helper.py | 6 +- test/test_YoutubeDL.py | 1 + test/test_all_urls.py | 6 +- test/test_download.py | 13 +- test/test_subtitles.py | 1 + test/test_utils.py | 6 +- test/test_write_annotations.py | 26 ++-- test/test_youtube_lists.py | 2 +- youtube_dl/YoutubeDL.py | 10 +- youtube_dl/__init__.py | 4 +- youtube_dl/aes.py | 125 ++++++++++-------- youtube_dl/compat.py | 32 ++--- youtube_dl/downloader/common.py | 6 +- youtube_dl/downloader/hls.py | 1 - youtube_dl/downloader/rtmp.py | 2 +- youtube_dl/extractor/adultswim.py | 1 + youtube_dl/extractor/aparat.py | 2 +- youtube_dl/extractor/appletrailers.py | 2 + youtube_dl/extractor/ard.py | 1 - youtube_dl/extractor/arte.py | 2 +- youtube_dl/extractor/audiomack.py | 8 +- youtube_dl/extractor/bambuser.py | 2 +- youtube_dl/extractor/bandcamp.py | 4 +- youtube_dl/extractor/bbccouk.py | 2 +- youtube_dl/extractor/beeg.py | 2 +- youtube_dl/extractor/bild.py | 2 +- youtube_dl/extractor/canalplus.py | 2 +- youtube_dl/extractor/cbsnews.py | 2 +- youtube_dl/extractor/ceskatelevize.py | 2 +- youtube_dl/extractor/channel9.py | 15 ++- youtube_dl/extractor/clipsyndicate.py | 1 + youtube_dl/extractor/common.py | 7 +- youtube_dl/extractor/cracked.py | 4 +- youtube_dl/extractor/crunchyroll.py | 6 +- youtube_dl/extractor/dailymotion.py | 4 +- youtube_dl/extractor/defense.py | 4 +- youtube_dl/extractor/dotsub.py | 2 +- youtube_dl/extractor/fc2.py | 2 +- youtube_dl/extractor/firsttv.py | 2 +- youtube_dl/extractor/flickr.py | 4 +- youtube_dl/extractor/fourtube.py | 2 +- youtube_dl/extractor/generic.py | 3 +- youtube_dl/extractor/globo.py | 2 +- youtube_dl/extractor/gorillavid.py | 2 +- youtube_dl/extractor/hornbunny.py | 2 +- youtube_dl/extractor/howcast.py | 2 +- youtube_dl/extractor/imdb.py | 2 +- youtube_dl/extractor/internetvideoarchive.py | 2 +- youtube_dl/extractor/ivi.py | 10 +- youtube_dl/extractor/jadorecettepub.py | 1 - youtube_dl/extractor/jeuxvideo.py | 4 +- youtube_dl/extractor/kankan.py | 2 +- youtube_dl/extractor/kontrtube.py | 2 +- youtube_dl/extractor/ku6.py | 1 - youtube_dl/extractor/laola1tv.py | 1 - youtube_dl/extractor/lifenews.py | 4 +- youtube_dl/extractor/lynda.py | 8 +- youtube_dl/extractor/m6.py | 2 +- youtube_dl/extractor/malemotion.py | 1 + youtube_dl/extractor/mdr.py | 2 +- youtube_dl/extractor/mojvideo.py | 2 +- youtube_dl/extractor/moniker.py | 2 +- youtube_dl/extractor/mooshare.py | 2 +- youtube_dl/extractor/motherless.py | 2 +- youtube_dl/extractor/moviezine.py | 2 +- youtube_dl/extractor/movshare.py | 2 +- youtube_dl/extractor/mpora.py | 2 +- youtube_dl/extractor/mtv.py | 6 +- youtube_dl/extractor/muenchentv.py | 1 - youtube_dl/extractor/musicplayon.py | 2 +- youtube_dl/extractor/muzu.py | 2 +- youtube_dl/extractor/myvideo.py | 7 +- youtube_dl/extractor/naver.py | 4 +- youtube_dl/extractor/nba.py | 1 - youtube_dl/extractor/ndr.py | 2 +- youtube_dl/extractor/newgrounds.py | 4 +- youtube_dl/extractor/newstube.py | 2 +- youtube_dl/extractor/nfb.py | 2 +- youtube_dl/extractor/nhl.py | 2 +- youtube_dl/extractor/noco.py | 2 +- youtube_dl/extractor/novamov.py | 2 +- youtube_dl/extractor/nowvideo.py | 2 +- youtube_dl/extractor/ntv.py | 2 +- youtube_dl/extractor/nuvid.py | 2 +- youtube_dl/extractor/nytimes.py | 2 +- youtube_dl/extractor/ooyala.py | 1 - youtube_dl/extractor/orf.py | 2 +- youtube_dl/extractor/podomatic.py | 1 + youtube_dl/extractor/pornhub.py | 2 +- youtube_dl/extractor/pornotube.py | 2 +- youtube_dl/extractor/prosiebensat1.py | 2 +- youtube_dl/extractor/rai.py | 2 +- youtube_dl/extractor/ringtv.py | 1 - youtube_dl/extractor/rtlnow.py | 4 +- youtube_dl/extractor/rtve.py | 1 - youtube_dl/extractor/rutv.py | 2 +- youtube_dl/extractor/scivee.py | 2 +- youtube_dl/extractor/servingsys.py | 2 - youtube_dl/extractor/shared.py | 2 +- youtube_dl/extractor/smotri.py | 2 +- youtube_dl/extractor/soundcloud.py | 4 +- youtube_dl/extractor/spiegeltv.py | 2 +- youtube_dl/extractor/sport5.py | 2 +- youtube_dl/extractor/sportdeutschland.py | 1 - youtube_dl/extractor/subtitles.py | 2 +- youtube_dl/extractor/teachertube.py | 2 +- youtube_dl/extractor/teamcoco.py | 2 +- youtube_dl/extractor/telecinco.py | 2 +- youtube_dl/extractor/theplatform.py | 4 +- youtube_dl/extractor/thisav.py | 4 +- youtube_dl/extractor/tinypic.py | 4 +- youtube_dl/extractor/tnaflix.py | 2 +- youtube_dl/extractor/traileraddict.py | 6 +- youtube_dl/extractor/trilulilu.py | 3 +- youtube_dl/extractor/tudou.py | 2 +- youtube_dl/extractor/tvigle.py | 2 +- youtube_dl/extractor/udemy.py | 2 +- youtube_dl/extractor/vesti.py | 2 +- youtube_dl/extractor/vgtv.py | 2 +- youtube_dl/extractor/vice.py | 1 - youtube_dl/extractor/videobam.py | 2 +- youtube_dl/extractor/videofyme.py | 5 +- youtube_dl/extractor/videott.py | 2 +- youtube_dl/extractor/videoweed.py | 2 +- youtube_dl/extractor/vidzi.py | 7 +- youtube_dl/extractor/vrt.py | 4 +- youtube_dl/extractor/wdr.py | 2 +- youtube_dl/extractor/worldstarhiphop.py | 1 - youtube_dl/extractor/xbef.py | 1 - youtube_dl/extractor/xhamster.py | 2 +- youtube_dl/extractor/ynet.py | 2 +- youtube_dl/extractor/youku.py | 14 +- youtube_dl/extractor/youporn.py | 4 +- youtube_dl/extractor/youtube.py | 30 +++-- youtube_dl/options.py | 2 +- youtube_dl/postprocessor/execafterdownload.py | 1 - youtube_dl/postprocessor/ffmpeg.py | 18 +-- youtube_dl/postprocessor/xattrpp.py | 1 - youtube_dl/swfinterp.py | 1 - youtube_dl/update.py | 45 +++++-- youtube_dl/utils.py | 32 ++++- 151 files changed, 419 insertions(+), 343 deletions(-) diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 49287724d6..70a3f552cf 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -9,16 +9,17 @@ BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" + def build_completion(opt_parser): opts_flag = [] for group in opt_parser.option_groups: for option in group.option_list: - #for every long flag + # for every long flag opts_flag.append(option.get_opt_string()) with open(BASH_COMPLETION_TEMPLATE) as f: template = f.read() with open(BASH_COMPLETION_FILE, "w") as f: - #just using the special char + # just using the special char filled_template = template.replace("{{flags}}", " ".join(opts_flag)) f.write(filled_template) diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py index e0c3cc83ed..42ee2b5cb6 100644 --- a/devscripts/buildserver.py +++ b/devscripts/buildserver.py @@ -233,6 +233,7 @@ def rmtree(path): #============================================================================== + class BuildError(Exception): def __init__(self, output, code=500): self.output = output @@ -369,7 +370,7 @@ class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, Clea class BuildHTTPRequestHandler(BaseHTTPRequestHandler): - actionDict = { 'build': Builder, 'download': Builder } # They're the same, no more caching. + actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching. def do_GET(self): path = urlparse.urlparse(self.path) diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index f4aaf0201f..2185d55229 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -23,6 +23,7 @@ 'batch-file': ['--require-parameter'], } + def build_completion(opt_parser): commands = [] diff --git a/devscripts/gh-pages/sign-versions.py b/devscripts/gh-pages/sign-versions.py index 8a824df56f..953a5162e5 100755 --- a/devscripts/gh-pages/sign-versions.py +++ b/devscripts/gh-pages/sign-versions.py @@ -11,22 +11,22 @@ versions_info = json.load(open('update/versions.json')) if 'signature' in versions_info: - del versions_info['signature'] + del versions_info['signature'] print('Enter the PKCS1 private key, followed by a blank line:') privkey = b'' while True: - try: - line = input() - except EOFError: - break - if line == '': - break - privkey += line.encode('ascii') + b'\n' + try: + line = input() + except EOFError: + break + if line == '': + break + privkey += line.encode('ascii') + b'\n' privkey = rsa.PrivateKey.load_pkcs1(privkey) signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode() print('signature: ' + signature) versions_info['signature'] = signature -json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True) \ No newline at end of file +json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True) diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py index 12c2a91949..5bc5c65147 100755 --- a/devscripts/gh-pages/update-copyright.py +++ b/devscripts/gh-pages/update-copyright.py @@ -5,7 +5,7 @@ import datetime import glob -import io # For Python 2 compatibilty +import io # For Python 2 compatibilty import os import re diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index 0ba15ae0f7..46373a8dbd 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -73,4 +73,3 @@ with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: atom_file.write(atom_template) - diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 153e15c8ab..0d526784d2 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -9,6 +9,7 @@ import youtube_dl + def main(): with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf: template = tmplf.read() diff --git a/devscripts/transition_helper.py b/devscripts/transition_helper.py index d5ca2d4bac..cfd21919bf 100644 --- a/devscripts/transition_helper.py +++ b/devscripts/transition_helper.py @@ -4,7 +4,7 @@ try: import urllib.request as compat_urllib_request -except ImportError: # Python 2 +except ImportError: # Python 2 import urllib2 as compat_urllib_request sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') @@ -12,9 +12,9 @@ sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') try: - raw_input() -except NameError: # Python 3 - input() + raw_input() +except NameError: # Python 3 + input() filename = sys.argv[0] diff --git a/devscripts/transition_helper_exe/setup.py b/devscripts/transition_helper_exe/setup.py index aaf5c2983e..bb3264af0a 100644 --- a/devscripts/transition_helper_exe/setup.py +++ b/devscripts/transition_helper_exe/setup.py @@ -9,4 +9,4 @@ "dll_excludes": ['w9xpopen.exe'] } -setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None) \ No newline at end of file +setup(console=['youtube-dl.py'], options={"py2exe": py2exe_options}, zipfile=None) diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py index 6297dfd40f..edb449fb30 100644 --- a/devscripts/transition_helper_exe/youtube-dl.py +++ b/devscripts/transition_helper_exe/youtube-dl.py @@ -4,13 +4,17 @@ import urllib2 import json, hashlib + def rsa_verify(message, signature, key): from struct import pack from hashlib import sha256 from sys import version_info + def b(x): - if version_info[0] == 2: return x - else: return x.encode('latin1') + if version_info[0] == 2: + return x + else: + return x.encode('latin1') assert(type(message) == type(b(''))) block_size = 0 n = key[0] @@ -23,13 +27,17 @@ def b(x): raw_bytes.insert(0, pack("B", signature & 0xFF)) signature >>= 8 signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) - if signature[0:2] != b('\x00\x01'): return False + if signature[0:2] != b('\x00\x01'): + return False signature = signature[2:] - if not b('\x00') in signature: return False + if not b('\x00') in signature: + return False signature = signature[signature.index(b('\x00'))+1:] - if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): + return False signature = signature[19:] - if signature != sha256(message).digest(): return False + if signature != sha256(message).digest(): + return False return True sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') @@ -92,7 +100,7 @@ def b(x): ping 127.0.0.1 -n 5 -w 1000 > NUL move /Y "%s.new" "%s" del "%s" - \n""" %(exe, exe, bat)) + \n""" % (exe, exe, bat)) b.close() os.startfile(bat) diff --git a/test/helper.py b/test/helper.py index 8be37a183a..91822935fa 100644 --- a/test/helper.py +++ b/test/helper.py @@ -59,7 +59,7 @@ def __init__(self, override=None): params = get_params(override=override) super(FakeYDL, self).__init__(params, auto_init=False) self.result = [] - + def to_screen(self, s, skip_eol=None): print(s) @@ -72,8 +72,10 @@ def download(self, x): def expect_warning(self, regex): # Silence an expected warning matching a regex old_report_warning = self.report_warning + def report_warning(self, message): - if re.match(regex, message): return + if re.match(regex, message): + return old_report_warning(message) self.report_warning = types.MethodType(report_warning, self) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index ab61e19768..f8e4f930eb 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -266,6 +266,7 @@ def test_prepare_filename(self): 'ext': 'mp4', 'width': None, } + def fname(templ): ydl = YoutubeDL({'outtmpl': templ}) return ydl.prepare_filename(info) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 965e5d8a58..bd4fe17bf2 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -32,19 +32,19 @@ def assertMatch(self, url, ie_list): def test_youtube_playlist_matching(self): assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist']) assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') - assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q') #585 + assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q') # 585 assertPlaylist('PL63F0C78739B09958') assertPlaylist('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q') assertPlaylist('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') assertPlaylist('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC') - assertPlaylist('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012') #668 + assertPlaylist('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012') # 668 self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M')) # Top tracks assertPlaylist('https://www.youtube.com/playlist?list=MCUS.20142101') def test_youtube_matching(self): self.assertTrue(YoutubeIE.suitable('PLtS2H6bU1M')) - self.assertFalse(YoutubeIE.suitable('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668 + self.assertFalse(YoutubeIE.suitable('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) # 668 self.assertMatch('http://youtu.be/BaW_jenozKc', ['youtube']) self.assertMatch('http://www.youtube.com/v/BaW_jenozKc', ['youtube']) self.assertMatch('https://youtube.googleapis.com/v/BaW_jenozKc', ['youtube']) diff --git a/test/test_download.py b/test/test_download.py index 12cfb5cbe4..b2615f338c 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -40,18 +40,22 @@ RETRIES = 3 + class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] super(YoutubeDL, self).__init__(*args, **kwargs) + def report_warning(self, message): # Don't accept warnings during tests raise ExtractorError(message) + def process_info(self, info_dict): self.processed_info_dicts.append(info_dict) return super(YoutubeDL, self).process_info(info_dict) + def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() @@ -61,10 +65,13 @@ def _file_md5(fn): class TestDownload(unittest.TestCase): maxDiff = None + def setUp(self): self.defs = defs -### Dynamically generate tests +# Dynamically generate tests + + def generator(test_case): def test_template(self): @@ -101,6 +108,7 @@ def print_skipping(reason): ydl = YoutubeDL(params, auto_init=False) ydl.add_default_info_extractors() finished_hook_called = set() + def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) @@ -111,6 +119,7 @@ def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) res_dict = None + def try_rm_tcs_files(tcs=None): if tcs is None: tcs = test_cases @@ -206,7 +215,7 @@ def try_rm_tcs_files(tcs=None): return test_template -### And add them to TestDownload +# And add them to TestDownload for n, test_case in enumerate(defs): test_method = generator(test_case) tname = 'test_' + str(test_case['name']) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 94e3290db6..2eeb31bc6e 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -23,6 +23,7 @@ class BaseTestSubtitles(unittest.TestCase): url = None IE = None + def setUp(self): self.DL = FakeYDL() self.ie = self.IE(self.DL) diff --git a/test/test_utils.py b/test/test_utils.py index 9a62322f01..380d1059d1 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -120,16 +120,16 @@ def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) self.assertEqual(orderedSet([1]), [1]) - #keep the list ordered + # keep the list ordered self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1]) def test_unescape_html(self): self.assertEqual(unescapeHTML('%20;'), '%20;') self.assertEqual( unescapeHTML('é'), 'é') - + def test_daterange(self): - _20century = DateRange("19000101","20000101") + _20century = DateRange("19000101", "20000101") self.assertFalse("17890714" in _20century) _ac = DateRange("00010101") self.assertTrue("19690721" in _ac) diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py index eac53b285a..852553ada5 100644 --- a/test/test_write_annotations.py +++ b/test/test_write_annotations.py @@ -31,19 +31,18 @@ def __init__(self, *args, **kwargs): }) - TEST_ID = 'gr51aVj-mLg' ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml' EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label'] + class TestAnnotations(unittest.TestCase): def setUp(self): # Clear old files self.tearDown() - def test_info_json(self): - expected = list(EXPECTED_ANNOTATIONS) #Two annotations could have the same text. + expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text. ie = youtube_dl.extractor.YoutubeIE() ydl = YoutubeDL(params) ydl.add_info_extractor(ie) @@ -51,7 +50,7 @@ def test_info_json(self): self.assertTrue(os.path.exists(ANNOTATIONS_FILE)) annoxml = None with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof: - annoxml = xml.etree.ElementTree.parse(annof) + annoxml = xml.etree.ElementTree.parse(annof) self.assertTrue(annoxml is not None, 'Failed to parse annotations XML') root = annoxml.getroot() self.assertEqual(root.tag, 'document') @@ -59,18 +58,17 @@ def test_info_json(self): self.assertEqual(annotationsTag.tag, 'annotations') annotations = annotationsTag.findall('annotation') - #Not all the annotations have TEXT children and the annotations are returned unsorted. + # Not all the annotations have TEXT children and the annotations are returned unsorted. for a in annotations: - self.assertEqual(a.tag, 'annotation') - if a.get('type') == 'text': - textTag = a.find('TEXT') - text = textTag.text - self.assertTrue(text in expected) #assertIn only added in python 2.7 - #remove the first occurance, there could be more than one annotation with the same text - expected.remove(text) - #We should have seen (and removed) all the expected annotation texts. + self.assertEqual(a.tag, 'annotation') + if a.get('type') == 'text': + textTag = a.find('TEXT') + text = textTag.text + self.assertTrue(text in expected) # assertIn only added in python 2.7 + # remove the first occurance, there could be more than one annotation with the same text + expected.remove(text) + # We should have seen (and removed) all the expected annotation texts. self.assertEqual(len(expected), 0, 'Not all expected annotations were found.') - def tearDown(self): try_rm(ANNOTATIONS_FILE) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 410f9edc29..452d342ae9 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -31,7 +31,7 @@ def test_youtube_playlist_noplaylist(self): result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') self.assertEqual(result['_type'], 'url') self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg') - + def test_youtube_course(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index bfa0c6d431..c1a529f134 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -552,7 +552,7 @@ def extract_info(self, url, download=True, ie_key=None, extra_info={}, try: ie_result = ie.extract(url) - if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) + if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) break if isinstance(ie_result, list): # Backwards compatibility: old IE result format @@ -565,7 +565,7 @@ def extract_info(self, url, download=True, ie_key=None, extra_info={}, return self.process_ie_result(ie_result, download, extra_info) else: return ie_result - except ExtractorError as de: # An error we somewhat expected + except ExtractorError as de: # An error we somewhat expected self.report_error(compat_str(de), de.format_traceback()) break except MaxDownloadsReached: @@ -700,6 +700,7 @@ def make_result(embedded_info): self.report_warning( 'Extractor %s returned a compat_list result. ' 'It needs to be updated.' % ie_result.get('extractor')) + def _fixup(r): self.add_extra_info(r, { @@ -1010,7 +1011,7 @@ def process_info(self, info_dict): else: self.to_screen('[info] Writing video subtitles to: ' + sub_filename) with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: - subfile.write(sub) + subfile.write(sub) except (OSError, IOError): self.report_error('Cannot write subtitles file ' + sub_filename) return @@ -1111,7 +1112,7 @@ def download(self, url_list): for url in url_list: try: - #It also downloads the videos + # It also downloads the videos res = self.extract_info(url) except UnavailableVideoError: self.report_error('unable to download video') @@ -1428,4 +1429,3 @@ def get_encoding(self): if encoding is None: encoding = preferredencoding() return encoding - diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index f519fae3e4..ee3067134a 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -128,7 +128,6 @@ def _real_main(argv=None): compat_print(desc) sys.exit(0) - # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error('using .netrc conflicts with giving username/password') @@ -197,7 +196,7 @@ def _real_main(argv=None): # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems) if opts.outtmpl is not None: opts.outtmpl = opts.outtmpl.decode(preferredencoding()) - outtmpl =((opts.outtmpl is not None and opts.outtmpl) + outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s') @@ -317,7 +316,6 @@ def _real_main(argv=None): ydl.add_post_processor(FFmpegAudioFixPP()) ydl.add_post_processor(AtomicParsleyPP()) - # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way. # So if the user is able to remove the file before your postprocessor runs it might cause a few problems. if opts.exec_cmd: diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py index e9c5e21521..ccfd737708 100644 --- a/youtube_dl/aes.py +++ b/youtube_dl/aes.py @@ -7,10 +7,11 @@ BLOCK_SIZE_BYTES = 16 + def aes_ctr_decrypt(data, key, counter): """ Decrypt with aes in counter mode - + @param {int[]} data cipher @param {int[]} key 16/24/32-Byte cipher key @param {instance} counter Instance whose next_value function (@returns {int[]} 16-Byte block) @@ -19,23 +20,24 @@ def aes_ctr_decrypt(data, key, counter): """ expanded_key = key_expansion(key) block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) - - decrypted_data=[] + + decrypted_data = [] for i in range(block_count): counter_block = counter.next_value() - block = data[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES] + block = data[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES] block += [0]*(BLOCK_SIZE_BYTES - len(block)) - + cipher_counter_block = aes_encrypt(counter_block, expanded_key) decrypted_data += xor(block, cipher_counter_block) decrypted_data = decrypted_data[:len(data)] - + return decrypted_data + def aes_cbc_decrypt(data, key, iv): """ Decrypt with aes in CBC mode - + @param {int[]} data cipher @param {int[]} key 16/24/32-Byte cipher key @param {int[]} iv 16-Byte IV @@ -43,60 +45,62 @@ def aes_cbc_decrypt(data, key, iv): """ expanded_key = key_expansion(key) block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) - - decrypted_data=[] + + decrypted_data = [] previous_cipher_block = iv for i in range(block_count): - block = data[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES] + block = data[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES] block += [0]*(BLOCK_SIZE_BYTES - len(block)) - + decrypted_block = aes_decrypt(block, expanded_key) decrypted_data += xor(decrypted_block, previous_cipher_block) previous_cipher_block = block decrypted_data = decrypted_data[:len(data)] - + return decrypted_data + def key_expansion(data): """ Generate key schedule - + @param {int[]} data 16/24/32-Byte cipher key - @returns {int[]} 176/208/240-Byte expanded key + @returns {int[]} 176/208/240-Byte expanded key """ - data = data[:] # copy + data = data[:] # copy rcon_iteration = 1 key_size_bytes = len(data) expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES - + while len(data) < expanded_key_size_bytes: temp = data[-4:] temp = key_schedule_core(temp, rcon_iteration) rcon_iteration += 1 - data += xor(temp, data[-key_size_bytes : 4-key_size_bytes]) - + data += xor(temp, data[-key_size_bytes: 4-key_size_bytes]) + for _ in range(3): temp = data[-4:] - data += xor(temp, data[-key_size_bytes : 4-key_size_bytes]) - + data += xor(temp, data[-key_size_bytes: 4-key_size_bytes]) + if key_size_bytes == 32: temp = data[-4:] temp = sub_bytes(temp) - data += xor(temp, data[-key_size_bytes : 4-key_size_bytes]) - + data += xor(temp, data[-key_size_bytes: 4-key_size_bytes]) + for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0): temp = data[-4:] - data += xor(temp, data[-key_size_bytes : 4-key_size_bytes]) + data += xor(temp, data[-key_size_bytes: 4-key_size_bytes]) data = data[:expanded_key_size_bytes] - + return data + def aes_encrypt(data, expanded_key): """ Encrypt one block with aes - + @param {int[]} data 16-Byte state - @param {int[]} expanded_key 176/208/240-Byte expanded key + @param {int[]} expanded_key 176/208/240-Byte expanded key @returns {int[]} 16-Byte cipher """ rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 @@ -107,30 +111,32 @@ def aes_encrypt(data, expanded_key): data = shift_rows(data) if i != rounds: data = mix_columns(data) - data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]) + data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES]) return data + def aes_decrypt(data, expanded_key): """ Decrypt one block with aes - + @param {int[]} data 16-Byte cipher @param {int[]} expanded_key 176/208/240-Byte expanded key @returns {int[]} 16-Byte state """ rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 - + for i in range(rounds, 0, -1): - data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]) + data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES]) if i != rounds: data = mix_columns_inv(data) data = shift_rows_inv(data) data = sub_bytes_inv(data) data = xor(data, expanded_key[:BLOCK_SIZE_BYTES]) - + return data + def aes_decrypt_text(data, password, key_size_bytes): """ Decrypt text @@ -138,33 +144,34 @@ def aes_decrypt_text(data, password, key_size_bytes): - The cipher key is retrieved by encrypting the first 16 Byte of 'password' with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's) - Mode of operation is 'counter' - + @param {str} data Base64 encoded string @param {str,unicode} password Password (will be encoded with utf-8) @param {int} key_size_bytes Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit @returns {str} Decrypted data """ NONCE_LENGTH_BYTES = 8 - + data = bytes_to_intlist(base64.b64decode(data)) password = bytes_to_intlist(password.encode('utf-8')) - + key = password[:key_size_bytes] + [0]*(key_size_bytes - len(password)) key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES) - + nonce = data[:NONCE_LENGTH_BYTES] cipher = data[NONCE_LENGTH_BYTES:] - + class Counter: __value = nonce + [0]*(BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES) + def next_value(self): temp = self.__value self.__value = inc(self.__value) return temp - + decrypted_data = aes_ctr_decrypt(cipher, key, Counter()) plaintext = intlist_to_bytes(decrypted_data) - + return plaintext RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36) @@ -200,14 +207,14 @@ def next_value(self): 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d) -MIX_COLUMN_MATRIX = ((0x2,0x3,0x1,0x1), - (0x1,0x2,0x3,0x1), - (0x1,0x1,0x2,0x3), - (0x3,0x1,0x1,0x2)) -MIX_COLUMN_MATRIX_INV = ((0xE,0xB,0xD,0x9), - (0x9,0xE,0xB,0xD), - (0xD,0x9,0xE,0xB), - (0xB,0xD,0x9,0xE)) +MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1), + (0x1, 0x2, 0x3, 0x1), + (0x1, 0x1, 0x2, 0x3), + (0x3, 0x1, 0x1, 0x2)) +MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9), + (0x9, 0xE, 0xB, 0xD), + (0xD, 0x9, 0xE, 0xB), + (0xB, 0xD, 0x9, 0xE)) RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35, 0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA, 0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31, @@ -241,30 +248,37 @@ def next_value(self): 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5, 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07) + def sub_bytes(data): return [SBOX[x] for x in data] + def sub_bytes_inv(data): return [SBOX_INV[x] for x in data] + def rotate(data): return data[1:] + [data[0]] + def key_schedule_core(data, rcon_iteration): data = rotate(data) data = sub_bytes(data) data[0] = data[0] ^ RCON[rcon_iteration] - + return data + def xor(data1, data2): return [x^y for x, y in zip(data1, data2)] + def rijndael_mul(a, b): - if(a==0 or b==0): + if(a == 0 or b == 0): return 0 return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF] + def mix_column(data, matrix): data_mixed = [] for row in range(4): @@ -275,33 +289,38 @@ def mix_column(data, matrix): data_mixed.append(mixed) return data_mixed + def mix_columns(data, matrix=MIX_COLUMN_MATRIX): data_mixed = [] for i in range(4): - column = data[i*4 : (i+1)*4] + column = data[i*4: (i+1)*4] data_mixed += mix_column(column, matrix) return data_mixed + def mix_columns_inv(data): return mix_columns(data, MIX_COLUMN_MATRIX_INV) + def shift_rows(data): data_shifted = [] for column in range(4): for row in range(4): - data_shifted.append( data[((column + row) & 0b11) * 4 + row] ) + data_shifted.append(data[((column + row) & 0b11) * 4 + row]) return data_shifted + def shift_rows_inv(data): data_shifted = [] for column in range(4): for row in range(4): - data_shifted.append( data[((column - row) & 0b11) * 4 + row] ) + data_shifted.append(data[((column - row) & 0b11) * 4 + row]) return data_shifted + def inc(data): - data = data[:] # copy - for i in range(len(data)-1,-1,-1): + data = data[:] # copy + for i in range(len(data)-1, -1, -1): if data[i] == 255: data[i] = 0 else: diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 5492065341..9087b4f857 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -10,47 +10,47 @@ try: import urllib.request as compat_urllib_request -except ImportError: # Python 2 +except ImportError: # Python 2 import urllib2 as compat_urllib_request try: import urllib.error as compat_urllib_error -except ImportError: # Python 2 +except ImportError: # Python 2 import urllib2 as compat_urllib_error try: import urllib.parse as compat_urllib_parse -except ImportError: # Python 2 +except ImportError: # Python 2 import urllib as compat_urllib_parse try: from urllib.parse import urlparse as compat_urllib_parse_urlparse -except ImportError: # Python 2 +except ImportError: # Python 2 from urlparse import urlparse as compat_urllib_parse_urlparse try: import urllib.parse as compat_urlparse -except ImportError: # Python 2 +except ImportError: # Python 2 import urlparse as compat_urlparse try: import http.cookiejar as compat_cookiejar -except ImportError: # Python 2 +except ImportError: # Python 2 import cookielib as compat_cookiejar try: import html.entities as compat_html_entities -except ImportError: # Python 2 +except ImportError: # Python 2 import htmlentitydefs as compat_html_entities try: import html.parser as compat_html_parser -except ImportError: # Python 2 +except ImportError: # Python 2 import HTMLParser as compat_html_parser try: import http.client as compat_http_client -except ImportError: # Python 2 +except ImportError: # Python 2 import httplib as compat_http_client try: @@ -111,7 +111,7 @@ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): try: from urllib.parse import parse_qs as compat_parse_qs -except ImportError: # Python 2 +except ImportError: # Python 2 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib. # Python 2's version is apparently totally broken @@ -157,12 +157,12 @@ def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False, return parsed_result try: - compat_str = unicode # Python 2 + compat_str = unicode # Python 2 except NameError: compat_str = str try: - compat_chr = unichr # Python 2 + compat_chr = unichr # Python 2 except NameError: compat_chr = chr @@ -182,8 +182,10 @@ def shlex_quote(s): def compat_ord(c): - if type(c) is int: return c - else: return ord(c) + if type(c) is int: + return c + else: + return ord(c) if sys.version_info >= (3, 0): @@ -254,7 +256,7 @@ def compat_expanduser(path): drive = '' userhome = os.path.join(drive, compat_getenv('HOMEPATH')) - if i != 1: #~user + if i != 1: # ~user userhome = os.path.join(os.path.dirname(userhome), path[1:i]) return userhome + path[i:] diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 7c33004b17..c0af50c591 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -81,7 +81,7 @@ def calc_eta(start, now, total, current): if total is None: return None dif = now - start - if current == 0 or dif < 0.001: # One millisecond + if current == 0 or dif < 0.001: # One millisecond return None rate = float(current) / dif return int((float(total) - float(current)) / rate) @@ -95,7 +95,7 @@ def format_eta(eta): @staticmethod def calc_speed(start, now, bytes): dif = now - start - if bytes == 0 or dif < 0.001: # One millisecond + if bytes == 0 or dif < 0.001: # One millisecond return None return float(bytes) / dif @@ -108,7 +108,7 @@ def format_speed(speed): @staticmethod def best_block_size(elapsed_time, bytes): new_min = max(bytes / 2.0, 1.0) - new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB + new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB if elapsed_time < 0.001: return int(new_max) rate = bytes / elapsed_time diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 68eafa403d..0c33d39eec 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -101,4 +101,3 @@ def real_download(self, filename, info_dict): }) self.try_rename(tmpfilename, filename) return True - diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py index 17d9631faa..b8e104c59b 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dl/downloader/rtmp.py @@ -180,7 +180,7 @@ def run_rtmpdump(args): while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live: prevsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen('[rtmpdump] %s bytes' % prevsize) - time.sleep(5.0) # This seems to be needed + time.sleep(5.0) # This seems to be needed retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED]) cursize = os.path.getsize(encodeFilename(tmpfilename)) if prevsize == cursize and retval == RD_FAILED: diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py index b4b40f2d4f..0d05cbb4b1 100644 --- a/youtube_dl/extractor/adultswim.py +++ b/youtube_dl/extractor/adultswim.py @@ -5,6 +5,7 @@ from .common import InfoExtractor + class AdultSwimIE(InfoExtractor): _VALID_URL = r'https?://video\.adultswim\.com/(?P.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$' _TEST = { diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py index 7486088262..56775289b5 100644 --- a/youtube_dl/extractor/aparat.py +++ b/youtube_dl/extractor/aparat.py @@ -1,4 +1,4 @@ -#coding: utf-8 +# coding: utf-8 from __future__ import unicode_literals diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index 4359b88d1b..567a76cf08 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -70,11 +70,13 @@ def _real_extract(self, url): uploader_id = mobj.group('company') playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc') + def fix_html(s): s = re.sub(r'(?s).*?', '', s) s = re.sub(r'', r'', s) # The ' in the onClick attributes are not escaped, it couldn't be parsed # like: http://trailers.apple.com/trailers/wb/gravity/ + def _clean_json(m): return 'iTunes.playURL(%s);' % m.group(1).replace('\'', ''') s = re.sub(self._JSON_RE, _clean_json, s) diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py index 630b1faa99..967bd865c5 100644 --- a/youtube_dl/extractor/ard.py +++ b/youtube_dl/extractor/ard.py @@ -192,4 +192,3 @@ def _real_extract(self, url): 'upload_date': upload_date, 'thumbnail': thumbnail, } - diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 3a57ce5278..219631b9b0 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -13,7 +13,7 @@ qualities, ) -# There are different sources of video in arte.tv, the extraction process +# There are different sources of video in arte.tv, the extraction process # is different for each one. The videos usually expire in 7 days, so we can't # add tests. diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py index 6232d2cd04..eeeec768fe 100644 --- a/youtube_dl/extractor/audiomack.py +++ b/youtube_dl/extractor/audiomack.py @@ -12,17 +12,17 @@ class AudiomackIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P[\w/-]+)' IE_NAME = 'audiomack' _TESTS = [ - #hosted on audiomack + # hosted on audiomack { 'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary', 'info_dict': { - 'id' : 'roosh-williams/extraordinary', + 'id': 'roosh-williams/extraordinary', 'ext': 'mp3', 'title': 'Roosh Williams - Extraordinary' } }, - #hosted on soundcloud via audiomack + # hosted on soundcloud via audiomack { 'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare', 'file': '172419696.mp3', @@ -49,7 +49,7 @@ def _real_extract(self, url): raise ExtractorError("Unable to deduce api url of song") realurl = api_response["url"] - #Audiomack wraps a lot of soundcloud tracks in their branded wrapper + # Audiomack wraps a lot of soundcloud tracks in their branded wrapper # - if so, pass the work off to the soundcloud extractor if SoundcloudIE.suitable(realurl): return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'} diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py index de5d4faf3b..eab99faaae 100644 --- a/youtube_dl/extractor/bambuser.py +++ b/youtube_dl/extractor/bambuser.py @@ -18,7 +18,7 @@ class BambuserIE(InfoExtractor): _TEST = { 'url': 'http://bambuser.com/v/4050584', # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388 - #u'md5': 'fba8f7693e48fd4e8641b3fd5539a641', + # u'md5': 'fba8f7693e48fd4e8641b3fd5539a641', 'info_dict': { 'id': '4050584', 'ext': 'flv', diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index 1b8da43cae..acddbc8f1d 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -83,12 +83,12 @@ def _real_extract(self, url): initial_url = mp3_info['url'] re_url = r'(?Phttp://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P.*?)&id=(?P.*?)&ts=(?P.*)$' m_url = re.match(re_url, initial_url) - #We build the url we will use to get the final track url + # We build the url we will use to get the final track url # This url is build in Bandcamp in the script download_bunde_*.js request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts')) final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url') # If we could correctly generate the .rand field the url would be - #in the "download_url" key + # in the "download_url" key final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1) return { diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py index 75e608f99d..89476f0325 100644 --- a/youtube_dl/extractor/bbccouk.py +++ b/youtube_dl/extractor/bbccouk.py @@ -220,4 +220,4 @@ def _real_extract(self, url): 'duration': duration, 'formats': formats, 'subtitles': subtitles, - } \ No newline at end of file + } diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py index 314e37f8be..4e79fea8f0 100644 --- a/youtube_dl/extractor/beeg.py +++ b/youtube_dl/extractor/beeg.py @@ -40,7 +40,7 @@ def _real_extract(self, url): title = self._html_search_regex( r'([^<]+)\s*-\s*beeg\.?', webpage, 'title') - + description = self._html_search_regex( r' 0: d = common.copy() - d.update({ 'title': title, 'formats': formats }) + d.update({'title': title, 'formats': formats}) result.append(d) return result @@ -270,5 +271,5 @@ def _real_extract(self, url): else: raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True) - else: # Assuming list + else: # Assuming list return self._extract_list(content_path) diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py index 02a1667fa3..d07d544eaf 100644 --- a/youtube_dl/extractor/clipsyndicate.py +++ b/youtube_dl/extractor/clipsyndicate.py @@ -39,6 +39,7 @@ def _real_extract(self, url): transform_source=fix_xml_ampersands) track_doc = pdoc.find('trackList/track') + def find_param(name): node = find_xpath_attr(track_doc, './/param', 'name', name) if node is not None: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 93a5a3d57b..f0489ede47 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -423,17 +423,18 @@ def report_login(self): """Report attempt to log in.""" self.to_screen('Logging in') - #Methods for following #608 + # Methods for following #608 @staticmethod def url_result(url, ie=None, video_id=None): """Returns a url that points to a page that should be processed""" - #TODO: ie should be the class used for getting the info + # TODO: ie should be the class used for getting the info video_info = {'_type': 'url', 'url': url, 'ie_key': ie} if video_id is not None: video_info['id'] = video_id return video_info + @staticmethod def playlist_result(entries, playlist_id=None, playlist_title=None): """Returns a playlist""" @@ -517,7 +518,7 @@ def _get_login_info(self): raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) except (IOError, netrc.NetrcParseError) as err: self._downloader.report_warning('parsing .netrc: %s' % compat_str(err)) - + return (username, password) def _get_tfa_info(self): diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py index 74b880ffce..cf763ee7e0 100644 --- a/youtube_dl/extractor/cracked.py +++ b/youtube_dl/extractor/cracked.py @@ -54,7 +54,7 @@ def _real_extract(self, url): return { 'id': video_id, - 'url':video_url, + 'url': video_url, 'title': title, 'description': description, 'timestamp': timestamp, @@ -62,4 +62,4 @@ def _real_extract(self, url): 'comment_count': comment_count, 'height': height, 'width': width, - } \ No newline at end of file + } diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index fe1324fe3c..c458581602 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -69,11 +69,9 @@ def _login(self): login_request.add_header('Content-Type', 'application/x-www-form-urlencoded') self._download_webpage(login_request, None, False, 'Wrong login info') - def _real_initialize(self): self._login() - def _decrypt_subtitles(self, data, iv, id): data = bytes_to_intlist(data) iv = bytes_to_intlist(iv) @@ -99,8 +97,10 @@ def obfuscate_key(key): return shaHash + [0] * 12 key = obfuscate_key(id) + class Counter: __value = iv + def next_value(self): temp = self.__value self.__value = inc(self.__value) @@ -183,7 +183,7 @@ def ass_bool(strvalue): return output - def _real_extract(self,url): + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('video_id') diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index fd4bc75b2a..22cdcdfa53 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -1,4 +1,4 @@ -#coding: utf-8 +# coding: utf-8 from __future__ import unicode_literals import re @@ -18,6 +18,7 @@ unescapeHTML, ) + class DailymotionBaseInfoExtractor(InfoExtractor): @staticmethod def _build_request(url): @@ -27,6 +28,7 @@ def _build_request(url): request.add_header('Cookie', 'ff=off') return request + class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): """Information Extractor for Dailymotion""" diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py index c5529f8d45..3ffed3d447 100644 --- a/youtube_dl/extractor/defense.py +++ b/youtube_dl/extractor/defense.py @@ -26,13 +26,13 @@ def _real_extract(self, url): video_id = self._search_regex( r"flashvars.pvg_id=\"(\d+)\";", webpage, 'ID') - + json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/' + video_id) info = self._download_webpage(json_url, title, 'Downloading JSON config') video_url = json.loads(info)['renditions'][0]['url'] - + return {'id': video_id, 'ext': 'mp4', 'url': video_url, diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py index 5ae0ad5b65..70e368021f 100644 --- a/youtube_dl/extractor/dotsub.py +++ b/youtube_dl/extractor/dotsub.py @@ -27,7 +27,7 @@ def _real_extract(self, url): video_id = mobj.group('id') info_url = "https://dotsub.com/api/media/%s/metadata" % video_id info = self._download_json(info_url, video_id) - date = time.gmtime(info['dateCreated']/1000) # The timestamp is in miliseconds + date = time.gmtime(info['dateCreated']/1000) # The timestamp is in miliseconds return { 'id': video_id, diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py index c663a0f81d..6f5d23559b 100644 --- a/youtube_dl/extractor/fc2.py +++ b/youtube_dl/extractor/fc2.py @@ -40,7 +40,7 @@ def _real_extract(self, url): info_url = ( "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&". - format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.','%2E'))) + format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.', '%2E'))) info_webpage = self._download_webpage( info_url, video_id, note='Downloading info page') diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py index c2e987ff72..3410daa988 100644 --- a/youtube_dl/extractor/firsttv.py +++ b/youtube_dl/extractor/firsttv.py @@ -57,4 +57,4 @@ def _real_extract(self, url): 'duration': int_or_none(duration), 'like_count': int_or_none(like_count), 'dislike_count': int_or_none(dislike_count), - } \ No newline at end of file + } diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py index e09982e88b..5b0bc9d219 100644 --- a/youtube_dl/extractor/flickr.py +++ b/youtube_dl/extractor/flickr.py @@ -17,8 +17,8 @@ class FlickrIE(InfoExtractor): 'info_dict': { 'id': '5645318632', 'ext': 'mp4', - "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.", - "uploader_id": "forestwander-nature-pictures", + "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.", + "uploader_id": "forestwander-nature-pictures", "title": "Dark Hollow Waterfalls" } } diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py index 7d56b9be93..24d4e97545 100644 --- a/youtube_dl/extractor/fourtube.py +++ b/youtube_dl/extractor/fourtube.py @@ -92,4 +92,4 @@ def _real_extract(self, url): 'duration': duration, 'age_limit': 18, 'webpage_url': webpage_url, - } \ No newline at end of file + } diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index c7a824c29b..2dc801ad95 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -733,7 +733,7 @@ def _playlist_from_matches(matches, getter, ie=None): 'title': video_title, 'id': video_id, } - + match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P[^"\']+)', webpage) if match: return { @@ -1025,4 +1025,3 @@ def filter_video(urls): '_type': 'playlist', 'entries': entries, } - diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py index 66ca37918a..6949a57c70 100644 --- a/youtube_dl/extractor/globo.py +++ b/youtube_dl/extractor/globo.py @@ -397,4 +397,4 @@ def _real_extract(self, url): 'uploader_id': uploader_id, 'like_count': like_count, 'formats': formats - } \ No newline at end of file + } diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py index e21e57510d..65b1534174 100644 --- a/youtube_dl/extractor/gorillavid.py +++ b/youtube_dl/extractor/gorillavid.py @@ -69,7 +69,7 @@ def _real_extract(self, url): (?:id="[^"]+"\s+)? value="([^"]*)" ''', webpage)) - + if fields['op'] == 'download1': post = compat_urllib_parse.urlencode(fields) diff --git a/youtube_dl/extractor/hornbunny.py b/youtube_dl/extractor/hornbunny.py index 7e7714438c..5b6efb27ee 100644 --- a/youtube_dl/extractor/hornbunny.py +++ b/youtube_dl/extractor/hornbunny.py @@ -37,7 +37,7 @@ def _real_extract(self, url): webpage2 = self._download_webpage(redirect_url, video_id) video_url = self._html_search_regex( r'flvMask:(.*?);', webpage2, 'video_url') - + duration = parse_duration(self._search_regex( r'Runtime:\s*([0-9:]+)
', webpage, 'duration', fatal=False)) diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py index 6ae04782c1..4ddf06409e 100644 --- a/youtube_dl/extractor/howcast.py +++ b/youtube_dl/extractor/howcast.py @@ -13,7 +13,7 @@ class HowcastIE(InfoExtractor): 'info_dict': { 'id': '390161', 'ext': 'mp4', - 'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.', + 'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.', 'title': 'How to Tie a Square Knot Properly', } } diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index 6108ed5526..f2c1c10f5c 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -71,7 +71,7 @@ class ImdbListIE(InfoExtractor): }, 'playlist_count': 7, } - + def _real_extract(self, url): list_id = self._match_id(url) webpage = self._download_webpage(url, list_id) diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py index 53f9a5f758..c137f4a5d9 100644 --- a/youtube_dl/extractor/internetvideoarchive.py +++ b/youtube_dl/extractor/internetvideoarchive.py @@ -32,7 +32,7 @@ def _build_url(query): def _clean_query(query): NEEDED_ARGS = ['publishedid', 'customerid'] query_dic = compat_urlparse.parse_qs(query) - cleaned_dic = dict((k,v[0]) for (k,v) in query_dic.items() if k in NEEDED_ARGS) + cleaned_dic = dict((k, v[0]) for (k, v) in query_dic.items() if k in NEEDED_ARGS) # Other player ids return m3u8 urls cleaned_dic['playerid'] = '247' cleaned_dic['videokbrate'] = '100000' diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index 75b543b7cf..06af734179 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -102,7 +102,7 @@ def _real_extract(self, url): compilation = result['compilation'] title = result['title'] - title = '%s - %s' % (compilation, title) if compilation is not None else title + title = '%s - %s' % (compilation, title) if compilation is not None else title previews = result['preview'] previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format'])) @@ -152,17 +152,17 @@ def _real_extract(self, url): compilation_id = mobj.group('compilationid') season_id = mobj.group('seasonid') - if season_id is not None: # Season link + if season_id is not None: # Season link season_page = self._download_webpage(url, compilation_id, 'Downloading season %s web page' % season_id) playlist_id = '%s/season%s' % (compilation_id, season_id) playlist_title = self._html_search_meta('title', season_page, 'title') entries = self._extract_entries(season_page, compilation_id) - else: # Compilation link + else: # Compilation link compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page') playlist_id = compilation_id playlist_title = self._html_search_meta('title', compilation_page, 'title') seasons = re.findall(r'[^<]+' % compilation_id, compilation_page) - if len(seasons) == 0: # No seasons in this compilation + if len(seasons) == 0: # No seasons in this compilation entries = self._extract_entries(compilation_page, compilation_id) else: entries = [] @@ -172,4 +172,4 @@ def _real_extract(self, url): compilation_id, 'Downloading season %s web page' % season_id) entries.extend(self._extract_entries(season_page, compilation_id)) - return self.playlist_result(entries, playlist_id, playlist_title) \ No newline at end of file + return self.playlist_result(entries, playlist_id, playlist_title) diff --git a/youtube_dl/extractor/jadorecettepub.py b/youtube_dl/extractor/jadorecettepub.py index ace08769bd..063e86de46 100644 --- a/youtube_dl/extractor/jadorecettepub.py +++ b/youtube_dl/extractor/jadorecettepub.py @@ -45,4 +45,3 @@ def _real_extract(self, url): 'title': title, 'description': description, } - diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py index 1881659665..8094cc2e48 100644 --- a/youtube_dl/extractor/jeuxvideo.py +++ b/youtube_dl/extractor/jeuxvideo.py @@ -29,7 +29,7 @@ def _real_extract(self, url): xml_link = self._html_search_regex( r'', webpage, 'config URL') - + video_id = self._search_regex( r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml', xml_link, 'video ID') @@ -38,7 +38,7 @@ def _real_extract(self, url): xml_link, title, 'Downloading XML config') info_json = config.find('format.json').text info = json.loads(info_json)['versions'][0] - + video_url = 'http://video720.jeuxvideo.com/' + info['file'] return { diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py index 23103b163f..dbfe4cc03f 100644 --- a/youtube_dl/extractor/kankan.py +++ b/youtube_dl/extractor/kankan.py @@ -10,7 +10,7 @@ class KankanIE(InfoExtractor): _VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P\d+)\.shtml' - + _TEST = { 'url': 'http://yinyue.kankan.com/vod/48/48863.shtml', 'file': '48863.flv', diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py index 8a73ecfa06..41fd62009a 100644 --- a/youtube_dl/extractor/kontrtube.py +++ b/youtube_dl/extractor/kontrtube.py @@ -63,4 +63,4 @@ def _real_extract(self, url): 'duration': duration, 'view_count': int_or_none(view_count), 'comment_count': int_or_none(comment_count), - } \ No newline at end of file + } diff --git a/youtube_dl/extractor/ku6.py b/youtube_dl/extractor/ku6.py index 89013e5223..a602980a14 100644 --- a/youtube_dl/extractor/ku6.py +++ b/youtube_dl/extractor/ku6.py @@ -30,4 +30,3 @@ def _real_extract(self, url): 'title': title, 'url': downloadUrl } - diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py index 102e29f7a8..2fd3b4699d 100644 --- a/youtube_dl/extractor/laola1tv.py +++ b/youtube_dl/extractor/laola1tv.py @@ -75,4 +75,3 @@ def _real_extract(self, url): 'categories': categories, 'ext': 'mp4', } - diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py index 8d9491f233..1f1e23dc3b 100644 --- a/youtube_dl/extractor/lifenews.py +++ b/youtube_dl/extractor/lifenews.py @@ -52,7 +52,7 @@ def _real_extract(self, url): r'
\s*(\d+)', webpage, 'comment count', fatal=False) upload_date = self._html_search_regex( - r'