MarcoBuster/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-07-29 10:03:32 +02:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'jtwaleson/master'

Browse Source
This commit is contained in:
Philipp Hagemeister 2014-11-23 21:33:31 +01:00
commit 784b6d3a9b
164 changed files with 574 additions and 570 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
devscripts/bash-completion.py
View File

@ -9,6 +9,7 @@
BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
def build_completion(opt_parser):
opts_flag = []
for group in opt_parser.option_groups:

1
devscripts/buildserver.py
View File

@ -233,6 +233,7 @@ def rmtree(path):
#==============================================================================
class BuildError(Exception):
def __init__(self, output, code=500):
self.output = output

1
devscripts/fish-completion.py
View File

@ -23,6 +23,7 @@
'batch-file': ['--require-parameter'],
}
def build_completion(opt_parser):
commands = []

3
devscripts/gh-pages/generate-download.py
View File

@ -1,8 +1,5 @@
#!/usr/bin/env python3
import hashlib
import shutil
import subprocess
import tempfile
import urllib.request
import json

1
devscripts/gh-pages/update-feed.py
View File

@ -73,4 +73,3 @@
with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
atom_file.write(atom_template)

1
devscripts/gh-pages/update-sites.py
View File

@ -9,6 +9,7 @@
import youtube_dl
def main():
with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
template = tmplf.read()

20
devscripts/transition_helper_exe/youtube-dl.py
View File

@ -4,13 +4,17 @@
import urllib2
import json, hashlib
def rsa_verify(message, signature, key):
from struct import pack
from hashlib import sha256
from sys import version_info
def b(x):
if version_info[0] == 2: return x
else: return x.encode('latin1')
if version_info[0] == 2:
return x
else:
return x.encode('latin1')
assert(type(message) == type(b('')))
block_size = 0
n = key[0]
@ -23,13 +27,17 @@ def b(x):
raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False
if signature[0:2] != b('\x00\x01'):
return False
signature = signature[2:]
if not b('\x00') in signature: return False
if not b('\x00') in signature:
return False
signature = signature[signature.index(b('\x00')) + 1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
return False
signature = signature[19:]
if signature != sha256(message).digest(): return False
if signature != sha256(message).digest():
return False
return True
sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')

4
test/helper.py
View File

@ -72,8 +72,10 @@ def download(self, x):
def expect_warning(self, regex):
# Silence an expected warning matching a regex
old_report_warning = self.report_warning
def report_warning(self, message):
if re.match(regex, message): return
if re.match(regex, message):
return
old_report_warning(message)
self.report_warning = types.MethodType(report_warning, self)

1
test/test_YoutubeDL.py
View File

@ -266,6 +266,7 @@ def test_prepare_filename(self):
'ext': 'mp4',
'width': None,
}
def fname(templ):
ydl = YoutubeDL({'outtmpl': templ})
return ydl.prepare_filename(info)

13
test/test_download.py
View File

@ -40,18 +40,22 @@
RETRIES = 3
class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs):
self.to_stderr = self.to_screen
self.processed_info_dicts = []
super(YoutubeDL, self).__init__(*args, **kwargs)
def report_warning(self, message):
# Don't accept warnings during tests
raise ExtractorError(message)
def process_info(self, info_dict):
self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict)
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
@ -61,10 +65,13 @@ def _file_md5(fn):
class TestDownload(unittest.TestCase):
maxDiff = None
def setUp(self):
self.defs = defs
### Dynamically generate tests
# Dynamically generate tests
def generator(test_case):
def test_template(self):
@ -101,6 +108,7 @@ def print_skipping(reason):
ydl = YoutubeDL(params, auto_init=False)
ydl.add_default_info_extractors()
finished_hook_called = set()
def _hook(status):
if status['status'] == 'finished':
finished_hook_called.add(status['filename'])
@ -111,6 +119,7 @@ def get_tc_filename(tc):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
res_dict = None
def try_rm_tcs_files(tcs=None):
if tcs is None:
tcs = test_cases
@ -206,7 +215,7 @@ def try_rm_tcs_files(tcs=None):
return test_template
### And add them to TestDownload
# And add them to TestDownload
for n, test_case in enumerate(defs):
test_method = generator(test_case)
tname = 'test_' + str(test_case['name'])

1
test/test_subtitles.py
View File

@ -23,6 +23,7 @@
class BaseTestSubtitles(unittest.TestCase):
url = None
IE = None
def setUp(self):
self.DL = FakeYDL()
self.ie = self.IE(self.DL)

1
test/test_utils.py
View File

@ -45,7 +45,6 @@
escape_rfc3986,
escape_url,
js_to_json,
get_filesystem_encoding,
intlist_to_bytes,
args_to_str,
)

4
test/test_write_annotations.py
View File

@ -31,17 +31,16 @@ def __init__(self, *args, **kwargs):
})
TEST_ID = 'gr51aVj-mLg'
ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'
EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']
class TestAnnotations(unittest.TestCase):
def setUp(self):
# Clear old files
self.tearDown()
def test_info_json(self):
expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text.
ie = youtube_dl.extractor.YoutubeIE()
@ -71,7 +70,6 @@ def test_info_json(self):
# We should have seen (and removed) all the expected annotation texts.
self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
def tearDown(self):
try_rm(ANNOTATIONS_FILE)

4
test/test_youtube_lists.py
View File

@ -12,10 +12,6 @@
from youtube_dl.extractor import (
YoutubePlaylistIE,
YoutubeIE,
YoutubeChannelIE,
YoutubeShowIE,
YoutubeTopListIE,
YoutubeSearchURLIE,
)

3
youtube_dl/YoutubeDL.py
View File

@ -29,7 +29,6 @@
compat_str,
compat_urllib_error,
compat_urllib_request,
shlex_quote,
)
from .utils import (
escape_url,
@ -700,6 +699,7 @@ def make_result(embedded_info):
self.report_warning(
'Extractor %s returned a compat_list result. '
'It needs to be updated.' % ie_result.get('extractor'))
def _fixup(r):
self.add_extra_info(r,
{
@ -1428,4 +1428,3 @@ def get_encoding(self):
if encoding is None:
encoding = preferredencoding()
return encoding

2
youtube_dl/__init__.py
View File

@ -128,7 +128,6 @@ def _real_main(argv=None):
compat_print(desc)
sys.exit(0)
# Conflicting, missing and erroneous options
if opts.usenetrc and (opts.username is not None or opts.password is not None):
parser.error('using .netrc conflicts with giving username/password')
@ -317,7 +316,6 @@ def _real_main(argv=None):
ydl.add_post_processor(FFmpegAudioFixPP())
ydl.add_post_processor(AtomicParsleyPP())
# Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
# So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
if opts.exec_cmd:

19
youtube_dl/aes.py
View File

@ -7,6 +7,7 @@
BLOCK_SIZE_BYTES = 16
def aes_ctr_decrypt(data, key, counter):
"""
Decrypt with aes in counter mode
@ -32,6 +33,7 @@ def aes_ctr_decrypt(data, key, counter):
return decrypted_data
def aes_cbc_decrypt(data, key, iv):
"""
Decrypt with aes in CBC mode
@ -57,6 +59,7 @@ def aes_cbc_decrypt(data, key, iv):
return decrypted_data
def key_expansion(data):
"""
Generate key schedule
@ -91,6 +94,7 @@ def key_expansion(data):
return data
def aes_encrypt(data, expanded_key):
"""
Encrypt one block with aes
@ -111,6 +115,7 @@ def aes_encrypt(data, expanded_key):
return data
def aes_decrypt(data, expanded_key):
"""
Decrypt one block with aes
@ -131,6 +136,7 @@ def aes_decrypt(data, expanded_key):
return data
def aes_decrypt_text(data, password, key_size_bytes):
"""
Decrypt text
@ -157,6 +163,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
class Counter:
__value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
def next_value(self):
temp = self.__value
self.__value = inc(self.__value)
@ -241,15 +248,19 @@ def next_value(self):
0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
def sub_bytes(data):
return [SBOX[x] for x in data]
def sub_bytes_inv(data):
return [SBOX_INV[x] for x in data]
def rotate(data):
return data[1:] + [data[0]]
def key_schedule_core(data, rcon_iteration):
data = rotate(data)
data = sub_bytes(data)
@ -257,14 +268,17 @@ def key_schedule_core(data, rcon_iteration):
return data
def xor(data1, data2):
return [x ^ y for x, y in zip(data1, data2)]
def rijndael_mul(a, b):
if(a == 0 or b == 0):
return 0
return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
def mix_column(data, matrix):
data_mixed = []
for row in range(4):
@ -275,6 +289,7 @@ def mix_column(data, matrix):
data_mixed.append(mixed)
return data_mixed
def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
data_mixed = []
for i in range(4):
@ -282,9 +297,11 @@ def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
data_mixed += mix_column(column, matrix)
return data_mixed
def mix_columns_inv(data):
return mix_columns(data, MIX_COLUMN_MATRIX_INV)
def shift_rows(data):
data_shifted = []
for column in range(4):
@ -292,6 +309,7 @@ def shift_rows(data):
data_shifted.append(data[((column + row) & 0b11) * 4 + row])
return data_shifted
def shift_rows_inv(data):
data_shifted = []
for column in range(4):
@ -299,6 +317,7 @@ def shift_rows_inv(data):
data_shifted.append(data[((column - row) & 0b11) * 4 + row])
return data_shifted
def inc(data):
data = data[:] # copy
for i in range(len(data) - 1, -1, -1):

6
youtube_dl/compat.py
View File

@ -182,8 +182,10 @@ def shlex_quote(s):
def compat_ord(c):
if type(c) is int: return c
else: return ord(c)
if type(c) is int:
return c
else:
return ord(c)
if sys.version_info >= (3, 0):

1
youtube_dl/downloader/hls.py
View File

@ -101,4 +101,3 @@ def real_download(self, filename, info_dict):
})
self.try_rename(tmpfilename, filename)
return True

1
youtube_dl/extractor/adultswim.py
View File

@ -5,6 +5,7 @@
from .common import InfoExtractor
class AdultSwimIE(InfoExtractor):
_VALID_URL = r'https?://video\.adultswim\.com/(?P<path>.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$'
_TEST = {

2
youtube_dl/extractor/appletrailers.py
View File

@ -70,11 +70,13 @@ def _real_extract(self, url):
uploader_id = mobj.group('company')
playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
def fix_html(s):
s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
# The ' in the onClick attributes are not escaped, it couldn't be parsed
# like: http://trailers.apple.com/trailers/wb/gravity/
def _clean_json(m):
return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
s = re.sub(self._JSON_RE, _clean_json, s)

1
youtube_dl/extractor/ard.py
View File

@ -192,4 +192,3 @@ def _real_extract(self, url):
'upload_date': upload_date,
'thumbnail': thumbnail,
}

1
youtube_dl/extractor/channel9.py
View File

@ -5,6 +5,7 @@
from .common import InfoExtractor
from ..utils import ExtractorError
class Channel9IE(InfoExtractor):
'''
Common extractor for channel9.msdn.com.

1
youtube_dl/extractor/clipsyndicate.py
View File

@ -39,6 +39,7 @@ def _real_extract(self, url):
transform_source=fix_xml_ampersands)
track_doc = pdoc.find('trackList/track')
def find_param(name):
node = find_xpath_attr(track_doc, './/param', 'name', name)
if node is not None:

1
youtube_dl/extractor/common.py
View File

@ -434,6 +434,7 @@ def url_result(url, ie=None, video_id=None):
if video_id is not None:
video_info['id'] = video_id
return video_info
@staticmethod
def playlist_result(entries, playlist_id=None, playlist_title=None):
"""Returns a playlist"""

4
youtube_dl/extractor/crunchyroll.py
View File

@ -69,11 +69,9 @@ def _login(self):
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self._download_webpage(login_request, None, False, 'Wrong login info')
def _real_initialize(self):
self._login()
def _decrypt_subtitles(self, data, iv, id):
data = bytes_to_intlist(data)
iv = bytes_to_intlist(iv)
@ -99,8 +97,10 @@ def obfuscate_key(key):
return shaHash + [0] * 12
key = obfuscate_key(id)
class Counter:
__value = iv
def next_value(self):
temp = self.__value
self.__value = inc(self.__value)

2
youtube_dl/extractor/dailymotion.py
View File

@ -18,6 +18,7 @@
unescapeHTML,
)
class DailymotionBaseInfoExtractor(InfoExtractor):
@staticmethod
def _build_request(url):
@ -27,6 +28,7 @@ def _build_request(url):
request.add_header('Cookie', 'ff=off')
return request
class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
"""Information Extractor for Dailymotion"""

1
youtube_dl/extractor/generic.py
View File

@ -1025,4 +1025,3 @@ def filter_video(urls):
'_type': 'playlist',
'entries': entries,
}

1
youtube_dl/extractor/jadorecettepub.py
View File

@ -45,4 +45,3 @@ def _real_extract(self, url):
'title': title,
'description': description,
}

1
youtube_dl/extractor/ku6.py
View File

@ -30,4 +30,3 @@ def _real_extract(self, url):
'title': title,
'url': downloadUrl
}

1
youtube_dl/extractor/laola1tv.py
View File

@ -75,4 +75,3 @@ def _real_extract(self, url):
'categories': categories,
'ext': 'mp4',
}

6
youtube_dl/extractor/liveleak.py
View File

@ -19,8 +19,7 @@ class LiveLeakIE(InfoExtractor):
'uploader': 'ljfriel2',
'title': 'Most unlucky car accident'
}
},
{
}, {
'url': 'http://www.liveleak.com/view?i=f93_1390833151',
'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
'info_dict': {
@ -30,8 +29,7 @@ class LiveLeakIE(InfoExtractor):
'uploader': 'ARD_Stinkt',
'title': 'German Television does first Edward Snowden Interview (ENGLISH)',
}
},
{
}, {
'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
'md5': '42c6d97d54f1db107958760788c5f48f',
'info_dict': {

1
youtube_dl/extractor/malemotion.py
View File

@ -7,6 +7,7 @@
compat_urllib_parse,
)
class MalemotionIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
_TEST = {

2
youtube_dl/extractor/mtv.py
View File

@ -245,7 +245,7 @@ def _real_extract(self, url):
m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
webpage, re.DOTALL)
if m_vevo:
vevo_id = m_vevo.group(1);
vevo_id = m_vevo.group(1)
self.to_screen('Vevo video detected: %s' % vevo_id)
return self.url_result('vevo:%s' % vevo_id, ie='Vevo')

1
youtube_dl/extractor/muenchentv.py
View File

@ -73,4 +73,3 @@ def _real_extract(self, url):
'is_live': True,
'thumbnail': thumbnail,
}

1
youtube_dl/extractor/myvideo.py
View File

@ -173,4 +173,3 @@ def _real_extract(self,url):
'play_path': video_playpath,
'player_url': video_swfobj,
}

1
youtube_dl/extractor/nba.py
View File

@ -39,7 +39,6 @@ def _real_extract(self, url):
duration = parse_duration(
self._html_search_meta('duration', webpage, 'duration', fatal=False))
return {
'id': shortened_video_id,
'url': video_url,

1
youtube_dl/extractor/ooyala.py
View File

@ -97,4 +97,3 @@ def _real_extract(self, url):
}
else:
return self._extract_result(videos_info[0], videos_more_info)

1
youtube_dl/extractor/podomatic.py
View File

@ -6,6 +6,7 @@
from .common import InfoExtractor
from ..utils import int_or_none
class PodomaticIE(InfoExtractor):
IE_NAME = 'podomatic'
_VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)'

1
youtube_dl/extractor/ringtv.py
View File

@ -41,4 +41,3 @@ def _real_extract(self, url):
'thumbnail': thumbnail_url,
'description': description,
}

2
youtube_dl/extractor/rtlnl.py
View File

@ -44,7 +44,7 @@ def _real_extract(self, url):
formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
video_urlpart = videopath.split('/flash/')[1][:-4]
video_urlpart = videopath.split('/flash/')[1][:-5]
PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
formats.extend([

1
youtube_dl/extractor/rtve.py
View File

@ -54,7 +54,6 @@ def _decrypt_url(png):
return url
class RTVEALaCartaIE(InfoExtractor):
IE_NAME = 'rtve.es:alacarta'
IE_DESC = 'RTVE a la carta'

2
youtube_dl/extractor/servingsys.py
View File

@ -67,5 +67,3 @@ def _real_extract(self, url):
'title': title,
'entries': entries,
}

149
youtube_dl/extractor/smotri.py
View File

@ -1,7 +1,6 @@
# encoding: utf-8
from __future__ import unicode_literals
import os.path
import re
import json
import hashlib
@ -12,15 +11,15 @@
compat_urllib_parse,
compat_urllib_request,
ExtractorError,
url_basename,
int_or_none,
unified_strdate,
)
class SmotriIE(InfoExtractor):
IE_DESC = 'Smotri.com'
IE_NAME = 'smotri'
_VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<videoid>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
_VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
_NETRC_MACHINE = 'smotri'
_TESTS = [
@ -35,7 +34,6 @@ class SmotriIE(InfoExtractor):
'uploader': 'rbc2008',
'uploader_id': 'rbc08',
'upload_date': '20131118',
'description': 'катастрофа с камер видеонаблюдения, видео катастрофа с камер видеонаблюдения',
'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg',
},
},
@ -50,7 +48,6 @@ class SmotriIE(InfoExtractor):
'uploader': 'Support Photofile@photofile',
'uploader_id': 'support-photofile',
'upload_date': '20070704',
'description': 'test, видео test',
'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
},
},
@ -66,7 +63,6 @@ class SmotriIE(InfoExtractor):
'uploader_id': 'timoxa40',
'upload_date': '20100404',
'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg',
'description': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1, видео TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1',
},
'params': {
'videopassword': 'qwerty',
@ -85,7 +81,6 @@ class SmotriIE(InfoExtractor):
'upload_date': '20101001',
'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg',
'age_limit': 18,
'description': 'этот ролик не покажут по ТВ, видео этот ролик не покажут по ТВ',
},
'params': {
'videopassword': '333'
@ -102,17 +97,11 @@ class SmotriIE(InfoExtractor):
'uploader': 'HannahL',
'uploader_id': 'lisaha95',
'upload_date': '20090331',
'description': 'Shakira - Don\'t Bother, видео Shakira - Don\'t Bother',
'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg',
},
},
]
_SUCCESS = 0
_PASSWORD_NOT_VERIFIED = 1
_PASSWORD_DETECTED = 2
_VIDEO_NOT_FOUND = 3
@classmethod
def _extract_url(cls, webpage):
mobj = re.search(
@ -137,44 +126,44 @@ def _search_meta(self, name, html, display_name=None):
return self._html_search_meta(name, html, display_name)
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('videoid')
real_video_id = mobj.group('realvideoid')
video_id = self._match_id(url)
# Download video JSON data
video_json_url = 'http://smotri.com/vt.php?id=%s' % real_video_id
video_json_page = self._download_webpage(video_json_url, video_id, 'Downloading video JSON')
video_json = json.loads(video_json_page)
video_form = {
'ticket': video_id,
'video_url': '1',
'frame_url': '1',
'devid': 'LoadupFlashPlayer',
'getvideoinfo': '1',
}
status = video_json['status']
if status == self._VIDEO_NOT_FOUND:
request = compat_urllib_request.Request(
'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
video = self._download_json(request, video_id, 'Downloading video JSON')
if video.get('_moderate_no') or not video.get('moderated'):
raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True)
if video.get('error'):
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with
# video-password set
video_password = self._downloader.params.get('videopassword', None)
if not video_password:
raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
video_json_url += '&md5pass=%s' % hashlib.md5(video_password.encode('utf-8')).hexdigest()
video_json_page = self._download_webpage(video_json_url, video_id, 'Downloading video JSON (video-password set)')
video_json = json.loads(video_json_page)
status = video_json['status']
if status == self._PASSWORD_NOT_VERIFIED:
raise ExtractorError('Video password is invalid', expected=True)
if status != self._SUCCESS:
raise ExtractorError('Unexpected status value %s' % status)
# Extract the URL of the video
video_url = video_json['file_data']
video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
title = video['title']
thumbnail = video['_imgURL']
upload_date = unified_strdate(video['added'])
uploader = video['userNick']
uploader_id = video['userLogin']
duration = int_or_none(video['duration'])
# Video JSON does not provide enough meta data
# We will extract some from the video web page instead
video_page_url = 'http://smotri.com/video/view/?id=%s' % video_id
video_page = self._download_webpage(video_page_url, video_id, 'Downloading video page')
webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id
webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page')
# Warning if video is unavailable
warning = self._html_search_regex(
r'<div class="videoUnModer">(.*?)</div>', video_page,
r'<div class="videoUnModer">(.*?)</div>', webpage,
'warning message', default=None)
if warning is not None:
self._downloader.report_warning(
@ -182,84 +171,32 @@ def _real_extract(self, url):
(video_id, warning))
# Adult content
if re.search('EroConfirmText">', video_page) is not None:
if re.search('EroConfirmText">', webpage) is not None:
self.report_age_confirmation()
confirm_string = self._html_search_regex(
r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id,
video_page, 'confirm string')
confirm_url = video_page_url + '&confirm=%s' % confirm_string
video_page = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
webpage, 'confirm string')
confirm_url = webpage_url + '&confirm=%s' % confirm_string
webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
adult_content = True
else:
adult_content = False
# Extract the rest of meta data
video_title = self._search_meta('name', video_page, 'title')
if not video_title:
video_title = os.path.splitext(url_basename(video_url))[0]
video_description = self._search_meta('description', video_page)
END_TEXT = ' на сайте Smotri.com'
if video_description and video_description.endswith(END_TEXT):
video_description = video_description[:-len(END_TEXT)]
START_TEXT = 'Смотреть онлайн ролик '
if video_description and video_description.startswith(START_TEXT):
video_description = video_description[len(START_TEXT):]
video_thumbnail = self._search_meta('thumbnail', video_page)
upload_date_str = self._search_meta('uploadDate', video_page, 'upload date')
if upload_date_str:
upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str)
video_upload_date = (
(
upload_date_m.group('year') +
upload_date_m.group('month') +
upload_date_m.group('day')
)
if upload_date_m else None
)
else:
video_upload_date = None
duration_str = self._search_meta('duration', video_page)
if duration_str:
duration_m = re.search(r'T(?P<hours>[0-9]{2})H(?P<minutes>[0-9]{2})M(?P<seconds>[0-9]{2})S', duration_str)
video_duration = (
(
(int(duration_m.group('hours')) * 60 * 60) +
(int(duration_m.group('minutes')) * 60) +
int(duration_m.group('seconds'))
)
if duration_m else None
)
else:
video_duration = None
video_uploader = self._html_search_regex(
'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>',
video_page, 'uploader', fatal=False, flags=re.MULTILINE|re.DOTALL)
video_uploader_id = self._html_search_regex(
'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info\\(.*?\'([^\']+)\'\\);">',
video_page, 'uploader id', fatal=False, flags=re.MULTILINE|re.DOTALL)
video_view_count = self._html_search_regex(
view_count = self._html_search_regex(
'Общее количество просмотров.*?<span class="Number">(\\d+)</span>',
video_page, 'view count', fatal=False, flags=re.MULTILINE|re.DOTALL)
webpage, 'view count', fatal=False, flags=re.MULTILINE | re.DOTALL)
return {
'id': video_id,
'url': video_url,
'title': video_title,
'thumbnail': video_thumbnail,
'description': video_description,
'uploader': video_uploader,
'upload_date': video_upload_date,
'uploader_id': video_uploader_id,
'duration': video_duration,
'view_count': int_or_none(video_view_count),
'title': title,
'thumbnail': thumbnail,
'uploader': uploader,
'upload_date': upload_date,
'uploader_id': uploader_id,
'duration': duration,
'view_count': int_or_none(view_count),
'age_limit': 18 if adult_content else 0,
'video_page_url': video_page_url
}

1
youtube_dl/extractor/sportdeutschland.py
View File

@ -93,4 +93,3 @@ def _real_extract(self, url):
'rtmp_live': asset.get('live'),
'timestamp': parse_iso8601(asset.get('date')),
}

2
youtube_dl/extractor/subtitles.py
View File

@ -50,7 +50,7 @@ def extract_subtitles(self, video_id, webpage):
sub_lang_list = {}
for sub_lang in requested_langs:
if not sub_lang in available_subs_list:
if sub_lang not in available_subs_list:
self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
continue
sub_lang_list[sub_lang] = available_subs_list[sub_lang]

2
youtube_dl/extractor/theplatform.py
View File

@ -35,6 +35,7 @@ class ThePlatformIE(InfoExtractor):
'skip_download': True,
},
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
@ -48,7 +49,6 @@ def _real_extract(self, url):
smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
'format=smil&mbr=true'.format(video_id))
meta = self._download_xml(smil_url, video_id)
try:
error_msg = next(

1
youtube_dl/extractor/trilulilu.py
View File

@ -63,4 +63,3 @@ def _real_extract(self, url):
'description': description,
'thumbnail': thumbnail,
}

1
youtube_dl/extractor/vice.py
View File

@ -35,4 +35,3 @@ def _real_extract(self, url):
except ExtractorError:
raise ExtractorError('The page doesn\'t contain a video', expected=True)
return self.url_result(ooyala_url, ie='Ooyala')

1
youtube_dl/extractor/videofyme.py
View File

@ -6,6 +6,7 @@
determine_ext,
)
class VideofyMeIE(InfoExtractor):
_VALID_URL = r'https?://(www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)'
IE_NAME = u'videofy.me'

1
youtube_dl/extractor/vidzi.py
View File

@ -30,4 +30,3 @@ def _real_extract(self, url):
'title': title,
'url': video_url,
}

1
youtube_dl/extractor/worldstarhiphop.py
View File

@ -51,4 +51,3 @@ def _real_extract(self, url):
'title': video_title,
'thumbnail': thumbnail,
}

1
youtube_dl/extractor/xbef.py
View File

@ -47,4 +47,3 @@ def _real_extract(self, url):
'thumbnail': thumbnail,
'age_limit': 18,
}

20
youtube_dl/extractor/youtube.py
View File

@ -33,6 +33,7 @@
uppercase_escape,
)
class YoutubeBaseInfoExtractor(InfoExtractor):
"""Provide base functions for Youtube extractors"""
_LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@ -180,8 +181,10 @@ def _confirm_age(self):
'next_url': '/',
'action_confirm': 'Confirm',
}
req = compat_urllib_request.Request(self._AGE_URL,
compat_urllib_parse.urlencode(age_form).encode('ascii'))
req = compat_urllib_request.Request(
self._AGE_URL,
compat_urllib_parse.urlencode(age_form).encode('ascii')
)
self._download_webpage(
req, None,
@ -651,6 +654,7 @@ def extract_id(cls, url):
def _extract_from_m3u8(self, manifest_url, video_id):
url_map = {}
def _get_urls(_manifest):
lines = _manifest.split('\n')
urls = filter(lambda l: l and not l.startswith('#'),
@ -974,6 +978,7 @@ def _map_to_format_list(urlmap):
dash_manifest_url = video_info.get('dashmpd')[0]
else:
dash_manifest_url = ytplayer_config['args']['dashmpd']
def decrypt_sig(mobj):
s = mobj.group(1)
dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@ -1033,6 +1038,7 @@ def decrypt_sig(mobj):
'formats': formats,
}
class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube.com playlists'
_VALID_URL = r"""(?x)(?:
@ -1333,8 +1339,10 @@ def suitable(cls, url):
# Don't return True if the url can be extracted with other youtube
# extractor, the regex would is too permissive and it would match.
other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
if any(ie.suitable(url) for ie in other_ies): return False
else: return super(YoutubeUserIE, cls).suitable(url)
if any(ie.suitable(url) for ie in other_ies):
return False
else:
return super(YoutubeUserIE, cls).suitable(url)
def _real_extract(self, url):
# Extract username
@ -1557,12 +1565,14 @@ def _real_extract(self, url):
paging = mobj.group('paging')
return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
_VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
_FEED_NAME = 'recommended'
_PLAYLIST_TITLE = 'Youtube Recommended videos'
class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
_VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
@ -1570,6 +1580,7 @@ class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
_PLAYLIST_TITLE = 'Youtube Watch Later'
_PERSONAL_FEED = True
class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
_VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
@ -1577,6 +1588,7 @@ class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
_PERSONAL_FEED = True
_PLAYLIST_TITLE = 'Youtube Watch History'
class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
IE_NAME = 'youtube:favorites'
IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'

1
youtube_dl/postprocessor/execafterdownload.py
View File

@ -26,4 +26,3 @@ def run(self, information):
'Command returned error code %d' % retCode)
return None, information # by default, keep file and do nothing

1
youtube_dl/postprocessor/xattrpp.py
View File

@ -108,4 +108,3 @@ def write_xattr(path, key, value):
except (subprocess.CalledProcessError, OSError):
self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
return False, info

1
youtube_dl/swfinterp.py
View File

@ -827,4 +827,3 @@ def resfunc(args):
avm_class.method_pyfunctions[func_name] = resfunc
return resfunc

43
youtube_dl/update.py
View File

@ -13,13 +13,17 @@
)
from .version import __version__
def rsa_verify(message, signature, key):
from struct import pack
from hashlib import sha256
from sys import version_info
def b(x):
if version_info[0] == 2: return x
else: return x.encode('latin1')
if version_info[0] == 2:
return x
else:
return x.encode('latin1')
assert(type(message) == type(b('')))
block_size = 0
n = key[0]
@ -32,13 +36,17 @@ def b(x):
raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False
if signature[0:2] != b('\x00\x01'):
return False
signature = signature[2:]
if not b('\x00') in signature: return False
if not b('\x00') in signature:
return False
signature = signature[signature.index(b('\x00')) + 1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
return False
signature = signature[19:]
if signature != sha256(message).digest(): return False
if signature != sha256(message).digest():
return False
return True
@ -58,7 +66,8 @@ def update_self(to_screen, verbose):
try:
newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
except:
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: can\'t find the current version. Please try again later.')
return
if newversion == __version__:
@ -70,7 +79,8 @@ def update_self(to_screen, verbose):
versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
versions_info = json.loads(versions_info)
except:
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
return
if not 'signature' in versions_info:
@ -118,7 +128,8 @@ def version_tuple(version_str):
newcontent = urlh.read()
urlh.close()
except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to download latest version')
return
@ -131,7 +142,8 @@ def version_tuple(version_str):
with open(exe + '.new', 'wb') as outf:
outf.write(newcontent)
except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to write the new version')
return
@ -150,7 +162,8 @@ def version_tuple(version_str):
subprocess.Popen([bat]) # Continues to run in the background
return # Do not show premature success messages
except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to overwrite current version')
return
@ -161,7 +174,8 @@ def version_tuple(version_str):
newcontent = urlh.read()
urlh.close()
except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to download latest version')
return
@ -174,12 +188,14 @@ def version_tuple(version_str):
with open(filename, 'wb') as outf:
outf.write(newcontent)
except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc()))
if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to overwrite current version')
return
to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
def get_notes(versions, fromVersion):
notes = []
for v, vdata in sorted(versions.items()):
@ -187,6 +203,7 @@ def get_notes(versions, fromVersion):
notes.extend(vdata.get('notes', []))
return notes
def print_notes(to_screen, versions, fromVersion=__version__):
notes = get_notes(versions, fromVersion)
if notes:

20
youtube_dl/utils.py
View File

@ -56,6 +56,7 @@
'Accept-Language': 'en-us,en;q=0.5',
}
def preferredencoding():
"""Get preferred encoding.
@ -146,6 +147,8 @@ def find_xpath_attr(node, xpath, key, val):
# On python2.6 the xml.etree.ElementTree.Element methods don't support
# the namespace parameter
def xpath_with_ns(path, ns_map):
components = [c.split(':') for c in path.split('/')]
replaced = []
@ -256,6 +259,7 @@ def timeconvert(timestr):
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
def sanitize_filename(s, restricted=False, is_id=False):
"""Sanitizes a string so it could be used as part of a filename.
If restricted is set, use a stricter subset of allowed characters.
@ -288,6 +292,7 @@ def replace_insane(char):
result = '_'
return result
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """
res = []
@ -372,6 +377,7 @@ def decodeOption(optval):
assert isinstance(optval, compat_str)
return optval
def formatSeconds(secs):
if secs > 3600:
return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
@ -424,6 +430,7 @@ def https_open(self, req):
class ExtractorError(Exception):
"""Error during info extraction."""
def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
""" tb, if given, is the original traceback (so that it can be printed out).
If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
@ -468,6 +475,7 @@ class DownloadError(Exception):
configured to continue on errors. They will contain the appropriate
error message.
"""
def __init__(self, msg, exc_info=None):
""" exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
super(DownloadError, self).__init__(msg)
@ -489,9 +497,11 @@ class PostProcessingError(Exception):
This exception may be raised by PostProcessor's .run() method to
indicate an error in the postprocessing task.
"""
def __init__(self, msg):
self.msg = msg
class MaxDownloadsReached(Exception):
""" --max-downloads limit has been reached. """
pass
@ -521,6 +531,7 @@ def __init__(self, downloaded, expected):
self.downloaded = downloaded
self.expected = expected
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
"""Handler for HTTP requests and responses.
@ -681,6 +692,7 @@ def unified_strdate(date_str):
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
return upload_date
def determine_ext(url, default_ext='unknown_video'):
if url is None:
return default_ext
@ -690,9 +702,11 @@ def determine_ext(url, default_ext='unknown_video'):
else:
return default_ext
def subtitles_filename(filename, sub_lang, sub_format):
return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
def date_from_str(date_str):
"""
Return a datetime object from a string in the format YYYYMMDD or
@ -719,6 +733,7 @@ def date_from_str(date_str):
return today + delta
return datetime.datetime.strptime(date_str, "%Y%m%d").date()
def hyphenate_date(date_str):
"""
Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
@ -728,8 +743,10 @@ def hyphenate_date(date_str):
else:
return date_str
class DateRange(object):
"""Represents a time interval between two dates"""
def __init__(self, start=None, end=None):
"""start and end must be strings in the format accepted by date"""
if start is not None:
@ -742,15 +759,18 @@ def __init__(self, start=None, end=None):
self.end = datetime.datetime.max.date()
if self.start > self.end:
raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
@classmethod
def day(cls, day):
"""Returns a range that only contains the given day"""
return cls(day, day)
def __contains__(self, date):
"""Check if the date is in the range"""
if not isinstance(date, datetime.date):
date = date_from_str(date)
return self.start <= date <= self.end
def __str__(self):
return '%s - %s' % (self.start.isoformat(), self.end.isoformat())