mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-08-24 16:21:40 +02:00
Switch codebase to use sanitized_Request instead of
compat_urllib_request.Request [downloader/dash] Use sanitized_Request [downloader/http] Use sanitized_Request [atresplayer] Use sanitized_Request [bambuser] Use sanitized_Request [bliptv] Use sanitized_Request [brightcove] Use sanitized_Request [cbs] Use sanitized_Request [ceskatelevize] Use sanitized_Request [collegerama] Use sanitized_Request [extractor/common] Use sanitized_Request [crunchyroll] Use sanitized_Request [dailymotion] Use sanitized_Request [dcn] Use sanitized_Request [dramafever] Use sanitized_Request [dumpert] Use sanitized_Request [eitb] Use sanitized_Request [escapist] Use sanitized_Request [everyonesmixtape] Use sanitized_Request [extremetube] Use sanitized_Request [facebook] Use sanitized_Request [fc2] Use sanitized_Request [flickr] Use sanitized_Request [4tube] Use sanitized_Request [gdcvault] Use sanitized_Request [extractor/generic] Use sanitized_Request [hearthisat] Use sanitized_Request [hotnewhiphop] Use sanitized_Request [hypem] Use sanitized_Request [iprima] Use sanitized_Request [ivi] Use sanitized_Request [keezmovies] Use sanitized_Request [letv] Use sanitized_Request [lynda] Use sanitized_Request [metacafe] Use sanitized_Request [minhateca] Use sanitized_Request [miomio] Use sanitized_Request [meovideo] Use sanitized_Request [mofosex] Use sanitized_Request [moniker] Use sanitized_Request [mooshare] Use sanitized_Request [movieclips] Use sanitized_Request [mtv] Use sanitized_Request [myvideo] Use sanitized_Request [neteasemusic] Use sanitized_Request [nfb] Use sanitized_Request [niconico] Use sanitized_Request [noco] Use sanitized_Request [nosvideo] Use sanitized_Request [novamov] Use sanitized_Request [nowness] Use sanitized_Request [nuvid] Use sanitized_Request [played] Use sanitized_Request [pluralsight] Use sanitized_Request [pornhub] Use sanitized_Request [pornotube] Use sanitized_Request [primesharetv] Use sanitized_Request [promptfile] Use sanitized_Request [qqmusic] Use sanitized_Request [rtve] Use sanitized_Request [safari] Use sanitized_Request [sandia] Use sanitized_Request [shared] Use sanitized_Request [sharesix] Use sanitized_Request [sina] Use sanitized_Request [smotri] Use sanitized_Request [sohu] Use sanitized_Request [spankwire] Use sanitized_Request [sportdeutschland] Use sanitized_Request [streamcloud] Use sanitized_Request [streamcz] Use sanitized_Request [tapely] Use sanitized_Request [tube8] Use sanitized_Request [tubitv] Use sanitized_Request [twitch] Use sanitized_Request [twitter] Use sanitized_Request [udemy] Use sanitized_Request [vbox7] Use sanitized_Request [veoh] Use sanitized_Request [vessel] Use sanitized_Request [vevo] Use sanitized_Request [viddler] Use sanitized_Request [videomega] Use sanitized_Request [viewvster] Use sanitized_Request [viki] Use sanitized_Request [vk] Use sanitized_Request [vodlocker] Use sanitized_Request [voicerepublic] Use sanitized_Request [wistia] Use sanitized_Request [xfileshare] Use sanitized_Request [xtube] Use sanitized_Request [xvideos] Use sanitized_Request [yandexmusic] Use sanitized_Request [youku] Use sanitized_Request [youporn] Use sanitized_Request [youtube] Use sanitized_Request [patreon] Use sanitized_Request [extractor/common] Remove unused import [nfb] PEP 8
This commit is contained in:
parent
67dda51722
commit
5c2266df4b
@ -1189,7 +1189,7 @@ def _calc_headers(self, info_dict):
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
def _calc_cookies(self, info_dict):
|
def _calc_cookies(self, info_dict):
|
||||||
pr = compat_urllib_request.Request(info_dict['url'])
|
pr = sanitized_Request(info_dict['url'])
|
||||||
self.cookiejar.add_cookie_header(pr)
|
self.cookiejar.add_cookie_header(pr)
|
||||||
return pr.get_header('Cookie')
|
return pr.get_header('Cookie')
|
||||||
|
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import FileDownloader
|
from .common import FileDownloader
|
||||||
from ..compat import compat_urllib_request
|
from ..utils import sanitized_Request
|
||||||
|
|
||||||
|
|
||||||
class DashSegmentsFD(FileDownloader):
|
class DashSegmentsFD(FileDownloader):
|
||||||
@ -22,7 +22,7 @@ def real_download(self, filename, info_dict):
|
|||||||
|
|
||||||
def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
|
def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
|
||||||
self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
|
self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
|
||||||
req = compat_urllib_request.Request(target_url)
|
req = sanitized_Request(target_url)
|
||||||
if remaining_bytes is not None:
|
if remaining_bytes is not None:
|
||||||
req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
|
req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
|
||||||
|
|
||||||
|
@ -7,14 +7,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import FileDownloader
|
from .common import FileDownloader
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_error
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_error,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ContentTooShortError,
|
ContentTooShortError,
|
||||||
encodeFilename,
|
encodeFilename,
|
||||||
sanitize_open,
|
sanitize_open,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -29,8 +27,8 @@ def real_download(self, filename, info_dict):
|
|||||||
add_headers = info_dict.get('http_headers')
|
add_headers = info_dict.get('http_headers')
|
||||||
if add_headers:
|
if add_headers:
|
||||||
headers.update(add_headers)
|
headers.update(add_headers)
|
||||||
basic_request = compat_urllib_request.Request(url, None, headers)
|
basic_request = sanitized_Request(url, None, headers)
|
||||||
request = compat_urllib_request.Request(url, None, headers)
|
request = sanitized_Request(url, None, headers)
|
||||||
|
|
||||||
is_test = self.params.get('test', False)
|
is_test = self.params.get('test', False)
|
||||||
|
|
||||||
|
@ -7,11 +7,11 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
|
sanitized_Request,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
)
|
)
|
||||||
@ -63,7 +63,7 @@ def _login(self):
|
|||||||
'j_password': password,
|
'j_password': password,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
response = self._download_webpage(
|
response = self._download_webpage(
|
||||||
@ -94,7 +94,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for fmt in ['windows', 'android_tablet']:
|
for fmt in ['windows', 'android_tablet']:
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
|
self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
|
||||||
request.add_header('User-Agent', self._USER_AGENT)
|
request.add_header('User-Agent', self._USER_AGENT)
|
||||||
|
|
||||||
|
@ -6,13 +6,13 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_str,
|
compat_str,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -57,7 +57,7 @@ def _login(self):
|
|||||||
'pass': password,
|
'pass': password,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
request.add_header('Referer', self._LOGIN_URL)
|
request.add_header('Referer', self._LOGIN_URL)
|
||||||
response = self._download_webpage(
|
response = self._download_webpage(
|
||||||
@ -126,7 +126,7 @@ def _real_extract(self, url):
|
|||||||
'&sort=created&access_mode=0%2C1%2C2&limit={count}'
|
'&sort=created&access_mode=0%2C1%2C2&limit={count}'
|
||||||
'&method=broadcast&format=json&vid_older_than={last}'
|
'&method=broadcast&format=json&vid_older_than={last}'
|
||||||
).format(user=user, count=self._STEP, last=last_id)
|
).format(user=user, count=self._STEP, last=last_id)
|
||||||
req = compat_urllib_request.Request(req_url)
|
req = sanitized_Request(req_url)
|
||||||
# Without setting this header, we wouldn't get any result
|
# Without setting this header, we wouldn't get any result
|
||||||
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
|
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
|
||||||
data = self._download_json(
|
data = self._download_json(
|
||||||
|
@ -4,14 +4,12 @@
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
|
||||||
from ..compat import (
|
from ..compat import compat_urlparse
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
xpath_with_ns,
|
xpath_with_ns,
|
||||||
@ -219,7 +217,7 @@ def _get_subtitles(self, video_id, subtitles_urls):
|
|||||||
for lang, url in subtitles_urls.items():
|
for lang, url in subtitles_urls.items():
|
||||||
# For some weird reason, blip.tv serves a video instead of subtitles
|
# For some weird reason, blip.tv serves a video instead of subtitles
|
||||||
# when we request with a common UA
|
# when we request with a common UA
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('User-Agent', 'youtube-dl')
|
req.add_header('User-Agent', 'youtube-dl')
|
||||||
subtitles[lang] = [{
|
subtitles[lang] = [{
|
||||||
# The extension is 'srt' but it's actually an 'ass' file
|
# The extension is 'srt' but it's actually an 'ass' file
|
||||||
|
@ -11,7 +11,6 @@
|
|||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
compat_xml_parse_error,
|
compat_xml_parse_error,
|
||||||
)
|
)
|
||||||
@ -24,6 +23,7 @@
|
|||||||
js_to_json,
|
js_to_json,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unsmuggle_url,
|
unsmuggle_url,
|
||||||
)
|
)
|
||||||
@ -250,7 +250,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
def _get_video_info(self, video_id, query_str, query, referer=None):
|
def _get_video_info(self, video_id, query_str, query, referer=None):
|
||||||
request_url = self._FEDERATED_URL_TEMPLATE % query_str
|
request_url = self._FEDERATED_URL_TEMPLATE % query_str
|
||||||
req = compat_urllib_request.Request(request_url)
|
req = sanitized_Request(request_url)
|
||||||
linkBase = query.get('linkBaseURL')
|
linkBase = query.get('linkBaseURL')
|
||||||
if linkBase is not None:
|
if linkBase is not None:
|
||||||
referer = linkBase[0]
|
referer = linkBase[0]
|
||||||
@ -443,7 +443,7 @@ def _real_extract(self, url):
|
|||||||
r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
|
r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
|
||||||
webpage, 'policy key', group='pk')
|
webpage, 'policy key', group='pk')
|
||||||
|
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
|
'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
|
||||||
% (account_id, video_id),
|
% (account_id, video_id),
|
||||||
headers={'Accept': 'application/json;pk=%s' % policy_key})
|
headers={'Accept': 'application/json;pk=%s' % policy_key})
|
||||||
|
@ -1,8 +1,10 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
from ..utils import (
|
||||||
from ..utils import smuggle_url
|
sanitized_Request,
|
||||||
|
smuggle_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class CBSIE(InfoExtractor):
|
class CBSIE(InfoExtractor):
|
||||||
@ -48,7 +50,7 @@ class CBSIE(InfoExtractor):
|
|||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
# Android UA is served with higher quality (720p) streams (see
|
# Android UA is served with higher quality (720p) streams (see
|
||||||
# https://github.com/rg3/youtube-dl/issues/7490)
|
# https://github.com/rg3/youtube-dl/issues/7490)
|
||||||
request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
|
request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
|
||||||
|
@ -5,7 +5,6 @@
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
@ -13,6 +12,7 @@
|
|||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -100,7 +100,7 @@ def _real_extract(self, url):
|
|||||||
'requestSource': 'iVysilani',
|
'requestSource': 'iVysilani',
|
||||||
}
|
}
|
||||||
|
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
|
'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
|
||||||
data=compat_urllib_parse.urlencode(data))
|
data=compat_urllib_parse.urlencode(data))
|
||||||
|
|
||||||
@ -115,7 +115,7 @@ def _real_extract(self, url):
|
|||||||
if playlist_url == 'error_region':
|
if playlist_url == 'error_region':
|
||||||
raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
|
raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
|
req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
|
||||||
req.add_header('Referer', url)
|
req.add_header('Referer', url)
|
||||||
|
|
||||||
playlist_title = self._og_search_title(webpage)
|
playlist_title = self._og_search_title(webpage)
|
||||||
|
@ -3,10 +3,10 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
float_or_none,
|
float_or_none,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -52,7 +52,7 @@ def _real_extract(self, url):
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
|
'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
|
||||||
json.dumps(player_options_request))
|
json.dumps(player_options_request))
|
||||||
request.add_header('Content-Type', 'application/json')
|
request.add_header('Content-Type', 'application/json')
|
||||||
|
@ -19,7 +19,6 @@
|
|||||||
compat_urllib_error,
|
compat_urllib_error,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_etree_fromstring,
|
compat_etree_fromstring,
|
||||||
@ -37,6 +36,7 @@
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
RegexNotFoundError,
|
RegexNotFoundError,
|
||||||
sanitize_filename,
|
sanitize_filename,
|
||||||
|
sanitized_Request,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
url_basename,
|
url_basename,
|
||||||
@ -1285,7 +1285,7 @@ def _set_cookie(self, domain, name, value, expire_time=None):
|
|||||||
|
|
||||||
def _get_cookies(self, url):
|
def _get_cookies(self, url):
|
||||||
""" Return a compat_cookies.SimpleCookie with the cookies for the url """
|
""" Return a compat_cookies.SimpleCookie with the cookies for the url """
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
self._downloader.cookiejar.add_cookie_header(req)
|
self._downloader.cookiejar.add_cookie_header(req)
|
||||||
return compat_cookies.SimpleCookie(req.get_header('Cookie'))
|
return compat_cookies.SimpleCookie(req.get_header('Cookie'))
|
||||||
|
|
||||||
|
@ -23,6 +23,7 @@
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
lowercase_escape,
|
lowercase_escape,
|
||||||
remove_end,
|
remove_end,
|
||||||
|
sanitized_Request,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
@ -46,7 +47,7 @@ def _login(self):
|
|||||||
'name': username,
|
'name': username,
|
||||||
'password': password,
|
'password': password,
|
||||||
})
|
})
|
||||||
login_request = compat_urllib_request.Request(login_url, data)
|
login_request = sanitized_Request(login_url, data)
|
||||||
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
self._download_webpage(login_request, None, False, 'Wrong login info')
|
self._download_webpage(login_request, None, False, 'Wrong login info')
|
||||||
|
|
||||||
@ -55,7 +56,7 @@ def _real_initialize(self):
|
|||||||
|
|
||||||
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
|
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
|
||||||
request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
|
request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
|
||||||
else compat_urllib_request.Request(url_or_request))
|
else sanitized_Request(url_or_request))
|
||||||
# Accept-Language must be set explicitly to accept any language to avoid issues
|
# Accept-Language must be set explicitly to accept any language to avoid issues
|
||||||
# similar to https://github.com/rg3/youtube-dl/issues/6797.
|
# similar to https://github.com/rg3/youtube-dl/issues/6797.
|
||||||
# Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
|
# Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
|
||||||
@ -307,7 +308,7 @@ def _real_extract(self, url):
|
|||||||
'video_uploader', fatal=False)
|
'video_uploader', fatal=False)
|
||||||
|
|
||||||
playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
|
playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
|
||||||
playerdata_req = compat_urllib_request.Request(playerdata_url)
|
playerdata_req = sanitized_Request(playerdata_url)
|
||||||
playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
|
playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
|
||||||
playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
|
playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
|
||||||
@ -319,7 +320,7 @@ def _real_extract(self, url):
|
|||||||
for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
|
for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
|
||||||
stream_quality, stream_format = self._FORMAT_IDS[fmt]
|
stream_quality, stream_format = self._FORMAT_IDS[fmt]
|
||||||
video_format = fmt + 'p'
|
video_format = fmt + 'p'
|
||||||
streamdata_req = compat_urllib_request.Request(
|
streamdata_req = sanitized_Request(
|
||||||
'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
|
'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
|
||||||
% (stream_id, stream_format, stream_quality),
|
% (stream_id, stream_format, stream_quality),
|
||||||
compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
|
compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
|
||||||
|
@ -7,15 +7,13 @@
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
|
||||||
from ..compat import (
|
from ..compat import compat_str
|
||||||
compat_str,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
)
|
)
|
||||||
@ -25,7 +23,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _build_request(url):
|
def _build_request(url):
|
||||||
"""Build a request with the family filter disabled"""
|
"""Build a request with the family filter disabled"""
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.add_header('Cookie', 'family_filter=off; ff=off')
|
request.add_header('Cookie', 'family_filter=off; ff=off')
|
||||||
return request
|
return request
|
||||||
|
|
||||||
|
@ -2,13 +2,11 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -36,7 +34,7 @@ class DCNIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
|
'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
|
||||||
headers={'Origin': 'http://www.dcndigital.ae'})
|
headers={'Origin': 'http://www.dcndigital.ae'})
|
||||||
|
|
||||||
|
@ -7,7 +7,6 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_HTTPError,
|
compat_HTTPError,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
@ -16,6 +15,7 @@
|
|||||||
determine_ext,
|
determine_ext,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -51,7 +51,7 @@ def _login(self):
|
|||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
response = self._download_webpage(
|
response = self._download_webpage(
|
||||||
request, None, 'Logging in as %s' % username)
|
request, None, 'Logging in as %s' % username)
|
||||||
|
@ -5,8 +5,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
from ..utils import (
|
||||||
from ..utils import qualities
|
qualities,
|
||||||
|
sanitized_Request,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DumpertIE(InfoExtractor):
|
class DumpertIE(InfoExtractor):
|
||||||
@ -32,7 +34,7 @@ def _real_extract(self, url):
|
|||||||
protocol = mobj.group('protocol')
|
protocol = mobj.group('protocol')
|
||||||
|
|
||||||
url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
|
url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'nsfw=1; cpc=10')
|
req.add_header('Cookie', 'nsfw=1; cpc=10')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -2,11 +2,11 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
float_or_none,
|
float_or_none,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -57,7 +57,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
hls_url = media.get('HLS_SURL')
|
hls_url = media.get('HLS_SURL')
|
||||||
if hls_url:
|
if hls_url:
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
|
'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
|
||||||
headers={'Referer': url})
|
headers={'Referer': url})
|
||||||
token_data = self._download_json(
|
token_data = self._download_json(
|
||||||
|
@ -3,13 +3,12 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
clean_html,
|
clean_html,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -75,7 +74,7 @@ def _real_extract(self, url):
|
|||||||
video_id = ims_video['videoID']
|
video_id = ims_video['videoID']
|
||||||
key = ims_video['hash']
|
key = ims_video['hash']
|
||||||
|
|
||||||
config_req = compat_urllib_request.Request(
|
config_req = sanitized_Request(
|
||||||
'http://www.escapistmagazine.com/videos/'
|
'http://www.escapistmagazine.com/videos/'
|
||||||
'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
|
'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
|
||||||
config_req.add_header('Referer', url)
|
config_req.add_header('Referer', url)
|
||||||
|
@ -3,11 +3,9 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -42,7 +40,7 @@ def _real_extract(self, url):
|
|||||||
playlist_id = mobj.group('id')
|
playlist_id = mobj.group('id')
|
||||||
|
|
||||||
pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
|
pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
|
||||||
pllist_req = compat_urllib_request.Request(pllist_url)
|
pllist_req = sanitized_Request(pllist_url)
|
||||||
pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
|
pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
|
||||||
|
|
||||||
playlist_list = self._download_json(
|
playlist_list = self._download_json(
|
||||||
@ -55,7 +53,7 @@ def _real_extract(self, url):
|
|||||||
raise ExtractorError('Playlist id not found')
|
raise ExtractorError('Playlist id not found')
|
||||||
|
|
||||||
pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
|
pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
|
||||||
pl_req = compat_urllib_request.Request(pl_url)
|
pl_req = sanitized_Request(pl_url)
|
||||||
pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
|
pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
|
||||||
playlist = self._download_json(
|
playlist = self._download_json(
|
||||||
pl_req, playlist_id, note='Downloading playlist info')
|
pl_req, playlist_id, note='Downloading playlist info')
|
||||||
|
@ -3,9 +3,9 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -37,7 +37,7 @@ class ExtremeTubeIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -10,11 +10,11 @@
|
|||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_error,
|
compat_urllib_error,
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
limit_length,
|
limit_length,
|
||||||
|
sanitized_Request,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
get_element_by_id,
|
get_element_by_id,
|
||||||
clean_html,
|
clean_html,
|
||||||
@ -73,7 +73,7 @@ def _login(self):
|
|||||||
if useremail is None:
|
if useremail is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
|
login_page_req = sanitized_Request(self._LOGIN_URL)
|
||||||
login_page_req.add_header('Cookie', 'locale=en_US')
|
login_page_req.add_header('Cookie', 'locale=en_US')
|
||||||
login_page = self._download_webpage(login_page_req, None,
|
login_page = self._download_webpage(login_page_req, None,
|
||||||
note='Downloading login page',
|
note='Downloading login page',
|
||||||
@ -94,7 +94,7 @@ def _login(self):
|
|||||||
'timezone': '-60',
|
'timezone': '-60',
|
||||||
'trynum': '1',
|
'trynum': '1',
|
||||||
}
|
}
|
||||||
request = compat_urllib_request.Request(self._LOGIN_URL, urlencode_postdata(login_form))
|
request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
try:
|
try:
|
||||||
login_results = self._download_webpage(request, None,
|
login_results = self._download_webpage(request, None,
|
||||||
@ -109,7 +109,7 @@ def _login(self):
|
|||||||
r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
|
r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
|
||||||
'name_action_selected': 'dont_save',
|
'name_action_selected': 'dont_save',
|
||||||
}
|
}
|
||||||
check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
|
check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
|
||||||
check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
check_response = self._download_webpage(check_req, None,
|
check_response = self._download_webpage(check_req, None,
|
||||||
note='Confirming login')
|
note='Confirming login')
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
from ..utils import (
|
from ..utils import (
|
||||||
encode_dict,
|
encode_dict,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -57,7 +58,7 @@ def _login(self):
|
|||||||
}
|
}
|
||||||
|
|
||||||
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
|
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
|
'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
|
||||||
|
|
||||||
login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
|
login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
|
||||||
@ -66,7 +67,7 @@ def _login(self):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
# this is also needed
|
# this is also needed
|
||||||
login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
|
login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
|
||||||
self._download_webpage(
|
self._download_webpage(
|
||||||
login_redir, None, note='Login redirect', errnote='Login redirect failed')
|
login_redir, None, note='Login redirect', errnote='Login redirect failed')
|
||||||
|
|
||||||
|
@ -3,10 +3,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
find_xpath_attr,
|
find_xpath_attr,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -30,7 +30,7 @@ def _real_extract(self, url):
|
|||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
video_uploader_id = mobj.group('uploader_id')
|
video_uploader_id = mobj.group('uploader_id')
|
||||||
webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
|
webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
|
||||||
req = compat_urllib_request.Request(webpage_url)
|
req = sanitized_Request(webpage_url)
|
||||||
req.add_header(
|
req.add_header(
|
||||||
'User-Agent',
|
'User-Agent',
|
||||||
# it needs a more recent version
|
# it needs a more recent version
|
||||||
|
@ -3,12 +3,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_duration,
|
parse_duration,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -93,7 +91,7 @@ def _real_extract(self, url):
|
|||||||
b'Content-Type': b'application/x-www-form-urlencoded',
|
b'Content-Type': b'application/x-www-form-urlencoded',
|
||||||
b'Origin': b'http://www.4tube.com',
|
b'Origin': b'http://www.4tube.com',
|
||||||
}
|
}
|
||||||
token_req = compat_urllib_request.Request(token_url, b'{}', headers)
|
token_req = sanitized_Request(token_url, b'{}', headers)
|
||||||
tokens = self._download_json(token_req, video_id)
|
tokens = self._download_json(token_req, video_id)
|
||||||
formats = [{
|
formats = [{
|
||||||
'url': tokens[format]['token'],
|
'url': tokens[format]['token'],
|
||||||
|
@ -3,13 +3,11 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
remove_end,
|
remove_end,
|
||||||
HEADRequest,
|
HEADRequest,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -125,7 +123,7 @@ def _login(self, webpage_url, display_id):
|
|||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form))
|
request = sanitized_Request(login_url, compat_urllib_parse.urlencode(login_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
self._download_webpage(request, display_id, 'Logging in')
|
self._download_webpage(request, display_id, 'Logging in')
|
||||||
start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
|
start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
|
||||||
|
@ -11,7 +11,6 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_etree_fromstring,
|
compat_etree_fromstring,
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
compat_xml_parse_error,
|
compat_xml_parse_error,
|
||||||
)
|
)
|
||||||
@ -22,6 +21,7 @@
|
|||||||
HEADRequest,
|
HEADRequest,
|
||||||
is_html,
|
is_html,
|
||||||
orderedSet,
|
orderedSet,
|
||||||
|
sanitized_Request,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
@ -1215,7 +1215,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
full_response = None
|
full_response = None
|
||||||
if head_response is False:
|
if head_response is False:
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.add_header('Accept-Encoding', '*')
|
request.add_header('Accept-Encoding', '*')
|
||||||
full_response = self._request_webpage(request, video_id)
|
full_response = self._request_webpage(request, video_id)
|
||||||
head_response = full_response
|
head_response = full_response
|
||||||
@ -1244,7 +1244,7 @@ def _real_extract(self, url):
|
|||||||
'%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
|
'%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
|
||||||
|
|
||||||
if not full_response:
|
if not full_response:
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
# Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
|
# Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
|
||||||
# making it impossible to download only chunk of the file (yet we need only 512kB to
|
# making it impossible to download only chunk of the file (yet we need only 512kB to
|
||||||
# test whether it's HTML or not). According to youtube-dl default Accept-Encoding
|
# test whether it's HTML or not). According to youtube-dl default Accept-Encoding
|
||||||
|
@ -4,12 +4,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urlparse
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
HEADRequest,
|
HEADRequest,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
urlhandle_detect_ext,
|
urlhandle_detect_ext,
|
||||||
@ -47,7 +45,7 @@ def _real_extract(self, url):
|
|||||||
r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
|
r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
|
||||||
|
|
||||||
payload = urlencode_postdata({'tracks[]': track_id})
|
payload = urlencode_postdata({'tracks[]': track_id})
|
||||||
req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
|
req = sanitized_Request(self._PLAYLIST_URL, payload)
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
track = self._download_json(req, track_id, 'Downloading playlist')[0]
|
track = self._download_json(req, track_id, 'Downloading playlist')[0]
|
||||||
|
@ -3,13 +3,11 @@
|
|||||||
import base64
|
import base64
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
HEADRequest,
|
HEADRequest,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -41,7 +39,7 @@ def _real_extract(self, url):
|
|||||||
('mediaType', 's'),
|
('mediaType', 's'),
|
||||||
('mediaId', video_id),
|
('mediaId', video_id),
|
||||||
])
|
])
|
||||||
r = compat_urllib_request.Request(
|
r = sanitized_Request(
|
||||||
'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
|
'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
|
||||||
r.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
r.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
mkd = self._download_json(
|
mkd = self._download_json(
|
||||||
|
@ -4,12 +4,10 @@
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -32,7 +30,7 @@ def _real_extract(self, url):
|
|||||||
data = {'ax': 1, 'ts': time.time()}
|
data = {'ax': 1, 'ts': time.time()}
|
||||||
data_encoded = compat_urllib_parse.urlencode(data)
|
data_encoded = compat_urllib_parse.urlencode(data)
|
||||||
complete_url = url + "?" + data_encoded
|
complete_url = url + "?" + data_encoded
|
||||||
request = compat_urllib_request.Request(complete_url)
|
request = sanitized_Request(complete_url)
|
||||||
response, urlh = self._download_webpage_handle(
|
response, urlh = self._download_webpage_handle(
|
||||||
request, track_id, 'Downloading webpage with the url')
|
request, track_id, 'Downloading webpage with the url')
|
||||||
cookie = urlh.headers.get('Set-Cookie', '')
|
cookie = urlh.headers.get('Set-Cookie', '')
|
||||||
@ -52,7 +50,7 @@ def _real_extract(self, url):
|
|||||||
title = track['song']
|
title = track['song']
|
||||||
|
|
||||||
serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
|
serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
serve_url, '', {'Content-Type': 'application/json'})
|
serve_url, '', {'Content-Type': 'application/json'})
|
||||||
request.add_header('cookie', cookie)
|
request.add_header('cookie', cookie)
|
||||||
song_data = self._download_json(request, track_id, 'Downloading metadata')
|
song_data = self._download_json(request, track_id, 'Downloading metadata')
|
||||||
|
@ -6,12 +6,10 @@
|
|||||||
from math import floor
|
from math import floor
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
remove_end,
|
remove_end,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -61,7 +59,7 @@ def _real_extract(self, url):
|
|||||||
(floor(random() * 1073741824), floor(random() * 1073741824))
|
(floor(random() * 1073741824), floor(random() * 1073741824))
|
||||||
)
|
)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(player_url)
|
req = sanitized_Request(player_url)
|
||||||
req.add_header('Referer', url)
|
req.add_header('Referer', url)
|
||||||
playerpage = self._download_webpage(req, video_id)
|
playerpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -5,11 +5,9 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -78,7 +76,7 @@ def _real_extract(self, url):
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(api_url, json.dumps(data))
|
request = sanitized_Request(api_url, json.dumps(data))
|
||||||
|
|
||||||
video_json_page = self._download_webpage(
|
video_json_page = self._download_webpage(
|
||||||
request, video_id, 'Downloading video JSON')
|
request, video_id, 'Downloading video JSON')
|
||||||
|
@ -4,10 +4,8 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse_urlparse
|
||||||
compat_urllib_parse_urlparse,
|
from ..utils import sanitized_Request
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class KeezMoviesIE(InfoExtractor):
|
class KeezMoviesIE(InfoExtractor):
|
||||||
@ -26,7 +24,7 @@ class KeezMoviesIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -8,13 +8,13 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_ord,
|
compat_ord,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
encode_data_uri,
|
encode_data_uri,
|
||||||
)
|
)
|
||||||
@ -114,7 +114,7 @@ def _real_extract(self, url):
|
|||||||
'tkey': self.calc_time_key(int(time.time())),
|
'tkey': self.calc_time_key(int(time.time())),
|
||||||
'domain': 'www.letv.com'
|
'domain': 'www.letv.com'
|
||||||
}
|
}
|
||||||
play_json_req = compat_urllib_request.Request(
|
play_json_req = sanitized_Request(
|
||||||
'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
|
'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
|
||||||
)
|
)
|
||||||
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
||||||
|
@ -7,12 +7,12 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
clean_html,
|
clean_html,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -35,7 +35,7 @@ def _login(self):
|
|||||||
'remember': 'false',
|
'remember': 'false',
|
||||||
'stayPut': 'false'
|
'stayPut': 'false'
|
||||||
}
|
}
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
login_page = self._download_webpage(
|
login_page = self._download_webpage(
|
||||||
request, None, 'Logging in as %s' % username)
|
request, None, 'Logging in as %s' % username)
|
||||||
@ -64,7 +64,7 @@ def _login(self):
|
|||||||
'remember': 'false',
|
'remember': 'false',
|
||||||
'stayPut': 'false',
|
'stayPut': 'false',
|
||||||
}
|
}
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
|
self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
|
||||||
login_page = self._download_webpage(
|
login_page = self._download_webpage(
|
||||||
request, None,
|
request, None,
|
||||||
|
@ -7,12 +7,12 @@
|
|||||||
compat_parse_qs,
|
compat_parse_qs,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -117,7 +117,7 @@ def _real_initialize(self):
|
|||||||
'filters': '0',
|
'filters': '0',
|
||||||
'submit': "Continue - I'm over 18",
|
'submit': "Continue - I'm over 18",
|
||||||
}
|
}
|
||||||
request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
|
request = sanitized_Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
self.report_age_confirmation()
|
self.report_age_confirmation()
|
||||||
self._download_webpage(request, None, False, 'Unable to confirm age')
|
self._download_webpage(request, None, False, 'Unable to confirm age')
|
||||||
@ -142,7 +142,7 @@ def _real_extract(self, url):
|
|||||||
return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
|
return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
|
||||||
|
|
||||||
# Retrieve video webpage to extract further information
|
# Retrieve video webpage to extract further information
|
||||||
req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
|
req = sanitized_Request('http://www.metacafe.com/watch/%s/' % video_id)
|
||||||
|
|
||||||
# AnyClip videos require the flashversion cookie so that we get the link
|
# AnyClip videos require the flashversion cookie so that we get the link
|
||||||
# to the mp4 file
|
# to the mp4 file
|
||||||
|
@ -2,14 +2,12 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
parse_filesize,
|
parse_filesize,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -39,7 +37,7 @@ def _real_extract(self, url):
|
|||||||
('fileId', video_id),
|
('fileId', video_id),
|
||||||
('__RequestVerificationToken', token),
|
('__RequestVerificationToken', token),
|
||||||
]
|
]
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
'http://minhateca.com.br/action/License/Download',
|
'http://minhateca.com.br/action/License/Download',
|
||||||
data=compat_urllib_parse.urlencode(token_data))
|
data=compat_urllib_parse.urlencode(token_data))
|
||||||
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
|
@ -4,11 +4,11 @@
|
|||||||
import random
|
import random
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
xpath_text,
|
xpath_text,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -63,7 +63,7 @@ def _real_extract(self, url):
|
|||||||
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
|
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
|
||||||
video_id)
|
video_id)
|
||||||
|
|
||||||
vid_config_request = compat_urllib_request.Request(
|
vid_config_request = sanitized_Request(
|
||||||
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
|
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
|
||||||
headers=http_headers)
|
headers=http_headers)
|
||||||
|
|
||||||
|
@ -5,13 +5,11 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -80,7 +78,7 @@ def _real_extract(self, url):
|
|||||||
]
|
]
|
||||||
r_json = json.dumps(r)
|
r_json = json.dumps(r)
|
||||||
post = compat_urllib_parse.urlencode({'r': r_json})
|
post = compat_urllib_parse.urlencode({'r': r_json})
|
||||||
req = compat_urllib_request.Request(self._API_URL, post)
|
req = sanitized_Request(self._API_URL, post)
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
response = self._download_json(req, video_id)
|
response = self._download_json(req, video_id)
|
||||||
|
@ -7,8 +7,8 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
|
from ..utils import sanitized_Request
|
||||||
|
|
||||||
|
|
||||||
class MofosexIE(InfoExtractor):
|
class MofosexIE(InfoExtractor):
|
||||||
@ -29,7 +29,7 @@ def _real_extract(self, url):
|
|||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
url = 'http://www.' + mobj.group('url')
|
url = 'http://www.' + mobj.group('url')
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -5,13 +5,11 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
remove_start,
|
remove_start,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -81,7 +79,7 @@ def _real_extract(self, url):
|
|||||||
orig_webpage, 'builtin URL', default=None, group='url')
|
orig_webpage, 'builtin URL', default=None, group='url')
|
||||||
|
|
||||||
if builtin_url:
|
if builtin_url:
|
||||||
req = compat_urllib_request.Request(builtin_url)
|
req = sanitized_Request(builtin_url)
|
||||||
req.add_header('Referer', url)
|
req.add_header('Referer', url)
|
||||||
webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
|
webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
|
||||||
title = self._og_search_title(orig_webpage).strip()
|
title = self._og_search_title(orig_webpage).strip()
|
||||||
@ -94,7 +92,7 @@ def _real_extract(self, url):
|
|||||||
headers = {
|
headers = {
|
||||||
b'Content-Type': b'application/x-www-form-urlencoded',
|
b'Content-Type': b'application/x-www-form-urlencoded',
|
||||||
}
|
}
|
||||||
req = compat_urllib_request.Request(url, post, headers)
|
req = sanitized_Request(url, post, headers)
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, video_id, note='Downloading video page ...')
|
req, video_id, note='Downloading video page ...')
|
||||||
|
|
||||||
|
@ -3,12 +3,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_parse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -59,7 +57,7 @@ def _real_extract(self, url):
|
|||||||
'hash': hash_key,
|
'hash': hash_key,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
|
'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
|
@ -2,9 +2,7 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..utils import sanitized_Request
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MovieClipsIE(InfoExtractor):
|
class MovieClipsIE(InfoExtractor):
|
||||||
@ -25,7 +23,7 @@ class MovieClipsIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
# it doesn't work if it thinks the browser it's too old
|
# it doesn't work if it thinks the browser it's too old
|
||||||
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
|
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
|
||||||
webpage = self._download_webpage(req, display_id)
|
webpage = self._download_webpage(req, display_id)
|
||||||
|
@ -5,7 +5,6 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_str,
|
compat_str,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
@ -13,6 +12,7 @@
|
|||||||
find_xpath_attr,
|
find_xpath_attr,
|
||||||
fix_xml_ampersands,
|
fix_xml_ampersands,
|
||||||
HEADRequest,
|
HEADRequest,
|
||||||
|
sanitized_Request,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
url_basename,
|
url_basename,
|
||||||
RegexNotFoundError,
|
RegexNotFoundError,
|
||||||
@ -53,7 +53,7 @@ def _get_thumbnail_url(self, uri, itemdoc):
|
|||||||
|
|
||||||
def _extract_mobile_video_formats(self, mtvn_id):
|
def _extract_mobile_video_formats(self, mtvn_id):
|
||||||
webpage_url = self._MOBILE_TEMPLATE % mtvn_id
|
webpage_url = self._MOBILE_TEMPLATE % mtvn_id
|
||||||
req = compat_urllib_request.Request(webpage_url)
|
req = sanitized_Request(webpage_url)
|
||||||
# Otherwise we get a webpage that would execute some javascript
|
# Otherwise we get a webpage that would execute some javascript
|
||||||
req.add_header('User-Agent', 'curl/7')
|
req.add_header('User-Agent', 'curl/7')
|
||||||
webpage = self._download_webpage(req, mtvn_id,
|
webpage = self._download_webpage(req, mtvn_id,
|
||||||
|
@ -11,10 +11,10 @@
|
|||||||
compat_ord,
|
compat_ord,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -83,7 +83,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
|
mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
|
||||||
if mobj is not None:
|
if mobj is not None:
|
||||||
request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
|
request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
|
||||||
response = self._download_webpage(request, video_id,
|
response = self._download_webpage(request, video_id,
|
||||||
'Downloading video info')
|
'Downloading video info')
|
||||||
info = json.loads(base64.b64decode(response).decode('utf-8'))
|
info = json.loads(base64.b64decode(response).decode('utf-8'))
|
||||||
|
@ -8,11 +8,11 @@
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_itertools_count,
|
compat_itertools_count,
|
||||||
)
|
)
|
||||||
|
from ..utils import sanitized_Request
|
||||||
|
|
||||||
|
|
||||||
class NetEaseMusicBaseIE(InfoExtractor):
|
class NetEaseMusicBaseIE(InfoExtractor):
|
||||||
@ -56,7 +56,7 @@ def convert_milliseconds(cls, ms):
|
|||||||
return int(round(ms / 1000.0))
|
return int(round(ms / 1000.0))
|
||||||
|
|
||||||
def query_api(self, endpoint, video_id, note):
|
def query_api(self, endpoint, video_id, note):
|
||||||
req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
|
req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
|
||||||
req.add_header('Referer', self._API_BASE)
|
req.add_header('Referer', self._API_BASE)
|
||||||
return self._download_json(req, video_id, note)
|
return self._download_json(req, video_id, note)
|
||||||
|
|
||||||
|
@ -1,10 +1,8 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_request,
|
from ..utils import sanitized_Request
|
||||||
compat_urllib_parse,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class NFBIE(InfoExtractor):
|
class NFBIE(InfoExtractor):
|
||||||
@ -40,8 +38,9 @@ def _real_extract(self, url):
|
|||||||
uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
|
uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
|
||||||
page, 'director name', fatal=False)
|
page, 'director name', fatal=False)
|
||||||
|
|
||||||
request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
|
request = sanitized_Request(
|
||||||
compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
|
'https://www.nfb.ca/film/%s/player_config' % video_id,
|
||||||
|
compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
|
request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
|
||||||
|
|
||||||
|
@ -8,7 +8,6 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
@ -17,6 +16,7 @@
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
)
|
)
|
||||||
@ -102,7 +102,7 @@ def _login(self):
|
|||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
|
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'https://secure.nicovideo.jp/secure/login', login_data)
|
'https://secure.nicovideo.jp/secure/login', login_data)
|
||||||
login_results = self._download_webpage(
|
login_results = self._download_webpage(
|
||||||
request, None, note='Logging in', errnote='Unable to log in')
|
request, None, note='Logging in', errnote='Unable to log in')
|
||||||
@ -145,7 +145,7 @@ def _real_extract(self, url):
|
|||||||
'k': thumb_play_key,
|
'k': thumb_play_key,
|
||||||
'v': video_id
|
'v': video_id
|
||||||
})
|
})
|
||||||
flv_info_request = compat_urllib_request.Request(
|
flv_info_request = sanitized_Request(
|
||||||
'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
|
'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
|
||||||
{'Content-Type': 'application/x-www-form-urlencoded'})
|
{'Content-Type': 'application/x-www-form-urlencoded'})
|
||||||
flv_info_webpage = self._download_webpage(
|
flv_info_webpage = self._download_webpage(
|
||||||
|
@ -9,7 +9,6 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
@ -17,6 +16,7 @@
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -74,7 +74,7 @@ def _login(self):
|
|||||||
'username': username,
|
'username': username,
|
||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
|
request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
|
||||||
|
|
||||||
login = self._download_json(request, None, 'Logging in as %s' % username)
|
login = self._download_json(request, None, 'Logging in as %s' % username)
|
||||||
|
@ -4,11 +4,9 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
xpath_with_ns,
|
xpath_with_ns,
|
||||||
@ -41,7 +39,7 @@ def _real_extract(self, url):
|
|||||||
'op': 'download1',
|
'op': 'download1',
|
||||||
'method_free': 'Continue to Video',
|
'method_free': 'Continue to Video',
|
||||||
}
|
}
|
||||||
req = compat_urllib_request.Request(url, urlencode_postdata(fields))
|
req = sanitized_Request(url, urlencode_postdata(fields))
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
webpage = self._download_webpage(req, video_id,
|
webpage = self._download_webpage(req, video_id,
|
||||||
'Downloading download page')
|
'Downloading download page')
|
||||||
|
@ -3,14 +3,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urlparse
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
NO_DEFAULT,
|
NO_DEFAULT,
|
||||||
encode_dict,
|
encode_dict,
|
||||||
|
sanitized_Request,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -65,7 +63,7 @@ def extract_filekey(default=NO_DEFAULT):
|
|||||||
'post url', default=url, group='url')
|
'post url', default=url, group='url')
|
||||||
if not post_url.startswith('http'):
|
if not post_url.startswith('http'):
|
||||||
post_url = compat_urlparse.urljoin(url, post_url)
|
post_url = compat_urlparse.urljoin(url, post_url)
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
post_url, urlencode_postdata(encode_dict(fields)))
|
post_url, urlencode_postdata(encode_dict(fields)))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
request.add_header('Referer', post_url)
|
request.add_header('Referer', post_url)
|
||||||
|
@ -3,10 +3,10 @@
|
|||||||
|
|
||||||
from .brightcove import BrightcoveLegacyIE
|
from .brightcove import BrightcoveLegacyIE
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import ExtractorError
|
from ..compat import compat_str
|
||||||
from ..compat import (
|
from ..utils import (
|
||||||
compat_str,
|
ExtractorError,
|
||||||
compat_urllib_request,
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -37,7 +37,7 @@ def _extract_url_result(self, post):
|
|||||||
|
|
||||||
def _api_request(self, url, request_path):
|
def _api_request(self, url, request_path):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://api.nowness.com/api/' + request_path % display_id,
|
'http://api.nowness.com/api/' + request_path % display_id,
|
||||||
headers={
|
headers={
|
||||||
'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
|
'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
|
||||||
|
@ -3,11 +3,9 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_duration,
|
parse_duration,
|
||||||
|
sanitized_Request,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -33,7 +31,7 @@ def _real_extract(self, url):
|
|||||||
formats = []
|
formats = []
|
||||||
|
|
||||||
for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
|
for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://m.nuvid.com/play/%s' % video_id)
|
'http://m.nuvid.com/play/%s' % video_id)
|
||||||
request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
|
request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
|
@ -2,9 +2,7 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import js_to_json
|
||||||
js_to_json,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class PatreonIE(InfoExtractor):
|
class PatreonIE(InfoExtractor):
|
||||||
@ -65,7 +63,7 @@ def _login(self):
|
|||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'https://www.patreon.com/processLogin',
|
'https://www.patreon.com/processLogin',
|
||||||
compat_urllib_parse.urlencode(login_form).encode('utf-8')
|
compat_urllib_parse.urlencode(login_form).encode('utf-8')
|
||||||
)
|
)
|
||||||
|
@ -5,12 +5,10 @@
|
|||||||
import os.path
|
import os.path
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -46,7 +44,7 @@ def _real_extract(self, url):
|
|||||||
headers = {
|
headers = {
|
||||||
b'Content-Type': b'application/x-www-form-urlencoded',
|
b'Content-Type': b'application/x-www-form-urlencoded',
|
||||||
}
|
}
|
||||||
req = compat_urllib_request.Request(url, post, headers)
|
req = sanitized_Request(url, post, headers)
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, video_id, note='Downloading video page ...')
|
req, video_id, note='Downloading video page ...')
|
||||||
|
|
||||||
|
@ -8,13 +8,13 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -73,7 +73,7 @@ def _login(self):
|
|||||||
if not post_url.startswith('http'):
|
if not post_url.startswith('http'):
|
||||||
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
|
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
@ -181,7 +181,7 @@ def guess_allowed_qualities():
|
|||||||
'mt': ext,
|
'mt': ext,
|
||||||
'q': '%dx%d' % (f['width'], f['height']),
|
'q': '%dx%d' % (f['width'], f['height']),
|
||||||
}
|
}
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'%s/training/Player/ViewClip' % self._API_BASE,
|
'%s/training/Player/ViewClip' % self._API_BASE,
|
||||||
json.dumps(clip_post).encode('utf-8'))
|
json.dumps(clip_post).encode('utf-8'))
|
||||||
request.add_header('Content-Type', 'application/json;charset=utf-8')
|
request.add_header('Content-Type', 'application/json;charset=utf-8')
|
||||||
|
@ -8,10 +8,10 @@
|
|||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_parse_unquote_plus,
|
compat_urllib_parse_unquote_plus,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
)
|
)
|
||||||
from ..aes import (
|
from ..aes import (
|
||||||
@ -53,7 +53,7 @@ def _extract_count(self, pattern, webpage, name):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
|
'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
@ -3,11 +3,9 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -46,7 +44,7 @@ def _real_extract(self, url):
|
|||||||
'authenticationSpaceKey': originAuthenticationSpaceKey,
|
'authenticationSpaceKey': originAuthenticationSpaceKey,
|
||||||
'credentials': 'Clip Application',
|
'credentials': 'Clip Application',
|
||||||
}
|
}
|
||||||
token_req = compat_urllib_request.Request(
|
token_req = sanitized_Request(
|
||||||
'https://api.aebn.net/auth/v1/token/primal',
|
'https://api.aebn.net/auth/v1/token/primal',
|
||||||
data=json.dumps(token_req_data).encode('utf-8'))
|
data=json.dumps(token_req_data).encode('utf-8'))
|
||||||
token_req.add_header('Content-Type', 'application/json')
|
token_req.add_header('Content-Type', 'application/json')
|
||||||
@ -56,7 +54,7 @@ def _real_extract(self, url):
|
|||||||
token = token_answer['tokenKey']
|
token = token_answer['tokenKey']
|
||||||
|
|
||||||
# Get video URL
|
# Get video URL
|
||||||
delivery_req = compat_urllib_request.Request(
|
delivery_req = sanitized_Request(
|
||||||
'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
|
'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
|
||||||
delivery_req.add_header('Authorization', token)
|
delivery_req.add_header('Authorization', token)
|
||||||
delivery_info = self._download_json(
|
delivery_info = self._download_json(
|
||||||
@ -64,7 +62,7 @@ def _real_extract(self, url):
|
|||||||
video_url = delivery_info['mediaUrl']
|
video_url = delivery_info['mediaUrl']
|
||||||
|
|
||||||
# Get additional info (title etc.)
|
# Get additional info (title etc.)
|
||||||
info_req = compat_urllib_request.Request(
|
info_req = sanitized_Request(
|
||||||
'https://api.aebn.net/content/v1/clips/%s?expand='
|
'https://api.aebn.net/content/v1/clips/%s?expand='
|
||||||
'title,description,primaryImageNumber,startSecond,endSecond,'
|
'title,description,primaryImageNumber,startSecond,endSecond,'
|
||||||
'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
|
'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
from ..utils import (
|
||||||
compat_urllib_request,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
from ..utils import ExtractorError
|
|
||||||
|
|
||||||
|
|
||||||
class PrimeShareTVIE(InfoExtractor):
|
class PrimeShareTVIE(InfoExtractor):
|
||||||
@ -41,7 +41,7 @@ def _real_extract(self, url):
|
|||||||
webpage, 'wait time', default=7)) + 1
|
webpage, 'wait time', default=7)) + 1
|
||||||
self._sleep(wait_time, video_id)
|
self._sleep(wait_time, video_id)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
url, compat_urllib_parse.urlencode(fields), headers)
|
url, compat_urllib_parse.urlencode(fields), headers)
|
||||||
video_page = self._download_webpage(
|
video_page = self._download_webpage(
|
||||||
req, video_id, 'Downloading video page')
|
req, video_id, 'Downloading video page')
|
||||||
|
@ -4,13 +4,11 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -37,7 +35,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
fields = self._hidden_inputs(webpage)
|
fields = self._hidden_inputs(webpage)
|
||||||
post = compat_urllib_parse.urlencode(fields)
|
post = compat_urllib_parse.urlencode(fields)
|
||||||
req = compat_urllib_request.Request(url, post)
|
req = sanitized_Request(url, post)
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, video_id, 'Downloading video page')
|
req, video_id, 'Downloading video page')
|
||||||
|
@ -7,11 +7,11 @@
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
sanitized_Request,
|
||||||
strip_jsonp,
|
strip_jsonp,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
clean_html,
|
clean_html,
|
||||||
)
|
)
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
|
|
||||||
|
|
||||||
class QQMusicIE(InfoExtractor):
|
class QQMusicIE(InfoExtractor):
|
||||||
@ -201,7 +201,7 @@ def _real_extract(self, url):
|
|||||||
singer_desc = None
|
singer_desc = None
|
||||||
|
|
||||||
if singer_id:
|
if singer_id:
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
|
'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
|
||||||
req.add_header(
|
req.add_header(
|
||||||
'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
|
'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
|
||||||
|
@ -6,11 +6,11 @@
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
remove_end,
|
remove_end,
|
||||||
|
sanitized_Request,
|
||||||
std_headers,
|
std_headers,
|
||||||
struct_unpack,
|
struct_unpack,
|
||||||
)
|
)
|
||||||
@ -102,7 +102,7 @@ def _real_extract(self, url):
|
|||||||
if info['state'] == 'DESPU':
|
if info['state'] == 'DESPU':
|
||||||
raise ExtractorError('The video is no longer available', expected=True)
|
raise ExtractorError('The video is no longer available', expected=True)
|
||||||
png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
|
png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
|
||||||
png_request = compat_urllib_request.Request(png_url)
|
png_request = sanitized_Request(png_url)
|
||||||
png_request.add_header('Referer', url)
|
png_request.add_header('Referer', url)
|
||||||
png = self._download_webpage(png_request, video_id, 'Downloading url information')
|
png = self._download_webpage(png_request, video_id, 'Downloading url information')
|
||||||
video_url = _decrypt_url(png)
|
video_url = _decrypt_url(png)
|
||||||
|
@ -6,12 +6,10 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from .brightcove import BrightcoveLegacyIE
|
from .brightcove import BrightcoveLegacyIE
|
||||||
|
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
std_headers,
|
std_headers,
|
||||||
)
|
)
|
||||||
@ -58,7 +56,7 @@ def _login(self):
|
|||||||
'next': '',
|
'next': '',
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
|
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
|
||||||
login_page = self._download_webpage(
|
login_page = self._download_webpage(
|
||||||
request, None, 'Logging in as %s' % username)
|
request, None, 'Logging in as %s' % username)
|
||||||
|
@ -6,14 +6,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urlparse
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
js_to_json,
|
js_to_json,
|
||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
|
sanitized_Request,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -37,7 +35,7 @@ class SandiaIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
|
req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -3,13 +3,11 @@
|
|||||||
import base64
|
import base64
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -46,7 +44,7 @@ def _real_extract(self, url):
|
|||||||
'Video %s does not exist' % video_id, expected=True)
|
'Video %s does not exist' % video_id, expected=True)
|
||||||
|
|
||||||
download_form = self._hidden_inputs(webpage)
|
download_form = self._hidden_inputs(webpage)
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
url, compat_urllib_parse.urlencode(download_form))
|
url, compat_urllib_parse.urlencode(download_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
|
@ -4,12 +4,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_duration,
|
parse_duration,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -50,7 +48,7 @@ def _real_extract(self, url):
|
|||||||
'method_free': 'Free'
|
'method_free': 'Free'
|
||||||
}
|
}
|
||||||
post = compat_urllib_parse.urlencode(fields)
|
post = compat_urllib_parse.urlencode(fields)
|
||||||
req = compat_urllib_request.Request(url, post)
|
req = sanitized_Request(url, post)
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
webpage = self._download_webpage(req, video_id,
|
webpage = self._download_webpage(req, video_id,
|
||||||
|
@ -4,10 +4,8 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_request,
|
from ..utils import sanitized_Request
|
||||||
compat_urllib_parse,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class SinaIE(InfoExtractor):
|
class SinaIE(InfoExtractor):
|
||||||
@ -61,7 +59,7 @@ def _real_extract(self, url):
|
|||||||
if mobj.group('token') is not None:
|
if mobj.group('token') is not None:
|
||||||
# The video id is in the redirected url
|
# The video id is in the redirected url
|
||||||
self.to_screen('Getting video id')
|
self.to_screen('Getting video id')
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.get_method = lambda: 'HEAD'
|
request.get_method = lambda: 'HEAD'
|
||||||
(_, urlh) = self._download_webpage_handle(request, 'NA', False)
|
(_, urlh) = self._download_webpage_handle(request, 'NA', False)
|
||||||
return self._real_extract(urlh.geturl())
|
return self._real_extract(urlh.geturl())
|
||||||
|
@ -7,13 +7,11 @@
|
|||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -176,7 +174,7 @@ def _real_extract(self, url):
|
|||||||
if video_password:
|
if video_password:
|
||||||
video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
|
video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
|
'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
@ -339,7 +337,7 @@ def _real_extract(self, url):
|
|||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
|
broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
broadcast_page = self._download_webpage(
|
broadcast_page = self._download_webpage(
|
||||||
|
@ -6,11 +6,11 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -96,7 +96,7 @@ def _fetch_data(vid_id, mytv=False):
|
|||||||
else:
|
else:
|
||||||
base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
|
base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
|
||||||
|
|
||||||
req = compat_urllib_request.Request(base_data_url + vid_id)
|
req = sanitized_Request(base_data_url + vid_id)
|
||||||
|
|
||||||
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
||||||
if cn_verification_proxy:
|
if cn_verification_proxy:
|
||||||
|
@ -6,9 +6,9 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
)
|
)
|
||||||
@ -51,7 +51,7 @@ def _real_extract(self, url):
|
|||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
|
|
||||||
req = compat_urllib_request.Request('http://www.' + mobj.group('url'))
|
req = sanitized_Request('http://www.' + mobj.group('url'))
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -4,11 +4,9 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -54,7 +52,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
|
api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
|
||||||
sport_id, video_id)
|
sport_id, video_id)
|
||||||
req = compat_urllib_request.Request(api_url, headers={
|
req = sanitized_Request(api_url, headers={
|
||||||
'Accept': 'application/vnd.vidibus.v2.html+json',
|
'Accept': 'application/vnd.vidibus.v2.html+json',
|
||||||
'Referer': url,
|
'Referer': url,
|
||||||
})
|
})
|
||||||
|
@ -4,10 +4,8 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
from ..utils import sanitized_Request
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class StreamcloudIE(InfoExtractor):
|
class StreamcloudIE(InfoExtractor):
|
||||||
@ -43,7 +41,7 @@ def _real_extract(self, url):
|
|||||||
headers = {
|
headers = {
|
||||||
b'Content-Type': b'application/x-www-form-urlencoded',
|
b'Content-Type': b'application/x-www-form-urlencoded',
|
||||||
}
|
}
|
||||||
req = compat_urllib_request.Request(url, post, headers)
|
req = sanitized_Request(url, post, headers)
|
||||||
|
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, video_id, note='Downloading video page ...')
|
req, video_id, note='Downloading video page ...')
|
||||||
|
@ -5,11 +5,9 @@
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -54,7 +52,7 @@ def _real_extract(self, url):
|
|||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
api_path = '/episode/%s' % video_id
|
api_path = '/episode/%s' % video_id
|
||||||
|
|
||||||
req = compat_urllib_request.Request(self._API_URL + api_path)
|
req = sanitized_Request(self._API_URL + api_path)
|
||||||
req.add_header('Api-Password', _get_api_key(api_path))
|
req.add_header('Api-Password', _get_api_key(api_path))
|
||||||
data = self._download_json(req, video_id)
|
data = self._download_json(req, video_id)
|
||||||
|
|
||||||
|
@ -4,14 +4,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -53,7 +51,7 @@ def _real_extract(self, url):
|
|||||||
display_id = mobj.group('id')
|
display_id = mobj.group('id')
|
||||||
|
|
||||||
playlist_url = self._API_URL.format(display_id)
|
playlist_url = self._API_URL.format(display_id)
|
||||||
request = compat_urllib_request.Request(playlist_url)
|
request = sanitized_Request(playlist_url)
|
||||||
request.add_header('X-Requested-With', 'XMLHttpRequest')
|
request.add_header('X-Requested-With', 'XMLHttpRequest')
|
||||||
request.add_header('Accept', 'application/json')
|
request.add_header('Accept', 'application/json')
|
||||||
request.add_header('Referer', url)
|
request.add_header('Referer', url)
|
||||||
|
@ -4,12 +4,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse_urlparse
|
||||||
compat_urllib_parse_urlparse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
)
|
)
|
||||||
from ..aes import aes_decrypt_text
|
from ..aes import aes_decrypt_text
|
||||||
@ -42,7 +40,7 @@ def _real_extract(self, url):
|
|||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
display_id = mobj.group('display_id')
|
display_id = mobj.group('display_id')
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, display_id)
|
webpage = self._download_webpage(req, display_id)
|
||||||
|
|
||||||
|
@ -5,13 +5,11 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -44,7 +42,7 @@ def _login(self):
|
|||||||
'password': password,
|
'password': password,
|
||||||
}
|
}
|
||||||
payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
|
payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
|
||||||
request = compat_urllib_request.Request(self._LOGIN_URL, payload)
|
request = sanitized_Request(self._LOGIN_URL, payload)
|
||||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
login_page = self._download_webpage(
|
login_page = self._download_webpage(
|
||||||
request, None, False, 'Wrong login info')
|
request, None, False, 'Wrong login info')
|
||||||
|
@ -11,7 +11,6 @@
|
|||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
@ -20,6 +19,7 @@
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -48,7 +48,7 @@ def _download_json(self, url, video_id, note='Downloading JSON metadata'):
|
|||||||
for cookie in self._downloader.cookiejar:
|
for cookie in self._downloader.cookiejar:
|
||||||
if cookie.name == 'api_token':
|
if cookie.name == 'api_token':
|
||||||
headers['Twitch-Api-Token'] = cookie.value
|
headers['Twitch-Api-Token'] = cookie.value
|
||||||
request = compat_urllib_request.Request(url, headers=headers)
|
request = sanitized_Request(url, headers=headers)
|
||||||
response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
|
response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
|
||||||
self._handle_error(response)
|
self._handle_error(response)
|
||||||
return response
|
return response
|
||||||
@ -80,7 +80,7 @@ def _login(self):
|
|||||||
if not post_url.startswith('http'):
|
if not post_url.startswith('http'):
|
||||||
post_url = compat_urlparse.urljoin(redirect_url, post_url)
|
post_url = compat_urlparse.urljoin(redirect_url, post_url)
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
|
post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
|
||||||
request.add_header('Referer', redirect_url)
|
request.add_header('Referer', redirect_url)
|
||||||
response = self._download_webpage(
|
response = self._download_webpage(
|
||||||
|
@ -4,13 +4,13 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
float_or_none,
|
float_or_none,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
remove_end,
|
remove_end,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -81,7 +81,7 @@ def _real_extract(self, url):
|
|||||||
config = None
|
config = None
|
||||||
formats = []
|
formats = []
|
||||||
for user_agent in USER_AGENTS:
|
for user_agent in USER_AGENTS:
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.add_header('User-Agent', user_agent)
|
request.add_header('User-Agent', user_agent)
|
||||||
webpage = self._download_webpage(request, video_id)
|
webpage = self._download_webpage(request, video_id)
|
||||||
|
|
||||||
|
@ -9,6 +9,7 @@
|
|||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -58,7 +59,7 @@ def _download_json(self, url_or_request, video_id, note='Downloading JSON metada
|
|||||||
for header, value in headers.items():
|
for header, value in headers.items():
|
||||||
url_or_request.add_header(header, value)
|
url_or_request.add_header(header, value)
|
||||||
else:
|
else:
|
||||||
url_or_request = compat_urllib_request.Request(url_or_request, headers=headers)
|
url_or_request = sanitized_Request(url_or_request, headers=headers)
|
||||||
|
|
||||||
response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
|
response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
|
||||||
self._handle_error(response)
|
self._handle_error(response)
|
||||||
@ -89,7 +90,7 @@ def is_logged(webpage):
|
|||||||
'password': password.encode('utf-8'),
|
'password': password.encode('utf-8'),
|
||||||
})
|
})
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
request.add_header('Referer', self._ORIGIN_URL)
|
request.add_header('Referer', self._ORIGIN_URL)
|
||||||
request.add_header('Origin', self._ORIGIN_URL)
|
request.add_header('Origin', self._ORIGIN_URL)
|
||||||
|
@ -4,11 +4,11 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -49,7 +49,7 @@ def get_session_id():
|
|||||||
|
|
||||||
info_url = "http://vbox7.com/play/magare.do"
|
info_url = "http://vbox7.com/play/magare.do"
|
||||||
data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
|
data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
|
||||||
info_request = compat_urllib_request.Request(info_url, data)
|
info_request = sanitized_Request(info_url, data)
|
||||||
info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||||
info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
|
info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
|
||||||
if info_response is None:
|
if info_response is None:
|
||||||
|
@ -4,12 +4,10 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -110,7 +108,7 @@ def _real_extract(self, url):
|
|||||||
if 'class="adultwarning-container"' in webpage:
|
if 'class="adultwarning-container"' in webpage:
|
||||||
self.report_age_confirmation()
|
self.report_age_confirmation()
|
||||||
age_limit = 18
|
age_limit = 18
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.add_header('Cookie', 'confirmedAdult=true')
|
request.add_header('Cookie', 'confirmedAdult=true')
|
||||||
webpage = self._download_webpage(request, video_id)
|
webpage = self._download_webpage(request, video_id)
|
||||||
|
|
||||||
|
@ -4,10 +4,10 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -33,7 +33,7 @@ class VesselIE(InfoExtractor):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def make_json_request(url, data):
|
def make_json_request(url, data):
|
||||||
payload = json.dumps(data).encode('utf-8')
|
payload = json.dumps(data).encode('utf-8')
|
||||||
req = compat_urllib_request.Request(url, payload)
|
req = sanitized_Request(url, payload)
|
||||||
req.add_header('Content-Type', 'application/json; charset=utf-8')
|
req.add_header('Content-Type', 'application/json; charset=utf-8')
|
||||||
return req
|
return req
|
||||||
|
|
||||||
|
@ -3,13 +3,11 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_etree_fromstring
|
||||||
compat_etree_fromstring,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -73,7 +71,7 @@ class VevoIE(InfoExtractor):
|
|||||||
_SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
|
_SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
|
||||||
|
|
||||||
def _real_initialize(self):
|
def _real_initialize(self):
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
'http://www.vevo.com/auth', data=b'')
|
'http://www.vevo.com/auth', data=b'')
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, None,
|
req, None,
|
||||||
|
@ -4,9 +4,7 @@
|
|||||||
from ..utils import (
|
from ..utils import (
|
||||||
float_or_none,
|
float_or_none,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
)
|
sanitized_Request,
|
||||||
from ..compat import (
|
|
||||||
compat_urllib_request
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -65,7 +63,7 @@ def _real_extract(self, url):
|
|||||||
'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
|
'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
|
||||||
video_id)
|
video_id)
|
||||||
headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
|
headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
|
||||||
request = compat_urllib_request.Request(json_url, None, headers)
|
request = sanitized_Request(json_url, None, headers)
|
||||||
data = self._download_json(request, video_id)['video']
|
data = self._download_json(request, video_id)['video']
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
|
@ -4,7 +4,7 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
from ..utils import sanitized_Request
|
||||||
|
|
||||||
|
|
||||||
class VideoMegaIE(InfoExtractor):
|
class VideoMegaIE(InfoExtractor):
|
||||||
@ -30,7 +30,7 @@ def _real_extract(self, url):
|
|||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
|
iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
|
||||||
req = compat_urllib_request.Request(iframe_url)
|
req = sanitized_Request(iframe_url)
|
||||||
req.add_header('Referer', url)
|
req.add_header('Referer', url)
|
||||||
req.add_header('Cookie', 'noadvtday=0')
|
req.add_header('Cookie', 'noadvtday=0')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
@ -4,7 +4,6 @@
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_HTTPError,
|
compat_HTTPError,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
)
|
)
|
||||||
@ -13,6 +12,7 @@
|
|||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
HEADRequest,
|
HEADRequest,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -76,7 +76,7 @@ class ViewsterIE(InfoExtractor):
|
|||||||
_ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
|
_ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
|
||||||
|
|
||||||
def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
|
def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.add_header('Accept', self._ACCEPT_HEADER)
|
request.add_header('Accept', self._ACCEPT_HEADER)
|
||||||
request.add_header('Auth-token', self._AUTH_TOKEN)
|
request.add_header('Auth-token', self._AUTH_TOKEN)
|
||||||
return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
|
return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
|
||||||
|
@ -7,14 +7,14 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import itertools
|
import itertools
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_age_limit,
|
parse_age_limit,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from .common import InfoExtractor
|
|
||||||
|
|
||||||
|
|
||||||
class VikiBaseIE(InfoExtractor):
|
class VikiBaseIE(InfoExtractor):
|
||||||
@ -43,7 +43,7 @@ def _prepare_call(self, path, timestamp=None, post_data=None):
|
|||||||
hashlib.sha1
|
hashlib.sha1
|
||||||
).hexdigest()
|
).hexdigest()
|
||||||
url = self._API_URL_TEMPLATE % (query, sig)
|
url = self._API_URL_TEMPLATE % (query, sig)
|
||||||
return compat_urllib_request.Request(
|
return sanitized_Request(
|
||||||
url, json.dumps(post_data).encode('utf-8')) if post_data else url
|
url, json.dumps(post_data).encode('utf-8')) if post_data else url
|
||||||
|
|
||||||
def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
|
def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
|
||||||
|
@ -8,11 +8,11 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
orderedSet,
|
orderedSet,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
@ -182,7 +182,7 @@ def _login(self):
|
|||||||
'pass': password.encode('cp1251'),
|
'pass': password.encode('cp1251'),
|
||||||
})
|
})
|
||||||
|
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'https://login.vk.com/?act=login',
|
'https://login.vk.com/?act=login',
|
||||||
compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
compat_urllib_parse.urlencode(login_form).encode('utf-8'))
|
||||||
login_page = self._download_webpage(
|
login_page = self._download_webpage(
|
||||||
|
@ -2,10 +2,8 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
from ..utils import sanitized_Request
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class VodlockerIE(InfoExtractor):
|
class VodlockerIE(InfoExtractor):
|
||||||
@ -31,7 +29,7 @@ def _real_extract(self, url):
|
|||||||
if fields['op'] == 'download1':
|
if fields['op'] == 'download1':
|
||||||
self._sleep(3, video_id) # they do detect when requests happen too fast!
|
self._sleep(3, video_id) # they do detect when requests happen too fast!
|
||||||
post = compat_urllib_parse.urlencode(fields)
|
post = compat_urllib_parse.urlencode(fields)
|
||||||
req = compat_urllib_request.Request(url, post)
|
req = sanitized_Request(url, post)
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, video_id, 'Downloading video page')
|
req, video_id, 'Downloading video page')
|
||||||
|
@ -3,14 +3,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urlparse
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -37,7 +35,7 @@ class VoiceRepublicIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(
|
req = sanitized_Request(
|
||||||
compat_urlparse.urljoin(url, '/talks/%s' % display_id))
|
compat_urlparse.urljoin(url, '/talks/%s' % display_id))
|
||||||
# Older versions of Firefox get redirected to an "upgrade browser" page
|
# Older versions of Firefox get redirected to an "upgrade browser" page
|
||||||
req.add_header('User-Agent', 'youtube-dl')
|
req.add_header('User-Agent', 'youtube-dl')
|
||||||
|
@ -1,8 +1,10 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
from ..utils import (
|
||||||
from ..utils import ExtractorError
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class WistiaIE(InfoExtractor):
|
class WistiaIE(InfoExtractor):
|
||||||
@ -23,7 +25,7 @@ class WistiaIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
request = compat_urllib_request.Request(self._API_URL.format(video_id))
|
request = sanitized_Request(self._API_URL.format(video_id))
|
||||||
request.add_header('Referer', url) # Some videos require this.
|
request.add_header('Referer', url) # Some videos require this.
|
||||||
data_json = self._download_json(request, video_id)
|
data_json = self._download_json(request, video_id)
|
||||||
if data_json.get('error'):
|
if data_json.get('error'):
|
||||||
|
@ -4,14 +4,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
encode_dict,
|
encode_dict,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -106,7 +104,7 @@ def _real_extract(self, url):
|
|||||||
|
|
||||||
post = compat_urllib_parse.urlencode(encode_dict(fields))
|
post = compat_urllib_parse.urlencode(encode_dict(fields))
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url, post)
|
req = sanitized_Request(url, post)
|
||||||
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
req.add_header('Content-type', 'application/x-www-form-urlencoded')
|
||||||
|
|
||||||
webpage = self._download_webpage(req, video_id, 'Downloading video page')
|
webpage = self._download_webpage(req, video_id, 'Downloading video page')
|
||||||
|
@ -3,12 +3,10 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse_unquote
|
||||||
compat_urllib_request,
|
|
||||||
compat_urllib_parse_unquote,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_duration,
|
parse_duration,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -32,7 +30,7 @@ class XTubeIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = sanitized_Request(url)
|
||||||
req.add_header('Cookie', 'age_verified=1')
|
req.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
|
@ -3,14 +3,12 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urllib_parse_unquote
|
||||||
compat_urllib_parse_unquote,
|
|
||||||
compat_urllib_request,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -48,7 +46,7 @@ def _real_extract(self, url):
|
|||||||
'url': video_url,
|
'url': video_url,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
android_req = compat_urllib_request.Request(url)
|
android_req = sanitized_Request(url)
|
||||||
android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
|
android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
|
||||||
android_webpage = self._download_webpage(android_req, video_id, fatal=False)
|
android_webpage = self._download_webpage(android_req, video_id, fatal=False)
|
||||||
|
|
||||||
|
@ -8,11 +8,11 @@
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_urllib_request,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -154,7 +154,7 @@ def _real_extract(self, url):
|
|||||||
if len(tracks) < len(track_ids):
|
if len(tracks) < len(track_ids):
|
||||||
present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
|
present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
|
||||||
missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
|
missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
|
||||||
request = compat_urllib_request.Request(
|
request = sanitized_Request(
|
||||||
'https://music.yandex.ru/handlers/track-entries.jsx',
|
'https://music.yandex.ru/handlers/track-entries.jsx',
|
||||||
compat_urllib_parse.urlencode({
|
compat_urllib_parse.urlencode({
|
||||||
'entries': ','.join(missing_track_ids),
|
'entries': ','.join(missing_track_ids),
|
||||||
|
@ -4,12 +4,13 @@
|
|||||||
import base64
|
import base64
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import ExtractorError
|
|
||||||
|
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
compat_ord,
|
compat_ord,
|
||||||
compat_urllib_request,
|
)
|
||||||
|
from ..utils import (
|
||||||
|
ExtractorError,
|
||||||
|
sanitized_Request,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -187,7 +188,7 @@ def _real_extract(self, url):
|
|||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
def retrieve_data(req_url, note):
|
def retrieve_data(req_url, note):
|
||||||
req = compat_urllib_request.Request(req_url)
|
req = sanitized_Request(req_url)
|
||||||
|
|
||||||
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
||||||
if cn_verification_proxy:
|
if cn_verification_proxy:
|
||||||
|
@ -3,9 +3,9 @@
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_request
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
sanitized_Request,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
@ -63,7 +63,7 @@ def _real_extract(self, url):
|
|||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
display_id = mobj.group('display_id')
|
display_id = mobj.group('display_id')
|
||||||
|
|
||||||
request = compat_urllib_request.Request(url)
|
request = sanitized_Request(url)
|
||||||
request.add_header('Cookie', 'age_verified=1')
|
request.add_header('Cookie', 'age_verified=1')
|
||||||
webpage = self._download_webpage(request, display_id)
|
webpage = self._download_webpage(request, display_id)
|
||||||
|
|
||||||
|
@ -20,7 +20,6 @@
|
|||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urllib_parse_unquote_plus,
|
compat_urllib_parse_unquote_plus,
|
||||||
compat_urllib_parse_urlparse,
|
compat_urllib_parse_urlparse,
|
||||||
compat_urllib_request,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
compat_str,
|
compat_str,
|
||||||
)
|
)
|
||||||
@ -35,6 +34,7 @@
|
|||||||
orderedSet,
|
orderedSet,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
remove_start,
|
remove_start,
|
||||||
|
sanitized_Request,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
@ -114,7 +114,7 @@ def _login(self):
|
|||||||
|
|
||||||
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
|
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
|
||||||
|
|
||||||
req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
|
req = sanitized_Request(self._LOGIN_URL, login_data)
|
||||||
login_results = self._download_webpage(
|
login_results = self._download_webpage(
|
||||||
req, None,
|
req, None,
|
||||||
note='Logging in', errnote='unable to log in', fatal=False)
|
note='Logging in', errnote='unable to log in', fatal=False)
|
||||||
@ -147,7 +147,7 @@ def _login(self):
|
|||||||
|
|
||||||
tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
|
tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
|
||||||
|
|
||||||
tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
|
tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
|
||||||
tfa_results = self._download_webpage(
|
tfa_results = self._download_webpage(
|
||||||
tfa_req, None,
|
tfa_req, None,
|
||||||
note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
|
note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
|
||||||
|
Loading…
Reference in New Issue
Block a user