From 6148833f5ceb7674142ddb8d761ffe03cee7df69 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 30 Jul 2023 03:36:17 +0530 Subject: [PATCH] [cleanup] Misc --- test/test_YoutubeDL.py | 1 + test/test_networking.py | 8 +++++--- yt_dlp/YoutubeDL.py | 14 ++++++-------- yt_dlp/extractor/common.py | 2 +- yt_dlp/extractor/lbry.py | 6 +++--- yt_dlp/extractor/netverse.py | 4 ++-- yt_dlp/extractor/ninenow.py | 2 +- yt_dlp/extractor/vk.py | 4 ++-- yt_dlp/extractor/wimbledon.py | 2 +- yt_dlp/networking/common.py | 9 +++++---- 10 files changed, 27 insertions(+), 25 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index ab1250848..3cfb61fb2 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -831,6 +831,7 @@ def expect_same_infodict(out): test('%(id&hi {:>10} {}|)s', 'hi 1234 1234') test(R'%(id&{0} {}|)s', 'NA') test(R'%(id&{0.1}|)s', 'NA') + test('%(height&{:,d})S', '1,080') # Laziness def gen(): diff --git a/test/test_networking.py b/test/test_networking.py index 684bf5f96..9c33b0d4c 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -29,6 +29,7 @@ from http.cookiejar import CookieJar from test.helper import FakeYDL, http_server_port +from yt_dlp.cookies import YoutubeDLCookieJar from yt_dlp.dependencies import brotli from yt_dlp.networking import ( HEADRequest, @@ -478,7 +479,7 @@ def test_request_cookie_header(self, handler): assert 'Cookie: test=test' not in res # Specified Cookie header should override global cookiejar for that request - cookiejar = http.cookiejar.CookieJar() + cookiejar = YoutubeDLCookieJar() cookiejar.set_cookie(http.cookiejar.Cookie( version=0, name='test', value='ytdlp', port=None, port_specified=False, domain='127.0.0.1', domain_specified=True, domain_initial_dot=False, path='/', @@ -505,7 +506,7 @@ def test_incompleteread(self, handler): @pytest.mark.parametrize('handler', ['Urllib'], indirect=True) def test_cookies(self, handler): - cookiejar = http.cookiejar.CookieJar() + cookiejar = YoutubeDLCookieJar() cookiejar.set_cookie(http.cookiejar.Cookie( 0, 'test', 'ytdlp', None, False, '127.0.0.1', True, False, '/headers', True, False, None, False, None, None, {})) @@ -903,7 +904,8 @@ class HTTPSupportedRH(ValidationRH): EXTENSION_TESTS = [ ('Urllib', [ ({'cookiejar': 'notacookiejar'}, AssertionError), - ({'cookiejar': CookieJar()}, False), + ({'cookiejar': YoutubeDLCookieJar()}, False), + ({'cookiejar': CookieJar()}, AssertionError), ({'timeout': 1}, False), ({'timeout': 'notatimeout'}, AssertionError), ({'unsupported': 'value'}, UnsupportedRequest), diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 6e8be40ba..db5932c44 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -256,8 +256,6 @@ class YoutubeDL: overwrites: Overwrite all video and metadata files if True, overwrite only non-video files if None and don't overwrite any file if False - For compatibility with youtube-dl, - "nooverwrites" may also be used instead playlist_items: Specific indices of playlist to download. playlistrandom: Download playlist items in random order. lazy_playlist: Process playlist entries as they are received. @@ -553,6 +551,7 @@ class YoutubeDL: You can reduce network I/O by disabling it if you don't care about HLS. (only for youtube) no_color: Same as `color='no_color'` + no_overwrites: Same as `overwrites=False` """ _NUMERIC_FIELDS = { @@ -604,6 +603,7 @@ def __init__(self, params=None, auto_init=True): self._playlist_level = 0 self._playlist_urls = set() self.cache = Cache(self) + self.__header_cookies = [] stdout = sys.stderr if self.params.get('logtostderr') else sys.stdout self._out_files = Namespace( @@ -632,7 +632,7 @@ def process_color_policy(stream): policy = traverse_obj(self.params, ('color', (stream_name, None), {str}), get_all=False) if policy in ('auto', None): return term_allow_color and supports_terminal_sequences(stream) - assert policy in ('always', 'never', 'no_color') + assert policy in ('always', 'never', 'no_color'), policy return {'always': True, 'never': False}.get(policy, policy) self._allow_colors = Namespace(**{ @@ -681,12 +681,10 @@ def process_color_policy(stream): self.params['compat_opts'] = set(self.params.get('compat_opts', ())) self.params['http_headers'] = HTTPHeaderDict(std_headers, self.params.get('http_headers')) - self.__header_cookies = [] self._load_cookies(self.params['http_headers'].get('Cookie')) # compat self.params['http_headers'].pop('Cookie', None) + self._request_director = self.build_request_director(_REQUEST_HANDLERS.values()) - self._request_director = self.build_request_director( - sorted(_REQUEST_HANDLERS.values(), key=lambda rh: rh.RH_NAME.lower())) if auto_init and auto_init != 'no_verbose_header': self.print_debug_header() @@ -3977,7 +3975,7 @@ def get_encoding(stream): })) or 'none')) write_debug(f'Proxy map: {self.proxies}') - # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers)}') + # write_debug(f'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}') for plugin_type, plugins in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}.items(): display_list = ['%s%s' % ( klass.__name__, '' if klass.__name__ == name else f' as {name}') @@ -4080,7 +4078,7 @@ def urlopen(self, req): def build_request_director(self, handlers): logger = _YDLLogger(self) - headers = self.params.get('http_headers').copy() + headers = self.params['http_headers'].copy() proxies = self.proxies.copy() clean_headers(headers) clean_proxies(proxies, headers) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index b69ac1d65..7deab995c 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -729,7 +729,7 @@ def extract(self, url): except UnsupportedError: raise except ExtractorError as e: - e.video_id = e.video_id or self.get_temp_id(url), + e.video_id = e.video_id or self.get_temp_id(url) e.ie = e.ie or self.IE_NAME, e.traceback = e.traceback or sys.exc_info()[2] raise diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py index 7dd3a4861..9a9f9256f 100644 --- a/yt_dlp/extractor/lbry.py +++ b/yt_dlp/extractor/lbry.py @@ -248,9 +248,9 @@ def _real_extract(self, url): # GET request to v3 API returns original video/audio file if available direct_url = re.sub(r'/api/v\d+/', '/api/v3/', streaming_url) - ext = urlhandle_detect_ext(self._request_webpage( - direct_url, display_id, 'Checking for original quality', headers=headers)) - if ext != 'm3u8': + urlh = self._request_webpage( + direct_url, display_id, 'Checking for original quality', headers=headers, fatal=False) + if urlh and urlhandle_detect_ext(urlh) != 'm3u8': formats.append({ 'url': direct_url, 'format_id': 'original', diff --git a/yt_dlp/extractor/netverse.py b/yt_dlp/extractor/netverse.py index 398198a1b..ef53e15da 100644 --- a/yt_dlp/extractor/netverse.py +++ b/yt_dlp/extractor/netverse.py @@ -160,7 +160,7 @@ class NetverseIE(NetverseBaseIE): 'uploader': 'Net Prime', 'comment_count': int, }, - 'params':{ + 'params': { 'getcomments': True } }, { @@ -187,7 +187,7 @@ class NetverseIE(NetverseBaseIE): 'season': 'Season 1', 'comment_count': int, }, - 'params':{ + 'params': { 'getcomments': True } }] diff --git a/yt_dlp/extractor/ninenow.py b/yt_dlp/extractor/ninenow.py index b970f8ccb..c655b75f4 100644 --- a/yt_dlp/extractor/ninenow.py +++ b/yt_dlp/extractor/ninenow.py @@ -53,7 +53,7 @@ class NineNowIE(InfoExtractor): 'upload_date': '20210421', }, 'expected_warnings': ['Ignoring subtitle tracks'], - 'params':{ + 'params': { 'skip_download': True, } }] diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py index 6b7379d46..915422817 100644 --- a/yt_dlp/extractor/vk.py +++ b/yt_dlp/extractor/vk.py @@ -765,7 +765,7 @@ def _extract_common_meta(self, stream_info): class VKPlayIE(VKPlayBaseIE): - _VALID_URL = r'https?://vkplay\.live/(?P[^/]+)/record/(?P[a-f0-9\-]+)' + _VALID_URL = r'https?://vkplay\.live/(?P[^/#?]+)/record/(?P[a-f0-9-]+)' _TESTS = [{ 'url': 'https://vkplay.live/zitsmann/record/f5e6e3b5-dc52-4d14-965d-0680dd2882da', 'info_dict': { @@ -802,7 +802,7 @@ def _real_extract(self, url): class VKPlayLiveIE(VKPlayBaseIE): - _VALID_URL = r'https?://vkplay\.live/(?P[^/]+)/?(?:[#?]|$)' + _VALID_URL = r'https?://vkplay\.live/(?P[^/#?]+)/?(?:[#?]|$)' _TESTS = [{ 'url': 'https://vkplay.live/bayda', 'info_dict': { diff --git a/yt_dlp/extractor/wimbledon.py b/yt_dlp/extractor/wimbledon.py index ee4872e88..0223e54f1 100644 --- a/yt_dlp/extractor/wimbledon.py +++ b/yt_dlp/extractor/wimbledon.py @@ -6,7 +6,7 @@ class WimbledonIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?wimbledon\.com/\w+/video/media/(?P\d+).html' + _VALID_URL = r'https?://(?:www\.)?wimbledon\.com/\w+/video/media/(?P\d+)\.html' _TESTS = [{ 'url': 'https://www.wimbledon.com/en_GB/video/media/6330247525112.html', 'info_dict': { diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py index 792e062fd..8fba8c1c5 100644 --- a/yt_dlp/networking/common.py +++ b/yt_dlp/networking/common.py @@ -12,7 +12,6 @@ from collections.abc import Iterable, Mapping from email.message import Message from http import HTTPStatus -from http.cookiejar import CookieJar from ._helper import make_ssl_context, wrap_request_errors from .exceptions import ( @@ -22,6 +21,7 @@ UnsupportedRequest, ) from ..compat.types import NoneType +from ..cookies import YoutubeDLCookieJar from ..utils import ( bug_reports_message, classproperty, @@ -194,7 +194,7 @@ def __init__( self, *, logger, # TODO(Grub4k): default logger headers: HTTPHeaderDict = None, - cookiejar: CookieJar = None, + cookiejar: YoutubeDLCookieJar = None, timeout: float | int | None = None, proxies: dict = None, source_address: str = None, @@ -208,7 +208,7 @@ def __init__( self._logger = logger self.headers = headers or {} - self.cookiejar = cookiejar if cookiejar is not None else CookieJar() + self.cookiejar = cookiejar if cookiejar is not None else YoutubeDLCookieJar() self.timeout = float(timeout or 20) self.proxies = proxies or {} self.source_address = source_address @@ -275,7 +275,7 @@ def _check_proxies(self, proxies): def _check_extensions(self, extensions): """Check extensions for unsupported extensions. Subclasses should extend this.""" - assert isinstance(extensions.get('cookiejar'), (CookieJar, NoneType)) + assert isinstance(extensions.get('cookiejar'), (YoutubeDLCookieJar, NoneType)) assert isinstance(extensions.get('timeout'), (float, int, NoneType)) def _validate(self, request): @@ -302,6 +302,7 @@ def send(self, request: Request) -> Response: @abc.abstractmethod def _send(self, request: Request): """Handle a request from start to finish. Redefine in subclasses.""" + pass def close(self): pass