From 01fe8e8fa6c0ca3dd83a109dbb8b51ce050c57aa Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sat, 6 Apr 2024 15:40:29 +1300 Subject: [PATCH] Handle urllib3 not being available --- test/test_http_proxy.py | 5 ++- yt_dlp/YoutubeDL.py | 6 ++-- yt_dlp/networking/_websockets.py | 61 +++++++++++++++++++++----------- yt_dlp/networking/websocket.py | 10 +++++- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/test/test_http_proxy.py b/test/test_http_proxy.py index 34dee4ab5..c6b7fbdf6 100644 --- a/test/test_http_proxy.py +++ b/test/test_http_proxy.py @@ -156,7 +156,10 @@ def __init__(self, request, *args, **kwargs): certfn = os.path.join(TEST_DIR, 'testcert.pem') sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) sslctx.load_cert_chain(certfn, None) - request = SSLTransport(request, ssl_context=sslctx, server_side=True) + if SSLTransport: + request = SSLTransport(request, ssl_context=sslctx, server_side=True) + else: + request = sslctx.wrap_socket(request, server_side=True) super().__init__(request, *args, **kwargs) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index e3d1db376..a97c272d9 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -4137,15 +4137,15 @@ def urlopen(self, req): 'Use --enable-file-urls to enable at your own risk.', cause=ue) from ue if ( 'unsupported proxy type: "https"' in ue.msg.lower() - and 'requests' not in self._request_director.handlers - and 'curl_cffi' not in self._request_director.handlers + and 'Requests' not in self._request_director.handlers + and 'CurlCFFI' not in self._request_director.handlers ): raise RequestError( 'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests, curl_cffi') elif ( re.match(r'unsupported url scheme: "wss?"', ue.msg.lower()) - and 'websockets' not in self._request_director.handlers + and 'Websockets' not in self._request_director.handlers ): raise RequestError( 'This request requires WebSocket support. ' diff --git a/yt_dlp/networking/_websockets.py b/yt_dlp/networking/_websockets.py index 37da6b102..b13e892ce 100644 --- a/yt_dlp/networking/_websockets.py +++ b/yt_dlp/networking/_websockets.py @@ -19,20 +19,18 @@ ProxyError, RequestError, SSLError, - TransportError, + TransportError, UnsupportedRequest, ) from .websocket import WebSocketRequestHandler, WebSocketResponse from ..compat import functools -from ..dependencies import websockets +from ..dependencies import websockets, urllib3 from ..socks import ProxyError as SocksProxyError -from ..utils import int_or_none, extract_basic_auth +from ..utils import int_or_none import io import urllib.parse import base64 -from http.client import HTTPResponse, HTTPConnection, HTTPSConnection - -from urllib3.util.ssltransport import SSLTransport +from http.client import HTTPResponse, HTTPConnection from ..utils.networking import HTTPHeaderDict @@ -45,6 +43,11 @@ if websockets_version < (12, 0): raise ImportError('Only websockets>=12.0 is supported') +urllib3_supported = False +urllib3_version = tuple(int_or_none(x, default=0) for x in urllib3.__version__.split('.')) if urllib3 else None +if urllib3_version and urllib3_version >= (1, 26, 17): + urllib3_supported = True + import websockets.sync.client from websockets.uri import parse_uri @@ -124,6 +127,17 @@ def __init__(self, *args, **kwargs): if self.verbose: logger.setLevel(logging.DEBUG) + def _validate(self, request): + super()._validate(request) + proxy = select_proxy(request.url, self._get_proxies(request)) + if ( + proxy + and urllib.parse.urlparse(proxy).scheme.lower() == 'https' + and urllib.parse.urlparse(request.url).scheme.lower() == 'wss' + and not urllib3_supported + ): + raise UnsupportedRequest('WSS over HTTPS proxies requires a supported version of urllib3') + def _check_extensions(self, extensions): super()._check_extensions(extensions) extensions.pop('timeout', None) @@ -178,6 +192,12 @@ def _send(self, request): proxy = select_proxy(request.url, self._get_proxies(request)) + ssl_context = None + if parse_uri(request.url).secure: + if WebsocketsSSLContext is not None: + ssl_context = WebsocketsSSLContext(self._make_sslcontext()) + else: + ssl_context = self._make_sslcontext() try: conn = websockets.sync.client.connect( sock=self._make_sock(proxy, request.url, timeout), @@ -185,10 +205,7 @@ def _send(self, request): additional_headers=headers, open_timeout=timeout, user_agent_header=None, - ssl_context=( - WebsocketsSSLContext(self._make_sslcontext()) - if parse_uri(request.url).secure else None - ), + ssl_context=ssl_context, close_timeout=0, # not ideal, but prevents yt-dlp hanging ) return WebsocketsResponseAdapter(conn, url=request.url) @@ -223,17 +240,21 @@ def begin(self): self.will_close = False -# todo: only define if urllib3 is available -class WebsocketsSSLTransport(SSLTransport): - """ - Modified version of urllib3 SSLTransport to support additional operations used by websockets - """ - def setsockopt(self, *args, **kwargs): - self.socket.setsockopt(*args, **kwargs) +if urllib3_supported: + from urllib3.util.ssltransport import SSLTransport - def shutdown(self, *args, **kwargs): - self.unwrap() - self.socket.shutdown(*args, **kwargs) + class WebsocketsSSLTransport(SSLTransport): + """ + Modified version of urllib3 SSLTransport to support additional operations used by websockets + """ + def setsockopt(self, *args, **kwargs): + self.socket.setsockopt(*args, **kwargs) + + def shutdown(self, *args, **kwargs): + self.unwrap() + self.socket.shutdown(*args, **kwargs) +else: + WebsocketsSSLTransport = None class WebsocketsSSLContext: diff --git a/yt_dlp/networking/websocket.py b/yt_dlp/networking/websocket.py index 0e7e73c9e..d407cadad 100644 --- a/yt_dlp/networking/websocket.py +++ b/yt_dlp/networking/websocket.py @@ -1,8 +1,9 @@ from __future__ import annotations import abc +import urllib.parse -from .common import RequestHandler, Response +from .common import RequestHandler, Response, register_preference class WebSocketResponse(Response): @@ -21,3 +22,10 @@ def recv(self): class WebSocketRequestHandler(RequestHandler, abc.ABC): pass + + +@register_preference(WebSocketRequestHandler) +def websocket_preference(_, request): + if urllib.parse.urlparse(request.url).scheme in ('ws', 'wss'): + return 200 + return 0