From e5a998f3684e7c56f9cf1c07c4e176e891d96509 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 05:31:54 +0530 Subject: [PATCH] [cleanup] Misc cleanup (#2173) Authored by: fstirlitz, pukkandan --- Changelog.md | 2 +- README.md | 2 +- devscripts/bash-completion.py | 4 ++-- devscripts/fish-completion.py | 4 ++-- devscripts/make_lazy_extractors.py | 3 +-- devscripts/zsh-completion.py | 4 ++-- test/test_overwrites.py | 5 ++--- yt_dlp/YoutubeDL.py | 4 +++- yt_dlp/compat.py | 2 +- yt_dlp/extractor/abematv.py | 1 + yt_dlp/extractor/sonyliv.py | 30 +++++++++++++++++++--------- yt_dlp/postprocessor/sponsorblock.py | 4 ++-- yt_dlp/utils.py | 14 ++++++------- 13 files changed, 46 insertions(+), 33 deletions(-) diff --git a/Changelog.md b/Changelog.md index a4cf0e92e..3fb6260b8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,7 +15,7 @@ ### 2022.04.08 * Use certificates from `certifi` if installed by [coletdjnz](https://github.com/coletdjnz) * Treat multiple `--match-filters` as OR -* File locking improvevemnts: +* File locking improvements: * Do not lock downloading file on Windows * Do not prevent download if locking is unsupported * Do not truncate files before locking by [jakeogh](https://github.com/jakeogh), [pukkandan](https://github.com/pukkandan) diff --git a/README.md b/README.md index f4b55f6d7..8a8477c9b 100644 --- a/README.md +++ b/README.md @@ -1748,7 +1748,7 @@ # EMBEDDING YT-DLP ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) ``` -Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L197). +Most likely, you'll want to use various options. For a list of options available, have a look at [`yt_dlp/YoutubeDL.py`](yt_dlp/YoutubeDL.py#L181). Here's a more complete example demonstrating various functionality: diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 73d698c39..27ec7ca7a 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os import sys -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp BASH_COMPLETION_FILE = "completions/bash/yt-dlp" diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index c318b69e4..dcb1d6582 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -2,9 +2,9 @@ import optparse import os import sys -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp from yt_dlp.utils import shell_quote diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 6d5f96cf0..5e2070602 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -2,9 +2,8 @@ import os import sys from inspect import getsource -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' if os.path.exists(lazy_extractors_filename): diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 2d5ac2a45..06660d8fd 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os import sys -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp" diff --git a/test/test_overwrites.py b/test/test_overwrites.py index 39741b65c..a6d5bae40 100644 --- a/test/test_overwrites.py +++ b/test/test_overwrites.py @@ -3,14 +3,13 @@ import subprocess import sys import unittest -from os.path import join sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import is_download_test, try_rm root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -download_file = join(root_dir, 'test.webm') +download_file = os.path.join(root_dir, 'test.webm') @is_download_test @@ -44,7 +43,7 @@ def test_yes_overwrites(self): self.assertTrue(os.path.getsize(download_file) > 1) def tearDown(self): - try_rm(join(root_dir, 'test.webm')) + try_rm(os.path.join(root_dir, 'test.webm')) if __name__ == '__main__': diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index a8bb7f45c..eaf2d9216 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -888,6 +888,7 @@ class Styles(Enum): SUPPRESS = 'light black' def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_encoding=False): + text = str(text) if test_encoding: original_text = text # handle.encoding can be None. See https://github.com/yt-dlp/yt-dlp/issues/2711 @@ -895,7 +896,7 @@ def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_enc text = text.encode(encoding, 'ignore').decode(encoding) if fallback is not None and text != original_text: text = fallback - if isinstance(f, self.Styles): + if isinstance(f, Enum): f = f.value return format_text(text, f) if allow_colors else text if fallback is None else fallback @@ -1708,6 +1709,7 @@ def get_entry(i): entries.append(entry) try: if entry is not None: + # TODO: Add auto-generated fields self._match_entry(entry, incomplete=True, silent=True) except (ExistingVideoReached, RejectedVideoReached): broken = True diff --git a/yt_dlp/compat.py b/yt_dlp/compat.py index 7a1500435..9d3a6bbfd 100644 --- a/yt_dlp/compat.py +++ b/yt_dlp/compat.py @@ -196,7 +196,7 @@ def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.pytho compat_urlparse = compat_urllib_parse = urllib.parse -# To be removed +# To be removed - Do not use compat_basestring = str compat_collections_abc = collections.abc diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py index 0dc8dea26..1b9deeae8 100644 --- a/yt_dlp/extractor/abematv.py +++ b/yt_dlp/extractor/abematv.py @@ -28,6 +28,7 @@ # NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862) + def add_opener(ydl, handler): ''' Add a handler for opening URLs, like _download_webpage ''' # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426 diff --git a/yt_dlp/extractor/sonyliv.py b/yt_dlp/extractor/sonyliv.py index 771f890cc..17d28478f 100644 --- a/yt_dlp/extractor/sonyliv.py +++ b/yt_dlp/extractor/sonyliv.py @@ -1,4 +1,5 @@ import datetime +import json import math import random import time @@ -82,21 +83,32 @@ def _perform_login(self, username, password): raise ExtractorError(f'Invalid username/password; {self._LOGIN_HINT}') self.report_login() - data = '''{"mobileNumber":"%s","channelPartnerID":"MSMIND","country":"IN","timestamp":"%s", - "otpSize":6,"loginType":"REGISTERORSIGNIN","isMobileMandatory":true} - ''' % (username, datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%MZ")) otp_request_json = self._download_json( 'https://apiv2.sonyliv.com/AGL/1.6/A/ENG/WEB/IN/HR/CREATEOTP-V2', - None, note='Sending OTP', data=data.encode(), headers=self._HEADERS) + None, note='Sending OTP', headers=self._HEADERS, data=json.dumps({ + 'mobileNumber': username, + 'channelPartnerID': 'MSMIND', + 'country': 'IN', + 'timestamp': datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%MZ'), + 'otpSize': 6, + 'loginType': 'REGISTERORSIGNIN', + 'isMobileMandatory': True, + }).encode()) if otp_request_json['resultCode'] == 'KO': raise ExtractorError(otp_request_json['message'], expected=True) - otp_code = self._get_tfa_info('OTP') - data = '''{"channelPartnerID":"MSMIND","mobileNumber":"%s","country":"IN","otp":"%s", - "dmaId":"IN","ageConfirmation":true,"timestamp":"%s","isMobileMandatory":true} - ''' % (username, otp_code, datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%MZ")) + otp_verify_json = self._download_json( 'https://apiv2.sonyliv.com/AGL/2.0/A/ENG/WEB/IN/HR/CONFIRMOTP-V2', - None, note='Verifying OTP', data=data.encode(), headers=self._HEADERS) + None, note='Verifying OTP', headers=self._HEADERS, data=json.dumps({ + 'channelPartnerID': 'MSMIND', + 'mobileNumber': username, + 'country': 'IN', + 'otp': self._get_tfa_info('OTP'), + 'dmaId': 'IN', + 'ageConfirmation': True, + 'timestamp': datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%MZ'), + 'isMobileMandatory': True, + }).encode()) if otp_verify_json['resultCode'] == 'KO': raise ExtractorError(otp_request_json['message'], expected=True) self._HEADERS['authorization'] = otp_verify_json['resultObj']['accessToken'] diff --git a/yt_dlp/postprocessor/sponsorblock.py b/yt_dlp/postprocessor/sponsorblock.py index 501e30320..7749ffe05 100644 --- a/yt_dlp/postprocessor/sponsorblock.py +++ b/yt_dlp/postprocessor/sponsorblock.py @@ -1,6 +1,6 @@ +import hashlib import json import re -from hashlib import sha256 from .ffmpeg import FFmpegPostProcessor from ..compat import compat_urllib_parse_urlencode @@ -84,7 +84,7 @@ def to_chapter(s): return sponsor_chapters def _get_sponsor_segments(self, video_id, service): - hash = sha256(video_id.encode('ascii')).hexdigest() + hash = hashlib.sha256(video_id.encode('ascii')).hexdigest() # SponsorBlock API recommends using first 4 hash characters. url = f'{self._API_URL}/api/skipSegments/{hash[:4]}?' + compat_urllib_parse_urlencode({ 'service': service, diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 25ac864f3..35e8d1d5b 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4793,12 +4793,12 @@ def random_birthday(year_field, month_field, day_field): # Templates for internet shortcut files, which are plain text files. -DOT_URL_LINK_TEMPLATE = ''' +DOT_URL_LINK_TEMPLATE = '''\ [InternetShortcut] URL=%(url)s -'''.lstrip() +''' -DOT_WEBLOC_LINK_TEMPLATE = ''' +DOT_WEBLOC_LINK_TEMPLATE = '''\ @@ -4807,16 +4807,16 @@ def random_birthday(year_field, month_field, day_field): \t%(url)s -'''.lstrip() +''' -DOT_DESKTOP_LINK_TEMPLATE = ''' +DOT_DESKTOP_LINK_TEMPLATE = '''\ [Desktop Entry] Encoding=UTF-8 Name=%(filename)s Type=Link URL=%(url)s Icon=text-html -'''.lstrip() +''' LINK_TEMPLATES = { 'url': DOT_URL_LINK_TEMPLATE, @@ -4872,7 +4872,7 @@ def iri_to_uri(iri): def to_high_limit_path(path): if sys.platform in ['win32', 'cygwin']: # Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited. - return r'\\?\ '.rstrip() + os.path.abspath(path) + return '\\\\?\\' + os.path.abspath(path) return path