[cleanup] Refactor some code

This commit is contained in:
pukkandan 2021-07-31 16:21:01 +05:30
parent d74a58a186
commit dbf5416a20
9 changed files with 67 additions and 69 deletions

View File

@ -126,6 +126,7 @@
) )
from .extractor.openload import PhantomJSwrapper from .extractor.openload import PhantomJSwrapper
from .downloader import ( from .downloader import (
FFmpegFD,
get_suitable_downloader, get_suitable_downloader,
shorten_protocol_name shorten_protocol_name
) )
@ -2690,20 +2691,15 @@ def correct_ext(filename):
info_dict['__real_download'] = False info_dict['__real_download'] = False
_protocols = set(determine_protocol(f) for f in requested_formats) _protocols = set(determine_protocol(f) for f in requested_formats)
if len(_protocols) == 1: if len(_protocols) == 1: # All requested formats have same protocol
info_dict['protocol'] = _protocols.pop() info_dict['protocol'] = _protocols.pop()
directly_mergable = ( directly_mergable = FFmpegFD.can_merge_formats(info_dict)
'no-direct-merge' not in self.params.get('compat_opts', []) if dl_filename is not None:
and info_dict.get('protocol') is not None # All requested formats have same protocol pass
and not self.params.get('allow_unplayable_formats') elif (directly_mergable and get_suitable_downloader(info_dict, self.params) == FFmpegFD):
and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD') info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
if directly_mergable: success, real_download = self.dl(temp_filename, info_dict)
info_dict['url'] = requested_formats[0]['url'] info_dict['__real_download'] = real_download
# Treat it as a single download
dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None:
success, real_download = self.dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download
else: else:
downloaded = [] downloaded = []
merger = FFmpegMergerPP(self) merger = FFmpegMergerPP(self)
@ -2717,28 +2713,25 @@ def correct_ext(filename):
'You have requested merging of multiple formats but ffmpeg is not installed. ' 'You have requested merging of multiple formats but ffmpeg is not installed. '
'The formats won\'t be merged.') 'The formats won\'t be merged.')
if dl_filename is None: for f in requested_formats:
for f in requested_formats: new_info = dict(info_dict)
new_info = dict(info_dict) del new_info['requested_formats']
del new_info['requested_formats'] new_info.update(f)
new_info.update(f) fname = prepend_extension(temp_filename, 'f%s' % f['format_id'], new_info['ext'])
fname = prepend_extension( if not self._ensure_dir_exists(fname):
self.prepare_filename(new_info, 'temp'), return
'f%s' % f['format_id'], new_info['ext']) downloaded.append(fname)
if not self._ensure_dir_exists(fname): partial_success, real_download = self.dl(fname, new_info)
return info_dict['__real_download'] = info_dict['__real_download'] or real_download
downloaded.append(fname) success = success and partial_success
partial_success, real_download = self.dl(fname, new_info) if merger.available and not self.params.get('allow_unplayable_formats'):
info_dict['__real_download'] = info_dict['__real_download'] or real_download info_dict['__postprocessors'].append(merger)
success = success and partial_success info_dict['__files_to_merge'] = downloaded
if merger.available and not self.params.get('allow_unplayable_formats'): # Even if there were no downloads, it is being merged only now
info_dict['__postprocessors'].append(merger) info_dict['__real_download'] = True
info_dict['__files_to_merge'] = downloaded else:
# Even if there were no downloads, it is being merged only now for file in downloaded:
info_dict['__real_download'] = True files_to_move[file] = None
else:
for file in downloaded:
files_to_move[file] = None
else: else:
# Just a single file # Just a single file
dl_filename = existing_file(full_filename, temp_filename) dl_filename = existing_file(full_filename, temp_filename)

View File

@ -3,17 +3,19 @@
from ..compat import compat_str from ..compat import compat_str
from ..utils import ( from ..utils import (
determine_protocol, determine_protocol,
NO_DEFAULT
) )
def _get_real_downloader(info_dict, protocol=None, *args, **kwargs): def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None):
info_dict['protocol'] = determine_protocol(info_dict)
info_copy = info_dict.copy() info_copy = info_dict.copy()
if protocol: if protocol:
info_copy['protocol'] = protocol info_copy['protocol'] = protocol
return get_suitable_downloader(info_copy, *args, **kwargs) return _get_suitable_downloader(info_copy, params, default)
# Some of these require _get_real_downloader # Some of these require get_suitable_downloader
from .common import FileDownloader from .common import FileDownloader
from .dash import DashSegmentsFD from .dash import DashSegmentsFD
from .f4m import F4mFD from .f4m import F4mFD
@ -69,14 +71,15 @@ def shorten_protocol_name(proto, simplify=False):
return short_protocol_names.get(proto, proto) return short_protocol_names.get(proto, proto)
def get_suitable_downloader(info_dict, params={}, default=HttpFD): def _get_suitable_downloader(info_dict, params, default):
"""Get the downloader class that can handle the info dict.""" """Get the downloader class that can handle the info dict."""
protocol = determine_protocol(info_dict) if default is NO_DEFAULT:
info_dict['protocol'] = protocol default = HttpFD
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict): # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
# return FFmpegFD # return FFmpegFD
protocol = info_dict['protocol']
downloaders = params.get('external_downloader') downloaders = params.get('external_downloader')
external_downloader = ( external_downloader = (
downloaders if isinstance(downloaders, compat_str) or downloaders is None downloaders if isinstance(downloaders, compat_str) or downloaders is None
@ -94,7 +97,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
return FFmpegFD return FFmpegFD
elif external_downloader == 'native': elif external_downloader == 'native':
return HlsFD return HlsFD
elif _get_real_downloader(info_dict, 'm3u8_frag_urls', params, None): elif get_suitable_downloader(info_dict, params, None, protocol='m3u8_frag_urls'):
return HlsFD return HlsFD
elif params.get('hls_prefer_native') is True: elif params.get('hls_prefer_native') is True:
return HlsFD return HlsFD

View File

@ -1,6 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from ..downloader import _get_real_downloader from ..downloader import get_suitable_downloader
from .fragment import FragmentFD from .fragment import FragmentFD
from ..utils import urljoin from ..utils import urljoin
@ -15,11 +15,14 @@ class DashSegmentsFD(FragmentFD):
FD_NAME = 'dashsegments' FD_NAME = 'dashsegments'
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
if info_dict.get('is_live'):
self.report_error('Live DASH videos are not supported')
fragment_base_url = info_dict.get('fragment_base_url') fragment_base_url = info_dict.get('fragment_base_url')
fragments = info_dict['fragments'][:1] if self.params.get( fragments = info_dict['fragments'][:1] if self.params.get(
'test', False) else info_dict['fragments'] 'test', False) else info_dict['fragments']
real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None) real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='dash_frag_urls')
ctx = { ctx = {
'filename': filename, 'filename': filename,
@ -54,9 +57,6 @@ def real_download(self, filename, info_dict):
info_copy = info_dict.copy() info_copy = info_dict.copy()
info_copy['fragments'] = fragments_to_download info_copy['fragments'] = fragments_to_download
fd = real_downloader(self.ydl, self.params) fd = real_downloader(self.ydl, self.params)
# TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks:
# fd.add_progress_hook(ph)
return fd.real_download(filename, info_copy) return fd.real_download(filename, info_copy)
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict) return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)

View File

@ -345,12 +345,22 @@ class FFmpegFD(ExternalFD):
@classmethod @classmethod
def available(cls, path=None): def available(cls, path=None):
# TODO: Fix path for ffmpeg # TODO: Fix path for ffmpeg
# Fixme: This may be wrong when --ffmpeg-location is used
return FFmpegPostProcessor().available return FFmpegPostProcessor().available
def on_process_started(self, proc, stdin): def on_process_started(self, proc, stdin):
""" Override this in subclasses """ """ Override this in subclasses """
pass pass
@classmethod
def can_merge_formats(cls, info_dict, params={}):
return (
info_dict.get('requested_formats')
and info_dict.get('protocol')
and not params.get('allow_unplayable_formats')
and 'no-direct-merge' not in params.get('compat_opts', [])
and cls.can_download(info_dict))
def _call_downloader(self, tmpfilename, info_dict): def _call_downloader(self, tmpfilename, info_dict):
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']] urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
ffpp = FFmpegPostProcessor(downloader=self) ffpp = FFmpegPostProcessor(downloader=self)

View File

@ -4,7 +4,7 @@
import io import io
import binascii import binascii
from ..downloader import _get_real_downloader from ..downloader import get_suitable_downloader
from .fragment import FragmentFD, can_decrypt_frag from .fragment import FragmentFD, can_decrypt_frag
from .external import FFmpegFD from .external import FFmpegFD
@ -80,16 +80,13 @@ def real_download(self, filename, info_dict):
fd = FFmpegFD(self.ydl, self.params) fd = FFmpegFD(self.ydl, self.params)
self.report_warning( self.report_warning(
'%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename())) '%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
# TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks:
# fd.add_progress_hook(ph)
return fd.real_download(filename, info_dict) return fd.real_download(filename, info_dict)
is_webvtt = info_dict['ext'] == 'vtt' is_webvtt = info_dict['ext'] == 'vtt'
if is_webvtt: if is_webvtt:
real_downloader = None # Packing the fragments is not currently supported for external downloader real_downloader = None # Packing the fragments is not currently supported for external downloader
else: else:
real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None) real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='m3u8_frag_urls')
if real_downloader and not real_downloader.supports_manifest(s): if real_downloader and not real_downloader.supports_manifest(s):
real_downloader = None real_downloader = None
if real_downloader: if real_downloader:

View File

@ -4,7 +4,7 @@
import threading import threading
from .common import FileDownloader from .common import FileDownloader
from ..downloader import _get_real_downloader from ..downloader import get_suitable_downloader
from ..extractor.niconico import NiconicoIE from ..extractor.niconico import NiconicoIE
from ..compat import compat_urllib_request from ..compat import compat_urllib_request
@ -20,7 +20,7 @@ def real_download(self, filename, info_dict):
ie = NiconicoIE(self.ydl) ie = NiconicoIE(self.ydl)
info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict) info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
fd = _get_real_downloader(info_dict, params=self.params)(self.ydl, self.params) fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
success = download_complete = False success = download_complete = False
timer = [None] timer = [None]

View File

@ -1298,7 +1298,7 @@ def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
# JSON-LD may be malformed and thus `fatal` should be respected. # JSON-LD may be malformed and thus `fatal` should be respected.
# At the same time `default` may be passed that assumes `fatal=False` # At the same time `default` may be passed that assumes `fatal=False`
# for _search_regex. Let's simulate the same behavior here as well. # for _search_regex. Let's simulate the same behavior here as well.
fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False
json_ld = [] json_ld = []
for mobj in json_ld_list: for mobj in json_ld_list:
json_ld_item = self._parse_json( json_ld_item = self._parse_json(
@ -1522,7 +1522,7 @@ class FormatSort:
'size': {'type': 'combined', 'same_limit': True, 'field': ('filesize', 'fs_approx')}, 'size': {'type': 'combined', 'same_limit': True, 'field': ('filesize', 'fs_approx')},
'ext': {'type': 'combined', 'field': ('vext', 'aext')}, 'ext': {'type': 'combined', 'field': ('vext', 'aext')},
'res': {'type': 'multiple', 'field': ('height', 'width'), 'res': {'type': 'multiple', 'field': ('height', 'width'),
'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))}, 'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
# Most of these exist only for compatibility reasons # Most of these exist only for compatibility reasons
'dimension': {'type': 'alias', 'field': 'res'}, 'dimension': {'type': 'alias', 'field': 'res'},

View File

@ -706,9 +706,8 @@ def _dict_from_options_callback(
callback_kwargs={ callback_kwargs={
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms', 'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
'default_key': 'default', 'default_key': 'default',
'process': lambda x: x.strip() 'process': str.strip
}, }, help=(
help=(
'Name or path of the external downloader to use (optionally) prefixed by ' 'Name or path of the external downloader to use (optionally) prefixed by '
'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. ' 'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
'Currently supports native, %s (Recommended: aria2c). ' 'Currently supports native, %s (Recommended: aria2c). '
@ -724,8 +723,7 @@ def _dict_from_options_callback(
'allowed_keys': '|'.join(list_external_downloaders()), 'allowed_keys': '|'.join(list_external_downloaders()),
'default_key': 'default', 'default_key': 'default',
'process': compat_shlex_split 'process': compat_shlex_split
}, }, help=(
help=(
'Give these arguments to the external downloader. ' 'Give these arguments to the external downloader. '
'Specify the downloader name and the arguments separated by a colon ":". ' 'Specify the downloader name and the arguments separated by a colon ":". '
'You can use this option multiple times to give different arguments to different downloaders ' 'You can use this option multiple times to give different arguments to different downloaders '
@ -944,8 +942,7 @@ def _dict_from_options_callback(
callback_kwargs={ callback_kwargs={
'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()), 'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()),
'default_key': 'default' 'default_key': 'default'
}, }, help='Output filename template; see "OUTPUT TEMPLATE" for details')
help='Output filename template; see "OUTPUT TEMPLATE" for details')
filesystem.add_option( filesystem.add_option(
'--output-na-placeholder', '--output-na-placeholder',
dest='outtmpl_na_placeholder', metavar='TEXT', default='NA', dest='outtmpl_na_placeholder', metavar='TEXT', default='NA',
@ -1191,8 +1188,7 @@ def _dict_from_options_callback(
'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat', 'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat',
'process': compat_shlex_split, 'process': compat_shlex_split,
'multiple_keys': False 'multiple_keys': False
}, }, help=(
help=(
'Give these arguments to the postprocessors. ' 'Give these arguments to the postprocessors. '
'Specify the postprocessor/executable name and the arguments separated by a colon ":" ' 'Specify the postprocessor/executable name and the arguments separated by a colon ":" '
'to give the argument to the specified postprocessor/executable. Supported PP are: ' 'to give the argument to the specified postprocessor/executable. Supported PP are: '
@ -1385,8 +1381,7 @@ def _dict_from_options_callback(
'multiple_keys': False, 'multiple_keys': False,
'process': lambda val: dict( 'process': lambda val: dict(
_extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';')) _extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';'))
}, }, help=(
help=(
'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. ' 'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. '
'You can use this option multiple times to give arguments for different extractors')) 'You can use this option multiple times to give arguments for different extractors'))
extractor.add_option( extractor.add_option(

View File

@ -4544,7 +4544,7 @@ def parse_codecs(codecs_str):
if not codecs_str: if not codecs_str:
return {} return {}
split_codecs = list(filter(None, map( split_codecs = list(filter(None, map(
lambda str: str.strip(), codecs_str.strip().strip(',').split(',')))) str.strip, codecs_str.strip().strip(',').split(','))))
vcodec, acodec = None, None vcodec, acodec = None, None
for full_codec in split_codecs: for full_codec in split_codecs:
codec = full_codec.split('.')[0] codec = full_codec.split('.')[0]
@ -6246,7 +6246,7 @@ def traverse_obj(
# TODO: Write tests # TODO: Write tests
''' '''
if not casesense: if not casesense:
_lower = lambda k: k.lower() if isinstance(k, str) else k _lower = lambda k: (k.lower() if isinstance(k, str) else k)
path_list = (map(_lower, variadic(path)) for path in path_list) path_list = (map(_lower, variadic(path)) for path in path_list)
def _traverse_obj(obj, path, _current_depth=0): def _traverse_obj(obj, path, _current_depth=0):