Add experimental option --check-formats to test the URLs before format selection

This commit is contained in:
pukkandan 2021-05-04 21:24:00 +05:30
parent e625be0d10
commit e8e738406a
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
5 changed files with 76 additions and 31 deletions

View File

@ -577,6 +577,8 @@ ## Video Format Options:
containers irrespective of quality containers irrespective of quality
--no-prefer-free-formats Don't give any special preference to free --no-prefer-free-formats Don't give any special preference to free
containers (default) containers (default)
--check-formats Check that the formats selected are
actually downloadable (Experimental)
-F, --list-formats List all available formats of requested -F, --list-formats List all available formats of requested
videos videos
--list-formats-as-table Present the output of -F in tabular form --list-formats-as-table Present the output of -F in tabular form

View File

@ -84,6 +84,7 @@
PostProcessingError, PostProcessingError,
preferredencoding, preferredencoding,
prepend_extension, prepend_extension,
random_uuidv4,
register_socks_protocols, register_socks_protocols,
render_table, render_table,
replace_extension, replace_extension,
@ -1521,6 +1522,8 @@ def syntax_error(note, start):
allow_multiple_streams = {'audio': self.params.get('allow_multiple_audio_streams', False), allow_multiple_streams = {'audio': self.params.get('allow_multiple_audio_streams', False),
'video': self.params.get('allow_multiple_video_streams', False)} 'video': self.params.get('allow_multiple_video_streams', False)}
check_formats = self.params.get('check_formats')
def _parse_filter(tokens): def _parse_filter(tokens):
filter_parts = [] filter_parts = []
for type, string, start, _, _ in tokens: for type, string, start, _, _ in tokens:
@ -1678,6 +1681,22 @@ def _merge(formats_pair):
return new_dict return new_dict
def _check_formats(formats):
for f in formats:
self.to_screen('[info] Testing format %s' % f['format_id'])
paths = self.params.get('paths', {})
temp_file = os.path.join(
expand_path(paths.get('home', '').strip()),
expand_path(paths.get('temp', '').strip()),
'ytdl.%s.f%s.check-format' % (random_uuidv4(), f['format_id']))
dl, _ = self.dl(temp_file, f, test=True)
if os.path.exists(temp_file):
os.remove(temp_file)
if dl:
yield f
else:
self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
def _build_selector_function(selector): def _build_selector_function(selector):
if isinstance(selector, list): # , if isinstance(selector, list): # ,
fs = [_build_selector_function(s) for s in selector] fs = [_build_selector_function(s) for s in selector]
@ -1708,12 +1727,13 @@ def selector_function(ctx):
if format_spec == 'all': if format_spec == 'all':
def selector_function(ctx): def selector_function(ctx):
formats = list(ctx['formats']) formats = list(ctx['formats'])
if formats: if check_formats:
for f in formats: formats = _check_formats(formats)
yield f for f in formats:
yield f
elif format_spec == 'mergeall': elif format_spec == 'mergeall':
def selector_function(ctx): def selector_function(ctx):
formats = list(ctx['formats']) formats = list(_check_formats(ctx['formats']))
if not formats: if not formats:
return return
merged_format = formats[-1] merged_format = formats[-1]
@ -1722,13 +1742,13 @@ def selector_function(ctx):
yield merged_format yield merged_format
else: else:
format_fallback = False format_fallback, format_reverse, format_idx = False, True, 1
mobj = re.match( mobj = re.match(
r'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$', r'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
format_spec) format_spec)
if mobj is not None: if mobj is not None:
format_idx = int_or_none(mobj.group('n'), default=1) format_idx = int_or_none(mobj.group('n'), default=1)
format_idx = format_idx - 1 if mobj.group('bw')[0] == 'w' else -format_idx format_reverse = mobj.group('bw')[0] == 'b'
format_type = (mobj.group('type') or [None])[0] format_type = (mobj.group('type') or [None])[0]
not_format_type = {'v': 'a', 'a': 'v'}.get(format_type) not_format_type = {'v': 'a', 'a': 'v'}.get(format_type)
format_modified = mobj.group('mod') is not None format_modified = mobj.group('mod') is not None
@ -1743,7 +1763,6 @@ def selector_function(ctx):
if not format_modified # b, w if not format_modified # b, w
else None) # b*, w* else None) # b*, w*
else: else:
format_idx = -1
filter_f = ((lambda f: f.get('ext') == format_spec) filter_f = ((lambda f: f.get('ext') == format_spec)
if format_spec in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension if format_spec in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension
else (lambda f: f.get('format_id') == format_spec)) # id else (lambda f: f.get('format_id') == format_spec)) # id
@ -1753,16 +1772,18 @@ def selector_function(ctx):
if not formats: if not formats:
return return
matches = list(filter(filter_f, formats)) if filter_f is not None else formats matches = list(filter(filter_f, formats)) if filter_f is not None else formats
n = len(matches) if format_fallback and ctx['incomplete_formats'] and not matches:
if -n <= format_idx < n:
yield matches[format_idx]
elif format_fallback and ctx['incomplete_formats']:
# for extractors with incomplete formats (audio only (soundcloud) # for extractors with incomplete formats (audio only (soundcloud)
# or video only (imgur)) best/worst will fallback to # or video only (imgur)) best/worst will fallback to
# best/worst {video,audio}-only format # best/worst {video,audio}-only format
n = len(formats) matches = formats
if -n <= format_idx < n: if format_reverse:
yield formats[format_idx] matches = matches[::-1]
if check_formats:
matches = list(itertools.islice(_check_formats(matches), format_idx))
n = len(matches)
if -n <= format_idx - 1 < n:
yield matches[format_idx - 1]
elif selector.type == MERGE: # + elif selector.type == MERGE: # +
selector_1, selector_2 = map(_build_selector_function, selector.selector) selector_1, selector_2 = map(_build_selector_function, selector.selector)
@ -2179,6 +2200,34 @@ def print_optional(field):
self.post_extract(info_dict) self.post_extract(info_dict)
self.to_stdout(json.dumps(info_dict, default=repr)) self.to_stdout(json.dumps(info_dict, default=repr))
def dl(self, name, info, subtitle=False, test=False):
if test:
verbose = self.params.get('verbose')
params = {
'test': True,
'quiet': not verbose,
'verbose': verbose,
'noprogress': not verbose,
'nopart': True,
'skip_unavailable_fragments': False,
'keep_fragments': False,
'overwrites': True,
'_no_ytdl_file': True,
}
else:
params = self.params
fd = get_suitable_downloader(info, params)(self, params)
if not test:
for ph in self._progress_hooks:
fd.add_progress_hook(ph)
if self.params.get('verbose'):
self.to_screen('[debug] Invoking downloader on %r' % info.get('url'))
new_info = dict(info)
if new_info.get('http_headers') is None:
new_info['http_headers'] = self._calc_headers(new_info)
return fd.download(name, new_info, subtitle)
def process_info(self, info_dict): def process_info(self, info_dict):
"""Process a single resolved IE result.""" """Process a single resolved IE result."""
@ -2264,17 +2313,6 @@ def process_info(self, info_dict):
self.report_error('Cannot write annotations file: ' + annofn) self.report_error('Cannot write annotations file: ' + annofn)
return return
def dl(name, info, subtitle=False):
fd = get_suitable_downloader(info, self.params)(self, self.params)
for ph in self._progress_hooks:
fd.add_progress_hook(ph)
if self.params.get('verbose'):
self.to_screen('[debug] Invoking downloader on %r' % info.get('url'))
new_info = dict(info)
if new_info.get('http_headers') is None:
new_info['http_headers'] = self._calc_headers(new_info)
return fd.download(name, new_info, subtitle)
subtitles_are_requested = any([self.params.get('writesubtitles', False), subtitles_are_requested = any([self.params.get('writesubtitles', False),
self.params.get('writeautomaticsub')]) self.params.get('writeautomaticsub')])
@ -2307,7 +2345,7 @@ def dl(name, info, subtitle=False):
return return
else: else:
try: try:
dl(sub_filename, sub_info.copy(), subtitle=True) self.dl(sub_filename, sub_info.copy(), subtitle=True)
sub_info['filepath'] = sub_filename sub_info['filepath'] = sub_filename
files_to_move[sub_filename] = sub_filename_final files_to_move[sub_filename] = sub_filename_final
except tuple([ExtractorError, IOError, OSError, ValueError] + network_exceptions) as err: except tuple([ExtractorError, IOError, OSError, ValueError] + network_exceptions) as err:
@ -2493,7 +2531,7 @@ def correct_ext(filename):
if not self._ensure_dir_exists(fname): if not self._ensure_dir_exists(fname):
return return
downloaded.append(fname) downloaded.append(fname)
partial_success, real_download = dl(fname, new_info) partial_success, real_download = self.dl(fname, new_info)
info_dict['__real_download'] = info_dict['__real_download'] or real_download info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success success = success and partial_success
if merger.available and not self.params.get('allow_unplayable_formats'): if merger.available and not self.params.get('allow_unplayable_formats'):
@ -2508,7 +2546,7 @@ def correct_ext(filename):
# Just a single file # Just a single file
dl_filename = existing_file(full_filename, temp_filename) dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None: if dl_filename is None:
success, real_download = dl(temp_filename, info_dict) success, real_download = self.dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download info_dict['__real_download'] = real_download
dl_filename = dl_filename or temp_filename dl_filename = dl_filename or temp_filename

View File

@ -471,6 +471,7 @@ def report_args_compat(arg, name):
'format_sort_force': opts.format_sort_force, 'format_sort_force': opts.format_sort_force,
'allow_multiple_video_streams': opts.allow_multiple_video_streams, 'allow_multiple_video_streams': opts.allow_multiple_video_streams,
'allow_multiple_audio_streams': opts.allow_multiple_audio_streams, 'allow_multiple_audio_streams': opts.allow_multiple_audio_streams,
'check_formats': opts.check_formats,
'listformats': opts.listformats, 'listformats': opts.listformats,
'listformats_table': opts.listformats_table, 'listformats_table': opts.listformats_table,
'outtmpl': outtmpl, 'outtmpl': outtmpl,

View File

@ -31,6 +31,7 @@ class FragmentFD(FileDownloader):
Skip unavailable fragments (DASH and hlsnative only) Skip unavailable fragments (DASH and hlsnative only)
keep_fragments: Keep downloaded fragments on disk after downloading is keep_fragments: Keep downloaded fragments on disk after downloading is
finished finished
_no_ytdl_file: Don't use .ytdl file
For each incomplete fragment download yt-dlp keeps on disk a special For each incomplete fragment download yt-dlp keeps on disk a special
bookkeeping file with download state and metadata (in future such files will bookkeeping file with download state and metadata (in future such files will
@ -69,9 +70,8 @@ def _prepare_and_start_frag_download(self, ctx):
self._prepare_frag_download(ctx) self._prepare_frag_download(ctx)
self._start_frag_download(ctx) self._start_frag_download(ctx)
@staticmethod def __do_ytdl_file(self, ctx):
def __do_ytdl_file(ctx): return not ctx['live'] and not ctx['tmpfilename'] == '-' and not self.params.get('_no_ytdl_file')
return not ctx['live'] and not ctx['tmpfilename'] == '-'
def _read_ytdl_file(self, ctx): def _read_ytdl_file(self, ctx):
assert 'ytdl_corrupt' not in ctx assert 'ytdl_corrupt' not in ctx

View File

@ -502,6 +502,10 @@ def _dict_from_multiple_values_options_callback(
'--no-prefer-free-formats', '--no-prefer-free-formats',
action='store_true', dest='prefer_free_formats', default=False, action='store_true', dest='prefer_free_formats', default=False,
help="Don't give any special preference to free containers (default)") help="Don't give any special preference to free containers (default)")
video_format.add_option(
'--check-formats',
action='store_true', dest='check_formats', default=False,
help="Check that the formats selected are actually downloadable (Experimental)")
video_format.add_option( video_format.add_option(
'-F', '--list-formats', '-F', '--list-formats',
action='store_true', dest='listformats', action='store_true', dest='listformats',