[downloader] Fix slow progress hooks

Closes #1301
This commit is contained in:
pukkandan 2021-10-16 18:31:00 +05:30
parent 48ee10ee8a
commit 03b4de722a
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
5 changed files with 23 additions and 21 deletions

View File

@ -950,13 +950,18 @@ def validate_outtmpl(cls, outtmpl):
except ValueError as err: except ValueError as err:
return err return err
@staticmethod
def _copy_infodict(info_dict):
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
return info_dict
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None): def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """ """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList info_dict = self._copy_infodict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
formatSeconds(info_dict['duration'], '-' if sanitize else ':') formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None if info_dict.get('duration', None) is not None
@ -2265,7 +2270,7 @@ def is_wellformed(f):
formats_dict[format_id].append(format) formats_dict[format_id].append(format)
# Make sure all formats have unique format_id # Make sure all formats have unique format_id
common_exts = set(ext for exts in self._format_selection_exts.values() for ext in exts) common_exts = set(itertools.chain(*self._format_selection_exts.values()))
for format_id, ambiguous_formats in formats_dict.items(): for format_id, ambiguous_formats in formats_dict.items():
ambigious_id = len(ambiguous_formats) > 1 ambigious_id = len(ambiguous_formats) > 1
for i, format in enumerate(ambiguous_formats): for i, format in enumerate(ambiguous_formats):
@ -2523,7 +2528,8 @@ def dl(self, name, info, subtitle=False, test=False):
fd.add_progress_hook(ph) fd.add_progress_hook(ph)
urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']]) urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']])
self.write_debug('Invoking downloader on "%s"' % urls) self.write_debug('Invoking downloader on "%s"' % urls)
new_info = dict(info)
new_info = copy.deepcopy(self._copy_infodict(info))
if new_info.get('http_headers') is None: if new_info.get('http_headers') is None:
new_info['http_headers'] = self._calc_headers(new_info) new_info['http_headers'] = self._calc_headers(new_info)
return fd.download(name, new_info, subtitle) return fd.download(name, new_info, subtitle)

View File

@ -405,13 +405,10 @@ def real_download(self, filename, info_dict):
def _hook_progress(self, status, info_dict): def _hook_progress(self, status, info_dict):
if not self._progress_hooks: if not self._progress_hooks:
return return
info_dict = dict(info_dict) status['info_dict'] = info_dict
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
# youtube-dl passes the same status object to all the hooks. # youtube-dl passes the same status object to all the hooks.
# Some third party scripts seems to be relying on this. # Some third party scripts seems to be relying on this.
# So keep this behavior if possible # So keep this behavior if possible
status['info_dict'] = copy.deepcopy(info_dict)
for ph in self._progress_hooks: for ph in self._progress_hooks:
ph(status) ph(status)

View File

@ -55,9 +55,8 @@ def real_download(self, filename, info_dict):
if real_downloader: if real_downloader:
self.to_screen( self.to_screen(
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename())) '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
info_copy = info_dict.copy() info_dict['fragments'] = fragments_to_download
info_copy['fragments'] = fragments_to_download
fd = real_downloader(self.ydl, self.params) fd = real_downloader(self.ydl, self.params)
return fd.real_download(filename, info_copy) return fd.real_download(filename, info_dict)
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict) return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)

View File

@ -245,13 +245,12 @@ def is_ad_fragment_end(s):
fragments = [fragments[0] if fragments else None] fragments = [fragments[0] if fragments else None]
if real_downloader: if real_downloader:
info_copy = info_dict.copy() info_dict['fragments'] = fragments
info_copy['fragments'] = fragments
fd = real_downloader(self.ydl, self.params) fd = real_downloader(self.ydl, self.params)
# TODO: Make progress updates work without hooking twice # TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks: # for ph in self._progress_hooks:
# fd.add_progress_hook(ph) # fd.add_progress_hook(ph)
return fd.real_download(filename, info_copy) return fd.real_download(filename, info_dict)
if is_webvtt: if is_webvtt:
def pack_fragment(frag_content, frag_index): def pack_fragment(frag_content, frag_index):

View File

@ -17,11 +17,12 @@ class PostProcessorMetaClass(type):
def run_wrapper(func): def run_wrapper(func):
@functools.wraps(func) @functools.wraps(func)
def run(self, info, *args, **kwargs): def run(self, info, *args, **kwargs):
self._hook_progress({'status': 'started'}, info) info_copy = copy.deepcopy(self._copy_infodict(info))
self._hook_progress({'status': 'started'}, info_copy)
ret = func(self, info, *args, **kwargs) ret = func(self, info, *args, **kwargs)
if ret is not None: if ret is not None:
_, info = ret _, info = ret
self._hook_progress({'status': 'finished'}, info) self._hook_progress({'status': 'finished'}, info_copy)
return ret return ret
return run return run
@ -93,6 +94,9 @@ def set_downloader(self, downloader):
for ph in getattr(downloader, '_postprocessor_hooks', []): for ph in getattr(downloader, '_postprocessor_hooks', []):
self.add_progress_hook(ph) self.add_progress_hook(ph)
def _copy_infodict(self, info_dict):
return getattr(self._downloader, '_copy_infodict', dict)(info_dict)
@staticmethod @staticmethod
def _restrict_to(*, video=True, audio=True, images=True): def _restrict_to(*, video=True, audio=True, images=True):
allowed = {'video': video, 'audio': audio, 'images': images} allowed = {'video': video, 'audio': audio, 'images': images}
@ -142,11 +146,8 @@ def _configuration_args(self, exe, *args, **kwargs):
def _hook_progress(self, status, info_dict): def _hook_progress(self, status, info_dict):
if not self._progress_hooks: if not self._progress_hooks:
return return
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
status.update({ status.update({
'info_dict': copy.deepcopy(info_dict), 'info_dict': info_dict,
'postprocessor': self.pp_key(), 'postprocessor': self.pp_key(),
}) })
for ph in self._progress_hooks: for ph in self._progress_hooks: