From 0c3d0f51778b153f65c21906031c2e091fcfb641 Mon Sep 17 00:00:00 2001 From: alxnull Date: Sun, 13 Oct 2019 18:00:48 +0200 Subject: [PATCH] Added `--force-overwrites` option (https://github.com/ytdl-org/youtube-dl/pull/20405) Co-authored by alxnull --- Makefile | 1 + devscripts/run_tests.sh | 2 +- test/parameters.json | 2 +- test/test_overwrites.py | 52 ++++++++++++++++++++++++++++++++ youtube_dlc/YoutubeDL.py | 32 +++++++++++++++----- youtube_dlc/__init__.py | 5 ++- youtube_dlc/downloader/common.py | 2 +- youtube_dlc/options.py | 12 ++++++-- 8 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 test/test_overwrites.py diff --git a/Makefile b/Makefile index fe0740582..357e53fdb 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ offlinetest: codetest --exclude test_age_restriction.py \ --exclude test_download.py \ --exclude test_iqiyi_sdk_interpreter.py \ + --exclude test_overwrites.py \ --exclude test_socks.py \ --exclude test_subtitles.py \ --exclude test_write_annotations.py \ diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index 2fa7d16e2..b5a56facb 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # Keep this list in sync with the `offlinetest` target in Makefile -DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|socks|subtitles|write_annotations|youtube_lists|youtube_signature|post_hooks" +DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|overwrites|socks|subtitles|write_annotations|youtube_lists|youtube_signature|post_hooks" test_set="" multiprocess_args="" diff --git a/test/parameters.json b/test/parameters.json index f8abed2dd..a342e2cac 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -14,7 +14,7 @@ "logtostderr": false, "matchtitle": null, "max_downloads": null, - "nooverwrites": false, + "overwrites": null, "nopart": false, "noprogress": false, "outtmpl": "%(id)s.%(ext)s", diff --git a/test/test_overwrites.py b/test/test_overwrites.py new file mode 100644 index 000000000..d5c866c83 --- /dev/null +++ b/test/test_overwrites.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import os +from os.path import join +import subprocess +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from test.helper import try_rm + + +root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +download_file = join(root_dir, 'test.webm') + + +class TestOverwrites(unittest.TestCase): + def setUp(self): + # create an empty file + open(download_file, 'a').close() + + def test_default_overwrites(self): + outp = subprocess.Popen( + [ + sys.executable, 'youtube_dlc/__main__.py', + '-o', 'test.webm', + 'https://www.youtube.com/watch?v=jNQXAC9IVRw' + ], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sout, serr = outp.communicate() + self.assertTrue(b'has already been downloaded' in sout) + # if the file has no content, it has not been redownloaded + self.assertTrue(os.path.getsize(download_file) < 1) + + def test_yes_overwrites(self): + outp = subprocess.Popen( + [ + sys.executable, 'youtube_dlc/__main__.py', '--yes-overwrites', + '-o', 'test.webm', + 'https://www.youtube.com/watch?v=jNQXAC9IVRw' + ], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sout, serr = outp.communicate() + self.assertTrue(b'has already been downloaded' not in sout) + # if the file has no content, it has not been redownloaded + self.assertTrue(os.path.getsize(download_file) > 1) + + def tearDown(self): + try_rm(join(root_dir, 'test.webm')) + + +if __name__ == '__main__': + unittest.main() diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 60986c58e..72c05339b 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -181,7 +181,9 @@ class YoutubeDL(object): trim_file_name: Limit length of filename (extension excluded). ignoreerrors: Do not stop on download errors. (Default True when running youtube-dlc, but False when directly accessing YoutubeDL class) force_generic_extractor: Force downloader to use the generic extractor - nooverwrites: Prevent overwriting files. + overwrites: Overwrite all video and metadata files if True, + overwrite only non-video files if None + and don't overwrite any file if False playliststart: Playlist item to start at. playlistend: Playlist item to end at. playlist_items: Specific indices of playlist to download. @@ -686,6 +688,13 @@ def report_file_already_downloaded(self, file_name): except UnicodeEncodeError: self.to_screen('[download] The file has already been downloaded') + def report_file_delete(self, file_name): + """Report that existing file will be deleted.""" + try: + self.to_screen('Deleting already existent file %s' % file_name) + except UnicodeEncodeError: + self.to_screen('Deleting already existent file') + def prepare_filename(self, info_dict): """Generate the output filename.""" try: @@ -1898,7 +1907,7 @@ def ensure_dir_exists(path): if self.params.get('writedescription', False): descfn = replace_extension(filename, 'description', info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): + if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)): self.to_screen('[info] Video description is already present') elif info_dict.get('description') is None: self.report_warning('There\'s no description to write.') @@ -1913,7 +1922,7 @@ def ensure_dir_exists(path): if self.params.get('writeannotations', False): annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): + if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)): self.to_screen('[info] Video annotations are already present') elif not info_dict.get('annotations'): self.report_warning('There are no annotations to write.') @@ -1947,7 +1956,7 @@ def dl(name, info, subtitle=False): for sub_lang, sub_info in subtitles.items(): sub_format = sub_info['ext'] sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): + if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)): self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format)) else: self.to_screen('[info] Writing video subtitles to: ' + sub_filename) @@ -2002,7 +2011,7 @@ def dl(name, info, subtitle=False): if self.params.get('writeinfojson', False): infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): + if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)): self.to_screen('[info] Video description metadata is already present') else: self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn) @@ -2110,11 +2119,15 @@ def compatible_formats(formats): 'Requested formats are incompatible for merge and will be merged into mkv.') # Ensure filename always has a correct extension for successful merge filename = '%s.%s' % (filename_wo_ext, info_dict['ext']) - if os.path.exists(encodeFilename(filename)): + file_exists = os.path.exists(encodeFilename(filename)) + if not self.params.get('overwrites', False) and file_exists: self.to_screen( '[download] %s has already been downloaded and ' 'merged' % filename) else: + if file_exists: + self.report_file_delete(filename) + os.remove(encodeFilename(filename)) for f in requested_formats: new_info = dict(info_dict) new_info.update(f) @@ -2131,6 +2144,11 @@ def compatible_formats(formats): # Even if there were no downloads, it is being merged only now info_dict['__real_download'] = True else: + # Delete existing file with --yes-overwrites + if self.params.get('overwrites', False): + if os.path.exists(encodeFilename(filename)): + self.report_file_delete(filename) + os.remove(encodeFilename(filename)) # Just a single file success, real_download = dl(filename, info_dict) info_dict['__real_download'] = real_download @@ -2661,7 +2679,7 @@ def _write_thumbnails(self, info_dict, filename): thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else '' t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext')) - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)): + if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)): self.to_screen('[%s] %s: Thumbnail %sis already present' % (info_dict['extractor'], info_dict['id'], thumb_display_id)) else: diff --git a/youtube_dlc/__init__.py b/youtube_dlc/__init__.py index e68942187..9c32d98b9 100644 --- a/youtube_dlc/__init__.py +++ b/youtube_dlc/__init__.py @@ -176,6 +176,9 @@ def _real_main(argv=None): opts.max_sleep_interval = opts.sleep_interval if opts.ap_mso and opts.ap_mso not in MSO_INFO: parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers') + if opts.overwrites: + # --yes-overwrites implies --no-continue + opts.continue_dl = False def parse_retries(retries): if retries in ('inf', 'infinite'): @@ -391,7 +394,7 @@ def parse_retries(retries): 'ignoreerrors': opts.ignoreerrors, 'force_generic_extractor': opts.force_generic_extractor, 'ratelimit': opts.ratelimit, - 'nooverwrites': opts.nooverwrites, + 'overwrites': opts.overwrites, 'retries': opts.retries, 'fragment_retries': opts.fragment_retries, 'skip_unavailable_fragments': opts.skip_unavailable_fragments, diff --git a/youtube_dlc/downloader/common.py b/youtube_dlc/downloader/common.py index a0acb6556..ff72f52d1 100644 --- a/youtube_dlc/downloader/common.py +++ b/youtube_dlc/downloader/common.py @@ -332,7 +332,7 @@ def download(self, filename, info_dict, subtitle=False): """ nooverwrites_and_exists = ( - self.params.get('nooverwrites', False) + not self.params.get('overwrites', True) and os.path.exists(encodeFilename(filename)) ) diff --git a/youtube_dlc/options.py b/youtube_dlc/options.py index 75e8db988..174290507 100644 --- a/youtube_dlc/options.py +++ b/youtube_dlc/options.py @@ -834,8 +834,16 @@ def _comma_separated_values_options_callback(option, opt_str, value, parser): help=optparse.SUPPRESS_HELP) filesystem.add_option( '-w', '--no-overwrites', - action='store_true', dest='nooverwrites', default=False, - help='Do not overwrite files') + action='store_false', dest='overwrites', default=None, + help='Do not overwrite any files') + filesystem.add_option( + '--force-overwrites', '--yes-overwrites', + action='store_true', dest='overwrites', + help='Overwrite all video and metadata files. This option includes --no-continue') + filesystem.add_option( + '--no-force-overwrites', + action='store_const', dest='overwrites', const=None, + help='Do not overwrite the video, but overwrite related files (default)') filesystem.add_option( '-c', '--continue', action='store_true', dest='continue_dl', default=True,