Co-authored by alxnull
This commit is contained in:
alxnull 2019-10-13 18:00:48 +02:00 committed by pukkandan
parent f5546c0b3c
commit 0c3d0f5177
8 changed files with 95 additions and 13 deletions

View File

@ -47,6 +47,7 @@ offlinetest: codetest
--exclude test_age_restriction.py \ --exclude test_age_restriction.py \
--exclude test_download.py \ --exclude test_download.py \
--exclude test_iqiyi_sdk_interpreter.py \ --exclude test_iqiyi_sdk_interpreter.py \
--exclude test_overwrites.py \
--exclude test_socks.py \ --exclude test_socks.py \
--exclude test_subtitles.py \ --exclude test_subtitles.py \
--exclude test_write_annotations.py \ --exclude test_write_annotations.py \

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
# Keep this list in sync with the `offlinetest` target in Makefile # Keep this list in sync with the `offlinetest` target in Makefile
DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|socks|subtitles|write_annotations|youtube_lists|youtube_signature|post_hooks" DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|overwrites|socks|subtitles|write_annotations|youtube_lists|youtube_signature|post_hooks"
test_set="" test_set=""
multiprocess_args="" multiprocess_args=""

View File

@ -14,7 +14,7 @@
"logtostderr": false, "logtostderr": false,
"matchtitle": null, "matchtitle": null,
"max_downloads": null, "max_downloads": null,
"nooverwrites": false, "overwrites": null,
"nopart": false, "nopart": false,
"noprogress": false, "noprogress": false,
"outtmpl": "%(id)s.%(ext)s", "outtmpl": "%(id)s.%(ext)s",

52
test/test_overwrites.py Normal file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env python
from __future__ import unicode_literals
import os
from os.path import join
import subprocess
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from test.helper import try_rm
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
download_file = join(root_dir, 'test.webm')
class TestOverwrites(unittest.TestCase):
def setUp(self):
# create an empty file
open(download_file, 'a').close()
def test_default_overwrites(self):
outp = subprocess.Popen(
[
sys.executable, 'youtube_dlc/__main__.py',
'-o', 'test.webm',
'https://www.youtube.com/watch?v=jNQXAC9IVRw'
], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sout, serr = outp.communicate()
self.assertTrue(b'has already been downloaded' in sout)
# if the file has no content, it has not been redownloaded
self.assertTrue(os.path.getsize(download_file) < 1)
def test_yes_overwrites(self):
outp = subprocess.Popen(
[
sys.executable, 'youtube_dlc/__main__.py', '--yes-overwrites',
'-o', 'test.webm',
'https://www.youtube.com/watch?v=jNQXAC9IVRw'
], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sout, serr = outp.communicate()
self.assertTrue(b'has already been downloaded' not in sout)
# if the file has no content, it has not been redownloaded
self.assertTrue(os.path.getsize(download_file) > 1)
def tearDown(self):
try_rm(join(root_dir, 'test.webm'))
if __name__ == '__main__':
unittest.main()

View File

@ -181,7 +181,9 @@ class YoutubeDL(object):
trim_file_name: Limit length of filename (extension excluded). trim_file_name: Limit length of filename (extension excluded).
ignoreerrors: Do not stop on download errors. (Default True when running youtube-dlc, but False when directly accessing YoutubeDL class) ignoreerrors: Do not stop on download errors. (Default True when running youtube-dlc, but False when directly accessing YoutubeDL class)
force_generic_extractor: Force downloader to use the generic extractor force_generic_extractor: Force downloader to use the generic extractor
nooverwrites: Prevent overwriting files. overwrites: Overwrite all video and metadata files if True,
overwrite only non-video files if None
and don't overwrite any file if False
playliststart: Playlist item to start at. playliststart: Playlist item to start at.
playlistend: Playlist item to end at. playlistend: Playlist item to end at.
playlist_items: Specific indices of playlist to download. playlist_items: Specific indices of playlist to download.
@ -686,6 +688,13 @@ def report_file_already_downloaded(self, file_name):
except UnicodeEncodeError: except UnicodeEncodeError:
self.to_screen('[download] The file has already been downloaded') self.to_screen('[download] The file has already been downloaded')
def report_file_delete(self, file_name):
"""Report that existing file will be deleted."""
try:
self.to_screen('Deleting already existent file %s' % file_name)
except UnicodeEncodeError:
self.to_screen('Deleting already existent file')
def prepare_filename(self, info_dict): def prepare_filename(self, info_dict):
"""Generate the output filename.""" """Generate the output filename."""
try: try:
@ -1898,7 +1907,7 @@ def ensure_dir_exists(path):
if self.params.get('writedescription', False): if self.params.get('writedescription', False):
descfn = replace_extension(filename, 'description', info_dict.get('ext')) descfn = replace_extension(filename, 'description', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
self.to_screen('[info] Video description is already present') self.to_screen('[info] Video description is already present')
elif info_dict.get('description') is None: elif info_dict.get('description') is None:
self.report_warning('There\'s no description to write.') self.report_warning('There\'s no description to write.')
@ -1913,7 +1922,7 @@ def ensure_dir_exists(path):
if self.params.get('writeannotations', False): if self.params.get('writeannotations', False):
annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext')) annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)):
self.to_screen('[info] Video annotations are already present') self.to_screen('[info] Video annotations are already present')
elif not info_dict.get('annotations'): elif not info_dict.get('annotations'):
self.report_warning('There are no annotations to write.') self.report_warning('There are no annotations to write.')
@ -1947,7 +1956,7 @@ def dl(name, info, subtitle=False):
for sub_lang, sub_info in subtitles.items(): for sub_lang, sub_info in subtitles.items():
sub_format = sub_info['ext'] sub_format = sub_info['ext']
sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext')) sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format)) self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
else: else:
self.to_screen('[info] Writing video subtitles to: ' + sub_filename) self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@ -2002,7 +2011,7 @@ def dl(name, info, subtitle=False):
if self.params.get('writeinfojson', False): if self.params.get('writeinfojson', False):
infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
self.to_screen('[info] Video description metadata is already present') self.to_screen('[info] Video description metadata is already present')
else: else:
self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn) self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
@ -2110,11 +2119,15 @@ def compatible_formats(formats):
'Requested formats are incompatible for merge and will be merged into mkv.') 'Requested formats are incompatible for merge and will be merged into mkv.')
# Ensure filename always has a correct extension for successful merge # Ensure filename always has a correct extension for successful merge
filename = '%s.%s' % (filename_wo_ext, info_dict['ext']) filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
if os.path.exists(encodeFilename(filename)): file_exists = os.path.exists(encodeFilename(filename))
if not self.params.get('overwrites', False) and file_exists:
self.to_screen( self.to_screen(
'[download] %s has already been downloaded and ' '[download] %s has already been downloaded and '
'merged' % filename) 'merged' % filename)
else: else:
if file_exists:
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
for f in requested_formats: for f in requested_formats:
new_info = dict(info_dict) new_info = dict(info_dict)
new_info.update(f) new_info.update(f)
@ -2131,6 +2144,11 @@ def compatible_formats(formats):
# Even if there were no downloads, it is being merged only now # Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True info_dict['__real_download'] = True
else: else:
# Delete existing file with --yes-overwrites
if self.params.get('overwrites', False):
if os.path.exists(encodeFilename(filename)):
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
# Just a single file # Just a single file
success, real_download = dl(filename, info_dict) success, real_download = dl(filename, info_dict)
info_dict['__real_download'] = real_download info_dict['__real_download'] = real_download
@ -2661,7 +2679,7 @@ def _write_thumbnails(self, info_dict, filename):
thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else '' thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext')) t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)): if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
self.to_screen('[%s] %s: Thumbnail %sis already present' % self.to_screen('[%s] %s: Thumbnail %sis already present' %
(info_dict['extractor'], info_dict['id'], thumb_display_id)) (info_dict['extractor'], info_dict['id'], thumb_display_id))
else: else:

View File

@ -176,6 +176,9 @@ def _real_main(argv=None):
opts.max_sleep_interval = opts.sleep_interval opts.max_sleep_interval = opts.sleep_interval
if opts.ap_mso and opts.ap_mso not in MSO_INFO: if opts.ap_mso and opts.ap_mso not in MSO_INFO:
parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers') parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
if opts.overwrites:
# --yes-overwrites implies --no-continue
opts.continue_dl = False
def parse_retries(retries): def parse_retries(retries):
if retries in ('inf', 'infinite'): if retries in ('inf', 'infinite'):
@ -391,7 +394,7 @@ def parse_retries(retries):
'ignoreerrors': opts.ignoreerrors, 'ignoreerrors': opts.ignoreerrors,
'force_generic_extractor': opts.force_generic_extractor, 'force_generic_extractor': opts.force_generic_extractor,
'ratelimit': opts.ratelimit, 'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites, 'overwrites': opts.overwrites,
'retries': opts.retries, 'retries': opts.retries,
'fragment_retries': opts.fragment_retries, 'fragment_retries': opts.fragment_retries,
'skip_unavailable_fragments': opts.skip_unavailable_fragments, 'skip_unavailable_fragments': opts.skip_unavailable_fragments,

View File

@ -332,7 +332,7 @@ def download(self, filename, info_dict, subtitle=False):
""" """
nooverwrites_and_exists = ( nooverwrites_and_exists = (
self.params.get('nooverwrites', False) not self.params.get('overwrites', True)
and os.path.exists(encodeFilename(filename)) and os.path.exists(encodeFilename(filename))
) )

View File

@ -834,8 +834,16 @@ def _comma_separated_values_options_callback(option, opt_str, value, parser):
help=optparse.SUPPRESS_HELP) help=optparse.SUPPRESS_HELP)
filesystem.add_option( filesystem.add_option(
'-w', '--no-overwrites', '-w', '--no-overwrites',
action='store_true', dest='nooverwrites', default=False, action='store_false', dest='overwrites', default=None,
help='Do not overwrite files') help='Do not overwrite any files')
filesystem.add_option(
'--force-overwrites', '--yes-overwrites',
action='store_true', dest='overwrites',
help='Overwrite all video and metadata files. This option includes --no-continue')
filesystem.add_option(
'--no-force-overwrites',
action='store_const', dest='overwrites', const=None,
help='Do not overwrite the video, but overwrite related files (default)')
filesystem.add_option( filesystem.add_option(
'-c', '--continue', '-c', '--continue',
action='store_true', dest='continue_dl', default=True, action='store_true', dest='continue_dl', default=True,