From c7667c2d7f602aecfd8a39f26d8151a363ba0b5e Mon Sep 17 00:00:00 2001 From: SyxbEaEQ2 Date: Thu, 31 Jul 2014 03:08:24 +0200 Subject: [PATCH] [downloader/(common/http)] Changes calculation of the rate-limit. (Fix #2297, fix #2140, fix #595, fix #2370) --- youtube_dl/downloader/common.py | 15 +++++++++------ youtube_dl/downloader/http.py | 31 ++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 917f3450e..6404e1928 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -77,8 +77,10 @@ def format_percent(percent): def calc_eta(start, now, total, current): if total is None: return None + if now is None: + now = time.time() dif = now - start - if current == 0 or dif < 0.001: # One millisecond + if current == 0 or dif < 0.001: # One millisecond return None rate = float(current) / dif return int((float(total) - float(current)) / rate) @@ -92,7 +94,7 @@ def format_eta(eta): @staticmethod def calc_speed(start, now, bytes): dif = now - start - if bytes == 0 or dif < 0.001: # One millisecond + if bytes == 0 or dif < 0.001: # One millisecond return None return float(bytes) / dif @@ -105,7 +107,7 @@ def format_speed(speed): @staticmethod def best_block_size(elapsed_time, bytes): new_min = max(bytes / 2.0, 1.0) - new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB + new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB if elapsed_time < 0.001: return int(new_max) rate = bytes / elapsed_time @@ -143,18 +145,19 @@ def report_warning(self, *args, **kargs): def report_error(self, *args, **kargs): self.ydl.report_error(*args, **kargs) - def slow_down(self, start_time, byte_counter): + def slow_down(self, start_time, now, byte_counter): """Sleep if the download speed is over the rate limit.""" rate_limit = self.params.get('ratelimit', None) if rate_limit is None or byte_counter == 0: return - now = time.time() + if now is None: + now = time.time() elapsed = now - start_time if elapsed <= 0.0: return speed = float(byte_counter) / elapsed if speed > rate_limit: - time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + time.sleep((byte_counter / rate_limit) - elapsed) def temp_name(self, filename): """Returns a temporary filename for the given filename.""" diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index f79e6a995..462be2739 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -128,16 +128,21 @@ def real_download(self, filename, info_dict): byte_counter = 0 + resume_len block_size = self.params.get('buffersize', 1024) start = time.time() + + # measure time over whole while-loop, so slow_down() and best_block_size() work together properly + now = None # needed for slow_down() in the first loop run + before = start # start measuring while True: + # Download and write - before = time.time() data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter)) - after = time.time() - if len(data_block) == 0: - break byte_counter += len(data_block) - # Open file just in time + # exit loop when download is finished + if len(data_block) == 0: + break + + # Open destination file just in time if stream is None: try: (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) @@ -153,11 +158,22 @@ def real_download(self, filename, info_dict): self.to_stderr(u"\n") self.report_error(u'unable to write data: %s' % str(err)) return False + + # Apply rate limit + self.slow_down(start, now, byte_counter - resume_len) + + # end measuring of one loop run + now = time.time() + after = now + + # Adjust block size if not self.params.get('noresizebuffer', False): block_size = self.best_block_size(after - before, len(data_block)) + before = after + # Progress message - speed = self.calc_speed(start, time.time(), byte_counter - resume_len) + speed = self.calc_speed(start, now, byte_counter - resume_len) if data_len is None: eta = percent = None else: @@ -178,9 +194,6 @@ def real_download(self, filename, info_dict): if is_test and byte_counter == data_len: break - # Apply rate limit - self.slow_down(start, byte_counter - resume_len) - if stream is None: self.to_stderr(u"\n") self.report_error(u'Did not get any data blocks')