[utils] Move FileDownloader.parse_bytes into utils

This commit is contained in:
pukkandan 2022-11-17 08:40:34 +05:30
parent 4de88a6a36
commit 64c464a144
No known key found for this signature in database
GPG Key ID: 7EEE9E1E817D0A39
3 changed files with 25 additions and 21 deletions

View File

@ -16,7 +16,6 @@
from .compat import compat_shlex_quote
from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
from .downloader import FileDownloader
from .downloader.external import get_external_downloader
from .extractor import list_extractor_classes
from .extractor.adobepass import MSO_INFO
@ -50,6 +49,7 @@
format_field,
int_or_none,
match_filter_func,
parse_bytes,
parse_duration,
preferredencoding,
read_batch_urls,
@ -281,19 +281,19 @@ def parse_sleep_func(expr):
raise ValueError(f'invalid {key} retry sleep expression {expr!r}')
# Bytes
def parse_bytes(name, value):
def validate_bytes(name, value):
if value is None:
return None
numeric_limit = FileDownloader.parse_bytes(value)
numeric_limit = parse_bytes(value)
validate(numeric_limit is not None, 'rate limit', value)
return numeric_limit
opts.ratelimit = parse_bytes('rate limit', opts.ratelimit)
opts.throttledratelimit = parse_bytes('throttled rate limit', opts.throttledratelimit)
opts.min_filesize = parse_bytes('min filesize', opts.min_filesize)
opts.max_filesize = parse_bytes('max filesize', opts.max_filesize)
opts.buffersize = parse_bytes('buffer size', opts.buffersize)
opts.http_chunk_size = parse_bytes('http chunk size', opts.http_chunk_size)
opts.ratelimit = validate_bytes('rate limit', opts.ratelimit)
opts.throttledratelimit = validate_bytes('throttled rate limit', opts.throttledratelimit)
opts.min_filesize = validate_bytes('min filesize', opts.min_filesize)
opts.max_filesize = validate_bytes('max filesize', opts.max_filesize)
opts.buffersize = validate_bytes('buffer size', opts.buffersize)
opts.http_chunk_size = validate_bytes('http chunk size', opts.http_chunk_size)
# Output templates
def validate_outtmpl(tmpl, msg):

View File

@ -15,7 +15,6 @@
from ..utils import (
IDENTITY,
NO_DEFAULT,
NUMBER_RE,
LockingUnsupportedError,
Namespace,
RetryManager,
@ -24,6 +23,7 @@
encodeFilename,
format_bytes,
join_nonempty,
parse_bytes,
remove_start,
sanitize_open,
shell_quote,
@ -180,12 +180,7 @@ def best_block_size(elapsed_time, bytes):
@staticmethod
def parse_bytes(bytestr):
"""Parse a string indicating a byte quantity into an integer."""
matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr)
if matchobj is None:
return None
number = float(matchobj.group(1))
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
return int(round(number * multiplier))
parse_bytes(bytestr)
def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit."""

View File

@ -2289,15 +2289,24 @@ def format_bytes(bytes):
return format_decimal_suffix(bytes, '%.2f%sB', factor=1024) or 'N/A'
def lookup_unit_table(unit_table, s):
def lookup_unit_table(unit_table, s, strict=False):
num_re = NUMBER_RE if strict else NUMBER_RE.replace(R'\.', '[,.]')
units_re = '|'.join(re.escape(u) for u in unit_table)
m = re.match(
r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
m = (re.fullmatch if strict else re.match)(
rf'(?P<num>{num_re})\s*(?P<unit>{units_re})\b', s)
if not m:
return None
num_str = m.group('num').replace(',', '.')
num = float(m.group('num').replace(',', '.'))
mult = unit_table[m.group('unit')]
return int(float(num_str) * mult)
return round(num * mult)
def parse_bytes(s):
"""Parse a string indicating a byte quantity into an integer"""
return lookup_unit_table(
{u: 1024**i for i, u in enumerate(['', *'KMGTPEZY'])},
s.upper(), strict=True)
def parse_filesize(s):