71 lines
2.7 KiB
Python
71 lines
2.7 KiB
Python
import logging
|
|
from typing import Dict
|
|
|
|
import aiohttp.client_exceptions
|
|
from aiohttp_socks import ProxyError
|
|
from nexus.pylon.drivers.base import BaseDriver
|
|
from nexus.pylon.exceptions import (
|
|
BadResponseError,
|
|
NotFoundError,
|
|
)
|
|
from nexus.pylon.prepared_request import PreparedRequest
|
|
from nexus.pylon.proto.file_pb2 import Chunk as ChunkPb
|
|
from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb
|
|
from python_socks import ProxyTimeoutError
|
|
from tenacity import (
|
|
retry,
|
|
retry_if_exception_type,
|
|
stop_after_attempt,
|
|
wait_random,
|
|
)
|
|
|
|
|
|
class DirectDriver(BaseDriver):
|
|
allowed_content_type = None
|
|
|
|
@retry(
|
|
reraise=True,
|
|
wait=wait_random(min=1, max=2),
|
|
stop=stop_after_attempt(3),
|
|
retry=retry_if_exception_type((ProxyError, aiohttp.client_exceptions.ClientPayloadError, ProxyTimeoutError)),
|
|
)
|
|
async def execute_prepared_file_request(self, prepared_file_request: PreparedRequest, params: Dict):
|
|
logging.debug({
|
|
'action': 'download',
|
|
'mode': 'pylon',
|
|
'params': params,
|
|
'source': str(self),
|
|
'url': prepared_file_request.url,
|
|
})
|
|
async with self.get_session() as session:
|
|
async with prepared_file_request.execute_with(session=session) as resp:
|
|
logging.debug({
|
|
'action': 'response',
|
|
'mode': 'pylon',
|
|
'params': params,
|
|
'source': str(self),
|
|
'url': prepared_file_request.url,
|
|
'status': resp.status,
|
|
})
|
|
if resp.status == 404:
|
|
raise NotFoundError(url=prepared_file_request.url)
|
|
elif (
|
|
resp.status != 200
|
|
or (
|
|
self.allowed_content_type
|
|
and resp.headers.get('Content-Type', '').lower() not in self.allowed_content_type
|
|
)
|
|
):
|
|
raise BadResponseError(
|
|
request_headers=prepared_file_request.headers,
|
|
url=prepared_file_request.url,
|
|
status=resp.status,
|
|
headers=str(resp.headers),
|
|
)
|
|
file_validator = self.validator(params)
|
|
yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_file_request.url)
|
|
async for content, _ in resp.content.iter_chunks():
|
|
file_validator.update(content)
|
|
yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_file_request.url)
|
|
file_validator.validate()
|