hyperboria/nexus/hub/services/submitter.py

189 lines
6.9 KiB
Python
Raw Normal View History

import hashlib
import logging
import time
from aiogrobid import GrobidClient
from aiogrobid.exceptions import BadRequestError
from grpc import (
Server,
ServicerContext,
)
from izihawa_utils.pb_to_json import MessageToDict
from library.aiogrpctools.base import aiogrpc_request_wrapper
from library.telegram.base import (
BaseTelegramClient,
RequestContext,
)
from nexus.hub.exceptions import (
FileTooBigError,
UnavailableMetadataError,
UnparsableDoiError,
)
from nexus.hub.proto.submitter_service_pb2 import \
SubmitRequest as SubmitRequestPb
from nexus.hub.proto.submitter_service_pb2 import \
SubmitResponse as SubmitResponsePb
from nexus.hub.proto.submitter_service_pb2_grpc import (
SubmitterServicer,
add_SubmitterServicer_to_server,
)
from nexus.hub.user_manager import UserManager
from nexus.meta_api.aioclient import MetaApiGrpcClient
from nexus.models.proto.operation_pb2 import \
DocumentOperation as DocumentOperationPb
from nexus.models.proto.operation_pb2 import UpdateDocument as UpdateDocumentPb
from nexus.models.proto.sharience_pb2 import Sharience as ShariencePb
from nexus.models.proto.typed_document_pb2 import \
TypedDocument as TypedDocumentPb
from nexus.translations import t
from nexus.views.telegram.common import close_button
from nexus.views.telegram.scimag import ScimagView
from telethon.extensions import BinaryReader
from .base import BaseHubService
async def operation_log(document_operation_pb):
logging.getLogger('operation').info(msg=MessageToDict(document_operation_pb))
class SubmitterService(SubmitterServicer, BaseHubService):
def __init__(
self,
server: Server,
service_name: str,
bot_external_name: str,
grobid_config: dict,
ipfs_config: dict,
meta_api_config: dict,
telegram_client: BaseTelegramClient,
):
super().__init__(
service_name=service_name,
bot_external_name=bot_external_name,
ipfs_config=ipfs_config,
telegram_client=telegram_client,
)
self.server = server
self.grobid_client = GrobidClient(base_url=grobid_config['url'])
self.meta_api_client = MetaApiGrpcClient(base_url=meta_api_config['url'])
self.telegram_client = telegram_client
self.bot_external_name = bot_external_name
self.user_manager = UserManager()
self.waits.extend([self.grobid_client, self.meta_api_client])
async def start(self):
add_SubmitterServicer_to_server(self, self.server)
async def stop(self):
await self.ipfs_client.close()
@aiogrpc_request_wrapper()
async def submit(
self,
request: SubmitRequestPb,
context: ServicerContext,
metadata: dict,
) -> SubmitResponsePb:
session_id = metadata.get('session-id')
request_context = RequestContext(
bot_name=self.service_name,
chat=request.chat,
request_id=metadata.get('request-id'),
)
request_context.add_default_fields(
mode='submit',
session_id=metadata.get('session-id'),
**self.get_default_service_fields(),
)
document = BinaryReader(request.telegram_document).tgread_object()
if document.size > 20 * 1024 * 1024:
request_context.error_log(FileTooBigError(size=document.size))
await self.telegram_client.send_message(
request_context.chat.id,
t('FILE_TOO_BIG_ERROR', language=request_context.chat.language),
buttons=[close_button()],
)
return SubmitResponsePb()
processing_message = await self.telegram_client.send_message(
request_context.chat.id,
t("PROCESSING_PAPER", language=request_context.chat.language).format(
filename=document.attributes[0].file_name,
),
)
try:
file = await self.telegram_client.download_document(document=document, file=bytes)
try:
processed_document = await self.grobid_client.process_fulltext_document(pdf_file=file)
except BadRequestError as e:
request_context.error_log(e)
await self.telegram_client.send_message(
request_context.chat.id,
t('UNPARSABLE_DOCUMENT_ERROR', language=request_context.chat.language),
buttons=[close_button()],
)
return SubmitResponsePb()
if not processed_document.get('doi'):
request_context.error_log(UnparsableDoiError())
await self.telegram_client.send_message(
request_context.chat.id,
t('UNPARSABLE_DOI_ERROR', language=request_context.chat.language),
buttons=[close_button()],
)
return SubmitResponsePb()
search_response_pb = await self.meta_api_client.search(
schemas=('scimag',),
query=processed_document['doi'],
page=0,
page_size=1,
request_id=request_context.request_id,
session_id=session_id,
user_id=request_context.chat.id,
language=request_context.chat.language,
)
if len(search_response_pb.scored_documents) == 0:
request_context.error_log(UnavailableMetadataError(doi=processed_document['doi']))
await self.telegram_client.send_message(
request_context.chat.id,
t(
'UNAVAILABLE_METADATA_ERROR',
language=request_context.chat.language
).format(doi=processed_document['doi']),
buttons=[close_button()],
)
return SubmitResponsePb()
document_view = ScimagView(search_response_pb.scored_documents[0].typed_document.scimag)
finally:
await processing_message.delete()
uploaded_message = await self.send_file(
document_view=document_view,
file=file,
request_context=request_context,
session_id=session_id,
voting=False,
)
document_operation_pb = DocumentOperationPb(
update_document=UpdateDocumentPb(
typed_document=TypedDocumentPb(sharience=ShariencePb(
parent_id=document_view.id,
uploader_id=request_context.chat.id,
updated_at=int(time.time()),
md5=hashlib.md5(file).hexdigest(),
filesize=document.size,
ipfs_multihashes=await self.get_ipfs_hashes(file=file),
telegram_file_id=uploaded_message.file.id,
)),
),
)
await operation_log(document_operation_pb)
return SubmitResponsePb()