From b4a624fa12e0daebd63de09c7799d5c3a5add7ff Mon Sep 17 00:00:00 2001 From: the-superpirate Date: Tue, 13 Apr 2021 10:39:00 +0300 Subject: [PATCH 1/3] - feat(nexus): Increase amount of logging 1 internal commit(s) GitOrigin-RevId: 1baac946df3ea7cdf452bc880bb0d4764f6fc65e --- nexus/bot/handlers/download.py | 2 +- nexus/bot/handlers/referencing_to.py | 3 ++- nexus/bot/handlers/view.py | 4 ++-- nexus/bot/handlers/vote.py | 11 +++++++---- nexus/hub/services/base.py | 3 +++ nexus/hub/services/delivery.py | 15 ++++++++++++--- nexus/views/telegram/common.py | 5 +++-- 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/nexus/bot/handlers/download.py b/nexus/bot/handlers/download.py index 12c15bf..b63f09f 100644 --- a/nexus/bot/handlers/download.py +++ b/nexus/bot/handlers/download.py @@ -22,7 +22,7 @@ class DownloadHandler(BaseCallbackQueryHandler): self.application.user_manager.last_widget[request_context.chat.chat_id] = None request_context.add_default_fields(mode='download', session_id=session_id) - request_context.statbox(action='get', query=str(document_id), position=position) + request_context.statbox(action='get', document_id=document_id, position=position, schema=schema) typed_document_pb = await self.get_typed_document_pb( schema=schema, diff --git a/nexus/bot/handlers/referencing_to.py b/nexus/bot/handlers/referencing_to.py index 00ac7d7..ef6160b 100644 --- a/nexus/bot/handlers/referencing_to.py +++ b/nexus/bot/handlers/referencing_to.py @@ -36,7 +36,8 @@ class ReferencingToHandler(BaseCallbackQueryHandler): request_context.statbox( action=action, duration=time.time() - start_time, - query=f'{document_id}', + document_id=document_id, + schema='scimag', ) serp, buttons = await document_list_widget.render() diff --git a/nexus/bot/handlers/view.py b/nexus/bot/handlers/view.py index 32f8440..bda0eb3 100644 --- a/nexus/bot/handlers/view.py +++ b/nexus/bot/handlers/view.py @@ -13,7 +13,7 @@ from .base import BaseHandler class ViewHandler(BaseHandler): - filter = events.NewMessage(incoming=True, pattern='^/v([abcm])([sr])?_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_' + filter = events.NewMessage(incoming=True, pattern='^/v([ab])([sr])?_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_' '([0-9]+)') should_reset_last_widget = False @@ -29,7 +29,7 @@ class ViewHandler(BaseHandler): page = int(position / self.application.config['application']['page_size']) request_context.add_default_fields(mode='view', session_id=session_id) - request_context.statbox(action='view', query=str(document_id), position=position) + request_context.statbox(action='view', document_id=document_id, position=position, schema=schema) found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id) try: diff --git a/nexus/bot/handlers/vote.py b/nexus/bot/handlers/vote.py index 4630558..520e517 100644 --- a/nexus/bot/handlers/vote.py +++ b/nexus/bot/handlers/vote.py @@ -13,12 +13,14 @@ from .base import BaseCallbackQueryHandler class VoteHandler(BaseCallbackQueryHandler): - filter = events.CallbackQuery(pattern='^/vote_([A-Za-z0-9]+)_([0-9]+)_([bo])$') + filter = events.CallbackQuery(pattern='^/vote([ab])?_([A-Za-z0-9]+)_([0-9]+)_([bo])$') async def handler(self, event: events.ChatAction, request_context: RequestContext): - session_id = event.pattern_match.group(1).decode() - document_id = int(event.pattern_match.group(2).decode()) - vote = event.pattern_match.group(3).decode() + short_schema = event.pattern_match.group(1) + schema = self.short_schema_to_schema(short_schema) if short_schema else None + session_id = event.pattern_match.group(2).decode() + document_id = int(event.pattern_match.group(3).decode()) + vote = event.pattern_match.group(4).decode() vote_value = {'b': -1, 'o': 1}[vote] request_context.add_default_fields(mode='vote', session_id=session_id) @@ -33,6 +35,7 @@ class VoteHandler(BaseCallbackQueryHandler): request_context.statbox( action='vote', document_id=document_id, + schema=schema, ) logging.getLogger('operation').info( msg=MessageToDict(document_operation_pb), diff --git a/nexus/hub/services/base.py b/nexus/hub/services/base.py index 45c8dcc..25423b5 100644 --- a/nexus/hub/services/base.py +++ b/nexus/hub/services/base.py @@ -54,12 +54,14 @@ class BaseHubService(BaseService): buttons = [ vote_button( case='broken', + schema=document_view.schema, document_id=document_id, language=request_context.chat.language, session_id=session_id, ), vote_button( case='ok', + schema=document_view.schema, document_id=document_id, language=request_context.chat.language, session_id=session_id, @@ -77,6 +79,7 @@ class BaseHubService(BaseService): request_context.statbox( action='sent', document_id=document_id, + schema=document_view.schema, voting=voting, ) return message diff --git a/nexus/hub/services/delivery.py b/nexus/hub/services/delivery.py index 1d61ad8..f07e6a1 100644 --- a/nexus/hub/services/delivery.py +++ b/nexus/hub/services/delivery.py @@ -123,6 +123,7 @@ class DownloadTask: action='missed', duration=time.time() - start_time, document_id=document_view.id, + schema=document_view.schema, ) is_served_from_sharience = False if self.delivery_service.is_sharience_enabled: @@ -135,6 +136,7 @@ class DownloadTask: action='not_found', document_id=document_view.id, duration=time.time() - start_time, + schema=document_view.schema, ) await self.respond_not_found( request_context=request_context, @@ -147,6 +149,7 @@ class DownloadTask: duration=time.time() - start_time, document_id=document_view.id, len=len(file), + schema=document_view.schema, ) progress_bar_upload = ProgressBar( @@ -171,6 +174,7 @@ class DownloadTask: action='uploaded', duration=time.time() - start_time, document_id=document_view.id, + schema=document_view.schema, ) if self.delivery_service.should_store_hashes: asyncio.create_task(self.store_hashes( @@ -185,6 +189,7 @@ class DownloadTask: action='user_canceled', duration=time.time() - start_time, document_id=document_view.id, + schema=document_view.schema, ) except asyncio.CancelledError: pass @@ -215,7 +220,7 @@ class DownloadTask: async def try_sharience(self, request_context, document_view): if document_view.doi: - request_context.statbox(action='try_sharience', doi=document_view.doi) + request_context.statbox(action='try_sharience', doi=document_view.doi, schema=document_view.schema) pg_data = await self.delivery_service.pool_holder.execute( ''' select sh.id, sh.telegram_file_id as vote_sum @@ -273,7 +278,11 @@ class DownloadTask: async def external_cancel(self): self.task.cancel() - self.request_context.statbox(action='externally_canceled') + self.request_context.statbox( + action='externally_canceled', + document_id=self.document_view.id, + schema=self.document_view.schema, + ) await self.delivery_service.telegram_client.send_message( self.request_context.chat.chat_id, t("DOWNLOAD_CANCELED", language=self.request_context.chat.language).format( @@ -372,7 +381,7 @@ class DeliveryService(DeliveryServicer, BaseHubService): request_context=request_context, voting=not is_group_or_channel(request_context.chat.chat_id), ) - request_context.statbox(action='cache_hit', document_id=document_view.id) + request_context.statbox(action='cache_hit', document_id=document_view.id, schema=document_view.schema) except ValueError: cache_hit = False if not cache_hit: diff --git a/nexus/views/telegram/common.py b/nexus/views/telegram/common.py index 5efceeb..dce660c 100644 --- a/nexus/views/telegram/common.py +++ b/nexus/views/telegram/common.py @@ -30,12 +30,13 @@ def close_button(session_id: str = None): ) -def vote_button(language: str, session_id: str, document_id: int, case: str): +def vote_button(language: str, session_id: str, schema: str, document_id: int, case: str): label = f"REPORT_{case.upper()}_FILE" case = {'broken': 'b', 'ok': 'o'}[case] + schema = {'scimag': 'a', 'scitech': 'b'}[schema] return Button.inline( text=t(label, language=language), - data=f'/vote_{session_id}_{document_id}_{case}', + data=f'/vote{schema}_{session_id}_{document_id}_{case}', ) From 422959914cdba51dbf589fe9bc8cd26f8da626dd Mon Sep 17 00:00:00 2001 From: the-superpirate Date: Tue, 13 Apr 2021 12:03:44 +0300 Subject: [PATCH 2/3] - feat(nexus): Slightly refactor bot - feat(nexus): Add vote to logging 1 internal commit(s) GitOrigin-RevId: 8686e431de9a7af6c6763fba2b50d62d6275667b --- nexus/bot/exceptions.py | 15 ++++ nexus/bot/handlers/ban.py | 7 +- nexus/bot/handlers/download.py | 7 +- nexus/bot/handlers/search.py | 122 ++++++++++++++++++--------------- nexus/bot/handlers/view.py | 101 ++++++++++++++++++--------- nexus/bot/handlers/vote.py | 22 +++--- 6 files changed, 177 insertions(+), 97 deletions(-) diff --git a/nexus/bot/exceptions.py b/nexus/bot/exceptions.py index 584ba33..f67e3bd 100644 --- a/nexus/bot/exceptions.py +++ b/nexus/bot/exceptions.py @@ -1,6 +1,21 @@ +import logging + from izihawa_utils.exceptions import BaseError +class BannedUserError(BaseError): + level = logging.WARNING + code = 'banned_user_error' + + def __init__(self, ban_timeout: int): + self.ban_timeout = ban_timeout + + +class MessageHasBeenDeletedError(BaseError): + level = logging.WARNING + code = 'message_has_been_deleted_error' + + class UnknownFileFormatError(BaseError): code = 'unknown_file_format_error' diff --git a/nexus/bot/handlers/ban.py b/nexus/bot/handlers/ban.py index f07a86b..fbdb92b 100644 --- a/nexus/bot/handlers/ban.py +++ b/nexus/bot/handlers/ban.py @@ -15,12 +15,17 @@ from .admin import BaseAdminHandler class BanHandler(BaseAdminHandler): filter = events.NewMessage(incoming=True, pattern='^/ban ([0-9]+) ([A-Za-z0-9]+)\\s?(.*)?$') - async def handler(self, event: events.ChatAction, request_context: RequestContext): + def parse_pattern(self, event: events.ChatAction): chat_id = int(event.pattern_match.group(1)) ban_duration = event.pattern_match.group(2) ban_message = event.pattern_match.group(3) ban_end_date = datetime.utcnow() + timedelta(seconds=timeparse(ban_duration)) + return chat_id, ban_duration, ban_message, ban_end_date + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + chat_id, ban_duration, ban_message, ban_end_date = self.parse_pattern(event) + try: await self.application.idm_client.update_chat( chat_id=chat_id, diff --git a/nexus/bot/handlers/download.py b/nexus/bot/handlers/download.py index b63f09f..64072cf 100644 --- a/nexus/bot/handlers/download.py +++ b/nexus/bot/handlers/download.py @@ -12,13 +12,18 @@ class DownloadHandler(BaseCallbackQueryHandler): filter = events.CallbackQuery(pattern='^/dl([abcm])_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$') is_group_handler = True - async def handler(self, event: events.ChatAction, request_context: RequestContext): + def parse_pattern(self, event: events.ChatAction): short_schema = event.pattern_match.group(1).decode() schema = self.short_schema_to_schema(short_schema) session_id = event.pattern_match.group(2).decode() document_id = int(event.pattern_match.group(3)) position = int(event.pattern_match.group(4).decode()) + return short_schema, schema, session_id, document_id, position + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + short_schema, schema, session_id, document_id, position = self.parse_pattern(event) + self.application.user_manager.last_widget[request_context.chat.chat_id] = None request_context.add_default_fields(mode='download', session_id=session_id) diff --git a/nexus/bot/handlers/search.py b/nexus/bot/handlers/search.py index d229942..255e57d 100644 --- a/nexus/bot/handlers/search.py +++ b/nexus/bot/handlers/search.py @@ -1,11 +1,14 @@ import asyncio -import logging import re import time from grpc import StatusCode from grpc.experimental.aio import AioRpcError from library.telegram.base import RequestContext +from nexus.bot.exceptions import ( + BannedUserError, + MessageHasBeenDeletedError, +) from nexus.bot.widgets.search_widget import SearchWidget from nexus.translations import t from nexus.views.telegram.common import close_button @@ -141,40 +144,41 @@ class SearchHandler(BaseSearchHandler): should_reset_last_widget = False is_subscription_required_for_handler = True - async def ban_handler(self, event: events.ChatAction, request_context: RequestContext, ban_timeout: float): - logging.getLogger('statbox').info({ - 'bot_name': self.application.config['telegram']['bot_name'], - 'action': 'user_flood_ban', - 'mode': 'search', - 'ban_timeout_seconds': ban_timeout, - 'chat_id': request_context.chat.chat_id, - }) - ban_reason = t( - 'BAN_MESSAGE_TOO_MANY_REQUESTS', - language=request_context.chat.language - ) - return await event.reply(t( - 'BANNED_FOR_SECONDS', - language=request_context.chat.language - ).format( - seconds=str(ban_timeout), - reason=ban_reason, - )) - - async def handler(self, event: events.ChatAction, request_context: RequestContext): - ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.chat_id) + def check_search_ban_timeout(self, chat_id: int): + ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=chat_id) if ban_timeout: - return await self.ban_handler(event, request_context, ban_timeout) - self.application.user_manager.add_search_time(user_id=request_context.chat.chat_id, search_time=time.time()) + raise BannedUserError(ban_timeout=ban_timeout) + self.application.user_manager.add_search_time(user_id=chat_id, search_time=time.time()) + def parse_pattern(self, event: events.ChatAction): search_prefix = event.pattern_match.group(1) query = event.pattern_match.group(2) is_group_mode = event.is_group or event.is_channel + return search_prefix, query, is_group_mode + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + try: + self.check_search_ban_timeout(chat_id=request_context.chat.chat_id) + except BannedUserError as e: + request_context.error_log(e) + return await event.reply(t( + 'BANNED_FOR_SECONDS', + language=request_context.chat.language + ).format( + seconds=e.ban_timeout, + reason=t( + 'BAN_MESSAGE_TOO_MANY_REQUESTS', + language=request_context.chat.language + ), + )) + search_prefix, query, is_group_mode = self.parse_pattern(event) + if is_group_mode and not search_prefix: return if not is_group_mode and search_prefix: query = event.raw_text + prefetch_message = await event.reply( t("SEARCHING", language=request_context.chat.language), ) @@ -199,38 +203,46 @@ class SearchEditHandler(BaseSearchHandler): is_group_handler = True should_reset_last_widget = False - async def handler(self, event: events.ChatAction, request_context: RequestContext): - request_context.add_default_fields(mode='search_edit') + def parse_pattern(self, event: events.ChatAction): search_prefix = event.pattern_match.group(1) query = event.pattern_match.group(2) is_group_mode = event.is_group or event.is_channel + return search_prefix, query, is_group_mode + + async def get_last_messages_in_chat(self, event: events.ChatAction): + return await self.application.telegram_client(functions.messages.GetMessagesRequest( + id=list(range(event.id + 1, event.id + 10))) + ) + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + search_prefix, query, is_group_mode = self.parse_pattern(event) + request_context.add_default_fields(mode='search_edit') + if is_group_mode and not search_prefix: return if not is_group_mode and search_prefix: query = event.raw_text - result = await self.application.telegram_client(functions.messages.GetMessagesRequest( - id=list(range(event.id + 1, event.id + 10))) - ) - if not result: - request_context.statbox(action='failed') + + last_messages = await self.get_last_messages_in_chat(event) + try: + if not last_messages: + raise MessageHasBeenDeletedError() + for next_message in last_messages.messages: + if next_message.is_reply and event.id == next_message.reply_to_msg_id: + request_context.statbox(action='resolved') + return await self.do_search( + event, + request_context, + prefetch_message=next_message, + query=query, + is_group_mode=is_group_mode, + ) + raise MessageHasBeenDeletedError() + except MessageHasBeenDeletedError as e: + request_context.error_log(e) return await event.reply( t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language), ) - for next_message in result.messages: - if next_message.is_reply and event.id == next_message.reply_to_msg_id: - request_context.statbox(action='resolved') - await self.do_search( - event, - request_context, - prefetch_message=next_message, - query=query, - is_group_mode=is_group_mode, - ) - return - request_context.statbox(action='failed') - return await event.reply( - t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language), - ) class SearchPagingHandler(BaseCallbackQueryHandler): @@ -243,19 +255,17 @@ class SearchPagingHandler(BaseCallbackQueryHandler): page = int(event.pattern_match.group(3).decode()) request_context.add_default_fields(mode='search_paging', session_id=session_id) + start_time = time.time() message = await event.get_message() if not message: return await event.answer() - reply_message = await message.get_reply_message() - if not reply_message: - return await event.respond( - t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language), - ) - start_time = time.time() - query = reply_message.raw_text + reply_message = await message.get_reply_message() try: + if not reply_message: + raise MessageHasBeenDeletedError() + query = reply_message.raw_text search_widget = await SearchWidget.create( application=self.application, chat=request_context.chat, @@ -265,6 +275,10 @@ class SearchPagingHandler(BaseCallbackQueryHandler): query=query, page=page, ) + except MessageHasBeenDeletedError: + return await event.respond( + t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language), + ) except AioRpcError as e: if e.code() == StatusCode.INVALID_ARGUMENT or e.code() == StatusCode.CANCELLED: request_context.error_log(e) diff --git a/nexus/bot/handlers/view.py b/nexus/bot/handlers/view.py index bda0eb3..0412c7c 100644 --- a/nexus/bot/handlers/view.py +++ b/nexus/bot/handlers/view.py @@ -2,6 +2,7 @@ import asyncio import re from library.telegram.base import RequestContext +from nexus.bot.exceptions import MessageHasBeenDeletedError from nexus.translations import t from telethon import ( events, @@ -17,7 +18,7 @@ class ViewHandler(BaseHandler): '([0-9]+)') should_reset_last_widget = False - async def handler(self, event: events.ChatAction, request_context: RequestContext): + def parse_pattern(self, event: events.ChatAction): short_schema = event.pattern_match.group(1) parent_view_type = event.pattern_match.group(2) or 's' schema = self.short_schema_to_schema(short_schema) @@ -28,26 +29,63 @@ class ViewHandler(BaseHandler): page = int(position / self.application.config['application']['page_size']) + return parent_view_type, schema, session_id, old_message_id, document_id, position, page + + async def process_widgeting(self, has_found_old_widget, old_message_id, request_context: RequestContext): + if has_found_old_widget: + message_id = old_message_id + link_preview = None + else: + old_message = (await self.application.telegram_client( + functions.messages.GetMessagesRequest(id=[old_message_id]) + )).messages[0] + prefetch_message = await self.application.telegram_client.send_message( + request_context.chat.chat_id, + t("SEARCHING", language=request_context.chat.language), + reply_to=old_message.reply_to_msg_id, + ) + self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id + message_id = prefetch_message.id + link_preview = True + return message_id, link_preview + + async def compose_back_command( + self, + parent_view_type, + session_id, + old_message_id, + message_id, + page, + ): + back_command = None + if parent_view_type == 's': + back_command = f'/search_{session_id}_{message_id}_{page}' + elif parent_view_type == 'r': + messages = (await self.application.telegram_client( + functions.messages.GetMessagesRequest(id=[old_message_id]) + )).messages + if not messages: + raise MessageHasBeenDeletedError() + message = messages[0] + referencing_to = re.search(r'Linked to: ([0-9]+)', message.raw_text).group(1) + back_command = f'/rp_{session_id}_{message_id}_{referencing_to}_{page}' + + return back_command + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + parent_view_type, schema, session_id, old_message_id, document_id, position, page = self.parse_pattern(event) + request_context.add_default_fields(mode='view', session_id=session_id) request_context.statbox(action='view', document_id=document_id, position=position, schema=schema) - found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id) + + has_found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id) try: - if found_old_widget: - message_id = old_message_id - link_preview = None - else: - old_message = (await self.application.telegram_client( - functions.messages.GetMessagesRequest(id=[old_message_id]) - )).messages[0] - prefetch_message = await self.application.telegram_client.send_message( - request_context.chat.chat_id, - t("SEARCHING", language=request_context.chat.language), - reply_to=old_message.reply_to_msg_id, - ) - self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id - message_id = prefetch_message.id - link_preview = True + message_id, link_preview = await self.process_widgeting( + has_found_old_widget=has_found_old_widget, + old_message_id=old_message_id, + request_context=request_context + ) document_view = await self.resolve_document( schema, @@ -56,21 +94,18 @@ class ViewHandler(BaseHandler): session_id, request_context, ) - - back_command = None - if parent_view_type == 's': - back_command = f'/search_{session_id}_{message_id}_{page}' - elif parent_view_type == 'r': - messages = (await self.application.telegram_client( - functions.messages.GetMessagesRequest(id=[old_message_id]) - )).messages - if not messages: - return await event.respond( - t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language), - ) - message = messages[0] - referencing_to = re.search(r'Linked to: ([0-9]+)', message.raw_text).group(1) - back_command = f'/rp_{session_id}_{message_id}_{referencing_to}_{page}' + try: + back_command = await self.compose_back_command( + parent_view_type=parent_view_type, + session_id=session_id, + old_message_id=old_message_id, + message_id=message_id, + page=page, + ) + except MessageHasBeenDeletedError: + return await event.respond( + t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language), + ) view, buttons = document_view.get_view( language=request_context.chat.language, @@ -89,7 +124,7 @@ class ViewHandler(BaseHandler): ), event.delete(), ] - if not found_old_widget: + if not has_found_old_widget: actions.append( self.application.telegram_client.delete_messages( request_context.chat.chat_id, diff --git a/nexus/bot/handlers/vote.py b/nexus/bot/handlers/vote.py index 520e517..da39355 100644 --- a/nexus/bot/handlers/vote.py +++ b/nexus/bot/handlers/vote.py @@ -15,14 +15,26 @@ from .base import BaseCallbackQueryHandler class VoteHandler(BaseCallbackQueryHandler): filter = events.CallbackQuery(pattern='^/vote([ab])?_([A-Za-z0-9]+)_([0-9]+)_([bo])$') - async def handler(self, event: events.ChatAction, request_context: RequestContext): + def parse_pattern(self, event: events.ChatAction): short_schema = event.pattern_match.group(1) - schema = self.short_schema_to_schema(short_schema) if short_schema else None + schema = self.short_schema_to_schema(short_schema.decode()) if short_schema else None session_id = event.pattern_match.group(2).decode() document_id = int(event.pattern_match.group(3).decode()) vote = event.pattern_match.group(4).decode() vote_value = {'b': -1, 'o': 1}[vote] + + return schema, session_id, document_id, vote, vote_value + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + schema, session_id, document_id, vote, vote_value = self.parse_pattern(event) + request_context.add_default_fields(mode='vote', session_id=session_id) + request_context.statbox( + action='vote', + document_id=document_id, + query=vote, + schema=schema, + ) document_operation_pb = DocumentOperationPb( vote=VotePb( @@ -31,12 +43,6 @@ class VoteHandler(BaseCallbackQueryHandler): voter_id=request_context.chat.chat_id, ), ) - - request_context.statbox( - action='vote', - document_id=document_id, - schema=schema, - ) logging.getLogger('operation').info( msg=MessageToDict(document_operation_pb), ) From 6de3fb1250f5f8cf765f293cdda887896e540905 Mon Sep 17 00:00:00 2001 From: the-superpirate Date: Tue, 13 Apr 2021 17:23:31 +0300 Subject: [PATCH 3/3] - feat(bot): Add top-missing cmd - feat(bot): Refactor referencing handler - fix(bot): Refactor DocumentListWidget 1 internal commit(s) GitOrigin-RevId: 9f388e98c7039711927abf4a048b5c45ea7a2fc3 --- nexus/bot/configs/base.yaml | 3 +- nexus/bot/handlers/__init__.py | 6 +- nexus/bot/handlers/referencing_to.py | 95 ------------- nexus/bot/handlers/search.py | 33 +++-- nexus/bot/handlers/top_missed.py | 54 +++++++ nexus/bot/handlers/view.py | 25 +--- nexus/bot/widgets/document_list_widget.py | 90 +++--------- nexus/meta_api/aioclient/aioclient.py | 24 ++++ nexus/meta_api/proto/documents_service.proto | 13 ++ nexus/meta_api/proto/documents_service_pb2.py | 133 +++++++++++++++++- .../proto/documents_service_pb2_grpc.py | 33 +++++ .../meta_api/proto/meta_search_service.proto | 4 +- .../meta_api/proto/meta_search_service_pb2.py | 19 +-- nexus/views/telegram/scimag.py | 15 +- 14 files changed, 318 insertions(+), 229 deletions(-) delete mode 100644 nexus/bot/handlers/referencing_to.py create mode 100644 nexus/bot/handlers/top_missed.py diff --git a/nexus/bot/configs/base.yaml b/nexus/bot/configs/base.yaml index f0c32c8..fb86766 100644 --- a/nexus/bot/configs/base.yaml +++ b/nexus/bot/configs/base.yaml @@ -79,8 +79,6 @@ telegram: - nexus.bot.handlers.download.DownloadHandler - nexus.bot.handlers.emoji.EmojiHandler - nexus.bot.handlers.help.HelpHandler - - nexus.bot.handlers.referencing_to.ReferencingToHandler - - nexus.bot.handlers.referencing_to.ReferencingToPagingHandler - nexus.bot.handlers.roll.RollHandler - nexus.bot.handlers.settings.SettingsButtonsHandler - nexus.bot.handlers.settings.SettingsHandler @@ -88,6 +86,7 @@ telegram: - nexus.bot.handlers.submit.SubmitHandler - nexus.bot.handlers.start.StartHandler - nexus.bot.handlers.stop.StopHandler + - nexus.bot.handlers.top_missed.TopMissedHandler - nexus.bot.handlers.view.ViewHandler - nexus.bot.handlers.vote.VoteHandler - nexus.bot.handlers.noop.NoopHandler diff --git a/nexus/bot/handlers/__init__.py b/nexus/bot/handlers/__init__.py index b96fa03..79072ec 100644 --- a/nexus/bot/handlers/__init__.py +++ b/nexus/bot/handlers/__init__.py @@ -10,7 +10,6 @@ from . import ( help, legacy, noop, - referencing_to, roll, search, settings, @@ -18,10 +17,11 @@ from . import ( start, stop, submit, + top_missed, view, vote, ) __all__ = ['admin', 'ban', 'contact', 'copyright', 'close', 'donate', 'download', 'emoji', 'help', - 'legacy', 'noop', 'referencing_to', 'roll', 'search', 'settings', - 'shortlink', 'start', 'stop', 'submit', 'view', 'vote'] + 'legacy', 'noop', 'roll', 'search', 'settings', + 'shortlink', 'start', 'stop', 'submit', 'top_missed', 'view', 'vote'] diff --git a/nexus/bot/handlers/referencing_to.py b/nexus/bot/handlers/referencing_to.py deleted file mode 100644 index ef6160b..0000000 --- a/nexus/bot/handlers/referencing_to.py +++ /dev/null @@ -1,95 +0,0 @@ -import re -import time - -from grpc.experimental.aio import AioRpcError -from library.telegram.base import RequestContext -from nexus.bot.widgets.document_list_widget import DocumentListWidget -from nexus.translations import t -from telethon import events - -from .base import BaseCallbackQueryHandler - - -class ReferencingToHandler(BaseCallbackQueryHandler): - filter = events.CallbackQuery(pattern=re.compile('^/r_([A-Za-z0-9]+)_([0-9]+)', re.DOTALL)) - should_reset_last_widget = False - - async def do_request(self, request_context: RequestContext, session_id: str, message_id: int, document_id: int, page: int): - start_time = time.time() - try: - document_list_widget = await DocumentListWidget.create( - application=self.application, - chat=request_context.chat, - session_id=session_id, - message_id=message_id, - request_id=request_context.request_id, - referencing_to=document_id, - page=page, - ) - except AioRpcError as e: - raise e - - action = 'referencing_to_found' - if len(document_list_widget.scored_documents) == 0: - action = 'referencing_to_not_found' - - request_context.statbox( - action=action, - duration=time.time() - start_time, - document_id=document_id, - schema='scimag', - ) - - serp, buttons = await document_list_widget.render() - return await self.application.telegram_client.edit_message( - request_context.chat.chat_id, - message_id, - serp, - buttons=buttons, - link_preview=False, - ) - - async def handler(self, event, request_context: RequestContext): - session_id = event.pattern_match.group(1).decode() - document_id = int(event.pattern_match.group(2).decode()) - - request_context.add_default_fields( - mode='referencing_to', - session_id=session_id, - ) - - prefetch_message = await event.respond( - t("SEARCHING", language=request_context.chat.language), - ) - message_id = prefetch_message.id - - return await self.do_request( - request_context=request_context, - session_id=session_id, - message_id=message_id, - document_id=document_id, - page=0, - ) - - -class ReferencingToPagingHandler(ReferencingToHandler): - filter = events.CallbackQuery(pattern=re.compile('^/rp_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)', re.DOTALL)) - - async def handler(self, event: events.ChatAction, request_context: RequestContext): - session_id = event.pattern_match.group(1).decode() - message_id = int(event.pattern_match.group(2).decode()) - document_id = int(event.pattern_match.group(3).decode()) - page = int(event.pattern_match.group(4).decode()) - - request_context.add_default_fields( - mode='referencing_to_paging', - session_id=session_id, - ) - - return await self.do_request( - request_context=request_context, - session_id=session_id, - message_id=message_id, - document_id=document_id, - page=page, - ) diff --git a/nexus/bot/handlers/search.py b/nexus/bot/handlers/search.py index 255e57d..e373e2a 100644 --- a/nexus/bot/handlers/search.py +++ b/nexus/bot/handlers/search.py @@ -1,6 +1,7 @@ import asyncio import re import time +from abc import ABC from grpc import StatusCode from grpc.experimental.aio import AioRpcError @@ -24,8 +25,9 @@ from .base import ( ) -class BaseSearchHandler(BaseHandler): - should_reset_last_widget = False +class BaseSearchHandler(BaseHandler, ABC): + def preprocess_query(self, query): + return query.replace(f'@{self.application.config["telegram"]["bot_external_name"]}', '').strip() async def do_search( self, @@ -38,10 +40,9 @@ class BaseSearchHandler(BaseHandler): ): session_id = self.generate_session_id() message_id = prefetch_message.id - request_context.add_default_fields(is_group_mode=is_group_mode, mode='search', session_id=session_id) - start_time = time.time() + try: search_widget = await SearchWidget.create( application=self.application, @@ -152,7 +153,7 @@ class SearchHandler(BaseSearchHandler): def parse_pattern(self, event: events.ChatAction): search_prefix = event.pattern_match.group(1) - query = event.pattern_match.group(2) + query = self.preprocess_query(event.pattern_match.group(2)) is_group_mode = event.is_group or event.is_channel return search_prefix, query, is_group_mode @@ -185,7 +186,9 @@ class SearchHandler(BaseSearchHandler): self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id try: await self.do_search( - event, request_context, prefetch_message, + event=event, + request_context=request_context, + prefetch_message=prefetch_message, query=query, is_group_mode=is_group_mode, is_shortpath_enabled=True, @@ -205,7 +208,7 @@ class SearchEditHandler(BaseSearchHandler): def parse_pattern(self, event: events.ChatAction): search_prefix = event.pattern_match.group(1) - query = event.pattern_match.group(2) + query = self.preprocess_query(event.pattern_match.group(2)) is_group_mode = event.is_group or event.is_channel return search_prefix, query, is_group_mode @@ -231,8 +234,8 @@ class SearchEditHandler(BaseSearchHandler): if next_message.is_reply and event.id == next_message.reply_to_msg_id: request_context.statbox(action='resolved') return await self.do_search( - event, - request_context, + event=event, + request_context=request_context, prefetch_message=next_message, query=query, is_group_mode=is_group_mode, @@ -249,11 +252,19 @@ class SearchPagingHandler(BaseCallbackQueryHandler): filter = events.CallbackQuery(pattern='^/search_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$') should_reset_last_widget = False - async def handler(self, event: events.ChatAction, request_context: RequestContext): + def preprocess_query(self, query): + return query.replace(f'@{self.application.config["telegram"]["bot_external_name"]}', '').strip() + + def parse_pattern(self, event: events.ChatAction): session_id = event.pattern_match.group(1).decode() message_id = int(event.pattern_match.group(2).decode()) page = int(event.pattern_match.group(3).decode()) + return session_id, message_id, page + + async def handler(self, event: events.ChatAction, request_context: RequestContext): + session_id, message_id, page = self.parse_pattern(event) + request_context.add_default_fields(mode='search_paging', session_id=session_id) start_time = time.time() @@ -265,7 +276,7 @@ class SearchPagingHandler(BaseCallbackQueryHandler): try: if not reply_message: raise MessageHasBeenDeletedError() - query = reply_message.raw_text + query = self.preprocess_query(reply_message.raw_text) search_widget = await SearchWidget.create( application=self.application, chat=request_context.chat, diff --git a/nexus/bot/handlers/top_missed.py b/nexus/bot/handlers/top_missed.py new file mode 100644 index 0000000..2dd8a38 --- /dev/null +++ b/nexus/bot/handlers/top_missed.py @@ -0,0 +1,54 @@ +from library.telegram.base import RequestContext +from nexus.bot.widgets.document_list_widget import DocumentListWidget +from nexus.translations import t +from telethon import events + +from .base import BaseHandler + + +class TopMissedHandler(BaseHandler): + filter = events.NewMessage(incoming=True, pattern='^/tm$') + is_group_handler = False + should_reset_last_widget = False + + async def do_request(self, request_context: RequestContext, session_id: str, message_id: int, page: int): + response = await self.application.meta_api_client.top_missed( + page=page, + page_size=10, + session_id=session_id, + request_id=request_context.request_id, + ) + document_list_widget = DocumentListWidget( + application=self.application, + chat=request_context.chat, + typed_documents=response.typed_documents, + cmd='tm', + has_next=response.has_next, + session_id=session_id, + message_id=message_id, + request_id=request_context.request_id, + page=page, + ) + serp, buttons = await document_list_widget.render() + return await self.application.telegram_client.edit_message( + request_context.chat.chat_id, + message_id, + serp, + buttons=buttons, + link_preview=False, + ) + + async def handler(self, event, request_context: RequestContext): + session_id = self.generate_session_id() + request_context.add_default_fields(mode='top_missed', session_id=session_id) + request_context.statbox() + + prefetch_message = await event.reply(t("SEARCHING", language=request_context.chat.language)) + message_id = prefetch_message.id + + return await self.do_request( + request_context=request_context, + session_id=session_id, + message_id=message_id, + page=0, + ) diff --git a/nexus/bot/handlers/view.py b/nexus/bot/handlers/view.py index 0412c7c..ca4d402 100644 --- a/nexus/bot/handlers/view.py +++ b/nexus/bot/handlers/view.py @@ -1,5 +1,4 @@ import asyncio -import re from library.telegram.base import RequestContext from nexus.bot.exceptions import MessageHasBeenDeletedError @@ -20,7 +19,6 @@ class ViewHandler(BaseHandler): def parse_pattern(self, event: events.ChatAction): short_schema = event.pattern_match.group(1) - parent_view_type = event.pattern_match.group(2) or 's' schema = self.short_schema_to_schema(short_schema) session_id = event.pattern_match.group(3) old_message_id = int(event.pattern_match.group(4)) @@ -29,7 +27,7 @@ class ViewHandler(BaseHandler): page = int(position / self.application.config['application']['page_size']) - return parent_view_type, schema, session_id, old_message_id, document_id, position, page + return schema, session_id, old_message_id, document_id, position, page async def process_widgeting(self, has_found_old_widget, old_message_id, request_context: RequestContext): if has_found_old_widget: @@ -51,29 +49,14 @@ class ViewHandler(BaseHandler): async def compose_back_command( self, - parent_view_type, session_id, - old_message_id, message_id, page, ): - back_command = None - if parent_view_type == 's': - back_command = f'/search_{session_id}_{message_id}_{page}' - elif parent_view_type == 'r': - messages = (await self.application.telegram_client( - functions.messages.GetMessagesRequest(id=[old_message_id]) - )).messages - if not messages: - raise MessageHasBeenDeletedError() - message = messages[0] - referencing_to = re.search(r'Linked to: ([0-9]+)', message.raw_text).group(1) - back_command = f'/rp_{session_id}_{message_id}_{referencing_to}_{page}' - - return back_command + return f'/search_{session_id}_{message_id}_{page}' async def handler(self, event: events.ChatAction, request_context: RequestContext): - parent_view_type, schema, session_id, old_message_id, document_id, position, page = self.parse_pattern(event) + schema, session_id, old_message_id, document_id, position, page = self.parse_pattern(event) request_context.add_default_fields(mode='view', session_id=session_id) request_context.statbox(action='view', document_id=document_id, position=position, schema=schema) @@ -96,9 +79,7 @@ class ViewHandler(BaseHandler): ) try: back_command = await self.compose_back_command( - parent_view_type=parent_view_type, session_id=session_id, - old_message_id=old_message_id, message_id=message_id, page=page, ) diff --git a/nexus/bot/widgets/document_list_widget.py b/nexus/bot/widgets/document_list_widget.py index 8ef1418..192708f 100644 --- a/nexus/bot/widgets/document_list_widget.py +++ b/nexus/bot/widgets/document_list_widget.py @@ -1,9 +1,12 @@ -from typing import Optional +from typing import ( + List, + Optional, +) from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.bot.application import TelegramApplication -from nexus.meta_api.proto.meta_search_service_pb2 import \ - ScoredDocument as ScoredDocumentPb +from nexus.models.proto.typed_document_pb2 import \ + TypedDocument as TypedDocumentPb from nexus.translations import t from nexus.views.telegram.common import close_button from nexus.views.telegram.registry import parse_typed_document_to_view @@ -15,87 +18,36 @@ class DocumentListWidget: self, application: TelegramApplication, chat: ChatPb, + typed_documents: List[TypedDocumentPb], + cmd: str, + has_next: bool, session_id: str, message_id: int, request_id: str, - referencing_to: int, page: int = 0, + page_size: int = 5, ): self.application = application self.chat = chat + self.typed_documents = typed_documents + self.cmd = cmd + self.has_next = has_next self.session_id = session_id self.message_id = message_id self.request_id = request_id - self.referencing_to = referencing_to self.page = page - - @staticmethod - async def create( - application: TelegramApplication, - chat: ChatPb, - session_id: str, - message_id: int, - request_id: str, - referencing_to: int, - page: int = 0, - ) -> 'DocumentListWidget': - document_list_view = DocumentListWidget( - application=application, - chat=chat, - session_id=session_id, - message_id=message_id, - request_id=request_id, - referencing_to=referencing_to, - page=page, - ) - await document_list_view._acquire_documents() - return document_list_view - - async def _acquire_documents(self): - typed_document_pb = await self.application.meta_api_client.get( - schema='scimag', - document_id=self.referencing_to, - position=0, - request_id=self.request_id, - session_id=self.session_id, - user_id=self.chat.chat_id, - ) - self._response = await self.application.meta_api_client.search( - schemas=('scimag',), - query=f'references:"{typed_document_pb.scimag.doi}"', - page=self.page, - request_id=self.request_id, - session_id=self.session_id, - user_id=self.chat.chat_id, - ) - - @property - def has_next(self) -> bool: - return self._response.has_next - - @property - def scored_documents(self) -> list[ScoredDocumentPb]: - return self._response.scored_documents + self.page_size = page_size async def render(self) -> tuple[str, Optional[list]]: - if not len(self.scored_documents): + if not len(self.typed_documents): return t('COULD_NOT_FIND_ANYTHING', language=self.chat.language), [close_button(self.session_id)] - serp_elements = [ - f'Linked to: {self.referencing_to}', - ] - for scored_document in self.scored_documents: - view = parse_typed_document_to_view(scored_document.typed_document) - view_command = view.get_view_command( - session_id=self.session_id, - message_id=self.message_id, - parent_view_type='r', - position=scored_document.position, - ) + serp_elements = [] + for position, typed_document in enumerate(self.typed_documents): + view = parse_typed_document_to_view(typed_document) serp_elements.append( view.get_snippet( language=self.chat.language, - view_command=view_command, limit=512 + 128, ) ) @@ -111,17 +63,17 @@ class DocumentListWidget: buttons = [ Button.inline( text='<<1' if self.page > 1 else ' ', - data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_0' + data=f'/{self.cmd}_{self.session_id}_{self.message_id}_0' if self.page > 1 else '/noop', ), Button.inline( text=f'<{self.page}' if self.page > 0 else ' ', - data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page - 1}' + data=f'/{self.cmd}_{self.session_id}_{self.message_id}_{self.page - 1}' if self.page > 0 else '/noop', ), Button.inline( text=f'{self.page + 2}>' if self.has_next else ' ', - data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page + 1}' + data=f'/{self.cmd}_{self.session_id}_{self.message_id}_{self.page + 1}' if self.has_next else '/noop', ) ] diff --git a/nexus/meta_api/aioclient/aioclient.py b/nexus/meta_api/aioclient/aioclient.py index 82b0daa..9cd8cb8 100644 --- a/nexus/meta_api/aioclient/aioclient.py +++ b/nexus/meta_api/aioclient/aioclient.py @@ -16,6 +16,10 @@ from nexus.meta_api.proto.documents_service_pb2 import \ RollRequest as RollRequestPb from nexus.meta_api.proto.documents_service_pb2 import \ RollResponse as RollResponsePb +from nexus.meta_api.proto.documents_service_pb2 import \ + TopMissedRequest as TopMissedRequestPb +from nexus.meta_api.proto.documents_service_pb2 import \ + TopMissedResponse as TopMissedResponsePb from nexus.meta_api.proto.documents_service_pb2 import \ TypedDocumentRequest as TypedDocumentRequestPb from nexus.meta_api.proto.documents_service_pb2_grpc import DocumentsStub @@ -130,3 +134,23 @@ class MetaApiGrpcClient(AioThing): ('request-id', request_id), ), ) + + async def top_missed( + self, + page: int, + page_size: int, + request_id: Optional[str] = None, + session_id: Optional[str] = None, + user_id: Optional[int] = None, + ) -> TopMissedResponsePb: + return await self.documents_stub.top_missed( + TopMissedRequestPb( + page=page, + page_size=page_size, + session_id=session_id, + user_id=user_id, + ), + metadata=( + ('request-id', request_id), + ), + ) diff --git a/nexus/meta_api/proto/documents_service.proto b/nexus/meta_api/proto/documents_service.proto index 3128c4f..c356690 100644 --- a/nexus/meta_api/proto/documents_service.proto +++ b/nexus/meta_api/proto/documents_service.proto @@ -13,6 +13,18 @@ message RollResponse { uint64 document_id = 1; } +message TopMissedRequest { + uint32 page = 1; + uint32 page_size = 2; + string session_id = 3; + int64 user_id = 4; +} + +message TopMissedResponse { + repeated nexus.models.proto.TypedDocument typed_documents = 1; + bool has_next = 2; +} + message TypedDocumentRequest { string schema = 1; uint64 document_id = 2; @@ -26,4 +38,5 @@ message PutTypedDocumentResponse {} service Documents { rpc get (TypedDocumentRequest) returns (nexus.models.proto.TypedDocument) {} rpc roll (RollRequest) returns (RollResponse) {} + rpc top_missed (TopMissedRequest) returns (TopMissedResponse) {} } diff --git a/nexus/meta_api/proto/documents_service_pb2.py b/nexus/meta_api/proto/documents_service_pb2.py index ce6ae59..1556ffd 100755 --- a/nexus/meta_api/proto/documents_service_pb2.py +++ b/nexus/meta_api/proto/documents_service_pb2.py @@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\"\x1a\n\x18PutTypedDocumentResponse2\xb4\x01\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x62\x06proto3' + serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"X\n\x10TopMissedRequest\x12\x0c\n\x04page\x18\x01 \x01(\r\x12\x11\n\tpage_size\x18\x02 \x01(\r\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x0f\n\x07user_id\x18\x04 \x01(\x03\"a\n\x11TopMissedResponse\x12:\n\x0ftyped_documents\x18\x01 \x03(\x0b\x32!.nexus.models.proto.TypedDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\"\x1a\n\x18PutTypedDocumentResponse2\x95\x02\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x12_\n\ntop_missed\x12&.nexus.meta_api.proto.TopMissedRequest\x1a\'.nexus.meta_api.proto.TopMissedResponse\"\x00\x62\x06proto3' , dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,]) @@ -106,6 +106,98 @@ _ROLLRESPONSE = _descriptor.Descriptor( ) +_TOPMISSEDREQUEST = _descriptor.Descriptor( + name='TopMissedRequest', + full_name='nexus.meta_api.proto.TopMissedRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='page', full_name='nexus.meta_api.proto.TopMissedRequest.page', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='page_size', full_name='nexus.meta_api.proto.TopMissedRequest.page_size', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='session_id', full_name='nexus.meta_api.proto.TopMissedRequest.session_id', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='user_id', full_name='nexus.meta_api.proto.TopMissedRequest.user_id', index=3, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=218, + serialized_end=306, +) + + +_TOPMISSEDRESPONSE = _descriptor.Descriptor( + name='TopMissedResponse', + full_name='nexus.meta_api.proto.TopMissedResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='typed_documents', full_name='nexus.meta_api.proto.TopMissedResponse.typed_documents', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='has_next', full_name='nexus.meta_api.proto.TopMissedResponse.has_next', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=308, + serialized_end=405, +) + + _TYPEDDOCUMENTREQUEST = _descriptor.Descriptor( name='TypedDocumentRequest', full_name='nexus.meta_api.proto.TypedDocumentRequest', @@ -161,8 +253,8 @@ _TYPEDDOCUMENTREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=218, - serialized_end=332, + serialized_start=407, + serialized_end=521, ) @@ -186,12 +278,15 @@ _PUTTYPEDDOCUMENTRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=334, - serialized_end=360, + serialized_start=523, + serialized_end=549, ) +_TOPMISSEDRESPONSE.fields_by_name['typed_documents'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT DESCRIPTOR.message_types_by_name['RollRequest'] = _ROLLREQUEST DESCRIPTOR.message_types_by_name['RollResponse'] = _ROLLRESPONSE +DESCRIPTOR.message_types_by_name['TopMissedRequest'] = _TOPMISSEDREQUEST +DESCRIPTOR.message_types_by_name['TopMissedResponse'] = _TOPMISSEDRESPONSE DESCRIPTOR.message_types_by_name['TypedDocumentRequest'] = _TYPEDDOCUMENTREQUEST DESCRIPTOR.message_types_by_name['PutTypedDocumentResponse'] = _PUTTYPEDDOCUMENTRESPONSE _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -210,6 +305,20 @@ RollResponse = _reflection.GeneratedProtocolMessageType('RollResponse', (_messag }) _sym_db.RegisterMessage(RollResponse) +TopMissedRequest = _reflection.GeneratedProtocolMessageType('TopMissedRequest', (_message.Message,), { + 'DESCRIPTOR' : _TOPMISSEDREQUEST, + '__module__' : 'nexus.meta_api.proto.documents_service_pb2' + # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.TopMissedRequest) + }) +_sym_db.RegisterMessage(TopMissedRequest) + +TopMissedResponse = _reflection.GeneratedProtocolMessageType('TopMissedResponse', (_message.Message,), { + 'DESCRIPTOR' : _TOPMISSEDRESPONSE, + '__module__' : 'nexus.meta_api.proto.documents_service_pb2' + # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.TopMissedResponse) + }) +_sym_db.RegisterMessage(TopMissedResponse) + TypedDocumentRequest = _reflection.GeneratedProtocolMessageType('TypedDocumentRequest', (_message.Message,), { 'DESCRIPTOR' : _TYPEDDOCUMENTREQUEST, '__module__' : 'nexus.meta_api.proto.documents_service_pb2' @@ -233,8 +342,8 @@ _DOCUMENTS = _descriptor.ServiceDescriptor( index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=363, - serialized_end=543, + serialized_start=552, + serialized_end=829, methods=[ _descriptor.MethodDescriptor( name='get', @@ -256,6 +365,16 @@ _DOCUMENTS = _descriptor.ServiceDescriptor( serialized_options=None, create_key=_descriptor._internal_create_key, ), + _descriptor.MethodDescriptor( + name='top_missed', + full_name='nexus.meta_api.proto.Documents.top_missed', + index=2, + containing_service=None, + input_type=_TOPMISSEDREQUEST, + output_type=_TOPMISSEDRESPONSE, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), ]) _sym_db.RegisterServiceDescriptor(_DOCUMENTS) diff --git a/nexus/meta_api/proto/documents_service_pb2_grpc.py b/nexus/meta_api/proto/documents_service_pb2_grpc.py index 1ae3a94..8935f94 100755 --- a/nexus/meta_api/proto/documents_service_pb2_grpc.py +++ b/nexus/meta_api/proto/documents_service_pb2_grpc.py @@ -27,6 +27,11 @@ class DocumentsStub(object): request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.SerializeToString, response_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString, ) + self.top_missed = channel.unary_unary( + '/nexus.meta_api.proto.Documents/top_missed', + request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedRequest.SerializeToString, + response_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedResponse.FromString, + ) class DocumentsServicer(object): @@ -44,6 +49,12 @@ class DocumentsServicer(object): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def top_missed(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_DocumentsServicer_to_server(servicer, server): rpc_method_handlers = { @@ -57,6 +68,11 @@ def add_DocumentsServicer_to_server(servicer, server): request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.FromString, response_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.SerializeToString, ), + 'top_missed': grpc.unary_unary_rpc_method_handler( + servicer.top_missed, + request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedRequest.FromString, + response_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'nexus.meta_api.proto.Documents', rpc_method_handlers) @@ -100,3 +116,20 @@ class Documents(object): nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def top_missed(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Documents/top_missed', + nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedRequest.SerializeToString, + nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/nexus/meta_api/proto/meta_search_service.proto b/nexus/meta_api/proto/meta_search_service.proto index 4264738..8032f91 100644 --- a/nexus/meta_api/proto/meta_search_service.proto +++ b/nexus/meta_api/proto/meta_search_service.proto @@ -5,8 +5,8 @@ import "nexus/models/proto/typed_document.proto"; message ScoredDocument { nexus.models.proto.TypedDocument typed_document = 1; - float score = 3; - uint32 position = 4; + float score = 2; + uint32 position = 3; } message SearchResponse { diff --git a/nexus/meta_api/proto/meta_search_service_pb2.py b/nexus/meta_api/proto/meta_search_service_pb2.py index 58a4d7c..313a537 100755 --- a/nexus/meta_api/proto/meta_search_service_pb2.py +++ b/nexus/meta_api/proto/meta_search_service_pb2.py @@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x03 \x01(\x02\x12\x10\n\x08position\x18\x04 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\xa3\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t\x12\x1a\n\x12skip_query_rewrite\x18\t \x01(\x08\x32\x63\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3' + serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2c\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3' , dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,]) @@ -45,14 +45,14 @@ _SCOREDDOCUMENT = _descriptor.Descriptor( serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='score', full_name='nexus.meta_api.proto.ScoredDocument.score', index=1, - number=3, type=2, cpp_type=6, label=1, + number=2, type=2, cpp_type=6, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='position', full_name='nexus.meta_api.proto.ScoredDocument.position', index=2, - number=4, type=13, cpp_type=3, label=1, + number=3, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -170,13 +170,6 @@ _SEARCHREQUEST = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='skip_query_rewrite', full_name='nexus.meta_api.proto.SearchRequest.skip_query_rewrite', index=7, - number=9, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -190,7 +183,7 @@ _SEARCHREQUEST = _descriptor.Descriptor( oneofs=[ ], serialized_start=324, - serialized_end=487, + serialized_end=459, ) _SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT @@ -230,8 +223,8 @@ _METASEARCH = _descriptor.ServiceDescriptor( index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=489, - serialized_end=588, + serialized_start=461, + serialized_end=560, methods=[ _descriptor.MethodDescriptor( name='search', diff --git a/nexus/views/telegram/scimag.py b/nexus/views/telegram/scimag.py index 1e37a81..ccef6fb 100644 --- a/nexus/views/telegram/scimag.py +++ b/nexus/views/telegram/scimag.py @@ -6,6 +6,7 @@ from typing import ( ) from urllib.parse import quote +from izihawa_types.safecast import safe_int from izihawa_utils.common import filter_none from nexus.models.proto.scimag_pb2 import Scimag as ScimagPb from nexus.nlptools.utils import ( @@ -204,16 +205,17 @@ class ScimagView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin): ) if self.ref_by_count: buttons[-1].append( - Button.inline( + Button.switch_inline( text=f'🔗 {self.ref_by_count or ""}', - data=f'/r_{session_id}_{self.id}', + query=f'references:"{self.doi}"', + same_peer=True, ) ) buttons[-1].append(close_button(session_id)) return '\n'.join(parts).strip()[:4096], buttons - def get_view_command(self, session_id: str, message_id: int, parent_view_type: str = 's', position: int = 0) -> str: - return f'/va{parent_view_type}_{session_id}_{message_id}_{self.id}_{position}' + def get_view_command(self, session_id: str, message_id: int, position: int = 0) -> str: + return f'/va_{session_id}_{message_id}_{self.id}_{position}' def get_robust_journal(self): if self.type != 'chapter' and self.type != 'book-chapter': @@ -232,4 +234,7 @@ class ScimagView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin): if self.issue: return f'vol. {self.volume}({self.issue})' else: - return self.volume + if safe_int(self.volume): + return f'vol. {self.volume}' + else: + return self.volume