Merge pull request #26 from the-superpirate/master

- feat(nexus): Increase amount of logging

nexus/bot/configs/base.yaml

@@ -79,8 +79,6 @@ telegram:
     - nexus.bot.handlers.download.DownloadHandler
     - nexus.bot.handlers.emoji.EmojiHandler
     - nexus.bot.handlers.help.HelpHandler
-    - nexus.bot.handlers.referencing_to.ReferencingToHandler
-    - nexus.bot.handlers.referencing_to.ReferencingToPagingHandler
     - nexus.bot.handlers.roll.RollHandler
     - nexus.bot.handlers.settings.SettingsButtonsHandler
     - nexus.bot.handlers.settings.SettingsHandler
@@ -88,6 +86,7 @@ telegram:
     - nexus.bot.handlers.submit.SubmitHandler
     - nexus.bot.handlers.start.StartHandler
     - nexus.bot.handlers.stop.StopHandler
+    - nexus.bot.handlers.top_missed.TopMissedHandler
     - nexus.bot.handlers.view.ViewHandler
     - nexus.bot.handlers.vote.VoteHandler
     - nexus.bot.handlers.noop.NoopHandler

nexus/bot/exceptions.py

@@ -1,6 +1,21 @@
+import logging
+
 from izihawa_utils.exceptions import BaseError
 
 
+class BannedUserError(BaseError):
+    level = logging.WARNING
+    code = 'banned_user_error'
+
+    def __init__(self, ban_timeout: int):
+        self.ban_timeout = ban_timeout
+
+
+class MessageHasBeenDeletedError(BaseError):
+    level = logging.WARNING
+    code = 'message_has_been_deleted_error'
+
+
 class UnknownFileFormatError(BaseError):
     code = 'unknown_file_format_error'
 

nexus/bot/handlers/__init__.py

@@ -10,7 +10,6 @@ from . import (
     help,
     legacy,
     noop,
-    referencing_to,
     roll,
     search,
     settings,
@@ -18,10 +17,11 @@ from . import (
     start,
     stop,
     submit,
+    top_missed,
     view,
     vote,
 )
 
 __all__ = ['admin', 'ban', 'contact', 'copyright', 'close', 'donate', 'download', 'emoji', 'help',
-           'legacy', 'noop', 'referencing_to', 'roll', 'search', 'settings',
+           'legacy', 'noop', 'roll', 'search', 'settings',
-           'shortlink', 'start', 'stop', 'submit', 'view', 'vote']
+           'shortlink', 'start', 'stop', 'submit', 'top_missed', 'view', 'vote']

nexus/bot/handlers/ban.py

@@ -15,12 +15,17 @@ from .admin import BaseAdminHandler
 class BanHandler(BaseAdminHandler):
     filter = events.NewMessage(incoming=True, pattern='^/ban ([0-9]+) ([A-Za-z0-9]+)\\s?(.*)?$')
 
-    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+    def parse_pattern(self, event: events.ChatAction):
         chat_id = int(event.pattern_match.group(1))
         ban_duration = event.pattern_match.group(2)
         ban_message = event.pattern_match.group(3)
         ban_end_date = datetime.utcnow() + timedelta(seconds=timeparse(ban_duration))
 
+        return chat_id, ban_duration, ban_message, ban_end_date
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        chat_id, ban_duration, ban_message, ban_end_date = self.parse_pattern(event)
+
         try:
             await self.application.idm_client.update_chat(
                 chat_id=chat_id,

nexus/bot/handlers/download.py

@@ -12,17 +12,22 @@ class DownloadHandler(BaseCallbackQueryHandler):
     filter = events.CallbackQuery(pattern='^/dl([abcm])_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$')
     is_group_handler = True
 
-    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+    def parse_pattern(self, event: events.ChatAction):
         short_schema = event.pattern_match.group(1).decode()
         schema = self.short_schema_to_schema(short_schema)
         session_id = event.pattern_match.group(2).decode()
         document_id = int(event.pattern_match.group(3))
         position = int(event.pattern_match.group(4).decode())
 
+        return short_schema, schema, session_id, document_id, position
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        short_schema, schema, session_id, document_id, position = self.parse_pattern(event)
+
         self.application.user_manager.last_widget[request_context.chat.chat_id] = None
 
         request_context.add_default_fields(mode='download', session_id=session_id)
-        request_context.statbox(action='get', query=str(document_id), position=position)
+        request_context.statbox(action='get', document_id=document_id, position=position, schema=schema)
 
         typed_document_pb = await self.get_typed_document_pb(
             schema=schema,

nexus/bot/handlers/referencing_to.py

@@ -1,94 +0,0 @@
-import re
-import time
-
-from grpc.experimental.aio import AioRpcError
-from library.telegram.base import RequestContext
-from nexus.bot.widgets.document_list_widget import DocumentListWidget
-from nexus.translations import t
-from telethon import events
-
-from .base import BaseCallbackQueryHandler
-
-
-class ReferencingToHandler(BaseCallbackQueryHandler):
-    filter = events.CallbackQuery(pattern=re.compile('^/r_([A-Za-z0-9]+)_([0-9]+)', re.DOTALL))
-    should_reset_last_widget = False
-
-    async def do_request(self, request_context: RequestContext, session_id: str, message_id: int, document_id: int, page: int):
-        start_time = time.time()
-        try:
-            document_list_widget = await DocumentListWidget.create(
-                application=self.application,
-                chat=request_context.chat,
-                session_id=session_id,
-                message_id=message_id,
-                request_id=request_context.request_id,
-                referencing_to=document_id,
-                page=page,
-            )
-        except AioRpcError as e:
-            raise e
-
-        action = 'referencing_to_found'
-        if len(document_list_widget.scored_documents) == 0:
-            action = 'referencing_to_not_found'
-
-        request_context.statbox(
-            action=action,
-            duration=time.time() - start_time,
-            query=f'{document_id}',
-        )
-
-        serp, buttons = await document_list_widget.render()
-        return await self.application.telegram_client.edit_message(
-            request_context.chat.chat_id,
-            message_id,
-            serp,
-            buttons=buttons,
-            link_preview=False,
-        )
-
-    async def handler(self, event, request_context: RequestContext):
-        session_id = event.pattern_match.group(1).decode()
-        document_id = int(event.pattern_match.group(2).decode())
-
-        request_context.add_default_fields(
-            mode='referencing_to',
-            session_id=session_id,
-        )
-
-        prefetch_message = await event.respond(
-            t("SEARCHING", language=request_context.chat.language),
-        )
-        message_id = prefetch_message.id
-
-        return await self.do_request(
-            request_context=request_context,
-            session_id=session_id,
-            message_id=message_id,
-            document_id=document_id,
-            page=0,
-        )
-
-
-class ReferencingToPagingHandler(ReferencingToHandler):
-    filter = events.CallbackQuery(pattern=re.compile('^/rp_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)', re.DOTALL))
-
-    async def handler(self, event: events.ChatAction, request_context: RequestContext):
-        session_id = event.pattern_match.group(1).decode()
-        message_id = int(event.pattern_match.group(2).decode())
-        document_id = int(event.pattern_match.group(3).decode())
-        page = int(event.pattern_match.group(4).decode())
-
-        request_context.add_default_fields(
-            mode='referencing_to_paging',
-            session_id=session_id,
-        )
-
-        return await self.do_request(
-            request_context=request_context,
-            session_id=session_id,
-            message_id=message_id,
-            document_id=document_id,
-            page=page,
-        )

nexus/bot/handlers/search.py

@@ -1,11 +1,15 @@
 import asyncio
-import logging
 import re
 import time
+from abc import ABC
 
 from grpc import StatusCode
 from grpc.experimental.aio import AioRpcError
 from library.telegram.base import RequestContext
+from nexus.bot.exceptions import (
+    BannedUserError,
+    MessageHasBeenDeletedError,
+)
 from nexus.bot.widgets.search_widget import SearchWidget
 from nexus.translations import t
 from nexus.views.telegram.common import close_button
@@ -21,8 +25,9 @@ from .base import (
 )
 
 
-class BaseSearchHandler(BaseHandler):
+class BaseSearchHandler(BaseHandler, ABC):
-    should_reset_last_widget = False
+    def preprocess_query(self, query):
+        return query.replace(f'@{self.application.config["telegram"]["bot_external_name"]}', '').strip()
 
     async def do_search(
         self,
@@ -35,10 +40,9 @@ class BaseSearchHandler(BaseHandler):
     ):
         session_id = self.generate_session_id()
         message_id = prefetch_message.id
-
         request_context.add_default_fields(is_group_mode=is_group_mode, mode='search', session_id=session_id)
-
         start_time = time.time()
+
         try:
             search_widget = await SearchWidget.create(
                 application=self.application,
@@ -141,47 +145,50 @@ class SearchHandler(BaseSearchHandler):
     should_reset_last_widget = False
     is_subscription_required_for_handler = True
 
-    async def ban_handler(self, event: events.ChatAction, request_context: RequestContext, ban_timeout: float):
+    def check_search_ban_timeout(self, chat_id: int):
-        logging.getLogger('statbox').info({
+        ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=chat_id)
-            'bot_name': self.application.config['telegram']['bot_name'],
+        if ban_timeout:
-            'action': 'user_flood_ban',
+            raise BannedUserError(ban_timeout=ban_timeout)
-            'mode': 'search',
+        self.application.user_manager.add_search_time(user_id=chat_id, search_time=time.time())
-            'ban_timeout_seconds': ban_timeout,
+
-            'chat_id': request_context.chat.chat_id,
+    def parse_pattern(self, event: events.ChatAction):
-        })
+        search_prefix = event.pattern_match.group(1)
-        ban_reason = t(
+        query = self.preprocess_query(event.pattern_match.group(2))
-            'BAN_MESSAGE_TOO_MANY_REQUESTS',
+        is_group_mode = event.is_group or event.is_channel
-            language=request_context.chat.language
+
-        )
+        return search_prefix, query, is_group_mode
-        return await event.reply(t(
-            'BANNED_FOR_SECONDS',
-            language=request_context.chat.language
-        ).format(
-            seconds=str(ban_timeout),
-            reason=ban_reason,
-        ))
 
     async def handler(self, event: events.ChatAction, request_context: RequestContext):
-        ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.chat_id)
+        try:
-        if ban_timeout:
+            self.check_search_ban_timeout(chat_id=request_context.chat.chat_id)
-            return await self.ban_handler(event, request_context, ban_timeout)
+        except BannedUserError as e:
-        self.application.user_manager.add_search_time(user_id=request_context.chat.chat_id, search_time=time.time())
+            request_context.error_log(e)
-
+            return await event.reply(t(
-        search_prefix = event.pattern_match.group(1)
+                'BANNED_FOR_SECONDS',
-        query = event.pattern_match.group(2)
+                language=request_context.chat.language
-        is_group_mode = event.is_group or event.is_channel
+            ).format(
+                seconds=e.ban_timeout,
+                reason=t(
+                    'BAN_MESSAGE_TOO_MANY_REQUESTS',
+                    language=request_context.chat.language
+                ),
+            ))
+        search_prefix, query, is_group_mode = self.parse_pattern(event)
 
         if is_group_mode and not search_prefix:
             return
         if not is_group_mode and search_prefix:
             query = event.raw_text
+
         prefetch_message = await event.reply(
             t("SEARCHING", language=request_context.chat.language),
         )
         self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
         try:
             await self.do_search(
-                event, request_context, prefetch_message,
+                event=event,
+                request_context=request_context,
+                prefetch_message=prefetch_message,
                 query=query,
                 is_group_mode=is_group_mode,
                 is_shortpath_enabled=True,
@@ -199,63 +206,77 @@ class SearchEditHandler(BaseSearchHandler):
     is_group_handler = True
     should_reset_last_widget = False
 
-    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+    def parse_pattern(self, event: events.ChatAction):
-        request_context.add_default_fields(mode='search_edit')
         search_prefix = event.pattern_match.group(1)
-        query = event.pattern_match.group(2)
+        query = self.preprocess_query(event.pattern_match.group(2))
         is_group_mode = event.is_group or event.is_channel
+        return search_prefix, query, is_group_mode
+
+    async def get_last_messages_in_chat(self, event: events.ChatAction):
+        return await self.application.telegram_client(functions.messages.GetMessagesRequest(
+            id=list(range(event.id + 1, event.id + 10)))
+        )
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        search_prefix, query, is_group_mode = self.parse_pattern(event)
+        request_context.add_default_fields(mode='search_edit')
+
         if is_group_mode and not search_prefix:
             return
         if not is_group_mode and search_prefix:
             query = event.raw_text
-        result = await self.application.telegram_client(functions.messages.GetMessagesRequest(
+
-            id=list(range(event.id + 1, event.id + 10)))
+        last_messages = await self.get_last_messages_in_chat(event)
-        )
+        try:
-        if not result:
+            if not last_messages:
-            request_context.statbox(action='failed')
+                raise MessageHasBeenDeletedError()
+            for next_message in last_messages.messages:
+                if next_message.is_reply and event.id == next_message.reply_to_msg_id:
+                    request_context.statbox(action='resolved')
+                    return await self.do_search(
+                        event=event,
+                        request_context=request_context,
+                        prefetch_message=next_message,
+                        query=query,
+                        is_group_mode=is_group_mode,
+                    )
+            raise MessageHasBeenDeletedError()
+        except MessageHasBeenDeletedError as e:
+            request_context.error_log(e)
             return await event.reply(
                 t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
             )
-        for next_message in result.messages:
-            if next_message.is_reply and event.id == next_message.reply_to_msg_id:
-                request_context.statbox(action='resolved')
-                await self.do_search(
-                    event,
-                    request_context,
-                    prefetch_message=next_message,
-                    query=query,
-                    is_group_mode=is_group_mode,
-                )
-                return
-        request_context.statbox(action='failed')
-        return await event.reply(
-            t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
-        )
 
 
 class SearchPagingHandler(BaseCallbackQueryHandler):
     filter = events.CallbackQuery(pattern='^/search_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$')
     should_reset_last_widget = False
 
-    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+    def preprocess_query(self, query):
+        return query.replace(f'@{self.application.config["telegram"]["bot_external_name"]}', '').strip()
+
+    def parse_pattern(self, event: events.ChatAction):
         session_id = event.pattern_match.group(1).decode()
         message_id = int(event.pattern_match.group(2).decode())
         page = int(event.pattern_match.group(3).decode())
 
+        return session_id, message_id, page
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        session_id, message_id, page = self.parse_pattern(event)
+
         request_context.add_default_fields(mode='search_paging', session_id=session_id)
+        start_time = time.time()
 
         message = await event.get_message()
         if not message:
             return await event.answer()
-        reply_message = await message.get_reply_message()
-        if not reply_message:
-            return await event.respond(
-                t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
-            )
 
-        start_time = time.time()
+        reply_message = await message.get_reply_message()
-        query = reply_message.raw_text
         try:
+            if not reply_message:
+                raise MessageHasBeenDeletedError()
+            query = self.preprocess_query(reply_message.raw_text)
             search_widget = await SearchWidget.create(
                 application=self.application,
                 chat=request_context.chat,
@@ -265,6 +286,10 @@ class SearchPagingHandler(BaseCallbackQueryHandler):
                 query=query,
                 page=page,
             )
+        except MessageHasBeenDeletedError:
+            return await event.respond(
+                t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
+            )
         except AioRpcError as e:
             if e.code() == StatusCode.INVALID_ARGUMENT or e.code() == StatusCode.CANCELLED:
                 request_context.error_log(e)

nexus/bot/handlers/top_missed.py

@@ -0,0 +1,54 @@
+from library.telegram.base import RequestContext
+from nexus.bot.widgets.document_list_widget import DocumentListWidget
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseHandler
+
+
+class TopMissedHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/tm$')
+    is_group_handler = False
+    should_reset_last_widget = False
+
+    async def do_request(self, request_context: RequestContext, session_id: str, message_id: int, page: int):
+        response = await self.application.meta_api_client.top_missed(
+            page=page,
+            page_size=10,
+            session_id=session_id,
+            request_id=request_context.request_id,
+        )
+        document_list_widget = DocumentListWidget(
+            application=self.application,
+            chat=request_context.chat,
+            typed_documents=response.typed_documents,
+            cmd='tm',
+            has_next=response.has_next,
+            session_id=session_id,
+            message_id=message_id,
+            request_id=request_context.request_id,
+            page=page,
+        )
+        serp, buttons = await document_list_widget.render()
+        return await self.application.telegram_client.edit_message(
+            request_context.chat.chat_id,
+            message_id,
+            serp,
+            buttons=buttons,
+            link_preview=False,
+        )
+
+    async def handler(self, event, request_context: RequestContext):
+        session_id = self.generate_session_id()
+        request_context.add_default_fields(mode='top_missed', session_id=session_id)
+        request_context.statbox()
+
+        prefetch_message = await event.reply(t("SEARCHING", language=request_context.chat.language))
+        message_id = prefetch_message.id
+
+        return await self.do_request(
+            request_context=request_context,
+            session_id=session_id,
+            message_id=message_id,
+            page=0,
+        )

nexus/bot/handlers/view.py

@@ -1,7 +1,7 @@
 import asyncio
-import re
 
 from library.telegram.base import RequestContext
+from nexus.bot.exceptions import MessageHasBeenDeletedError
 from nexus.translations import t
 from telethon import (
     events,
@@ -13,13 +13,12 @@ from .base import BaseHandler
 
 
 class ViewHandler(BaseHandler):
-    filter = events.NewMessage(incoming=True, pattern='^/v([abcm])([sr])?_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_'
+    filter = events.NewMessage(incoming=True, pattern='^/v([ab])([sr])?_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_'
                                                       '([0-9]+)')
     should_reset_last_widget = False
 
-    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+    def parse_pattern(self, event: events.ChatAction):
         short_schema = event.pattern_match.group(1)
-        parent_view_type = event.pattern_match.group(2) or 's'
         schema = self.short_schema_to_schema(short_schema)
         session_id = event.pattern_match.group(3)
         old_message_id = int(event.pattern_match.group(4))
@@ -28,26 +27,48 @@ class ViewHandler(BaseHandler):
 
         page = int(position / self.application.config['application']['page_size'])
 
+        return schema, session_id, old_message_id, document_id, position, page
+
+    async def process_widgeting(self, has_found_old_widget, old_message_id, request_context: RequestContext):
+        if has_found_old_widget:
+            message_id = old_message_id
+            link_preview = None
+        else:
+            old_message = (await self.application.telegram_client(
+                functions.messages.GetMessagesRequest(id=[old_message_id])
+            )).messages[0]
+            prefetch_message = await self.application.telegram_client.send_message(
+                request_context.chat.chat_id,
+                t("SEARCHING", language=request_context.chat.language),
+                reply_to=old_message.reply_to_msg_id,
+            )
+            self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
+            message_id = prefetch_message.id
+            link_preview = True
+        return message_id, link_preview
+
+    async def compose_back_command(
+        self,
+        session_id,
+        message_id,
+        page,
+    ):
+        return f'/search_{session_id}_{message_id}_{page}'
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        schema, session_id, old_message_id, document_id, position, page = self.parse_pattern(event)
+
         request_context.add_default_fields(mode='view', session_id=session_id)
-        request_context.statbox(action='view', query=str(document_id), position=position)
+        request_context.statbox(action='view', document_id=document_id, position=position, schema=schema)
-        found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id)
+
+        has_found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id)
 
         try:
-            if found_old_widget:
+            message_id, link_preview = await self.process_widgeting(
-                message_id = old_message_id
+                has_found_old_widget=has_found_old_widget,
-                link_preview = None
+                old_message_id=old_message_id,
-            else:
+                request_context=request_context
-                old_message = (await self.application.telegram_client(
+            )
-                    functions.messages.GetMessagesRequest(id=[old_message_id])
-                )).messages[0]
-                prefetch_message = await self.application.telegram_client.send_message(
-                    request_context.chat.chat_id,
-                    t("SEARCHING", language=request_context.chat.language),
-                    reply_to=old_message.reply_to_msg_id,
-                )
-                self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
-                message_id = prefetch_message.id
-                link_preview = True
 
             document_view = await self.resolve_document(
                 schema,
@@ -56,21 +77,16 @@ class ViewHandler(BaseHandler):
                 session_id,
                 request_context,
             )
-
+            try:
-            back_command = None
+                back_command = await self.compose_back_command(
-            if parent_view_type == 's':
+                    session_id=session_id,
-                back_command = f'/search_{session_id}_{message_id}_{page}'
+                    message_id=message_id,
-            elif parent_view_type == 'r':
+                    page=page,
-                messages = (await self.application.telegram_client(
+                )
-                    functions.messages.GetMessagesRequest(id=[old_message_id])
+            except MessageHasBeenDeletedError:
-                )).messages
+                return await event.respond(
-                if not messages:
+                    t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
-                    return await event.respond(
+                )
-                        t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
-                    )
-                message = messages[0]
-                referencing_to = re.search(r'Linked to: ([0-9]+)', message.raw_text).group(1)
-                back_command = f'/rp_{session_id}_{message_id}_{referencing_to}_{page}'
 
             view, buttons = document_view.get_view(
                 language=request_context.chat.language,
@@ -89,7 +105,7 @@ class ViewHandler(BaseHandler):
                 ),
                 event.delete(),
             ]
-            if not found_old_widget:
+            if not has_found_old_widget:
                 actions.append(
                     self.application.telegram_client.delete_messages(
                         request_context.chat.chat_id,

nexus/bot/handlers/vote.py

@@ -13,14 +13,28 @@ from .base import BaseCallbackQueryHandler
 
 
 class VoteHandler(BaseCallbackQueryHandler):
-    filter = events.CallbackQuery(pattern='^/vote_([A-Za-z0-9]+)_([0-9]+)_([bo])$')
+    filter = events.CallbackQuery(pattern='^/vote([ab])?_([A-Za-z0-9]+)_([0-9]+)_([bo])$')
+
+    def parse_pattern(self, event: events.ChatAction):
+        short_schema = event.pattern_match.group(1)
+        schema = self.short_schema_to_schema(short_schema.decode()) if short_schema else None
+        session_id = event.pattern_match.group(2).decode()
+        document_id = int(event.pattern_match.group(3).decode())
+        vote = event.pattern_match.group(4).decode()
+        vote_value = {'b': -1, 'o': 1}[vote]
+
+        return schema, session_id, document_id, vote, vote_value
 
     async def handler(self, event: events.ChatAction, request_context: RequestContext):
-        session_id = event.pattern_match.group(1).decode()
+        schema, session_id, document_id, vote, vote_value = self.parse_pattern(event)
-        document_id = int(event.pattern_match.group(2).decode())
+
-        vote = event.pattern_match.group(3).decode()
-        vote_value = {'b': -1, 'o': 1}[vote]
         request_context.add_default_fields(mode='vote', session_id=session_id)
+        request_context.statbox(
+            action='vote',
+            document_id=document_id,
+            query=vote,
+            schema=schema,
+        )
 
         document_operation_pb = DocumentOperationPb(
             vote=VotePb(
@@ -29,11 +43,6 @@ class VoteHandler(BaseCallbackQueryHandler):
                 voter_id=request_context.chat.chat_id,
             ),
         )
-
-        request_context.statbox(
-            action='vote',
-            document_id=document_id,
-        )
         logging.getLogger('operation').info(
             msg=MessageToDict(document_operation_pb),
         )

nexus/bot/widgets/document_list_widget.py

@@ -1,9 +1,12 @@
-from typing import Optional
+from typing import (
+    List,
+    Optional,
+)
 
 from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
-from nexus.meta_api.proto.meta_search_service_pb2 import \
+from nexus.models.proto.typed_document_pb2 import \
-    ScoredDocument as ScoredDocumentPb
+    TypedDocument as TypedDocumentPb
 from nexus.translations import t
 from nexus.views.telegram.common import close_button
 from nexus.views.telegram.registry import parse_typed_document_to_view
@@ -15,87 +18,36 @@ class DocumentListWidget:
         self,
         application: TelegramApplication,
         chat: ChatPb,
+        typed_documents: List[TypedDocumentPb],
+        cmd: str,
+        has_next: bool,
         session_id: str,
         message_id: int,
         request_id: str,
-        referencing_to: int,
         page: int = 0,
+        page_size: int = 5,
     ):
         self.application = application
         self.chat = chat
+        self.typed_documents = typed_documents
+        self.cmd = cmd
+        self.has_next = has_next
         self.session_id = session_id
         self.message_id = message_id
         self.request_id = request_id
-        self.referencing_to = referencing_to
         self.page = page
-
+        self.page_size = page_size
-    @staticmethod
-    async def create(
-        application: TelegramApplication,
-        chat: ChatPb,
-        session_id: str,
-        message_id: int,
-        request_id: str,
-        referencing_to: int,
-        page: int = 0,
-    ) -> 'DocumentListWidget':
-        document_list_view = DocumentListWidget(
-            application=application,
-            chat=chat,
-            session_id=session_id,
-            message_id=message_id,
-            request_id=request_id,
-            referencing_to=referencing_to,
-            page=page,
-        )
-        await document_list_view._acquire_documents()
-        return document_list_view
-
-    async def _acquire_documents(self):
-        typed_document_pb = await self.application.meta_api_client.get(
-            schema='scimag',
-            document_id=self.referencing_to,
-            position=0,
-            request_id=self.request_id,
-            session_id=self.session_id,
-            user_id=self.chat.chat_id,
-        )
-        self._response = await self.application.meta_api_client.search(
-            schemas=('scimag',),
-            query=f'references:"{typed_document_pb.scimag.doi}"',
-            page=self.page,
-            request_id=self.request_id,
-            session_id=self.session_id,
-            user_id=self.chat.chat_id,
-        )
-
-    @property
-    def has_next(self) -> bool:
-        return self._response.has_next
-
-    @property
-    def scored_documents(self) -> list[ScoredDocumentPb]:
-        return self._response.scored_documents
 
     async def render(self) -> tuple[str, Optional[list]]:
-        if not len(self.scored_documents):
+        if not len(self.typed_documents):
             return t('COULD_NOT_FIND_ANYTHING', language=self.chat.language), [close_button(self.session_id)]
 
-        serp_elements = [
+        serp_elements = []
-            f'Linked to: {self.referencing_to}',
+        for position, typed_document in enumerate(self.typed_documents):
-        ]
+            view = parse_typed_document_to_view(typed_document)
-        for scored_document in self.scored_documents:
-            view = parse_typed_document_to_view(scored_document.typed_document)
-            view_command = view.get_view_command(
-                session_id=self.session_id,
-                message_id=self.message_id,
-                parent_view_type='r',
-                position=scored_document.position,
-            )
             serp_elements.append(
                 view.get_snippet(
                     language=self.chat.language,
-                    view_command=view_command,
                     limit=512 + 128,
                 )
             )
@@ -111,17 +63,17 @@ class DocumentListWidget:
             buttons = [
                 Button.inline(
                     text='<<1' if self.page > 1 else ' ',
-                    data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_0'
+                    data=f'/{self.cmd}_{self.session_id}_{self.message_id}_0'
                     if self.page > 1 else '/noop',
                 ),
                 Button.inline(
                     text=f'<{self.page}' if self.page > 0 else ' ',
-                    data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page - 1}'
+                    data=f'/{self.cmd}_{self.session_id}_{self.message_id}_{self.page - 1}'
                     if self.page > 0 else '/noop',
                 ),
                 Button.inline(
                     text=f'{self.page + 2}>' if self.has_next else ' ',
-                    data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page + 1}'
+                    data=f'/{self.cmd}_{self.session_id}_{self.message_id}_{self.page + 1}'
                     if self.has_next else '/noop',
                 )
             ]

nexus/hub/services/base.py

@@ -54,12 +54,14 @@ class BaseHubService(BaseService):
             buttons = [
                 vote_button(
                     case='broken',
+                    schema=document_view.schema,
                     document_id=document_id,
                     language=request_context.chat.language,
                     session_id=session_id,
                 ),
                 vote_button(
                     case='ok',
+                    schema=document_view.schema,
                     document_id=document_id,
                     language=request_context.chat.language,
                     session_id=session_id,
@@ -77,6 +79,7 @@ class BaseHubService(BaseService):
         request_context.statbox(
             action='sent',
             document_id=document_id,
+            schema=document_view.schema,
             voting=voting,
         )
         return message

nexus/hub/services/delivery.py

@@ -123,6 +123,7 @@ class DownloadTask:
                         action='missed',
                         duration=time.time() - start_time,
                         document_id=document_view.id,
+                        schema=document_view.schema,
                     )
                     is_served_from_sharience = False
                     if self.delivery_service.is_sharience_enabled:
@@ -135,6 +136,7 @@ class DownloadTask:
                             action='not_found',
                             document_id=document_view.id,
                             duration=time.time() - start_time,
+                            schema=document_view.schema,
                         )
                         await self.respond_not_found(
                             request_context=request_context,
@@ -147,6 +149,7 @@ class DownloadTask:
                         duration=time.time() - start_time,
                         document_id=document_view.id,
                         len=len(file),
+                        schema=document_view.schema,
                     )
 
                 progress_bar_upload = ProgressBar(
@@ -171,6 +174,7 @@ class DownloadTask:
                     action='uploaded',
                     duration=time.time() - start_time,
                     document_id=document_view.id,
+                    schema=document_view.schema,
                 )
                 if self.delivery_service.should_store_hashes:
                     asyncio.create_task(self.store_hashes(
@@ -185,6 +189,7 @@ class DownloadTask:
                     action='user_canceled',
                     duration=time.time() - start_time,
                     document_id=document_view.id,
+                    schema=document_view.schema,
                 )
             except asyncio.CancelledError:
                 pass
@@ -215,7 +220,7 @@ class DownloadTask:
 
     async def try_sharience(self, request_context, document_view):
         if document_view.doi:
-            request_context.statbox(action='try_sharience', doi=document_view.doi)
+            request_context.statbox(action='try_sharience', doi=document_view.doi, schema=document_view.schema)
             pg_data = await self.delivery_service.pool_holder.execute(
                 '''
                 select sh.id, sh.telegram_file_id as vote_sum
@@ -273,7 +278,11 @@ class DownloadTask:
 
     async def external_cancel(self):
         self.task.cancel()
-        self.request_context.statbox(action='externally_canceled')
+        self.request_context.statbox(
+            action='externally_canceled',
+            document_id=self.document_view.id,
+            schema=self.document_view.schema,
+        )
         await self.delivery_service.telegram_client.send_message(
             self.request_context.chat.chat_id,
             t("DOWNLOAD_CANCELED", language=self.request_context.chat.language).format(
@@ -372,7 +381,7 @@ class DeliveryService(DeliveryServicer, BaseHubService):
                     request_context=request_context,
                     voting=not is_group_or_channel(request_context.chat.chat_id),
                 )
-                request_context.statbox(action='cache_hit', document_id=document_view.id)
+                request_context.statbox(action='cache_hit', document_id=document_view.id, schema=document_view.schema)
             except ValueError:
                 cache_hit = False
         if not cache_hit:

nexus/meta_api/aioclient/aioclient.py

@@ -16,6 +16,10 @@ from nexus.meta_api.proto.documents_service_pb2 import \
     RollRequest as RollRequestPb
 from nexus.meta_api.proto.documents_service_pb2 import \
     RollResponse as RollResponsePb
+from nexus.meta_api.proto.documents_service_pb2 import \
+    TopMissedRequest as TopMissedRequestPb
+from nexus.meta_api.proto.documents_service_pb2 import \
+    TopMissedResponse as TopMissedResponsePb
 from nexus.meta_api.proto.documents_service_pb2 import \
     TypedDocumentRequest as TypedDocumentRequestPb
 from nexus.meta_api.proto.documents_service_pb2_grpc import DocumentsStub
@@ -130,3 +134,23 @@ class MetaApiGrpcClient(AioThing):
                 ('request-id', request_id),
             ),
         )
+
+    async def top_missed(
+        self,
+        page: int,
+        page_size: int,
+        request_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+    ) -> TopMissedResponsePb:
+        return await self.documents_stub.top_missed(
+            TopMissedRequestPb(
+                page=page,
+                page_size=page_size,
+                session_id=session_id,
+                user_id=user_id,
+            ),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )

nexus/meta_api/proto/documents_service.proto

@@ -13,6 +13,18 @@ message RollResponse {
   uint64 document_id = 1;
 }
 
+message TopMissedRequest {
+  uint32 page = 1;
+  uint32 page_size = 2;
+  string session_id = 3;
+  int64 user_id = 4;
+}
+
+message TopMissedResponse {
+  repeated nexus.models.proto.TypedDocument typed_documents = 1;
+  bool has_next = 2;
+}
+
 message TypedDocumentRequest {
   string schema = 1;
   uint64 document_id = 2;
@@ -26,4 +38,5 @@ message PutTypedDocumentResponse {}
 service Documents {
   rpc get (TypedDocumentRequest) returns (nexus.models.proto.TypedDocument) {}
   rpc roll (RollRequest) returns (RollResponse) {}
+  rpc top_missed (TopMissedRequest) returns (TopMissedResponse) {}
 }

nexus/meta_api/proto/documents_service_pb2.py

@@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\"\x1a\n\x18PutTypedDocumentResponse2\xb4\x01\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x62\x06proto3'
+  serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"X\n\x10TopMissedRequest\x12\x0c\n\x04page\x18\x01 \x01(\r\x12\x11\n\tpage_size\x18\x02 \x01(\r\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x0f\n\x07user_id\x18\x04 \x01(\x03\"a\n\x11TopMissedResponse\x12:\n\x0ftyped_documents\x18\x01 \x03(\x0b\x32!.nexus.models.proto.TypedDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\"\x1a\n\x18PutTypedDocumentResponse2\x95\x02\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x12_\n\ntop_missed\x12&.nexus.meta_api.proto.TopMissedRequest\x1a\'.nexus.meta_api.proto.TopMissedResponse\"\x00\x62\x06proto3'
   ,
   dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
 
@@ -106,6 +106,98 @@ _ROLLRESPONSE = _descriptor.Descriptor(
 )
 
 
+_TOPMISSEDREQUEST = _descriptor.Descriptor(
+  name='TopMissedRequest',
+  full_name='nexus.meta_api.proto.TopMissedRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='page', full_name='nexus.meta_api.proto.TopMissedRequest.page', index=0,
+      number=1, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='page_size', full_name='nexus.meta_api.proto.TopMissedRequest.page_size', index=1,
+      number=2, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='session_id', full_name='nexus.meta_api.proto.TopMissedRequest.session_id', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='user_id', full_name='nexus.meta_api.proto.TopMissedRequest.user_id', index=3,
+      number=4, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=218,
+  serialized_end=306,
+)
+
+
+_TOPMISSEDRESPONSE = _descriptor.Descriptor(
+  name='TopMissedResponse',
+  full_name='nexus.meta_api.proto.TopMissedResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='typed_documents', full_name='nexus.meta_api.proto.TopMissedResponse.typed_documents', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='has_next', full_name='nexus.meta_api.proto.TopMissedResponse.has_next', index=1,
+      number=2, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=308,
+  serialized_end=405,
+)
+
+
 _TYPEDDOCUMENTREQUEST = _descriptor.Descriptor(
   name='TypedDocumentRequest',
   full_name='nexus.meta_api.proto.TypedDocumentRequest',
@@ -161,8 +253,8 @@ _TYPEDDOCUMENTREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=218,
+  serialized_start=407,
-  serialized_end=332,
+  serialized_end=521,
 )
 
 
@@ -186,12 +278,15 @@ _PUTTYPEDDOCUMENTRESPONSE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=334,
+  serialized_start=523,
-  serialized_end=360,
+  serialized_end=549,
 )
 
+_TOPMISSEDRESPONSE.fields_by_name['typed_documents'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
 DESCRIPTOR.message_types_by_name['RollRequest'] = _ROLLREQUEST
 DESCRIPTOR.message_types_by_name['RollResponse'] = _ROLLRESPONSE
+DESCRIPTOR.message_types_by_name['TopMissedRequest'] = _TOPMISSEDREQUEST
+DESCRIPTOR.message_types_by_name['TopMissedResponse'] = _TOPMISSEDRESPONSE
 DESCRIPTOR.message_types_by_name['TypedDocumentRequest'] = _TYPEDDOCUMENTREQUEST
 DESCRIPTOR.message_types_by_name['PutTypedDocumentResponse'] = _PUTTYPEDDOCUMENTRESPONSE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -210,6 +305,20 @@ RollResponse = _reflection.GeneratedProtocolMessageType('RollResponse', (_messag
   })
 _sym_db.RegisterMessage(RollResponse)
 
+TopMissedRequest = _reflection.GeneratedProtocolMessageType('TopMissedRequest', (_message.Message,), {
+  'DESCRIPTOR' : _TOPMISSEDREQUEST,
+  '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.TopMissedRequest)
+  })
+_sym_db.RegisterMessage(TopMissedRequest)
+
+TopMissedResponse = _reflection.GeneratedProtocolMessageType('TopMissedResponse', (_message.Message,), {
+  'DESCRIPTOR' : _TOPMISSEDRESPONSE,
+  '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.TopMissedResponse)
+  })
+_sym_db.RegisterMessage(TopMissedResponse)
+
 TypedDocumentRequest = _reflection.GeneratedProtocolMessageType('TypedDocumentRequest', (_message.Message,), {
   'DESCRIPTOR' : _TYPEDDOCUMENTREQUEST,
   '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
@@ -233,8 +342,8 @@ _DOCUMENTS = _descriptor.ServiceDescriptor(
   index=0,
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_start=363,
+  serialized_start=552,
-  serialized_end=543,
+  serialized_end=829,
   methods=[
   _descriptor.MethodDescriptor(
     name='get',
@@ -256,6 +365,16 @@ _DOCUMENTS = _descriptor.ServiceDescriptor(
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
   ),
+  _descriptor.MethodDescriptor(
+    name='top_missed',
+    full_name='nexus.meta_api.proto.Documents.top_missed',
+    index=2,
+    containing_service=None,
+    input_type=_TOPMISSEDREQUEST,
+    output_type=_TOPMISSEDRESPONSE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
 ])
 _sym_db.RegisterServiceDescriptor(_DOCUMENTS)
 

nexus/meta_api/proto/documents_service_pb2_grpc.py

@@ -27,6 +27,11 @@ class DocumentsStub(object):
                 request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.SerializeToString,
                 response_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString,
                 )
+        self.top_missed = channel.unary_unary(
+                '/nexus.meta_api.proto.Documents/top_missed',
+                request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedRequest.SerializeToString,
+                response_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedResponse.FromString,
+                )
 
 
 class DocumentsServicer(object):
@@ -44,6 +49,12 @@ class DocumentsServicer(object):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def top_missed(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_DocumentsServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -57,6 +68,11 @@ def add_DocumentsServicer_to_server(servicer, server):
                     request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.FromString,
                     response_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.SerializeToString,
             ),
+            'top_missed': grpc.unary_unary_rpc_method_handler(
+                    servicer.top_missed,
+                    request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedRequest.FromString,
+                    response_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedResponse.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'nexus.meta_api.proto.Documents', rpc_method_handlers)
@@ -100,3 +116,20 @@ class Documents(object):
             nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def top_missed(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Documents/top_missed',
+            nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedRequest.SerializeToString,
+            nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TopMissedResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

nexus/meta_api/proto/meta_search_service.proto

@@ -5,8 +5,8 @@ import "nexus/models/proto/typed_document.proto";
 
 message ScoredDocument {
   nexus.models.proto.TypedDocument typed_document = 1;
-  float score = 3;
+  float score = 2;
-  uint32 position = 4;
+  uint32 position = 3;
 }
 
 message SearchResponse {

nexus/meta_api/proto/meta_search_service_pb2.py

@@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x03 \x01(\x02\x12\x10\n\x08position\x18\x04 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\xa3\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t\x12\x1a\n\x12skip_query_rewrite\x18\t \x01(\x08\x32\x63\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
+  serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2c\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
   ,
   dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
 
@@ -45,14 +45,14 @@ _SCOREDDOCUMENT = _descriptor.Descriptor(
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='score', full_name='nexus.meta_api.proto.ScoredDocument.score', index=1,
-      number=3, type=2, cpp_type=6, label=1,
+      number=2, type=2, cpp_type=6, label=1,
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='position', full_name='nexus.meta_api.proto.ScoredDocument.position', index=2,
-      number=4, type=13, cpp_type=3, label=1,
+      number=3, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
@@ -170,13 +170,6 @@ _SEARCHREQUEST = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='skip_query_rewrite', full_name='nexus.meta_api.proto.SearchRequest.skip_query_rewrite', index=7,
-      number=9, type=8, cpp_type=7, label=1,
-      has_default_value=False, default_value=False,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -190,7 +183,7 @@ _SEARCHREQUEST = _descriptor.Descriptor(
   oneofs=[
   ],
   serialized_start=324,
-  serialized_end=487,
+  serialized_end=459,
 )
 
 _SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
@@ -230,8 +223,8 @@ _METASEARCH = _descriptor.ServiceDescriptor(
   index=0,
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_start=489,
+  serialized_start=461,
-  serialized_end=588,
+  serialized_end=560,
   methods=[
   _descriptor.MethodDescriptor(
     name='search',

nexus/views/telegram/common.py

@@ -30,12 +30,13 @@ def close_button(session_id: str = None):
         )
 
 
-def vote_button(language: str, session_id: str, document_id: int, case: str):
+def vote_button(language: str, session_id: str, schema: str, document_id: int, case: str):
     label = f"REPORT_{case.upper()}_FILE"
     case = {'broken': 'b', 'ok': 'o'}[case]
+    schema = {'scimag': 'a', 'scitech': 'b'}[schema]
     return Button.inline(
         text=t(label, language=language),
-        data=f'/vote_{session_id}_{document_id}_{case}',
+        data=f'/vote{schema}_{session_id}_{document_id}_{case}',
     )
 
 

nexus/views/telegram/scimag.py

@@ -6,6 +6,7 @@ from typing import (
 )
 from urllib.parse import quote
 
+from izihawa_types.safecast import safe_int
 from izihawa_utils.common import filter_none
 from nexus.models.proto.scimag_pb2 import Scimag as ScimagPb
 from nexus.nlptools.utils import (
@@ -204,16 +205,17 @@ class ScimagView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin):
             )
             if self.ref_by_count:
                 buttons[-1].append(
-                    Button.inline(
+                    Button.switch_inline(
                         text=f'🔗 {self.ref_by_count or ""}',
-                        data=f'/r_{session_id}_{self.id}',
+                        query=f'references:"{self.doi}"',
+                        same_peer=True,
                     )
                 )
             buttons[-1].append(close_button(session_id))
         return '\n'.join(parts).strip()[:4096], buttons
 
-    def get_view_command(self, session_id: str, message_id: int, parent_view_type: str = 's', position: int = 0) -> str:
+    def get_view_command(self, session_id: str, message_id: int, position: int = 0) -> str:
-        return f'/va{parent_view_type}_{session_id}_{message_id}_{self.id}_{position}'
+        return f'/va_{session_id}_{message_id}_{self.id}_{position}'
 
     def get_robust_journal(self):
         if self.type != 'chapter' and self.type != 'book-chapter':
@@ -232,4 +234,7 @@ class ScimagView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin):
             if self.issue:
                 return f'vol. {self.volume}({self.issue})'
             else:
-                return self.volume
+                if safe_int(self.volume):
+                    return f'vol. {self.volume}'
+                else:
+                    return self.volume