No description

GitOrigin-RevId: 83514338be1d662518bab9fe5ab6eefef98f229f

WORKSPACE

@@ -68,6 +68,36 @@ http_archive(
     ],
 )
 
+_configure_python_based_on_os = """
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
+else
+    ./configure --prefix=$(pwd)/bazel_install
+fi
+"""
+
+http_archive(
+    name = "python_interpreter",
+    build_file_content = """
+exports_files(["python_bin"])
+filegroup(
+    name = "files",
+    srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
+    visibility = ["//visibility:public"],
+)
+""",
+    patch_cmds = [
+        "mkdir $(pwd)/bazel_install",
+        _configure_python_based_on_os,
+        "make",
+        "make install",
+        "ln -s bazel_install/bin/python3 python_bin",
+    ],
+    sha256 = "4b0e6644a76f8df864ae24ac500a51bbf68bd098f6a173e27d3b61cdca9aa134",
+    strip_prefix = "Python-3.9.4",
+    urls = ["https://www.python.org/ftp/python/3.9.4/Python-3.9.4.tar.xz"],
+)
+
 http_archive(
     name = "rules_python",
     sha256 = "b228318a786d99b665bc83bd6cdb81512cae5f8eb15e8cd19f9956604b8939f5",
@@ -189,12 +219,13 @@ py3_image_repos()
 rust_image_repos()
 
 # Python
-register_toolchains("//rules/python:py_toolchain")
+register_toolchains("//rules/python:py_3_toolchain")
 
 load("@rules_python//python:pip.bzl", "pip_install")
 
 pip_install(
     name = "pip_modules",
+    python_interpreter_target = "@python_interpreter//:python_bin",
     requirements = "//rules/python:requirements.txt",
 )
 

idm/api2/aioclient/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("@pip_modules//:requirements.bzl", "requirement")
+
+py_library(
+    name = "aioclient",
+    srcs = glob(["**/*.py"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("cachetools"),
+        requirement("lru-dict"),
+        requirement("tenacity"),
+        "//idm/api2/proto:idm_grpc_py",
+        "//idm/api2/proto:idm_proto_py",
+    ],
+)

idm/api2/aioclient/__init__.py

@@ -0,0 +1,3 @@
+from .aioclient import IdmApi2GrpcClient
+
+__all__ = ['IdmApi2GrpcClient']

idm/api2/aioclient/aioclient.py

@@ -0,0 +1,125 @@
+from aiokit import AioThing
+from grpc import StatusCode
+from grpc.experimental.aio import (
+    AioRpcError,
+    insecure_channel,
+)
+from idm.api2.proto.chats_service_pb2 import (
+    CreateChatRequest,
+    GetChatRequest,
+    ListChatsRequest,
+    UpdateChatRequest,
+)
+from idm.api2.proto.chats_service_pb2_grpc import ChatsStub
+from lru import LRU
+from tenacity import (
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_fixed,
+)
+
+
+class IdmApi2GrpcClient(AioThing):
+    def __init__(
+        self,
+        base_url,
+    ):
+        super().__init__()
+        self.channel = insecure_channel(base_url, [
+            ('grpc.dns_min_time_between_resolutions_ms', 1000),
+            ('grpc.initial_reconnect_backoff_ms', 1000),
+            ('grpc.lb_policy_name', 'round_robin'),
+            ('grpc.min_reconnect_backoff_ms', 1000),
+            ('grpc.max_reconnect_backoff_ms', 2000),
+        ])
+        self.chats_stub = ChatsStub(self.channel)
+        self.cache = LRU(4096)
+
+    async def start(self):
+        await self.channel.channel_ready()
+
+    async def stop(self):
+        await self.channel.close()
+
+    async def create_chat(
+        self,
+        chat_id,
+        username,
+        language,
+        request_id: str = None,
+    ):
+        response = await self.chats_stub.create_chat(
+            CreateChatRequest(
+                chat_id=chat_id,
+                username=username,
+                language=language,
+            ),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )
+        return response
+
+    @retry(
+        retry=retry_if_exception(
+            lambda e: isinstance(e, AioRpcError) and e.code() == StatusCode.UNAVAILABLE
+        ),
+        reraise=True,
+        stop=stop_after_attempt(10),
+        wait=wait_fixed(5),
+    )
+    async def get_chat(
+        self,
+        chat_id,
+        request_id: str = None,
+    ):
+        response = await self.chats_stub.get_chat(
+            GetChatRequest(chat_id=chat_id),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )
+        return response
+
+    async def list_chats(
+        self,
+        request_id: str = None,
+        banned_at_moment=None,
+    ):
+        response = await self.chats_stub.list_chats(
+            ListChatsRequest(banned_at_moment=banned_at_moment),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )
+        return response
+
+    async def update_chat(
+        self,
+        chat_id,
+        request_id: str = None,
+        language=None,
+        is_system_messaging_enabled=None,
+        is_discovery_enabled=None,
+        ban_until=None,
+        ban_message=None,
+        is_admin=None,
+        last_location=None,
+    ):
+        response = await self.chats_stub.update_chat(
+            UpdateChatRequest(
+                chat_id=chat_id,
+                language=language,
+                is_system_messaging_enabled=is_system_messaging_enabled,
+                is_discovery_enabled=is_discovery_enabled,
+                ban_until=ban_until,
+                ban_message=ban_message,
+                is_admin=is_admin,
+                last_location=last_location,
+            ),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )
+        return response

idm/api2/proto/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_library(
+    name = "idm_proto",
+    srcs = [
+        "chats_service.proto",
+        "location.proto",
+    ],
+    deps = [
+        "@com_google_protobuf//:wrappers_proto",
+    ],
+)
+
+py_proto_library(
+    name = "idm_proto_py",
+    deps = [":idm_proto"],
+)
+
+py_grpc_library(
+    name = "idm_grpc_py",
+    srcs = [":idm_proto"],
+    deps = [":idm_proto_py"],
+)

idm/api2/proto/chats_service.proto

@@ -0,0 +1,70 @@
+syntax = "proto3";
+package idm.api2.proto;
+
+import "idm/api2/proto/location.proto";
+
+message ChatData {
+    int64 id = 1;
+    string username = 2;
+    string language = 3;
+    bool is_system_messaging_enabled = 4;
+    bool is_discovery_enabled = 5;
+    int32 ban_until = 6;
+    string ban_message = 7;
+    bool is_admin = 8;
+    string tzinfo = 9;
+    bool is_subscribed = 10;
+    int64 created_at = 11;
+}
+
+message ChatsData {
+    repeated ChatData chats = 1;
+}
+
+message CreateChatRequest {
+    int64 chat_id = 1;
+    string username = 2;
+    string language = 3;
+}
+
+message GetChatRequest {
+    int64 chat_id = 1;
+}
+
+message ListChatsRequest {
+    oneof banned_at_moment_oneof {
+        int32 banned_at_moment = 1;
+    }
+}
+
+message UpdateChatRequest {
+    int64 chat_id = 1;
+    oneof language_oneof {
+        string language = 2;
+    }
+    oneof is_system_messaging_enabled_oneof {
+        bool is_system_messaging_enabled = 3;
+    }
+    oneof is_discovery_enabled_oneof {
+        bool is_discovery_enabled = 4;
+    }
+    oneof ban_until_oneof {
+        int32 ban_until = 5;
+    }
+    oneof ban_message_oneof {
+        string ban_message = 6;
+    }
+    oneof is_admin_oneof {
+        bool is_admin = 7;
+    }
+    oneof last_location_oneof {
+        idm.api2.proto.Location last_location = 8;
+    }
+}
+
+service Chats {
+    rpc create_chat(CreateChatRequest) returns (ChatData) {};
+    rpc get_chat(GetChatRequest) returns (ChatData) {};
+    rpc list_chats(ListChatsRequest) returns (ChatsData) {};
+    rpc update_chat(UpdateChatRequest) returns (ChatData) {};
+}

idm/api2/proto/chats_service_pb2.py

@@ -0,0 +1,528 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: idm/api2/proto/chats_service.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from idm.api2.proto import \
+    location_pb2 as idm_dot_api2_dot_proto_dot_location__pb2
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='idm/api2/proto/chats_service.proto',
+  package='idm.api2.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\"idm/api2/proto/chats_service.proto\x12\x0eidm.api2.proto\x1a\x1didm/api2/proto/location.proto\"\xf2\x01\n\x08\x43hatData\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x0e\n\x06tzinfo\x18\t \x01(\t\x12\x15\n\ris_subscribed\x18\n \x01(\x08\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\"4\n\tChatsData\x12\'\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x18.idm.api2.proto.ChatData\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"H\n\x10ListChatsRequest\x12\x1a\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x42\x18\n\x16\x62\x61nned_at_moment_oneof\"\x98\x03\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x12\n\x08language\x18\x02 \x01(\tH\x00\x12%\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x12\x1e\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x12\x13\n\tban_until\x18\x05 \x01(\x05H\x03\x12\x15\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x12\x12\n\x08is_admin\x18\x07 \x01(\x08H\x05\x12\x31\n\rlast_location\x18\x08 \x01(\x0b\x32\x18.idm.api2.proto.LocationH\x06\x42\x10\n\x0elanguage_oneofB#\n!is_system_messaging_enabled_oneofB\x1c\n\x1ais_discovery_enabled_oneofB\x11\n\x0f\x62\x61n_until_oneofB\x13\n\x11\x62\x61n_message_oneofB\x10\n\x0eis_admin_oneofB\x15\n\x13last_location_oneof2\xb8\x02\n\x05\x43hats\x12L\n\x0b\x63reate_chat\x12!.idm.api2.proto.CreateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12\x46\n\x08get_chat\x12\x1e.idm.api2.proto.GetChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12K\n\nlist_chats\x12 .idm.api2.proto.ListChatsRequest\x1a\x19.idm.api2.proto.ChatsData\"\x00\x12L\n\x0bupdate_chat\x12!.idm.api2.proto.UpdateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x62\x06proto3'
+  ,
+  dependencies=[idm_dot_api2_dot_proto_dot_location__pb2.DESCRIPTOR,])
+
+
+
+
+_CHATDATA = _descriptor.Descriptor(
+  name='ChatData',
+  full_name='idm.api2.proto.ChatData',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='idm.api2.proto.ChatData.id', index=0,
+      number=1, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='username', full_name='idm.api2.proto.ChatData.username', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='language', full_name='idm.api2.proto.ChatData.language', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_system_messaging_enabled', full_name='idm.api2.proto.ChatData.is_system_messaging_enabled', index=3,
+      number=4, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_discovery_enabled', full_name='idm.api2.proto.ChatData.is_discovery_enabled', index=4,
+      number=5, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='ban_until', full_name='idm.api2.proto.ChatData.ban_until', index=5,
+      number=6, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='ban_message', full_name='idm.api2.proto.ChatData.ban_message', index=6,
+      number=7, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_admin', full_name='idm.api2.proto.ChatData.is_admin', index=7,
+      number=8, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='tzinfo', full_name='idm.api2.proto.ChatData.tzinfo', index=8,
+      number=9, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_subscribed', full_name='idm.api2.proto.ChatData.is_subscribed', index=9,
+      number=10, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='created_at', full_name='idm.api2.proto.ChatData.created_at', index=10,
+      number=11, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=86,
+  serialized_end=328,
+)
+
+
+_CHATSDATA = _descriptor.Descriptor(
+  name='ChatsData',
+  full_name='idm.api2.proto.ChatsData',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='chats', full_name='idm.api2.proto.ChatsData.chats', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=330,
+  serialized_end=382,
+)
+
+
+_CREATECHATREQUEST = _descriptor.Descriptor(
+  name='CreateChatRequest',
+  full_name='idm.api2.proto.CreateChatRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='chat_id', full_name='idm.api2.proto.CreateChatRequest.chat_id', index=0,
+      number=1, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='username', full_name='idm.api2.proto.CreateChatRequest.username', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='language', full_name='idm.api2.proto.CreateChatRequest.language', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=384,
+  serialized_end=456,
+)
+
+
+_GETCHATREQUEST = _descriptor.Descriptor(
+  name='GetChatRequest',
+  full_name='idm.api2.proto.GetChatRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='chat_id', full_name='idm.api2.proto.GetChatRequest.chat_id', index=0,
+      number=1, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=458,
+  serialized_end=491,
+)
+
+
+_LISTCHATSREQUEST = _descriptor.Descriptor(
+  name='ListChatsRequest',
+  full_name='idm.api2.proto.ListChatsRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='banned_at_moment', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='banned_at_moment_oneof', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment_oneof',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=493,
+  serialized_end=565,
+)
+
+
+_UPDATECHATREQUEST = _descriptor.Descriptor(
+  name='UpdateChatRequest',
+  full_name='idm.api2.proto.UpdateChatRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='chat_id', full_name='idm.api2.proto.UpdateChatRequest.chat_id', index=0,
+      number=1, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='language', full_name='idm.api2.proto.UpdateChatRequest.language', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_system_messaging_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled', index=2,
+      number=3, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_discovery_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled', index=3,
+      number=4, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='ban_until', full_name='idm.api2.proto.UpdateChatRequest.ban_until', index=4,
+      number=5, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='ban_message', full_name='idm.api2.proto.UpdateChatRequest.ban_message', index=5,
+      number=6, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_admin', full_name='idm.api2.proto.UpdateChatRequest.is_admin', index=6,
+      number=7, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='last_location', full_name='idm.api2.proto.UpdateChatRequest.last_location', index=7,
+      number=8, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='language_oneof', full_name='idm.api2.proto.UpdateChatRequest.language_oneof',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+    _descriptor.OneofDescriptor(
+      name='is_system_messaging_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled_oneof',
+      index=1, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+    _descriptor.OneofDescriptor(
+      name='is_discovery_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled_oneof',
+      index=2, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+    _descriptor.OneofDescriptor(
+      name='ban_until_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_until_oneof',
+      index=3, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+    _descriptor.OneofDescriptor(
+      name='ban_message_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_message_oneof',
+      index=4, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+    _descriptor.OneofDescriptor(
+      name='is_admin_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_admin_oneof',
+      index=5, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+    _descriptor.OneofDescriptor(
+      name='last_location_oneof', full_name='idm.api2.proto.UpdateChatRequest.last_location_oneof',
+      index=6, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=568,
+  serialized_end=976,
+)
+
+_CHATSDATA.fields_by_name['chats'].message_type = _CHATDATA
+_LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof'].fields.append(
+  _LISTCHATSREQUEST.fields_by_name['banned_at_moment'])
+_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof']
+_UPDATECHATREQUEST.fields_by_name['last_location'].message_type = idm_dot_api2_dot_proto_dot_location__pb2._LOCATION
+_UPDATECHATREQUEST.oneofs_by_name['language_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['language'])
+_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['language_oneof']
+_UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'])
+_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof']
+_UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'])
+_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof']
+_UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['ban_until'])
+_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof']
+_UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['ban_message'])
+_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof']
+_UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['is_admin'])
+_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof']
+_UPDATECHATREQUEST.oneofs_by_name['last_location_oneof'].fields.append(
+  _UPDATECHATREQUEST.fields_by_name['last_location'])
+_UPDATECHATREQUEST.fields_by_name['last_location'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['last_location_oneof']
+DESCRIPTOR.message_types_by_name['ChatData'] = _CHATDATA
+DESCRIPTOR.message_types_by_name['ChatsData'] = _CHATSDATA
+DESCRIPTOR.message_types_by_name['CreateChatRequest'] = _CREATECHATREQUEST
+DESCRIPTOR.message_types_by_name['GetChatRequest'] = _GETCHATREQUEST
+DESCRIPTOR.message_types_by_name['ListChatsRequest'] = _LISTCHATSREQUEST
+DESCRIPTOR.message_types_by_name['UpdateChatRequest'] = _UPDATECHATREQUEST
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ChatData = _reflection.GeneratedProtocolMessageType('ChatData', (_message.Message,), {
+  'DESCRIPTOR' : _CHATDATA,
+  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.ChatData)
+  })
+_sym_db.RegisterMessage(ChatData)
+
+ChatsData = _reflection.GeneratedProtocolMessageType('ChatsData', (_message.Message,), {
+  'DESCRIPTOR' : _CHATSDATA,
+  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.ChatsData)
+  })
+_sym_db.RegisterMessage(ChatsData)
+
+CreateChatRequest = _reflection.GeneratedProtocolMessageType('CreateChatRequest', (_message.Message,), {
+  'DESCRIPTOR' : _CREATECHATREQUEST,
+  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.CreateChatRequest)
+  })
+_sym_db.RegisterMessage(CreateChatRequest)
+
+GetChatRequest = _reflection.GeneratedProtocolMessageType('GetChatRequest', (_message.Message,), {
+  'DESCRIPTOR' : _GETCHATREQUEST,
+  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.GetChatRequest)
+  })
+_sym_db.RegisterMessage(GetChatRequest)
+
+ListChatsRequest = _reflection.GeneratedProtocolMessageType('ListChatsRequest', (_message.Message,), {
+  'DESCRIPTOR' : _LISTCHATSREQUEST,
+  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.ListChatsRequest)
+  })
+_sym_db.RegisterMessage(ListChatsRequest)
+
+UpdateChatRequest = _reflection.GeneratedProtocolMessageType('UpdateChatRequest', (_message.Message,), {
+  'DESCRIPTOR' : _UPDATECHATREQUEST,
+  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.UpdateChatRequest)
+  })
+_sym_db.RegisterMessage(UpdateChatRequest)
+
+
+
+_CHATS = _descriptor.ServiceDescriptor(
+  name='Chats',
+  full_name='idm.api2.proto.Chats',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=979,
+  serialized_end=1291,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='create_chat',
+    full_name='idm.api2.proto.Chats.create_chat',
+    index=0,
+    containing_service=None,
+    input_type=_CREATECHATREQUEST,
+    output_type=_CHATDATA,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='get_chat',
+    full_name='idm.api2.proto.Chats.get_chat',
+    index=1,
+    containing_service=None,
+    input_type=_GETCHATREQUEST,
+    output_type=_CHATDATA,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='list_chats',
+    full_name='idm.api2.proto.Chats.list_chats',
+    index=2,
+    containing_service=None,
+    input_type=_LISTCHATSREQUEST,
+    output_type=_CHATSDATA,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='update_chat',
+    full_name='idm.api2.proto.Chats.update_chat',
+    index=3,
+    containing_service=None,
+    input_type=_UPDATECHATREQUEST,
+    output_type=_CHATDATA,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_CHATS)
+
+DESCRIPTOR.services_by_name['Chats'] = _CHATS
+
+# @@protoc_insertion_point(module_scope)

idm/api2/proto/chats_service_pb2_grpc.py

@@ -0,0 +1,165 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+from idm.api2.proto import \
+    chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
+
+
+class ChatsStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.create_chat = channel.unary_unary(
+                '/idm.api2.proto.Chats/create_chat',
+                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
+                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+                )
+        self.get_chat = channel.unary_unary(
+                '/idm.api2.proto.Chats/get_chat',
+                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
+                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+                )
+        self.list_chats = channel.unary_unary(
+                '/idm.api2.proto.Chats/list_chats',
+                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
+                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
+                )
+        self.update_chat = channel.unary_unary(
+                '/idm.api2.proto.Chats/update_chat',
+                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
+                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+                )
+
+
+class ChatsServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def create_chat(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def get_chat(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def list_chats(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def update_chat(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ChatsServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'create_chat': grpc.unary_unary_rpc_method_handler(
+                    servicer.create_chat,
+                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.FromString,
+                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
+            ),
+            'get_chat': grpc.unary_unary_rpc_method_handler(
+                    servicer.get_chat,
+                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.FromString,
+                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
+            ),
+            'list_chats': grpc.unary_unary_rpc_method_handler(
+                    servicer.list_chats,
+                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.FromString,
+                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.SerializeToString,
+            ),
+            'update_chat': grpc.unary_unary_rpc_method_handler(
+                    servicer.update_chat,
+                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.FromString,
+                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'idm.api2.proto.Chats', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Chats(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def create_chat(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/create_chat',
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def get_chat(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/get_chat',
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def list_chats(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/list_chats',
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def update_chat(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/update_chat',
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
+            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

idm/api2/proto/location.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+package idm.api2.proto;
+
+message Location {
+    float lat = 1;
+    float lon = 2;
+}

idm/api2/proto/location_pb2.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: idm/api2/proto/location.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='idm/api2/proto/location.proto',
+  package='idm.api2.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x1didm/api2/proto/location.proto\x12\x0eidm.api2.proto\"$\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x02\x12\x0b\n\x03lon\x18\x02 \x01(\x02\x62\x06proto3'
+)
+
+
+
+
+_LOCATION = _descriptor.Descriptor(
+  name='Location',
+  full_name='idm.api2.proto.Location',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='lat', full_name='idm.api2.proto.Location.lat', index=0,
+      number=1, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='lon', full_name='idm.api2.proto.Location.lon', index=1,
+      number=2, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=49,
+  serialized_end=85,
+)
+
+DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), {
+  'DESCRIPTOR' : _LOCATION,
+  '__module__' : 'idm.api2.proto.location_pb2'
+  # @@protoc_insertion_point(class_scope:idm.api2.proto.Location)
+  })
+_sym_db.RegisterMessage(Location)
+
+
+# @@protoc_insertion_point(module_scope)

idm/api2/proto/location_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

images/install.bzl

@@ -11,8 +11,8 @@ def images_install():
 
     container_pull(
         name = "ubuntu",
-        digest = "sha256:c65d2b75a62135c95e2c595822af9b6f6cf0f32c11bcd4a38368d7b7c36b66f5",
+        digest = "sha256:45ff0162921e61c004010a67db1bee7d039a677bed0cb294e61f2b47346d42b3",
         registry = "index.docker.io",
         repository = "library/ubuntu",
-        tag = "20.04",
+        tag = "20.10",
     )

images/production/BUILD.bazel

@@ -13,7 +13,7 @@ download_pkgs(
         "libev4",
         "libgomp1",
         "libgoogle-perftools-dev",
-        "libprotobuf17",
+        "libprotobuf23",
         "libssl1.1",
     ],
 )
@@ -28,7 +28,10 @@ install_pkgs(
 container_image(
     name = "base-production-image",
     base = ":install-base-production-image",
-    env = {"LANG": "C.UTF-8"},
+    env = {
+        "LANG": "C.UTF-8",
+        "LC_CTYPE": "en_US.UTF-8",
+    },
 )
 
 download_pkgs(

library/aiopostgres/pool_holder.py

@@ -1,3 +1,4 @@
+import aiopg
 import psycopg2.extras
 from aiokit import AioThing
 from psycopg2 import OperationalError
@@ -10,7 +11,7 @@ from tenacity import (
 
 
 class AioPostgresPoolHolder(AioThing):
-    def __init__(self, fn, *args, **kwargs):
+    def __init__(self, fn=aiopg.create_pool, *args, **kwargs):
         super().__init__()
         self.fn = fn
         self.args = args

nexus/actions/__init__.py

@@ -9,6 +9,10 @@ from .update_document_scitech import (
     CleanDocumentOperationUpdateDocumentScitechPbAction,
     SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction,
 )
+from .update_document_sharience import (
+    SendDocumentOperationUpdateDocumentShariencePbToGoldenPostgresAction,
+)
+from .vote import SendDocumentOperationVotePbToGoldenPostgresAction
 
 __all__ = [
     'CleanDocumentOperationUpdateDocumentScimagPbAction',
@@ -18,4 +22,6 @@ __all__ = [
     'SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction',
     'SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction',
     'SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction',
+    'SendDocumentOperationUpdateDocumentShariencePbToGoldenPostgresAction',
+    'SendDocumentOperationVotePbToGoldenPostgresAction',
 ]

nexus/actions/update_document_sharience.py

@@ -0,0 +1,86 @@
+from typing import (
+    Optional,
+    Set,
+)
+
+import aiopg
+from library.aiopostgres.pool_holder import AioPostgresPoolHolder
+from nexus.models.proto.operation_pb2 import \
+    DocumentOperation as DocumentOperationPb
+from nexus.models.proto.sharience_pb2 import Sharience as ShariencePb
+from pypika import (
+    PostgreSQLQuery,
+    Table,
+)
+from pypika.terms import Array
+
+from .base import BaseAction
+
+
+class SendDocumentOperationUpdateDocumentShariencePbToGoldenPostgresAction(BaseAction):
+    sharience_table = Table('sharience')
+    db_multi_fields = {
+        'ipfs_multihashes',
+    }
+    db_single_fields = {
+        'id',
+        'parent_id',
+        'uploader_id',
+        'filesize',
+        'md5',
+        'telegram_file_id',
+        'updated_at',
+    }
+    db_fields = db_single_fields | db_multi_fields
+
+    def __init__(self, database):
+        super().__init__()
+        self.pool_holder = AioPostgresPoolHolder(
+            fn=aiopg.create_pool,
+            dsn=f'dbname={database["database"]} '
+            f'user={database["username"]} '
+            f'password={database["password"]} '
+            f'host={database["host"]}',
+            timeout=30,
+            pool_recycle=60,
+            maxsize=4,
+        )
+        self.waits.append(self.pool_holder)
+
+    def cast_field_value(self, field_name: str, field_value):
+        if field_name in self.db_multi_fields:
+            field_value = Array(*field_value)
+        return field_name, field_value
+
+    def is_field_set(self, sharience_pb: ShariencePb, field_name: str):
+        field_value = getattr(sharience_pb, field_name)
+        return field_value
+
+    def generate_insert_sql(self, sharience_pb: ShariencePb, fields: Optional[Set[str]] = None):
+        columns = []
+        inserts = []
+
+        fields = fields or self.db_fields
+        for field_name in fields:
+            if self.is_field_set(sharience_pb, field_name):
+                field_value = getattr(sharience_pb, field_name)
+                field_name, field_value = self.cast_field_value(field_name, field_value)
+                columns.append(field_name)
+                inserts.append(field_value)
+
+        query = PostgreSQLQuery.into(self.sharience_table).columns(*columns).insert(*inserts)
+        return query.returning(self.sharience_table.id).get_sql()
+
+    async def do(self, document_operation_pb: DocumentOperationPb) -> DocumentOperationPb:
+        update_document_pb = document_operation_pb.update_document
+        sharience_pb = update_document_pb.typed_document.sharience
+        fields = update_document_pb.fields or self.db_fields
+
+        sql = self.generate_insert_sql(
+            sharience_pb=sharience_pb,
+            fields=fields,
+        )
+        result = await self.pool_holder.execute(sql, fetch=True)
+        sharience_pb.id = result[0][0]
+
+        return document_operation_pb

nexus/actions/vote.py

@@ -0,0 +1,51 @@
+import aiopg
+from library.aiopostgres.pool_holder import AioPostgresPoolHolder
+from nexus.models.proto.operation_pb2 import \
+    DocumentOperation as DocumentOperationPb
+from pypika import (
+    PostgreSQLQuery,
+    Table,
+)
+
+from .base import BaseAction
+
+
+class SendDocumentOperationVotePbToGoldenPostgresAction(BaseAction):
+    votes_table = Table('votes')
+
+    def __init__(self, database):
+        super().__init__()
+        self.pool_holder = AioPostgresPoolHolder(
+            fn=aiopg.create_pool,
+            dsn=f'dbname={database["database"]} '
+            f'user={database["username"]} '
+            f'password={database["password"]} '
+            f'host={database["host"]}',
+            timeout=30,
+            pool_recycle=60,
+            maxsize=2,
+        )
+        self.waits.append(self.pool_holder)
+
+    def generate_insert_sql(self, document_id: int, value: int, voter_id: int):
+        query = PostgreSQLQuery.into(self.votes_table).columns(
+            'document_id',
+            'value',
+            'voter_id',
+        ).insert(
+            document_id,
+            value,
+            voter_id,
+        )
+        query = query.on_conflict('document_id', 'voter_id').do_update('value', value)
+        return query.get_sql()
+
+    async def do(self, document_operation_pb: DocumentOperationPb) -> DocumentOperationPb:
+        vote_pb = document_operation_pb.vote
+        sql = self.generate_insert_sql(
+            document_id=vote_pb.document_id,
+            value=vote_pb.value,
+            voter_id=vote_pb.voter_id,
+        )
+        await self.pool_holder.execute(sql)
+        return vote_pb

nexus/bot/BUILD.bazel

@@ -0,0 +1,71 @@
+load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_push")
+
+load("@pip_modules//:requirements.bzl", "requirement")
+
+alias(
+    name = "binary",
+    actual = ":image.binary",
+    visibility = ["//visibility:public"],
+)
+
+py3_image(
+    name = "image",
+    srcs = glob(["**/*.py"]),
+    base = "//images/production:base-python-image",
+    data = [
+        "configs/base.yaml",
+        "configs/custom.yaml",
+        "configs/development.yaml",
+        "configs/logging.yaml",
+        "configs/metrics.yaml",
+        "configs/production.yaml",
+        "configs/promotions.yaml",
+        "configs/testing.yaml",
+    ],
+    main = "main.py",
+    srcs_version = "PY3ONLY",
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("aiodns"),
+        requirement("aiohttp"),
+        requirement("aiohttp_socks"),
+        requirement("psycopg2-binary"),
+        requirement("pytimeparse"),
+        requirement("python_socks"),
+        requirement("tenacity"),
+        requirement("uvloop"),
+        "//idm/api2/aioclient",
+        "//library/aiobaseclient",
+        requirement("aiocrossref"),
+        requirement("aiokit"),
+        "//library/configurator",
+        "//library/logging",
+        "//library/metrics_server",
+        "//library/telegram",
+        "//nexus/hub/aioclient",
+        "//nexus/meta_api/aioclient",
+        "//nexus/models/proto:models_proto_py",
+        "//nexus/nlptools",
+        "//nexus/views/telegram",
+        requirement("izihawa_utils"),
+    ],
+)
+
+container_push(
+    name = "push-testing",
+    format = "Docker",
+    image = ":image",
+    registry = "registry.example.com",
+    repository = "nexus-bot",
+    tag = "testing",
+)
+
+container_push(
+    name = "push-latest",
+    format = "Docker",
+    image = ":image",
+    registry = "registry.example.com",
+    repository = "nexus-bot",
+    tag = "latest",
+)

nexus/bot/README.md

@@ -0,0 +1,8 @@
+# Nexus Search: Telegram Bot
+
+The bot requires three other essential parts: 
+- IDM API (auth)
+- Nexus Hub API (managing files)
+- Nexus Meta API (rescoring API for Summa Search server)
+
+or their substitutions

nexus/bot/__init__.py

@@ -0,0 +1,3 @@
+from . import handlers
+
+__all__ = ['handlers']

nexus/bot/application.py

@@ -0,0 +1,44 @@
+from aiokit import AioRootThing
+from idm.api2.aioclient import IdmApi2GrpcClient
+from izihawa_utils.importlib import import_object
+from library.telegram.base import BaseTelegramClient
+from nexus.bot.promotioner import Promotioner
+from nexus.bot.user_manager import UserManager
+from nexus.hub.aioclient import HubGrpcClient
+from nexus.meta_api.aioclient import MetaApiGrpcClient
+
+
+class TelegramApplication(AioRootThing):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.telegram_client = BaseTelegramClient(
+            app_id=self.config['telegram']['app_id'],
+            app_hash=self.config['telegram']['app_hash'],
+            bot_token=self.config['telegram']['bot_token'],
+            database=self.config['telegram'].get('database'),
+            mtproxy=self.config['telegram'].get('mtproxy'),
+        )
+
+        self.hub_client = HubGrpcClient(base_url=self.config['hub']['url'])
+        self.idm_client = IdmApi2GrpcClient(base_url=self.config['idm']['url'])
+        self.meta_api_client = MetaApiGrpcClient(base_url=self.config['meta_api']['url'])
+
+        self.promotioner = Promotioner(promotions=self.config['promotions'])
+        self.user_manager = UserManager()
+        self._handlers = []
+
+        self.starts.extend([self.hub_client, self.idm_client, self.meta_api_client])
+
+    def set_handlers(self, telegram_client):
+        for handler in self.config['telegram']['handlers']:
+            import_object(handler)(self).register_for(telegram_client)
+
+    async def start(self):
+        self.set_handlers(self.telegram_client)
+        await self.telegram_client.start_and_wait()
+        await self.telegram_client.run_until_disconnected()
+
+    async def stop(self):
+        self.telegram_client.remove_event_handlers()
+        await self.telegram_client.stop()

nexus/bot/configs/__init__.py

@@ -0,0 +1,16 @@
+from izihawa_utils import env
+from library.configurator import Configurator
+
+
+def get_config():
+    return Configurator([
+        'nexus/bot/configs/base.yaml',
+        'nexus/bot/configs/custom.yaml?',
+        'nexus/bot/configs/metrics.yaml?',
+        'nexus/bot/configs/%s.yaml?' % env.type,
+        'nexus/bot/configs/logging.yaml?',
+        'nexus/bot/configs/promotions.yaml?',
+    ])
+
+
+config = get_config()

nexus/bot/configs/base.yaml

@@ -0,0 +1,101 @@
+---
+
+application:
+  # Amazon Recipient Email in /donate message
+  amazon_gift_card_recipient: pirate@ship.space
+  # Amazon URL for buying card in /donate message
+  amazon_gift_card_url: https://www.amazon.com/dp/B07TMNGSN4
+  bot_version: 1.6.0
+  # Bitcoin Donation address in /donate message
+  btc_donate_address: 3QbF3zRQVjn3qMJBSbmLC1gb6VUc555xkw
+  # List of chat IDs that is allowed to bypass maintenance mode
+  bypass_maintenance: []
+  # Debugging mode
+  debug: true
+  # All users (except `bypass_maintenance` ones) will get UPGRADE_MAINTENANCE message in response
+  is_maintenance_mode: false
+  # Disable /settings, uploading new articles (can be used while vacuuming backend Postgres)
+  # and preventing creation of new users
+  is_read_only_mode: false
+  # Require subscription to `related_channel` before allowing to use the bot
+  is_subscription_required: false
+  # Libera Pay URL in /donate message
+  libera_pay_url:
+  maintenance_picture_url:
+  nexus_version: InterCom
+  # Default page size for SERP
+  page_size: 5
+  # Length of generated Request-ID used for tracking requests across all backends
+  request_id_length: 12
+  # Enabled schemas (passed to Nexus Meta API)
+  schemas:
+    - scimag
+    - scitech
+  # Length of generated Session-ID used in commands to clue user sessions
+  session_id_length: 8
+  too_difficult_picture_url:
+  upgrade_maintenance_picture_url:
+  # Configuring behaviour of the bot in some cases
+  views:
+    settings:
+      has_discovery_button: true
+      has_language_buttons: true
+      has_location_button: false
+      has_router: false
+      has_system_messaging_button: true
+hub:
+  url:
+idm:
+  url:
+log_path: '/var/log/nexus-bot/{{ ENV_TYPE }}'
+meta_api:
+  url:
+telegram:
+  # Telegram App Hash from https://my.telegram.org/
+  app_hash: '{{ APP_HASH }}'
+  # Telegram App ID from https://my.telegram.org/
+  app_id: 00000
+  # External bot name shown in messages to users
+  bot_external_name: libgen_scihub_bot
+  # Internal bot name used in logging
+  bot_name: nexus-bot
+  bot_token: '{{ BOT_TOKEN }}'
+  # WARNING! Potentially buggy telethon option. Sometimes it goes mad and overload users with tons of messages
+  # Collect missed messages at startup time and answer to them
+  catch_up: false
+  # Telegram account for forwarding copyright infringements from /copyright command
+  copyright_infringement_account:
+  # Telethon database for keeping cache
+  database:
+    session_id: '/usr/lib/nexus-bot/{{ ENV_TYPE }}/session.db'
+  # Enabled handlers
+  handlers:
+    - nexus.bot.handlers.admin.AdminHandler
+    - nexus.bot.handlers.ban.BanHandler
+    - nexus.bot.handlers.ban.BanlistHandler
+    - nexus.bot.handlers.ban.UnbanHandler
+    - nexus.bot.handlers.contact.ContactHandler
+    - nexus.bot.handlers.copyright.CopyrightHandler
+    - nexus.bot.handlers.close.CloseHandler
+    - nexus.bot.handlers.donate.DonateHandler
+    - nexus.bot.handlers.download.DownloadHandler
+    - nexus.bot.handlers.emoji.EmojiHandler
+    - nexus.bot.handlers.help.HelpHandler
+    - nexus.bot.handlers.referencing_to.ReferencingToHandler
+    - nexus.bot.handlers.referencing_to.ReferencingToPagingHandler
+    - nexus.bot.handlers.roll.RollHandler
+    - nexus.bot.handlers.settings.SettingsButtonsHandler
+    - nexus.bot.handlers.settings.SettingsRouterHandler
+    - nexus.bot.handlers.settings.SettingsManualHandler
+    - nexus.bot.handlers.shortlink.ShortlinkHandler
+    - nexus.bot.handlers.submit.SubmitHandler
+    - nexus.bot.handlers.start.StartHandler
+    - nexus.bot.handlers.stop.StopHandler
+    - nexus.bot.handlers.view.ViewHandler
+    - nexus.bot.handlers.vote.VoteHandler
+    - nexus.bot.handlers.noop.NoopHandler
+    - nexus.bot.handlers.search.SearchHandler
+    - nexus.bot.handlers.search.SearchEditHandler
+    - nexus.bot.handlers.search.SearchPagingHandler
+  # Channel that will be shown in /help, /donate, /contact and in promotions
+  related_channel: '@nexus_search'

nexus/bot/configs/promotions.yaml

@@ -0,0 +1,13 @@
+---
+
+promotions:
+  - texts:
+      en: 🎁 Help us at /donate to accelerate knowledge unchaining
+    weight: 3.0
+  - texts:
+      en: ⤴️ Stay tuned with us at {related_channel}
+      es: ⤴️ Mantente en contacto con nosotros en {related_channel}
+      it: ⤴️ Resta aggiornato con noi su {related_channel}
+      pb: ⤴️ Fique ligado conosco em {related_channel}
+      ru: ⤴️ Оставайся на связи с нами на {related_channel}
+    weight: 1.0

nexus/bot/exceptions.py

@@ -0,0 +1,9 @@
+from izihawa_utils.exceptions import BaseError
+
+
+class UnknownFileFormatError(BaseError):
+    code = 'unknown_file_format_error'
+
+
+class UnknownSchemaError(BaseError):
+    code = 'unknown_schema_error'

nexus/bot/handlers/__init__.py

@@ -0,0 +1,27 @@
+from . import (
+    admin,
+    ban,
+    close,
+    contact,
+    copyright,
+    donate,
+    download,
+    emoji,
+    help,
+    legacy,
+    noop,
+    referencing_to,
+    roll,
+    search,
+    settings,
+    shortlink,
+    start,
+    stop,
+    submit,
+    view,
+    vote,
+)
+
+__all__ = ['admin', 'ban', 'contact', 'copyright', 'close', 'donate', 'download', 'emoji', 'help',
+           'legacy', 'noop', 'referencing_to', 'roll', 'search', 'settings',
+           'shortlink', 'start', 'stop', 'submit', 'view', 'vote']

nexus/bot/handlers/admin.py

@@ -0,0 +1,22 @@
+from abc import ABC
+
+from library.telegram.base import RequestContext
+from nexus.bot.widgets.admin_widget import AdminWidget
+from telethon import events
+
+from .base import BaseHandler
+
+
+class BaseAdminHandler(BaseHandler, ABC):
+    def _has_access(self, chat):
+        return chat.is_admin
+
+
+class AdminHandler(BaseAdminHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/admin$')
+
+    async def handler(self, event, request_context: RequestContext):
+        request_context.statbox(action='show', mode='admin')
+        admin_widget_view = AdminWidget(application=self.application, chat=request_context.chat)
+        text = await admin_widget_view.render()
+        await event.reply(text)

nexus/bot/handlers/ban.py

@@ -0,0 +1,82 @@
+from datetime import (
+    datetime,
+    timedelta,
+)
+
+from aiobaseclient.exceptions import ClientError
+from library.telegram.base import RequestContext
+from nexus.bot.widgets.banlist_widget import BanlistWidget
+from pytimeparse.timeparse import timeparse
+from telethon import events
+
+from .admin import BaseAdminHandler
+
+
+class BanHandler(BaseAdminHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/ban ([0-9]+) ([A-Za-z0-9]+)\\s?(.*)?$')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        chat_id = int(event.pattern_match.group(1))
+        ban_duration = event.pattern_match.group(2)
+        ban_message = event.pattern_match.group(3)
+        ban_end_date = datetime.utcnow() + timedelta(seconds=timeparse(ban_duration))
+
+        try:
+            await self.application.idm_client.update_chat(
+                chat_id=chat_id,
+                ban_until=int(ban_end_date.timestamp()),
+                ban_message=ban_message,
+                request_id=request_context.request_id,
+            )
+            request_context.statbox(
+                action='banned',
+                ban_message=ban_message,
+                ban_until=ban_end_date.timestamp(),
+                banned_chat_id=chat_id,
+            )
+        except ClientError as e:
+            if e.code == 'nonexistent_entity_error':
+                await event.reply('Chat not found')
+                return
+            raise
+
+        await event.reply('User banned until ' + ban_end_date.strftime("%Y-%m-%d %H:%M") + ' UTC')
+
+
+class UnbanHandler(BaseAdminHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/unban(?:_|\\s)([0-9]+)$')
+
+    async def handler(self, event, request_context: RequestContext):
+        chat_id = int(event.pattern_match.group(1))
+
+        try:
+            await self.application.idm_client.update_chat(
+                chat_id=chat_id,
+                ban_until=0,
+                request_id=request_context.request_id,
+            )
+            request_context.statbox(
+                action='unbanned',
+                unbanned_chat_id=chat_id,
+            )
+        except ClientError as e:
+            if e.code == 'nonexistent_entity_error':
+                await event.reply('Chat not found')
+                return
+            raise
+
+        await event.reply('User unbanned')
+
+
+class BanlistHandler(BaseAdminHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/banlist$')
+
+    async def handler(self, event, request_context: RequestContext):
+        request_context.statbox(action='show', mode='banlist')
+        chat_list = (await self.application.idm_client.list_chats(
+            banned_at_moment=int(datetime.utcnow().timestamp()),
+            request_id=request_context.request_id,
+        )).chats
+        banlist_widget_view = BanlistWidget(application=self.application, chat=request_context.chat)
+        widget_content = await banlist_widget_view.render(chat_list=chat_list)
+        await event.reply(widget_content)

nexus/bot/handlers/base.py

@@ -0,0 +1,332 @@
+import logging
+import time
+from abc import ABC
+from datetime import datetime
+from typing import Union
+
+from grpc import StatusCode
+from grpc.experimental.aio import AioRpcError
+from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from izihawa_utils.exceptions import BaseError
+from izihawa_utils.random import random_string
+from library.logging import error_log
+from library.telegram.base import RequestContext
+from library.telegram.utils import safe_execution
+from nexus.bot.application import TelegramApplication
+from nexus.bot.exceptions import UnknownSchemaError
+from nexus.models.proto.typed_document_pb2 import \
+    TypedDocument as TypedDocumentPb
+from nexus.translations import t
+from nexus.views.telegram.common import close_button
+from nexus.views.telegram.scimag import ScimagView
+from nexus.views.telegram.scitech import ScitechView
+from telethon import (
+    TelegramClient,
+    events,
+)
+from telethon.errors import QueryIdInvalidError
+
+
+def get_username(event: events.ChatAction, chat):
+    if event.is_group or event.is_channel:
+        return str(event.chat_id)
+    else:
+        return chat.username
+
+
+def get_language(event: events.ChatAction, chat):
+    if event.is_group or event.is_channel:
+        return 'en'
+    return chat.lang_code
+
+
+def is_banned(chat: Chat) -> bool:
+    return chat.ban_until is not None and datetime.utcnow().timestamp() < chat.ban_until
+
+
+def is_subscribed(chat: Chat) -> bool:
+    return chat.is_subscribed or chat.id < 0 or chat.created_at > time.time() - 10 * 60
+
+
+class ReadOnlyModeError(BaseError):
+    level = logging.WARNING
+    code = 'read_only_mode_error'
+
+
+class BaseHandler(ABC):
+    # Is handler working in the groups
+    is_group_handler = False
+    # Is subscription to the `config['telegram']['related_channel'] required to use this handler
+    is_subscription_required_for_handler = False
+    # Telethon filter
+    filter = events.NewMessage(incoming=True)
+    should_reset_last_widget = True
+    # Raises StopPropagation in the end of handling. It means this handler would be the last one in chain
+    stop_propagation = True
+    # If set to True then read_only mode will disable handler
+    writing_handler = False
+
+    def __init__(self, application: TelegramApplication):
+        self.application = application
+        self.schema_to_resolver = {
+            'scimag': self.resolve_scimag,
+            'scitech': self.resolve_scitech,
+        }
+        self.short_schema_to_schema_dict = {
+            'a': 'scimag',
+            'b': 'scitech',
+        }
+
+    def generate_session_id(self) -> str:
+        return random_string(self.application.config['application']['session_id_length'])
+
+    def short_schema_to_schema(self, short_schema: str) -> str:
+        return self.short_schema_to_schema_dict[short_schema]
+
+    async def get_typed_document_pb(
+        self,
+        schema: str,
+        document_id: int,
+        request_context: RequestContext,
+        session_id: str,
+        position: int,
+    ) -> TypedDocumentPb:
+        return await self.application.meta_api_client.get(
+            schema=schema,
+            document_id=document_id,
+            session_id=session_id,
+            position=position,
+            request_id=request_context.request_id,
+            user_id=request_context.chat.id,
+        )
+
+    async def resolve_scimag(
+        self,
+        document_id: int,
+        position: int,
+        request_context: RequestContext,
+        session_id: str,
+    ) -> ScimagView:
+        typed_document_pb = await self.get_typed_document_pb(
+            schema='scimag',
+            document_id=document_id,
+            position=position,
+            request_context=request_context,
+            session_id=session_id,
+        )
+        return ScimagView(document_pb=typed_document_pb.scimag)
+
+    async def resolve_scitech(
+        self,
+        document_id: int,
+        position: int,
+        request_context: RequestContext,
+        session_id: str,
+    ) -> ScitechView:
+        typed_document_pb = await self.get_typed_document_pb(
+            schema='scitech',
+            document_id=document_id,
+            position=position,
+            request_context=request_context,
+            session_id=session_id,
+        )
+        search_response_duplicates = await self.application.meta_api_client.search(
+            schemas=('scitech',),
+            query=f'original_id:{document_id}',
+            page_size=16,
+            request_id=request_context.request_id,
+            session_id=session_id,
+            user_id=request_context.chat.id,
+        )
+        duplicates = [
+            scored_document.typed_document.scitech
+            for scored_document in search_response_duplicates.scored_documents
+        ]
+        return ScitechView(
+            document_pb=typed_document_pb.scitech,
+            duplicates=duplicates,
+        )
+
+    async def resolve_document(
+        self,
+        schema: str,
+        document_id: int,
+        position: int,
+        session_id: str,
+        request_context: RequestContext
+    ) -> Union[ScimagView, ScitechView]:
+        if schema not in self.schema_to_resolver:
+            raise UnknownSchemaError()
+
+        resolver = self.schema_to_resolver[schema]
+        return await resolver(
+            document_id=document_id,
+            position=position,
+            request_context=request_context,
+            session_id=session_id,
+        )
+
+    def reset_last_widget(self, chat_id: int):
+        self.application.user_manager.last_widget[chat_id] = None
+
+    def register_for(self, telegram_client: TelegramClient):
+        telegram_client.add_event_handler(self._wrapped_handler, self.filter)
+        return self._wrapped_handler
+
+    async def _send_fail_response(self, event: events.ChatAction, request_context: RequestContext):
+        try:
+            await event.reply(
+                t('MAINTENANCE', language=request_context.chat.language).format(
+                    maintenance_picture_url=self.application.config['application']['maintenance_picture_url'],
+                ),
+                buttons=[close_button()]
+            )
+        except (ConnectionError, QueryIdInvalidError) as e:
+            request_context.error_log(e)
+
+    async def _put_chat(self, event: events.ChatAction, request_id: str):
+        try:
+            chat = await self.application.idm_client.get_chat(
+                chat_id=event.chat_id,
+                request_id=request_id,
+            )
+            return chat
+        except AioRpcError as e:
+            if e.code() != StatusCode.NOT_FOUND:
+                raise
+            if self.application.config['application']['is_read_only_mode']:
+                raise ReadOnlyModeError()
+            event_chat = await event.get_chat()
+            username = get_username(event, event_chat)
+            language = get_language(event, event_chat)
+            if language not in {'en', 'ru'}:
+                language = 'en'
+            chat = await self.application.idm_client.create_chat(
+                chat_id=event.chat_id,
+                username=username,
+                language=language,
+                request_id=request_id,
+            )
+            return chat
+
+    async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
+        if is_banned(chat):
+            if chat.ban_message is not None:
+                async with safe_execution(
+                    request_context=request_context,
+                    on_fail=lambda: self._send_fail_response(event, request_context),
+                ):
+                    await event.reply(t(
+                        'BANNED',
+                        language=chat.language
+                    ).format(
+                        datetime=str(time.ctime(chat.ban_until)),
+                        reason=chat.ban_message,
+                    ))
+            raise events.StopPropagation()
+
+    async def _check_maintenance(self, event: events.ChatAction):
+        if (
+            self.application.config['application']['is_maintenance_mode']
+            and event.chat_id not in self.application.config['application']['bypass_maintenance']
+        ):
+            await event.reply(
+                t('UPGRADE_MAINTENANCE', language='en').format(
+                    upgrade_maintenance_picture_url=self.application.config['application']
+                    ['upgrade_maintenance_picture_url']
+                ),
+            )
+            raise events.StopPropagation()
+
+    async def _check_read_only(self, event: events.ChatAction):
+        if self.application.config['application']['is_read_only_mode']:
+            await event.reply(
+                t("READ_ONLY_MODE", language='en'),
+            )
+            raise events.StopPropagation()
+
+    async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
+        if (
+            self.application.config['application']['is_subscription_required']
+            and self.is_subscription_required_for_handler
+            and not is_subscribed(chat)
+        ):
+            async with safe_execution(
+                request_context=request_context,
+                on_fail=lambda: self._send_fail_response(event, request_context),
+            ):
+                await event.reply(t(
+                    'SUBSCRIBE_TO_CHANNEL',
+                    language=chat.language
+                ).format(related_channel=self.application.config['telegram']['related_channel']))
+            raise events.StopPropagation()
+
+    def _has_access(self, chat: Chat) -> bool:
+        return True
+
+    async def _process_chat(self, event: events.ChatAction, request_id: str):
+        try:
+            chat = await self._put_chat(event, request_id=request_id)
+        except (AioRpcError, BaseError) as e:
+            error_log(e)
+            event_chat = await event.get_chat()
+            username = get_username(event, event_chat)
+            chat = Chat(
+                id=event.chat_id,
+                is_system_messaging_enabled=True,
+                is_discovery_enabled=True,
+                language='en',
+                username=username,
+                is_admin=False,
+                is_subscribed=True,
+            )
+        return chat
+
+    async def _wrapped_handler(self, event: events.ChatAction) -> None:
+        # Checking group permissions
+        if (event.is_group or event.is_channel) and not self.is_group_handler:
+            return
+
+        await self._check_maintenance(event=event)
+        await self._check_read_only(event=event)
+
+        request_id = RequestContext.generate_request_id(self.application.config['application']['request_id_length'])
+        chat = await self._process_chat(event=event, request_id=request_id)
+
+        request_context = RequestContext(
+            bot_name=self.application.config['telegram']['bot_name'],
+            chat=chat,
+            request_id=request_id,
+            request_id_length=self.application.config['application']['request_id_length'],
+        )
+
+        if not self._has_access(chat):
+            return
+
+        await self._check_subscription(event=event, request_context=request_context, chat=chat)
+        await self._check_ban(event=event, request_context=request_context, chat=chat)
+
+        if self.should_reset_last_widget:
+            self.reset_last_widget(request_context.chat.id)
+
+        async with safe_execution(
+            request_context=request_context,
+            on_fail=lambda: self._send_fail_response(event, request_context),
+        ):
+            await self.handler(
+                event,
+                request_context=request_context,
+            )
+        if self.stop_propagation:
+            raise events.StopPropagation()
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        raise NotImplementedError()
+
+
+class BaseCallbackQueryHandler(BaseHandler, ABC):
+    async def _send_fail_response(self, event, request_context: RequestContext):
+        try:
+            await event.answer(t('MAINTENANCE_WO_PIC', language=request_context.chat.language))
+        except (ConnectionError, QueryIdInvalidError) as e:
+            request_context.error_log(e)

nexus/bot/handlers/close.py

@@ -0,0 +1,31 @@
+import asyncio
+
+from library.telegram.base import RequestContext
+from telethon import events
+
+from .base import BaseCallbackQueryHandler
+
+
+class CloseHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern='^/close(?:_([A-Za-z0-9]+))?(?:_([0-9]+))?$')
+
+    async def handler(self, event, request_context: RequestContext):
+        session_id = event.pattern_match.group(1)
+        if session_id:
+            session_id = session_id.decode()
+        request_context.add_default_fields(mode='close')
+
+        target_events = [event.answer()]
+        message = await event.get_message()
+
+        if message:
+            request_context.statbox(
+                action='close',
+                message_id=message.id,
+                session_id=session_id,
+            )
+            reply_message = await message.get_reply_message()
+            if reply_message:
+                target_events.append(reply_message.delete())
+            target_events.append(message.delete())
+        await asyncio.gather(*target_events)

nexus/bot/handlers/contact.py

@@ -0,0 +1,33 @@
+import re
+
+from library.telegram.base import RequestContext
+from nexus.bot.configs import config
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseHandler
+
+
+class ContactHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern=re.compile('^/contact\\s?(.*)', re.DOTALL))
+    is_group_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        query = event.pattern_match.group(1)
+        if query:
+            request_context.statbox(action='show', mode='contact', query=query)
+            await event.reply(
+                t('THANK_YOU_FOR_CONTACT', language=request_context.chat.language).format(
+                    related_channel=self.application.config['telegram']['related_channel'],
+                ),
+            )
+        else:
+            request_context.statbox(action='show', mode='contact')
+            await event.reply(
+                t('CONTACT', language=request_context.chat.language).format(
+                    btc_donate_address=config['application']['btc_donate_address'],
+                    libera_pay_url=config['application']['libera_pay_url'],
+                    related_channel=config['telegram']['related_channel'],
+                ),
+                link_preview=False,
+            )

nexus/bot/handlers/copyright.py

@@ -0,0 +1,31 @@
+import re
+
+from library.telegram.base import RequestContext
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseHandler
+
+
+class CopyrightHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern=re.compile('^/copyright\\s?(.*)', re.DOTALL))
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        query = event.pattern_match.group(1)
+        if query:
+            request_context.statbox(
+                action='show',
+                mode='copyright',
+                query=query,
+            )
+            await self.application.telegram_client.forward_messages(
+                self.application.config['telegram']['copyright_infringement_account'],
+                event.message,
+            )
+            await event.reply(t(
+                'COPYRIGHT_INFRINGEMENT_ACCEPTED',
+                language=request_context.chat.language,
+            ))
+        else:
+            request_context.statbox(action='show', mode='copyright')
+            await event.reply(t('COPYRIGHT_DESCRIPTION', language=request_context.chat.language,))

nexus/bot/handlers/donate.py

@@ -0,0 +1,25 @@
+from library.telegram.base import RequestContext
+from nexus.bot.configs import config
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseHandler
+
+
+class DonateHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/donate(@[A-Za-z0-9_]+)?$')
+    is_group_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='show', mode='donate')
+        await event.reply(
+            t(
+                'DONATE',
+                language=request_context.chat.language
+            ).format(
+                amazon_gift_card_recipient=config['application'].get('amazon_gift_card_recipient', '🚫'),
+                amazon_gift_card_url=config['application'].get('amazon_gift_card_url', '🚫'),
+                btc_donate_address=config['application'].get('btc_donate_address', '🚫'),
+                libera_pay_url=config['application'].get('libera_pay_url', '🚫'),
+                related_channel=config['telegram'].get('related_channel', '🚫'),
+            ))

nexus/bot/handlers/download.py

@@ -0,0 +1,51 @@
+from library.telegram.base import RequestContext
+from nexus.hub.proto.delivery_service_pb2 import \
+    StartDeliveryResponse as StartDeliveryResponsePb
+from nexus.translations import t
+from nexus.views.telegram.common import remove_button
+from telethon import events
+
+from .base import BaseCallbackQueryHandler
+
+
+class DownloadHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern='^/dl([abcm])_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$')
+    is_group_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        short_schema = event.pattern_match.group(1).decode()
+        schema = self.short_schema_to_schema(short_schema)
+        session_id = event.pattern_match.group(2).decode()
+        document_id = int(event.pattern_match.group(3))
+        position = int(event.pattern_match.group(4).decode())
+
+        self.application.user_manager.last_widget[request_context.chat.id] = None
+
+        request_context.add_default_fields(mode='download', session_id=session_id)
+        request_context.statbox(action='get', query=str(document_id), position=position)
+
+        typed_document_pb = await self.get_typed_document_pb(
+            schema=schema,
+            document_id=document_id,
+            request_context=request_context,
+            session_id=session_id,
+            position=position,
+        )
+        start_delivery_response_pb = await self.application.hub_client.start_delivery(
+            typed_document_pb=typed_document_pb,
+            chat=request_context.chat,
+            request_id=request_context.request_id,
+            session_id=session_id,
+        )
+        if start_delivery_response_pb.status == StartDeliveryResponsePb.Status.ALREADY_DOWNLOADING:
+            await event.answer(
+                f'{t("ALREADY_DOWNLOADING", language=request_context.chat.language)}',
+            )
+            await remove_button(event, '⬇️', and_empty_too=True)
+        elif start_delivery_response_pb.status == StartDeliveryResponsePb.Status.TOO_MANY_DOWNLOADS:
+            await event.answer(
+                f'{t("TOO_MANY_DOWNLOADS", language=request_context.chat.language)}',
+            )
+        else:
+            await remove_button(event, '⬇️', and_empty_too=True)
+            self.application.user_manager.last_widget[request_context.chat.id] = None

nexus/bot/handlers/emoji.py

@@ -0,0 +1,17 @@
+from library.telegram.base import RequestContext
+from nexus.nlptools.regex import STICKER_REGEX
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseHandler
+
+
+class EmojiHandler(BaseHandler):
+    filter = events.NewMessage(
+        incoming=True,
+        pattern=STICKER_REGEX,
+    )
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='show', mode='emoji')
+        await event.reply(t('TANKS_BRUH', language=request_context.chat.language))

nexus/bot/handlers/help.py

@@ -0,0 +1,20 @@
+from library.telegram.base import RequestContext
+from nexus.translations import t
+from telethon import (
+    Button,
+    events,
+)
+
+from .base import BaseHandler
+
+
+class HelpHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/help(@[A-Za-z0-9_]+)?$')
+    is_group_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='show', mode='help')
+        if event.is_group or event.is_channel:
+            await event.reply(t('HELP_FOR_GROUPS', language=request_context.chat.language), buttons=Button.clear())
+        else:
+            await event.reply(t('HELP', language=request_context.chat.language), buttons=Button.clear())

nexus/bot/handlers/legacy.py

@@ -0,0 +1,28 @@
+from library.telegram.base import RequestContext
+from nexus.translations import t
+from telethon import events
+
+from .base import (
+    BaseCallbackQueryHandler,
+    BaseHandler,
+)
+
+
+class LegacyHandler(BaseHandler):
+    filter = events.NewMessage(
+        incoming=True,
+        pattern='^/v_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)$')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='show', mode='legacy')
+        await event.reply(t('LEGACY', language=request_context.chat.language))
+
+
+class LegacyCallbackHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(
+        pattern='^/dl_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)$'
+    )
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='show', mode='legacy')
+        return await event.answer(t('LEGACY', language=request_context.chat.language))

nexus/bot/handlers/noop.py

@@ -0,0 +1,12 @@
+from library.telegram.base import RequestContext
+from telethon import events
+
+from .base import BaseCallbackQueryHandler
+
+
+class NoopHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern='^/noop$')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='start', mode='noop')
+        await event.answer()

nexus/bot/handlers/referencing_to.py

@@ -0,0 +1,94 @@
+import re
+import time
+
+from grpc.experimental.aio import AioRpcError
+from library.telegram.base import RequestContext
+from nexus.bot.widgets.document_list_widget import DocumentListWidget
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseCallbackQueryHandler
+
+
+class ReferencingToHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern=re.compile('^/r_([A-Za-z0-9]+)_([0-9]+)', re.DOTALL))
+    should_reset_last_widget = False
+
+    async def do_request(self, request_context: RequestContext, session_id: str, message_id: int, document_id: int, page: int):
+        start_time = time.time()
+        try:
+            document_list_widget = await DocumentListWidget.create(
+                application=self.application,
+                chat=request_context.chat,
+                session_id=session_id,
+                message_id=message_id,
+                request_id=request_context.request_id,
+                referencing_to=document_id,
+                page=page,
+            )
+        except AioRpcError as e:
+            raise e
+
+        action = 'referencing_to_found'
+        if len(document_list_widget.scored_documents) == 0:
+            action = 'referencing_to_not_found'
+
+        request_context.statbox(
+            action=action,
+            duration=time.time() - start_time,
+            query=f'{document_id}',
+        )
+
+        serp, buttons = await document_list_widget.render()
+        return await self.application.telegram_client.edit_message(
+            request_context.chat.id,
+            message_id,
+            serp,
+            buttons=buttons,
+            link_preview=False,
+        )
+
+    async def handler(self, event, request_context: RequestContext):
+        session_id = event.pattern_match.group(1).decode()
+        document_id = int(event.pattern_match.group(2).decode())
+
+        request_context.add_default_fields(
+            mode='referencing_to',
+            session_id=session_id,
+        )
+
+        prefetch_message = await event.respond(
+            t("SEARCHING", language=request_context.chat.language),
+        )
+        message_id = prefetch_message.id
+
+        return await self.do_request(
+            request_context=request_context,
+            session_id=session_id,
+            message_id=message_id,
+            document_id=document_id,
+            page=0,
+        )
+
+
+class ReferencingToPagingHandler(ReferencingToHandler):
+    filter = events.CallbackQuery(pattern=re.compile('^/rp_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)', re.DOTALL))
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        session_id = event.pattern_match.group(1).decode()
+        message_id = int(event.pattern_match.group(2).decode())
+        document_id = int(event.pattern_match.group(3).decode())
+        page = int(event.pattern_match.group(4).decode())
+
+        request_context.add_default_fields(
+            mode='referencing_to_paging',
+            session_id=session_id,
+        )
+
+        return await self.do_request(
+            request_context=request_context,
+            session_id=session_id,
+            message_id=message_id,
+            document_id=document_id,
+            page=page,
+        )

nexus/bot/handlers/roll.py

@@ -0,0 +1,44 @@
+import asyncio
+import re
+
+from library.telegram.base import RequestContext
+from telethon import events
+
+from .base import BaseHandler
+
+
+class RollHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern=re.compile('^/roll(@[A-Za-z0-9_]+)?$', re.DOTALL))
+    is_group_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        session_id = self.generate_session_id()
+        request_context.add_default_fields(mode='roll', session_id=session_id)
+        request_context.statbox(action='show')
+
+        roll_response_pb = await self.application.meta_api_client.roll(
+            language=request_context.chat.language,
+            session_id=session_id,
+            request_id=request_context.request_id,
+            user_id=request_context.chat.id,
+        )
+        scitech_view = await self.resolve_scitech(
+            document_id=roll_response_pb.document_id,
+            position=0,
+            request_context=request_context,
+            session_id=session_id,
+        )
+        view, buttons = scitech_view.get_view(
+            language=request_context.chat.language,
+            session_id=session_id,
+            bot_external_name=self.application.config['telegram']['bot_external_name'],
+        )
+        actions = [
+            self.application.telegram_client.send_message(
+                request_context.chat.id,
+                view,
+                buttons=buttons,
+            ),
+            event.delete(),
+        ]
+        return await asyncio.gather(*actions)

nexus/bot/handlers/search.py

@@ -0,0 +1,289 @@
+import asyncio
+import logging
+import re
+import time
+
+from grpc import StatusCode
+from grpc.experimental.aio import AioRpcError
+from library.telegram.base import RequestContext
+from nexus.bot.widgets.search_widget import SearchWidget
+from nexus.translations import t
+from nexus.views.telegram.common import close_button
+from nexus.views.telegram.registry import parse_typed_document_to_view
+from telethon import (
+    events,
+    functions,
+)
+
+from .base import (
+    BaseCallbackQueryHandler,
+    BaseHandler,
+)
+
+
+class BaseSearchHandler(BaseHandler):
+    should_reset_last_widget = False
+
+    async def do_search(
+        self,
+        event: events.ChatAction,
+        request_context: RequestContext,
+        prefetch_message,
+        query: str,
+        is_group_mode: bool = False,
+        is_shortpath_enabled: bool = False,
+    ):
+        session_id = self.generate_session_id()
+        message_id = prefetch_message.id
+
+        request_context.add_default_fields(is_group_mode=is_group_mode, mode='search', session_id=session_id)
+
+        start_time = time.time()
+        try:
+            search_widget = await SearchWidget.create(
+                application=self.application,
+                chat=request_context.chat,
+                session_id=session_id,
+                message_id=message_id,
+                request_id=request_context.request_id,
+                query=query,
+                is_group_mode=is_group_mode,
+            )
+        except AioRpcError as e:
+            actions = [
+                self.application.telegram_client.delete_messages(
+                    request_context.chat.id,
+                    [message_id],
+                )
+            ]
+            if e.code() == StatusCode.INVALID_ARGUMENT:
+                too_difficult_picture_url = self.application.config['application'].get('too_difficult_picture_url', '')
+                if e.details() == 'url_query_error':
+                    actions.append(
+                        event.reply(
+                            t('INVALID_QUERY_ERROR', language=request_context.chat.language).format(
+                                too_difficult_picture_url=too_difficult_picture_url,
+                            ),
+                            buttons=[close_button()],
+                        )
+                    )
+                elif e.details() == 'invalid_query_error':
+                    actions.append(
+                        event.reply(
+                            t('INVALID_SYNTAX_ERROR', language=request_context.chat.language).format(
+                                too_difficult_picture_url=too_difficult_picture_url,
+                            ),
+                            buttons=[close_button()],
+                        )
+                    )
+                return await asyncio.gather(*actions)
+            elif e.code() == StatusCode.CANCELLED:
+                maintenance_picture_url = self.application.config['application'].get('maintenance_picture_url', '')
+                request_context.error_log(e)
+                actions.append(event.reply(
+                    t('MAINTENANCE', language=request_context.chat.language).format(
+                        maintenance_picture_url=maintenance_picture_url,
+                    ),
+                    buttons=[close_button()],
+                ))
+                return await asyncio.gather(*actions)
+            raise e
+
+        action = 'documents_found'
+        if len(search_widget.scored_documents) == 0:
+            action = 'documents_not_found'
+
+        request_context.statbox(
+            action=action,
+            duration=time.time() - start_time,
+            query=f'page:0 query:{query}',
+        )
+
+        if len(search_widget.scored_documents) == 1 and is_shortpath_enabled:
+            scored_document = search_widget.scored_documents[0]
+            document_view = parse_typed_document_to_view(scored_document.typed_document)
+            # Second (re-)fetching is required to retrieve duplicates
+            document_view = await self.resolve_document(
+                schema=scored_document.typed_document.WhichOneof('document'),
+                document_id=document_view.id,
+                position=0,
+                session_id=session_id,
+                request_context=request_context,
+            )
+            view, buttons = document_view.get_view(
+                language=request_context.chat.language,
+                session_id=session_id,
+                bot_external_name=self.application.config['telegram']['bot_external_name'],
+                with_buttons=not is_group_mode,
+            )
+            return await asyncio.gather(
+                self.application.telegram_client.edit_message(
+                    request_context.chat.id,
+                    message_id,
+                    view,
+                    buttons=buttons,
+                ),
+            )
+
+        serp, buttons = await search_widget.render()
+        return await self.application.telegram_client.edit_message(
+            request_context.chat.id,
+            message_id,
+            serp,
+            buttons=buttons,
+            link_preview=False,
+        )
+
+
+class SearchHandler(BaseSearchHandler):
+    filter = events.NewMessage(incoming=True, pattern=re.compile('^(/search\\s+)?(.*)', flags=re.DOTALL))
+    is_group_handler = True
+    should_reset_last_widget = False
+    is_subscription_required_for_handler = True
+
+    async def ban_handler(self, event: events.ChatAction, request_context: RequestContext, ban_timeout: float):
+        logging.getLogger('statbox').info({
+            'bot_name': self.application.config['telegram']['bot_name'],
+            'action': 'user_flood_ban',
+            'mode': 'search',
+            'ban_timeout_seconds': ban_timeout,
+            'chat_id': request_context.chat.id,
+        })
+        ban_reason = t(
+            'BAN_MESSAGE_TOO_MANY_REQUESTS',
+            language=request_context.chat.language
+        )
+        return await event.reply(t(
+            'BANNED_FOR_SECONDS',
+            language=request_context.chat.language
+        ).format(
+            seconds=str(ban_timeout),
+            reason=ban_reason,
+        ))
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.id)
+        if ban_timeout:
+            return await self.ban_handler(event, request_context, ban_timeout)
+        self.application.user_manager.add_search_time(user_id=request_context.chat.id, search_time=time.time())
+
+        search_prefix = event.pattern_match.group(1)
+        query = event.pattern_match.group(2)
+        is_group_mode = event.is_group or event.is_channel
+
+        if is_group_mode and not search_prefix:
+            return
+        if not is_group_mode and search_prefix:
+            query = event.raw_text
+        prefetch_message = await event.reply(
+            t("SEARCHING", language=request_context.chat.language),
+        )
+        self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
+        try:
+            await self.do_search(
+                event, request_context, prefetch_message,
+                query=query,
+                is_group_mode=is_group_mode,
+                is_shortpath_enabled=True,
+            )
+        except (AioRpcError, asyncio.CancelledError) as e:
+            await asyncio.gather(
+                event.delete(),
+                prefetch_message.delete(),
+            )
+            raise e
+
+
+class SearchEditHandler(BaseSearchHandler):
+    filter = events.MessageEdited(incoming=True, pattern=re.compile('^(/search\\s+)?(.*)', flags=re.DOTALL))
+    is_group_handler = True
+    should_reset_last_widget = False
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.add_default_fields(mode='search_edit')
+        search_prefix = event.pattern_match.group(1)
+        query = event.pattern_match.group(2)
+        is_group_mode = event.is_group or event.is_channel
+        if is_group_mode and not search_prefix:
+            return
+        if not is_group_mode and search_prefix:
+            query = event.raw_text
+        result = await self.application.telegram_client(functions.messages.GetMessagesRequest(
+            id=list(range(event.id + 1, event.id + 10)))
+        )
+        if not result:
+            request_context.statbox(action='failed')
+            return await event.reply(
+                t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
+            )
+        for next_message in result.messages:
+            if next_message.is_reply and event.id == next_message.reply_to_msg_id:
+                request_context.statbox(action='resolved')
+                await self.do_search(
+                    event,
+                    request_context,
+                    prefetch_message=next_message,
+                    query=query,
+                    is_group_mode=is_group_mode,
+                )
+                return
+        request_context.statbox(action='failed')
+        return await event.reply(
+            t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
+        )
+
+
+class SearchPagingHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern='^/search_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$')
+    should_reset_last_widget = False
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        session_id = event.pattern_match.group(1).decode()
+        message_id = int(event.pattern_match.group(2).decode())
+        page = int(event.pattern_match.group(3).decode())
+
+        request_context.add_default_fields(mode='search_paging', session_id=session_id)
+
+        message = await event.get_message()
+        if not message:
+            return await event.answer()
+        reply_message = await message.get_reply_message()
+        if not reply_message:
+            return await event.respond(
+                t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
+            )
+
+        start_time = time.time()
+        query = reply_message.raw_text
+        try:
+            search_widget = await SearchWidget.create(
+                application=self.application,
+                chat=request_context.chat,
+                session_id=session_id,
+                message_id=message_id,
+                request_id=request_context.request_id,
+                query=query,
+                page=page,
+            )
+        except AioRpcError as e:
+            if e.code() == StatusCode.INVALID_ARGUMENT or e.code() == StatusCode.CANCELLED:
+                request_context.error_log(e)
+                return await event.answer(
+                    t('MAINTENANCE_WO_PIC', language=request_context.chat.language),
+                )
+            raise e
+
+        action = 'documents_found'
+        if len(search_widget.scored_documents) == 0:
+            action = 'documents_not_found'
+
+        request_context.statbox(
+            action=action,
+            duration=time.time() - start_time,
+            query=f'page:{page} query:{query}',
+        )
+        serp, buttons = await search_widget.render()
+        return await asyncio.gather(
+            event.answer(),
+            message.edit(serp, buttons=buttons, link_preview=False)
+        )

nexus/bot/handlers/settings.py

@@ -0,0 +1,129 @@
+from izihawa_utils.podolsky_encoding import encode
+from library.telegram.base import RequestContext
+from nexus.bot.widgets.settings_widget import (
+    SettingsManualWidget,
+    SettingsRouterWidget,
+)
+from nexus.translations import t
+from telethon import (
+    Button,
+    events,
+    functions,
+)
+
+from .base import (
+    BaseCallbackQueryHandler,
+    BaseHandler,
+)
+
+
+class SettingsRouterHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/settings(@[A-Za-z0-9_]+)?$')
+    is_group_handler = True
+    writing_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.add_default_fields(mode='settings_router')
+        request_context.statbox(action='show')
+        if self.application.config['application']['views']['settings']['has_router']:
+            settings_router_widget = SettingsRouterWidget(
+                application=self.application,
+                chat=request_context.chat,
+                request_id=request_context.request_id,
+            )
+            text, buttons = await settings_router_widget.render()
+        else:
+            settings_widget = SettingsManualWidget(
+                application=self.application,
+                chat=request_context.chat,
+                is_group_mode=event.is_group or event.is_channel,
+                request_id=request_context.request_id,
+            )
+            text, buttons = await settings_widget.render()
+        await event.reply(text, buttons=buttons)
+
+
+class SettingsAutomaticHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern=f'^🌎{encode("1")}')
+
+    async def handler(self, event, request_context: RequestContext):
+        request_context.add_default_fields(mode='settings_automatic')
+        request_context.statbox(action='show')
+        await event.reply(
+            f'{t("SEND_YOUR_LOCATION", language=request_context.chat.language)}{encode("sg")}',
+            buttons=Button.clear()
+        )
+
+
+class SettingsManualHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern=f'^👇{encode("1")}')
+
+    async def handler(self, event, request_context: RequestContext):
+        request_context.add_default_fields(mode='settings_manual')
+        request_context.statbox(action='show')
+        settings_widget = SettingsManualWidget(
+            application=self.application,
+            chat=request_context.chat,
+            is_group_mode=event.is_group or event.is_channel,
+            request_id=request_context.request_id,
+        )
+        text, buttons = await settings_widget.render()
+        await event.reply(text, buttons=buttons)
+
+
+class SettingsButtonsHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern='^/settings_([A-Za-z0-9]+)_([A-Za-z0-9]+)$')
+    is_group_handler = True
+
+    async def handler(self, event, request_context: RequestContext):
+        request_context.add_default_fields(mode='settings')
+        action_id = event.pattern_match.group(1).decode()
+        data = event.pattern_match.group(2).decode()
+
+        request_context.statbox(action='change', query=f'action_id: {action_id} data: {data}')
+
+        settings_manual_widget = SettingsManualWidget(
+            application=self.application,
+            chat=request_context.chat,
+            is_group_mode=event.is_group or event.is_channel,
+            request_id=request_context.request_id,
+        )
+        is_changed = await settings_manual_widget.process_action(action_id=action_id, data=data)
+        text, buttons = await settings_manual_widget.render()
+        if not is_changed and not (event.is_group or event.is_channel):
+            await event.answer()
+            return
+        if event.is_group or event.is_channel:
+            buttons = None
+        await event.edit(text, buttons=buttons)
+
+
+class GeoHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True)
+    stop_propagation = False
+
+    async def handler(self, event, request_context: RequestContext):
+        request_context.add_default_fields(mode='geo')
+
+        if not event.geo:
+            return
+        request_context.statbox(action='geo', query=f'lon:{event.geo.long} lat:{event.geo.lat}')
+
+        result = await self.application.telegram_client(functions.messages.GetMessagesRequest(id=[event.id - 1]))
+        if not result.messages:
+            return
+
+        previous_message = result.messages[0]
+        if previous_message.message.endswith(encode("sg")):
+            request_context.statbox(action='catched_settings')
+            settings_manual_widget = SettingsManualWidget(
+                application=self.application,
+                chat=request_context.chat,
+                has_language_buttons=False,
+                is_group_mode=event.is_group or event.is_channel,
+                request_id=request_context.request_id,
+            )
+            await settings_manual_widget.set_last_location(lon=event.geo.long, lat=event.geo.lat)
+            text, buttons = await settings_manual_widget.render()
+            await event.reply(text, buttons=buttons or Button.clear())
+            raise events.StopPropagation()

nexus/bot/handlers/shortlink.py

@@ -0,0 +1,25 @@
+from library.telegram.base import RequestContext
+from nexus.translations import t
+from nexus.views.telegram.common import (
+    TooLongQueryError,
+    encode_query_to_deep_link,
+)
+from telethon import events
+
+from .base import BaseHandler
+
+
+class ShortlinkHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/shortlink\\s?(.*)?')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        query = event.pattern_match.group(1)
+        request_context.statbox(action='start', mode='shortlink', query=query)
+
+        try:
+            bot_name = self.application.config["telegram"]["bot_external_name"]
+            text = encode_query_to_deep_link(query, bot_name)
+        except TooLongQueryError:
+            text = t('TOO_LONG_QUERY_FOR_SHORTLINK', language=request_context.chat.language),
+
+        return await event.reply(f'`{text}`', link_preview=False)

nexus/bot/handlers/start.py

@@ -0,0 +1,43 @@
+import asyncio
+
+from library.telegram.base import RequestContext
+from nexus.translations import t
+from nexus.views.telegram.common import (
+    DecodeDeepQueryError,
+    decode_deep_query,
+)
+from telethon import events
+
+from .search import BaseSearchHandler
+
+
+class StartHandler(BaseSearchHandler):
+    should_reset_last_widget = False
+    filter = events.NewMessage(incoming=True, pattern='^/start\\s?(.*)?')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        raw_query = event.pattern_match.group(1)
+        query = None
+
+        request_context.statbox(action='start', mode='start')
+
+        try:
+            query = decode_deep_query(raw_query)
+        except DecodeDeepQueryError as e:
+            request_context.error_log(e, mode='start', raw_query=raw_query)
+
+        if query:
+            request_context.statbox(action='query', mode='start', query=query)
+            request_message = await self.application.telegram_client.send_message(event.chat, query)
+            prefetch_message = await request_message.reply(
+                t("SEARCHING", language=request_context.chat.language),
+            )
+            self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
+            await asyncio.gather(
+                event.delete(),
+                self.do_search(event, request_context, prefetch_message, query=query,
+                               is_shortpath_enabled=True),
+            )
+        else:
+            request_context.statbox(action='show', mode='start')
+            await event.reply(t('HELP', language=request_context.chat.language))

nexus/bot/handlers/stop.py

@@ -0,0 +1,11 @@
+from library.telegram.base import RequestContext
+from telethon import events
+
+from .base import BaseHandler
+
+
+class StopHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/stop$')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        request_context.statbox(action='show', mode='stop')

nexus/bot/handlers/submit.py

@@ -0,0 +1,42 @@
+import asyncio
+
+from library.telegram.base import RequestContext
+from nexus.bot.exceptions import UnknownFileFormatError
+from nexus.translations import t
+from nexus.views.telegram.common import close_button
+from telethon import events
+
+from .base import BaseHandler
+
+
+class SubmitHandler(BaseHandler):
+    filter = events.NewMessage(func=lambda e: e.document, incoming=True)
+    is_group_handler = False
+    writing_handler = True
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        session_id = self.generate_session_id()
+
+        request_context.add_default_fields(session_id=session_id)
+        request_context.statbox(action='show', mode='submit')
+
+        if event.document.mime_type != 'application/pdf':
+            request_context.error_log(UnknownFileFormatError(format=event.document.mime_type))
+            return await asyncio.gather(
+                event.reply(
+                    t('UNKNOWN_FILE_FORMAT_ERROR', language=request_context.chat.language),
+                    buttons=[close_button()],
+                ),
+                event.delete(),
+            )
+
+        return await asyncio.gather(
+            self.application.hub_client.submit(
+                telegram_document=bytes(event.document),
+                telegram_file_id=event.file.id,
+                chat=request_context.chat,
+                request_id=request_context.request_id,
+                session_id=session_id,
+            ),
+            event.delete(),
+        )

nexus/bot/handlers/view.py

@@ -0,0 +1,101 @@
+import asyncio
+import re
+
+from library.telegram.base import RequestContext
+from nexus.translations import t
+from telethon import (
+    events,
+    functions,
+)
+from telethon.errors import MessageIdInvalidError
+
+from .base import BaseHandler
+
+
+class ViewHandler(BaseHandler):
+    filter = events.NewMessage(incoming=True, pattern='^/v([abcm])([sr])?_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_'
+                                                      '([0-9]+)')
+    should_reset_last_widget = False
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        short_schema = event.pattern_match.group(1)
+        parent_view_type = event.pattern_match.group(2) or 's'
+        schema = self.short_schema_to_schema(short_schema)
+        session_id = event.pattern_match.group(3)
+        old_message_id = int(event.pattern_match.group(4))
+        document_id = int(event.pattern_match.group(5))
+        position = int(event.pattern_match.group(6))
+
+        page = int(position / self.application.config['application']['page_size'])
+
+        request_context.add_default_fields(mode='view', session_id=session_id)
+        request_context.statbox(action='view', query=str(document_id), position=position)
+        found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.id)
+
+        try:
+            if found_old_widget:
+                message_id = old_message_id
+                link_preview = None
+            else:
+                old_message = (await self.application.telegram_client(
+                    functions.messages.GetMessagesRequest(id=[old_message_id])
+                )).messages[0]
+                prefetch_message = await self.application.telegram_client.send_message(
+                    request_context.chat.id,
+                    t("SEARCHING", language=request_context.chat.language),
+                    reply_to=old_message.reply_to_msg_id,
+                )
+                self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
+                message_id = prefetch_message.id
+                link_preview = True
+
+            document_view = await self.resolve_document(
+                schema,
+                document_id,
+                position,
+                session_id,
+                request_context,
+            )
+
+            back_command = None
+            if parent_view_type == 's':
+                back_command = f'/search_{session_id}_{message_id}_{page}'
+            elif parent_view_type == 'r':
+                messages = (await self.application.telegram_client(
+                    functions.messages.GetMessagesRequest(id=[old_message_id])
+                )).messages
+                if not messages:
+                    return await event.respond(
+                        t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
+                    )
+                message = messages[0]
+                referencing_to = re.search(r'Linked to: ([0-9]+)', message.raw_text).group(1)
+                back_command = f'/rp_{session_id}_{message_id}_{referencing_to}_{page}'
+
+            view, buttons = document_view.get_view(
+                language=request_context.chat.language,
+                session_id=session_id,
+                bot_external_name=self.application.config['telegram']['bot_external_name'],
+                position=position,
+                back_command=back_command,
+            )
+            actions = [
+                self.application.telegram_client.edit_message(
+                    request_context.chat.id,
+                    message_id,
+                    view,
+                    buttons=buttons,
+                    link_preview=link_preview,
+                ),
+                event.delete(),
+            ]
+            if not found_old_widget:
+                actions.append(
+                    self.application.telegram_client.delete_messages(
+                        request_context.chat.id,
+                        [old_message_id],
+                    )
+                )
+            return await asyncio.gather(*actions)
+        except MessageIdInvalidError:
+            await event.reply(t("VIEWS_CANNOT_BE_SHARED", language=request_context.chat.language))

nexus/bot/handlers/vote.py

@@ -0,0 +1,52 @@
+import asyncio
+import logging
+
+from izihawa_utils.pb_to_json import MessageToDict
+from library.telegram.base import RequestContext
+from nexus.models.proto.operation_pb2 import \
+    DocumentOperation as DocumentOperationPb
+from nexus.models.proto.operation_pb2 import Vote as VotePb
+from nexus.translations import t
+from telethon import events
+
+from .base import BaseCallbackQueryHandler
+
+
+class VoteHandler(BaseCallbackQueryHandler):
+    filter = events.CallbackQuery(pattern='^/vote_([A-Za-z0-9]+)_([0-9]+)_([bo])$')
+
+    async def handler(self, event: events.ChatAction, request_context: RequestContext):
+        session_id = event.pattern_match.group(1).decode()
+        document_id = int(event.pattern_match.group(2).decode())
+        vote = event.pattern_match.group(3).decode()
+        vote_value = {'b': -1, 'o': 1}[vote]
+        request_context.add_default_fields(mode='vote', session_id=session_id)
+
+        document_operation_pb = DocumentOperationPb(
+            vote=VotePb(
+                document_id=document_id,
+                value=vote_value,
+                voter_id=request_context.chat.id,
+            ),
+        )
+
+        request_context.statbox(
+            action='vote',
+            document_id=document_id,
+        )
+        logging.getLogger('operation').info(
+            msg=MessageToDict(document_operation_pb),
+        )
+
+        message = await event.get_message()
+
+        # ToDo: Generalize nexus.views.telegram.common.remove_button and use it here
+        return await asyncio.gather(
+            self.application.telegram_client.edit_message(
+                request_context.chat.id,
+                message.id,
+                message.text,
+                buttons=None,
+            ),
+            event.answer(t('TANKS_BRUH')),
+        )

nexus/bot/main.py

@@ -0,0 +1,19 @@
+import asyncio
+
+import uvloop
+from library.logging import configure_logging
+from nexus.bot.application import TelegramApplication
+from nexus.bot.configs import get_config
+
+
+def main(config):
+    uvloop.install()
+    configure_logging(config)
+    if config['metrics']['enabled']:
+        from library.metrics_server import MetricsServer
+        MetricsServer(config['metrics']).fork_process()
+    asyncio.get_event_loop().run_until_complete(TelegramApplication(config=config).start_and_wait())
+
+
+if __name__ == '__main__':
+    main(config=get_config())

nexus/bot/promotioner.py

@@ -0,0 +1,21 @@
+import random
+
+
+class Promotioner:
+    """
+    Promotioner is used to select promotion randomly based on weights of every promotion.
+    """
+    def __init__(self, promotions: list[dict]):
+        self.promotions = promotions
+        self.partial_sums: list = [self.promotions[0]['weight']]
+        for promotion in self.promotions[1:]:
+            self.partial_sums.append(promotion['weight'] + self.partial_sums[-1])
+
+    def choose_promotion(self, language: str = 'en') -> str:
+        pivot = random.randrange(self.partial_sums[-1])
+        for partial_sum, promotion in zip(self.partial_sums, self.promotions):
+            if partial_sum <= pivot:
+                continue
+            if language in promotion['texts']:
+                return promotion['texts'][language]
+            return promotion['texts']['en']

nexus/bot/user_manager/__init__.py

@@ -0,0 +1,3 @@
+from .user_manager import UserManager
+
+__all__ = ['UserManager']

nexus/bot/user_manager/user_manager.py

@@ -0,0 +1,41 @@
+import time
+
+
+class UserManager:
+    def __init__(self):
+        self.last_widget = {}
+        self.search_times = {}
+        self.search_ban_times = {}
+
+    def add_search_time(self, user_id: int, search_time: float):
+        current_time = time.time()
+        search_times = self.search_times.get(user_id, [])
+        search_times.append(search_time)
+        counter = 0
+
+        for i in reversed(search_times):
+            if i > current_time - 10:
+                counter = counter + 1
+                if counter > 5:
+                    self.search_ban_times[user_id] = current_time + int(60)
+                    del self.search_times[user_id]
+                    return
+            else:
+                if counter == 1:
+                    del self.search_times[user_id]
+                    return
+
+        if len(search_times) > 20:
+            self.search_ban_times[user_id] = current_time + int(120)
+            del self.search_times[user_id]
+            return
+
+        self.search_times[user_id] = search_times
+
+    def check_search_ban_timeout(self, user_id: int):
+        ban_time = self.search_ban_times.get(user_id)
+        if ban_time:
+            timeout = int(ban_time - time.time())
+            if timeout > 0:
+                return timeout
+            del self.search_ban_times[user_id]

nexus/bot/widgets/admin_widget.py

@@ -0,0 +1,16 @@
+from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from nexus.bot.application import TelegramApplication
+
+
+class AdminWidget:
+    def __init__(self, application: TelegramApplication, chat: Chat):
+        self.application = application
+        self.chat = chat
+
+    async def render(self):
+        return (
+            'Ban: `/ban 12345 20d Spam`\n'
+            'Ban (silent): `/ban 12345 100h`\n'
+            'List of banned chats: `/banlist`\n'
+            'Unban chat: `/unban 12345`'
+        )

nexus/bot/widgets/banlist_widget.py

@@ -0,0 +1,26 @@
+import time
+
+from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from nexus.bot.application import TelegramApplication
+
+
+class BanlistWidget:
+    def __init__(self, application: TelegramApplication, chat: Chat):
+        self.application = application
+        self.chat = chat
+
+    async def render(self, chat_list: list[Chat]):
+        if not chat_list:
+            return 'Nobody is banned'
+        separator = '------------\n'
+        return separator.join(
+            map(
+                lambda chat: (
+                    f'```{chat.username} ({chat.id})\n'
+                    f'Until: {time.ctime(chat.ban_until)}\n'
+                    f'Message: {chat.ban_message}```\n'
+                    f'/unban_{chat.id}\n'
+                ),
+                chat_list
+            )
+        )

nexus/bot/widgets/document_list_widget.py

@@ -0,0 +1,130 @@
+from typing import Optional
+
+from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from nexus.bot.application import TelegramApplication
+from nexus.meta_api.proto.meta_search_service_pb2 import \
+    ScoredDocument as ScoredDocumentPb
+from nexus.translations import t
+from nexus.views.telegram.common import close_button
+from nexus.views.telegram.registry import parse_typed_document_to_view
+from telethon import Button
+
+
+class DocumentListWidget:
+    def __init__(
+        self,
+        application: TelegramApplication,
+        chat: Chat,
+        session_id: str,
+        message_id: int,
+        request_id: str,
+        referencing_to: int,
+        page: int = 0,
+    ):
+        self.application = application
+        self.chat = chat
+        self.session_id = session_id
+        self.message_id = message_id
+        self.request_id = request_id
+        self.referencing_to = referencing_to
+        self.page = page
+
+    @staticmethod
+    async def create(
+        application: TelegramApplication,
+        chat: Chat,
+        session_id: str,
+        message_id: int,
+        request_id: str,
+        referencing_to: int,
+        page: int = 0,
+    ) -> 'DocumentListWidget':
+        document_list_view = DocumentListWidget(
+            application=application,
+            chat=chat,
+            session_id=session_id,
+            message_id=message_id,
+            request_id=request_id,
+            referencing_to=referencing_to,
+            page=page,
+        )
+        await document_list_view._acquire_documents()
+        return document_list_view
+
+    async def _acquire_documents(self):
+        typed_document_pb = await self.application.meta_api_client.get(
+            schema='scimag',
+            document_id=self.referencing_to,
+            position=0,
+            request_id=self.request_id,
+            session_id=self.session_id,
+            user_id=self.chat.id,
+        )
+        self._response = await self.application.meta_api_client.search(
+            schemas=('scimag',),
+            query=f'references:"{typed_document_pb.scimag.doi}"',
+            page=self.page,
+            request_id=self.request_id,
+            session_id=self.session_id,
+            user_id=self.chat.id,
+        )
+
+    @property
+    def has_next(self) -> bool:
+        return self._response.has_next
+
+    @property
+    def scored_documents(self) -> list[ScoredDocumentPb]:
+        return self._response.scored_documents
+
+    async def render(self) -> tuple[str, Optional[list]]:
+        if not len(self.scored_documents):
+            return t('COULD_NOT_FIND_ANYTHING', language=self.chat.language), [close_button(self.session_id)]
+
+        serp_elements = [
+            f'Linked to: {self.referencing_to}',
+        ]
+        for scored_document in self.scored_documents:
+            view = parse_typed_document_to_view(scored_document.typed_document)
+            view_command = view.get_view_command(
+                session_id=self.session_id,
+                message_id=self.message_id,
+                parent_view_type='r',
+                position=scored_document.position,
+            )
+            serp_elements.append(
+                view.get_snippet(
+                    language=self.chat.language,
+                    view_command=view_command,
+                    limit=512 + 128,
+                )
+            )
+
+        promo = self.application.promotioner.choose_promotion(language=self.chat.language).format(
+            related_channel=self.application.config['telegram']['related_channel'],
+        )
+        serp_elements.append(promo)
+        serp = '\n\n'.join(serp_elements)
+
+        buttons = []
+        if self.has_next or self.page > 0:
+            buttons = [
+                Button.inline(
+                    text='<<1' if self.page > 1 else ' ',
+                    data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_0'
+                    if self.page > 1 else '/noop',
+                ),
+                Button.inline(
+                    text=f'<{self.page}' if self.page > 0 else ' ',
+                    data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page - 1}'
+                    if self.page > 0 else '/noop',
+                ),
+                Button.inline(
+                    text=f'{self.page + 2}>' if self.has_next else ' ',
+                    data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page + 1}'
+                    if self.has_next else '/noop',
+                )
+            ]
+        buttons.append(close_button(self.session_id))
+
+        return serp, buttons

nexus/bot/widgets/search_widget.py

@@ -0,0 +1,157 @@
+from typing import Optional
+
+from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from nexus.bot.application import TelegramApplication
+from nexus.meta_api.proto.meta_search_service_pb2 import \
+    ScoredDocument as ScoredDocumentPb
+from nexus.translations import t
+from nexus.views.telegram.common import (
+    TooLongQueryError,
+    close_button,
+    encode_query_to_deep_link,
+)
+from nexus.views.telegram.registry import parse_typed_document_to_view
+from telethon import Button
+
+
+class SearchWidget:
+    """
+    Presents markup for the SERP.
+    """
+    def __init__(
+        self,
+        application: TelegramApplication,
+        chat: Chat,
+        session_id: str,
+        message_id: int,
+        request_id: str,
+        query: str,
+        page: int = 0,
+        is_group_mode: bool = False,
+    ):
+        self.application = application
+        self.chat = chat
+        self.session_id = session_id
+        self.message_id = message_id
+        self.request_id = request_id
+        self.query = query
+        self.page = page
+        self.is_group_mode = is_group_mode
+
+    @staticmethod
+    async def create(
+        application: TelegramApplication,
+        chat: Chat,
+        session_id: str,
+        message_id: int,
+        request_id: str,
+        query: str,
+        page: int = 0,
+        is_group_mode: bool = False,
+    ) -> 'SearchWidget':
+        search_widget_view = SearchWidget(
+            application=application,
+            chat=chat,
+            session_id=session_id,
+            message_id=message_id,
+            request_id=request_id,
+            query=query,
+            page=page,
+            is_group_mode=is_group_mode,
+        )
+        await search_widget_view._acquire_documents()
+        return search_widget_view
+
+    async def _acquire_documents(self):
+        self._search_response = await self.application.meta_api_client.search(
+            schemas=self.application.config['application']['schemas'],
+            query=self.query,
+            page=self.page,
+            page_size=self.application.config['application']['page_size'],
+            request_id=self.request_id,
+            session_id=self.session_id,
+            user_id=self.chat.id,
+            language=self.chat.language,
+        )
+
+    @property
+    def has_next(self) -> bool:
+        return self._search_response.has_next
+
+    @property
+    def scored_documents(self) -> list[ScoredDocumentPb]:
+        return self._search_response.scored_documents
+
+    async def render(self) -> tuple[str, Optional[list]]:
+        if not len(self.scored_documents):
+            return t('COULD_NOT_FIND_ANYTHING', language=self.chat.language), [close_button(self.session_id)]
+
+        serp_elements = []
+        bot_external_name = self.application.config['telegram']['bot_external_name']
+
+        for scored_document in self.scored_documents:
+            view = parse_typed_document_to_view(scored_document.typed_document)
+            if not self.is_group_mode:
+                view_command = view.get_view_command(
+                    session_id=self.session_id,
+                    message_id=self.message_id,
+                    position=scored_document.position,
+                )
+            else:
+                view_command = view.get_deep_link(bot_external_name, text='⬇️')
+            serp_elements.append(
+                view.get_snippet(
+                    language=self.chat.language,
+                    view_command=view_command,
+                    limit=512 + 128,
+                )
+            )
+        serp = '\n\n'.join(serp_elements)
+
+        if self.is_group_mode:
+            try:
+                encoded_query = encode_query_to_deep_link(
+                    self.query,
+                    bot_external_name,
+                )
+                serp = (
+                    f"{serp}\n\n**{t('DOWNLOAD_AND_SEARCH_MORE', language=self.chat.language)}: **"
+                    f'[@{bot_external_name}]'
+                    f'({encoded_query})'
+                )
+            except TooLongQueryError:
+                serp = (
+                    f"{serp}\n\n**{t('DOWNLOAD_AND_SEARCH_MORE', language=self.chat.language)}: **"
+                    f'[@{bot_external_name}]'
+                    f'(https://t.me/{bot_external_name})'
+                )
+
+        if not self.is_group_mode:
+            promo = self.application.promotioner.choose_promotion(language=self.chat.language).format(
+                related_channel=self.application.config['telegram']['related_channel'],
+            )
+            serp = f'{serp}\n\n{promo}\n'
+
+        buttons = None
+        if not self.is_group_mode:
+            buttons = []
+            if self.has_next or self.page > 0:
+                buttons = [
+                    Button.inline(
+                        text='<<1' if self.page > 1 else ' ',
+                        data=f'/search_{self.session_id}_{self.message_id}_0' if self.page > 1 else '/noop',
+                    ),
+                    Button.inline(
+                        text=f'<{self.page}' if self.page > 0 else ' ',
+                        data=f'/search_{self.session_id}_{self.message_id}_{self.page - 1}'
+                        if self.page > 0 else '/noop',
+                    ),
+                    Button.inline(
+                        text=f'{self.page + 2}>' if self.has_next else ' ',
+                        data=f'/search_{self.session_id}_{self.message_id}_{self.page + 1}'
+                        if self.has_next else '/noop',
+                    )
+                ]
+            buttons.append(close_button(self.session_id))
+
+        return serp, buttons

nexus/bot/widgets/settings_widget.py

@@ -0,0 +1,158 @@
+from typing import Optional
+
+from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api2.proto.location_pb2 import Location
+from izihawa_utils.podolsky_encoding import encode
+from nexus.bot.application import TelegramApplication
+from nexus.translations import t
+from telethon import Button
+
+top_languages = {
+    'am': '🇪🇹',
+    'ar': '🇦🇪',
+    'de': '🇩🇪',
+    'en': '🇬🇧',
+    'es': '🇪🇸',
+    'fa': '🇮🇷',
+    'hi': '🇮🇳',
+    'id': '🇮🇩',
+    'it': '🇮🇹',
+    'ja': '🇯🇵',
+    'ms': '🇲🇾',
+    'pb': '🇧🇷',
+    'ru': '🇷🇺',
+    'tg': '🇹🇯',
+    'uk': '🇺🇦',
+    'uz': '🇺🇿',
+}
+
+boolean_emoji = {
+    False: '❎',
+    True: '✅️',
+}
+
+
+class SettingsRouterWidget:
+    def __init__(self, application: TelegramApplication, chat: Chat, request_id: str = None):
+        self.application = application
+        self.chat = chat
+        self.request_id = request_id
+
+    async def render(self):
+        sa = f'🌎{encode("1")}{t("SETUP_AUTOMATICALLY", language=self.chat.language)}'
+        sm = f'👇{encode("1")}{t("SETUP_MANUALLY", language=self.chat.language)}'
+        return t("SETTINGS_ROUTER_HELP", language=self.chat.language), [[
+            Button.text(sa, resize=True, single_use=True),
+            Button.text(sm, resize=True, single_use=True),
+            Button.force_reply(),
+        ]]
+
+
+class SettingsManualWidget:
+    def __init__(
+        self,
+        application: TelegramApplication,
+        chat: Chat,
+        has_language_buttons: Optional[bool] = None,
+        is_group_mode: bool = False,
+        request_id: Optional[str] = None,
+    ):
+        self.application = application
+        self.chat = chat
+        self.has_language_buttons = has_language_buttons
+        if self.has_language_buttons is None:
+            self.has_language_buttons = self.application.config['application']['views']['settings']['has_language_buttons']
+        self.is_group_mode = is_group_mode
+        self.request_id = request_id
+        self._actions = {
+            'sl': self._switch_language,
+            'ssm': self._switch_system_messaging,
+            'sd': self._switch_discovery,
+        }
+
+    async def _switch_language(self, target_language: str):
+        return await self.application.idm_client.update_chat(
+            chat_id=self.chat.id,
+            language=target_language,
+            request_id=self.request_id,
+        )
+
+    async def _switch_system_messaging(self, is_system_messaging_enabled: str):
+        self.chat = await self.application.idm_client.update_chat(
+            chat_id=self.chat.id,
+            is_system_messaging_enabled=bool(int(is_system_messaging_enabled)),
+            request_id=self.request_id,
+        )
+        return self.chat
+
+    async def _switch_discovery(self, is_discovery_enabled: str):
+        self.chat = await self.application.idm_client.update_chat(
+            chat_id=self.chat.id,
+            is_discovery_enabled=bool(int(is_discovery_enabled)),
+            request_id=self.request_id,
+        )
+        return self.chat
+
+    async def set_last_location(self, lon: float, lat: float):
+        self.chat = await self.application.idm_client.update_chat(
+            chat_id=self.chat.id,
+            last_location=Location(lon=lon, lat=lat),
+            request_id=self.request_id,
+        )
+        return
+
+    async def process_action(self, action_id: str, data: str):
+        old_chat = self.chat
+        await self._actions[action_id](data)
+        return old_chat != self.chat
+
+    async def render(self):
+        text = t('SETTINGS_TEMPLATE', language=self.chat.language).format(
+            bot_version=self.application.config['application']['bot_version'],
+            nexus_version=self.application.config['application']['nexus_version'],
+            language=top_languages.get(self.chat.language, self.chat.language),
+            tzinfo=self.chat.tzinfo or 'UTC',
+        )
+        if not self.is_group_mode and self.application.config['application']['views']['settings']['has_discovery_button']:
+            text = f"{text}\n\n{t('NEXUS_DISCOVERY_DESCRIPTION', language=self.chat.language)}"
+        buttons = []
+        if self.has_language_buttons:
+            buttons.append([])
+            for language in sorted(top_languages):
+                if len(buttons[-1]) >= 4:
+                    buttons.append([])
+                buttons[-1].append(
+                    Button.inline(
+                        text=top_languages[language],
+                        data=f'/settings_sl_{language}'
+                    )
+                )
+
+        if self.is_group_mode:
+            return text, buttons
+
+        if self.application.config['application']['views']['settings']['has_system_messaging_button']:
+            buttons.append([
+                Button.inline(
+                    text=(
+                        f'{t("SYSTEM_MESSAGING_OPTION", language=self.chat.language)}: '
+                        f'{boolean_emoji[self.chat.is_system_messaging_enabled]}'
+                    ),
+                    data=f'/settings_ssm_{1 - int(self.chat.is_system_messaging_enabled)}'
+                )
+            ])
+        if self.application.config['application']['views']['settings']['has_discovery_button']:
+            buttons.append([
+                Button.inline(
+                    text=(
+                        f'{t("DISCOVERY_OPTION", language=self.chat.language)}: '
+                        f'{boolean_emoji[self.chat.is_discovery_enabled]}'
+                    ),
+                    data=f'/settings_sd_{1 - int(self.chat.is_discovery_enabled)}'
+                )
+            ])
+        if self.application.config['application']['views']['settings']['has_location_button']:
+            buttons.append([
+                Button.request_location('Setup preferences automatically', resize=True)
+            ])
+        return text, buttons

nexus/hub/aioclient/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@pip_modules//:requirements.bzl", "requirement")
+
+py_library(
+    name = "aioclient",
+    srcs = glob(["**/*.py"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("grpcio"),
+        "//idm/api2/proto:idm_proto_py",
+        requirement("aiokit"),
+        "//nexus/hub/proto:hub_grpc_py",
+        "//nexus/hub/proto:hub_proto_py",
+        "//nexus/models/proto:models_proto_py",
+    ],
+)

nexus/hub/aioclient/__init__.py

@@ -0,0 +1,3 @@
+from .aioclient import HubGrpcClient
+
+__all__ = ['HubGrpcClient']

nexus/hub/aioclient/aioclient.py

@@ -0,0 +1,72 @@
+from typing import Optional
+
+from aiokit import AioThing
+from grpc.experimental.aio import insecure_channel
+from idm.api2.proto.chats_service_pb2 import ChatData as ChatDataPb
+from nexus.hub.proto.delivery_service_pb2 import \
+    StartDeliveryRequest as StartDeliveryRequestPb
+from nexus.hub.proto.delivery_service_pb2 import \
+    StartDeliveryResponse as StartDeliveryResponsePb
+from nexus.hub.proto.delivery_service_pb2_grpc import DeliveryStub
+from nexus.hub.proto.submitter_service_pb2 import \
+    SubmitRequest as SubmitRequestPb
+from nexus.hub.proto.submitter_service_pb2 import \
+    SubmitResponse as SubmitResponsePb
+from nexus.hub.proto.submitter_service_pb2_grpc import SubmitterStub
+from nexus.models.proto.typed_document_pb2 import \
+    TypedDocument as TypedDocumentPb
+
+
+class HubGrpcClient(AioThing):
+    def __init__(
+        self,
+        base_url: str,
+    ):
+        super().__init__()
+        self.channel = insecure_channel(base_url, [
+            ('grpc.dns_min_time_between_resolutions_ms', 1000),
+            ('grpc.initial_reconnect_backoff_ms', 1000),
+            ('grpc.lb_policy_name', 'round_robin'),
+            ('grpc.min_reconnect_backoff_ms', 1000),
+            ('grpc.max_reconnect_backoff_ms', 2000),
+        ])
+        self.delivery_stub = DeliveryStub(self.channel)
+        self.submitter_stub = SubmitterStub(self.channel)
+
+    async def start(self):
+        await self.channel.channel_ready()
+
+    async def stop(self):
+        await self.channel.close()
+
+    async def start_delivery(
+        self,
+        typed_document_pb: TypedDocumentPb,
+        chat: ChatDataPb,
+        request_id: Optional[str],
+        session_id: Optional[str],
+    ) -> StartDeliveryResponsePb:
+        return await self.delivery_stub.start_delivery(
+            StartDeliveryRequestPb(
+                typed_document=typed_document_pb,
+                chat=chat,
+            ),
+            metadata=(('request-id', request_id), ('session-id', session_id))
+        )
+
+    async def submit(
+        self,
+        telegram_document: bytes,
+        telegram_file_id: str,
+        chat: ChatDataPb,
+        request_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+    ) -> SubmitResponsePb:
+        return await self.submitter_stub.submit(
+            SubmitRequestPb(
+                telegram_document=telegram_document,
+                telegram_file_id=telegram_file_id,
+                chat=chat,
+            ),
+            metadata=(('request-id', request_id), ('session-id', session_id))
+        )

nexus/hub/proto/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_library(
+    name = "hub_proto",
+    srcs = glob([
+        "*.proto",
+    ]),
+    deps = [
+        "//idm/api2/proto:idm_proto",
+        "//nexus/models/proto:models_proto",
+        "@com_google_protobuf//:wrappers_proto",
+    ],
+)
+
+py_proto_library(
+    name = "hub_proto_py",
+    deps = [":hub_proto"],
+)
+
+py_grpc_library(
+    name = "hub_grpc_py",
+    srcs = [":hub_proto"],
+    deps = [":hub_proto_py"],
+)

nexus/hub/proto/delivery_service.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+package nexus.hub.proto;
+
+import "nexus/models/proto/typed_document.proto";
+import "idm/api2/proto/chats_service.proto";
+
+message StartDeliveryRequest {
+  nexus.models.proto.TypedDocument typed_document = 1;
+  idm.api2.proto.ChatData chat = 2;
+}
+
+message StartDeliveryResponse {
+  enum Status {
+    OK = 0;
+    TOO_MANY_DOWNLOADS = 1;
+    ALREADY_DOWNLOADING = 2;
+  }
+  Status status = 1;
+}
+
+service Delivery {
+    rpc start_delivery(StartDeliveryRequest) returns (StartDeliveryResponse) {};
+}

nexus/hub/proto/delivery_service_pb2.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nexus/hub/proto/delivery_service.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from idm.api2.proto import \
+    chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
+from nexus.models.proto import \
+    typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nexus/hub/proto/delivery_service.proto',
+  package='nexus.hub.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a\"idm/api2/proto/chats_service.proto\"y\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x03\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3'
+  ,
+  dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
+
+
+
+_STARTDELIVERYRESPONSE_STATUS = _descriptor.EnumDescriptor(
+  name='Status',
+  full_name='nexus.hub.proto.StartDeliveryResponse.Status',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='OK', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='TOO_MANY_DOWNLOADS', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='ALREADY_DOWNLOADING', index=2, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=348,
+  serialized_end=413,
+)
+_sym_db.RegisterEnumDescriptor(_STARTDELIVERYRESPONSE_STATUS)
+
+
+_STARTDELIVERYREQUEST = _descriptor.Descriptor(
+  name='StartDeliveryRequest',
+  full_name='nexus.hub.proto.StartDeliveryRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='typed_document', full_name='nexus.hub.proto.StartDeliveryRequest.typed_document', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chat', full_name='nexus.hub.proto.StartDeliveryRequest.chat', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=136,
+  serialized_end=257,
+)
+
+
+_STARTDELIVERYRESPONSE = _descriptor.Descriptor(
+  name='StartDeliveryResponse',
+  full_name='nexus.hub.proto.StartDeliveryResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='status', full_name='nexus.hub.proto.StartDeliveryResponse.status', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _STARTDELIVERYRESPONSE_STATUS,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=260,
+  serialized_end=413,
+)
+
+_STARTDELIVERYREQUEST.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
+_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
+_STARTDELIVERYRESPONSE.fields_by_name['status'].enum_type = _STARTDELIVERYRESPONSE_STATUS
+_STARTDELIVERYRESPONSE_STATUS.containing_type = _STARTDELIVERYRESPONSE
+DESCRIPTOR.message_types_by_name['StartDeliveryRequest'] = _STARTDELIVERYREQUEST
+DESCRIPTOR.message_types_by_name['StartDeliveryResponse'] = _STARTDELIVERYRESPONSE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+StartDeliveryRequest = _reflection.GeneratedProtocolMessageType('StartDeliveryRequest', (_message.Message,), {
+  'DESCRIPTOR' : _STARTDELIVERYREQUEST,
+  '__module__' : 'nexus.hub.proto.delivery_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.hub.proto.StartDeliveryRequest)
+  })
+_sym_db.RegisterMessage(StartDeliveryRequest)
+
+StartDeliveryResponse = _reflection.GeneratedProtocolMessageType('StartDeliveryResponse', (_message.Message,), {
+  'DESCRIPTOR' : _STARTDELIVERYRESPONSE,
+  '__module__' : 'nexus.hub.proto.delivery_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.hub.proto.StartDeliveryResponse)
+  })
+_sym_db.RegisterMessage(StartDeliveryResponse)
+
+
+
+_DELIVERY = _descriptor.ServiceDescriptor(
+  name='Delivery',
+  full_name='nexus.hub.proto.Delivery',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=415,
+  serialized_end=524,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='start_delivery',
+    full_name='nexus.hub.proto.Delivery.start_delivery',
+    index=0,
+    containing_service=None,
+    input_type=_STARTDELIVERYREQUEST,
+    output_type=_STARTDELIVERYRESPONSE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_DELIVERY)
+
+DESCRIPTOR.services_by_name['Delivery'] = _DELIVERY
+
+# @@protoc_insertion_point(module_scope)

nexus/hub/proto/delivery_service_pb2_grpc.py

@@ -0,0 +1,66 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+from nexus.hub.proto import \
+    delivery_service_pb2 as nexus_dot_hub_dot_proto_dot_delivery__service__pb2
+
+
+class DeliveryStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.start_delivery = channel.unary_unary(
+                '/nexus.hub.proto.Delivery/start_delivery',
+                request_serializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryRequest.SerializeToString,
+                response_deserializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryResponse.FromString,
+                )
+
+
+class DeliveryServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def start_delivery(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_DeliveryServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'start_delivery': grpc.unary_unary_rpc_method_handler(
+                    servicer.start_delivery,
+                    request_deserializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryRequest.FromString,
+                    response_serializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'nexus.hub.proto.Delivery', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Delivery(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def start_delivery(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/nexus.hub.proto.Delivery/start_delivery',
+            nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryRequest.SerializeToString,
+            nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

nexus/hub/proto/submitter_service.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+package nexus.hub.proto;
+
+import "idm/api2/proto/chats_service.proto";
+
+message SubmitRequest {
+    bytes telegram_document = 1;
+    string telegram_file_id = 2;
+    idm.api2.proto.ChatData chat = 3;
+}
+message SubmitResponse { }
+
+service Submitter {
+    rpc submit(SubmitRequest) returns (SubmitResponse) {};
+}

nexus/hub/proto/submitter_service_pb2.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nexus/hub/proto/submitter_service.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from idm.api2.proto import \
+    chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nexus/hub/proto/submitter_service.proto',
+  package='nexus.hub.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a\"idm/api2/proto/chats_service.proto\"R\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3'
+  ,
+  dependencies=[idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
+
+
+
+
+_SUBMITREQUEST = _descriptor.Descriptor(
+  name='SubmitRequest',
+  full_name='nexus.hub.proto.SubmitRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='telegram_document', full_name='nexus.hub.proto.SubmitRequest.telegram_document', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=96,
+  serialized_end=178,
+)
+
+
+_SUBMITRESPONSE = _descriptor.Descriptor(
+  name='SubmitResponse',
+  full_name='nexus.hub.proto.SubmitResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=180,
+  serialized_end=196,
+)
+
+_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
+DESCRIPTOR.message_types_by_name['SubmitRequest'] = _SUBMITREQUEST
+DESCRIPTOR.message_types_by_name['SubmitResponse'] = _SUBMITRESPONSE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+SubmitRequest = _reflection.GeneratedProtocolMessageType('SubmitRequest', (_message.Message,), {
+  'DESCRIPTOR' : _SUBMITREQUEST,
+  '__module__' : 'nexus.hub.proto.submitter_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.hub.proto.SubmitRequest)
+  })
+_sym_db.RegisterMessage(SubmitRequest)
+
+SubmitResponse = _reflection.GeneratedProtocolMessageType('SubmitResponse', (_message.Message,), {
+  'DESCRIPTOR' : _SUBMITRESPONSE,
+  '__module__' : 'nexus.hub.proto.submitter_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.hub.proto.SubmitResponse)
+  })
+_sym_db.RegisterMessage(SubmitResponse)
+
+
+
+_SUBMITTER = _descriptor.ServiceDescriptor(
+  name='Submitter',
+  full_name='nexus.hub.proto.Submitter',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=198,
+  serialized_end=286,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='submit',
+    full_name='nexus.hub.proto.Submitter.submit',
+    index=0,
+    containing_service=None,
+    input_type=_SUBMITREQUEST,
+    output_type=_SUBMITRESPONSE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_SUBMITTER)
+
+DESCRIPTOR.services_by_name['Submitter'] = _SUBMITTER
+
+# @@protoc_insertion_point(module_scope)

nexus/hub/proto/submitter_service_pb2_grpc.py

@@ -0,0 +1,67 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+from nexus.hub.proto import \
+    submitter_service_pb2 as \
+    nexus_dot_hub_dot_proto_dot_submitter__service__pb2
+
+
+class SubmitterStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.submit = channel.unary_unary(
+                '/nexus.hub.proto.Submitter/submit',
+                request_serializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitRequest.SerializeToString,
+                response_deserializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitResponse.FromString,
+                )
+
+
+class SubmitterServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def submit(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_SubmitterServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'submit': grpc.unary_unary_rpc_method_handler(
+                    servicer.submit,
+                    request_deserializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitRequest.FromString,
+                    response_serializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'nexus.hub.proto.Submitter', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Submitter(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def submit(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/nexus.hub.proto.Submitter/submit',
+            nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitRequest.SerializeToString,
+            nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

nexus/meta_api/aioclient/BUILD.bazel

@@ -0,0 +1,17 @@
+load("@pip_modules//:requirements.bzl", "requirement")
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "aioclient",
+    srcs = glob(["**/*.py"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("grpcio"),
+        requirement("lru-dict"),
+        requirement("tenacity"),
+        requirement("aiokit"),
+        "//nexus/meta_api/proto:meta_api_grpc_py",
+        "//nexus/meta_api/proto:meta_api_proto_py",
+        "//nexus/models/proto:models_proto_py",
+    ],
+)

nexus/meta_api/aioclient/__init__.py

@@ -0,0 +1,3 @@
+from .aioclient import MetaApiGrpcClient
+
+__all__ = ['MetaApiGrpcClient']

nexus/meta_api/aioclient/aioclient.py

@@ -0,0 +1,132 @@
+from typing import (
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from aiokit import AioThing
+from grpc import StatusCode
+from grpc.experimental.aio import (
+    AioRpcError,
+    insecure_channel,
+)
+from lru import LRU
+from nexus.meta_api.proto.documents_service_pb2 import \
+    RollRequest as RollRequestPb
+from nexus.meta_api.proto.documents_service_pb2 import \
+    RollResponse as RollResponsePb
+from nexus.meta_api.proto.documents_service_pb2 import \
+    TypedDocumentRequest as TypedDocumentRequestPb
+from nexus.meta_api.proto.documents_service_pb2_grpc import DocumentsStub
+from nexus.meta_api.proto.meta_search_service_pb2 import \
+    SearchRequest as SearchRequestPb
+from nexus.meta_api.proto.meta_search_service_pb2 import \
+    SearchResponse as SearchResponsePb
+from nexus.meta_api.proto.meta_search_service_pb2_grpc import MetaSearchStub
+from nexus.models.proto.typed_document_pb2 import \
+    TypedDocument as TypedDocumentPb
+from tenacity import (
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_fixed,
+)
+
+
+class MetaApiGrpcClient(AioThing):
+    def __init__(self, base_url):
+        super().__init__()
+        self.channel = insecure_channel(base_url, [
+            ('grpc.dns_min_time_between_resolutions_ms', 1000),
+            ('grpc.initial_reconnect_backoff_ms', 1000),
+            ('grpc.lb_policy_name', 'round_robin'),
+            ('grpc.min_reconnect_backoff_ms', 1000),
+            ('grpc.max_reconnect_backoff_ms', 2000),
+        ])
+        self.meta_search_stub = MetaSearchStub(self.channel)
+        self.documents_stub = DocumentsStub(self.channel)
+        self.cache = LRU(4096)
+
+    async def start(self):
+        await self.channel.channel_ready()
+
+    async def stop(self):
+        await self.channel.close()
+
+    async def get(
+        self,
+        schema: str,
+        document_id: int,
+        position: Optional[int] = None,
+        request_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+    ) -> TypedDocumentPb:
+        return await self.documents_stub.get(
+            TypedDocumentRequestPb(
+                schema=schema,
+                document_id=document_id,
+                position=position,
+                session_id=session_id,
+                user_id=user_id,
+            ),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )
+
+    async def roll(
+        self,
+        language: Optional[str] = None,
+        request_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+    ) -> RollResponsePb:
+        return await self.documents_stub.roll(
+            RollRequestPb(
+                language=language,
+                session_id=session_id,
+                user_id=user_id,
+            ),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )
+
+    @retry(
+        retry=retry_if_exception(
+            lambda e: isinstance(e, AioRpcError) and (
+                e.code() == StatusCode.CANCELLED
+                or e.code() == StatusCode.UNAVAILABLE
+            )
+        ),
+        reraise=True,
+        stop=stop_after_attempt(5),
+        wait=wait_fixed(2),
+    )
+    async def search(
+        self,
+        schemas: Union[List[str], Tuple[str]],
+        query: str,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
+        language: Optional[str] = None,
+        request_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+    ) -> SearchResponsePb:
+        return await self.meta_search_stub.search(
+            SearchRequestPb(
+                schemas=schemas,
+                query=query,
+                page=page,
+                page_size=page_size,
+                language=language,
+                session_id=session_id,
+                user_id=user_id,
+            ),
+            metadata=(
+                ('request-id', request_id),
+            ),
+        )

nexus/meta_api/proto/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
+load("@rules_rust//proto:proto.bzl", "rust_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_library(
+    name = "meta_api_proto",
+    srcs = glob([
+        "*.proto",
+    ]),
+    deps = [
+        "//nexus/models/proto:models_proto",
+        "@com_google_protobuf//:wrappers_proto",
+    ],
+)
+
+py_proto_library(
+    name = "meta_api_proto_py",
+    deps = [":meta_api_proto"],
+)
+
+py_grpc_library(
+    name = "meta_api_grpc_py",
+    srcs = [":meta_api_proto"],
+    deps = [":meta_api_proto_py"],
+)
+
+rust_proto_library(
+    name = "meta_api_proto_rust",
+    rust_deps = ["//rules/rust/cargo:protobuf"],
+    visibility = ["//visibility:public"],
+    deps = [":meta_api_proto"],
+)

nexus/meta_api/proto/documents_service.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+package nexus.meta_api.proto;
+
+import "nexus/models/proto/typed_document.proto";
+
+message RollRequest {
+  string language = 1;
+  string session_id = 2;
+  int64 user_id = 3;
+}
+
+message RollResponse {
+  uint64 document_id = 1;
+}
+
+message TypedDocumentRequest {
+  string schema = 1;
+  uint64 document_id = 2;
+  uint32 position = 3;
+  string session_id = 4;
+  int64 user_id = 5;
+}
+
+message PutTypedDocumentResponse {}
+
+service Documents {
+  rpc get (TypedDocumentRequest) returns (nexus.models.proto.TypedDocument) {}
+  rpc roll (RollRequest) returns (RollResponse) {}
+}

nexus/meta_api/proto/documents_service_pb2.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nexus/meta_api/proto/documents_service.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from nexus.models.proto import \
+    typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nexus/meta_api/proto/documents_service.proto',
+  package='nexus.meta_api.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\"\x1a\n\x18PutTypedDocumentResponse2\xb4\x01\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x62\x06proto3'
+  ,
+  dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
+
+
+
+
+_ROLLREQUEST = _descriptor.Descriptor(
+  name='RollRequest',
+  full_name='nexus.meta_api.proto.RollRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='language', full_name='nexus.meta_api.proto.RollRequest.language', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='session_id', full_name='nexus.meta_api.proto.RollRequest.session_id', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='user_id', full_name='nexus.meta_api.proto.RollRequest.user_id', index=2,
+      number=3, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=111,
+  serialized_end=179,
+)
+
+
+_ROLLRESPONSE = _descriptor.Descriptor(
+  name='RollResponse',
+  full_name='nexus.meta_api.proto.RollResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='document_id', full_name='nexus.meta_api.proto.RollResponse.document_id', index=0,
+      number=1, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=181,
+  serialized_end=216,
+)
+
+
+_TYPEDDOCUMENTREQUEST = _descriptor.Descriptor(
+  name='TypedDocumentRequest',
+  full_name='nexus.meta_api.proto.TypedDocumentRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='schema', full_name='nexus.meta_api.proto.TypedDocumentRequest.schema', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='document_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.document_id', index=1,
+      number=2, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='position', full_name='nexus.meta_api.proto.TypedDocumentRequest.position', index=2,
+      number=3, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='session_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.session_id', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='user_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.user_id', index=4,
+      number=5, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=218,
+  serialized_end=332,
+)
+
+
+_PUTTYPEDDOCUMENTRESPONSE = _descriptor.Descriptor(
+  name='PutTypedDocumentResponse',
+  full_name='nexus.meta_api.proto.PutTypedDocumentResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=334,
+  serialized_end=360,
+)
+
+DESCRIPTOR.message_types_by_name['RollRequest'] = _ROLLREQUEST
+DESCRIPTOR.message_types_by_name['RollResponse'] = _ROLLRESPONSE
+DESCRIPTOR.message_types_by_name['TypedDocumentRequest'] = _TYPEDDOCUMENTREQUEST
+DESCRIPTOR.message_types_by_name['PutTypedDocumentResponse'] = _PUTTYPEDDOCUMENTRESPONSE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+RollRequest = _reflection.GeneratedProtocolMessageType('RollRequest', (_message.Message,), {
+  'DESCRIPTOR' : _ROLLREQUEST,
+  '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.RollRequest)
+  })
+_sym_db.RegisterMessage(RollRequest)
+
+RollResponse = _reflection.GeneratedProtocolMessageType('RollResponse', (_message.Message,), {
+  'DESCRIPTOR' : _ROLLRESPONSE,
+  '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.RollResponse)
+  })
+_sym_db.RegisterMessage(RollResponse)
+
+TypedDocumentRequest = _reflection.GeneratedProtocolMessageType('TypedDocumentRequest', (_message.Message,), {
+  'DESCRIPTOR' : _TYPEDDOCUMENTREQUEST,
+  '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.TypedDocumentRequest)
+  })
+_sym_db.RegisterMessage(TypedDocumentRequest)
+
+PutTypedDocumentResponse = _reflection.GeneratedProtocolMessageType('PutTypedDocumentResponse', (_message.Message,), {
+  'DESCRIPTOR' : _PUTTYPEDDOCUMENTRESPONSE,
+  '__module__' : 'nexus.meta_api.proto.documents_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.PutTypedDocumentResponse)
+  })
+_sym_db.RegisterMessage(PutTypedDocumentResponse)
+
+
+
+_DOCUMENTS = _descriptor.ServiceDescriptor(
+  name='Documents',
+  full_name='nexus.meta_api.proto.Documents',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=363,
+  serialized_end=543,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='get',
+    full_name='nexus.meta_api.proto.Documents.get',
+    index=0,
+    containing_service=None,
+    input_type=_TYPEDDOCUMENTREQUEST,
+    output_type=nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='roll',
+    full_name='nexus.meta_api.proto.Documents.roll',
+    index=1,
+    containing_service=None,
+    input_type=_ROLLREQUEST,
+    output_type=_ROLLRESPONSE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_DOCUMENTS)
+
+DESCRIPTOR.services_by_name['Documents'] = _DOCUMENTS
+
+# @@protoc_insertion_point(module_scope)

nexus/meta_api/proto/documents_service_pb2_grpc.py

@@ -0,0 +1,102 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+from nexus.meta_api.proto import \
+    documents_service_pb2 as \
+    nexus_dot_meta__api_dot_proto_dot_documents__service__pb2
+from nexus.models.proto import \
+    typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
+
+
+class DocumentsStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.get = channel.unary_unary(
+                '/nexus.meta_api.proto.Documents/get',
+                request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TypedDocumentRequest.SerializeToString,
+                response_deserializer=nexus_dot_models_dot_proto_dot_typed__document__pb2.TypedDocument.FromString,
+                )
+        self.roll = channel.unary_unary(
+                '/nexus.meta_api.proto.Documents/roll',
+                request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.SerializeToString,
+                response_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString,
+                )
+
+
+class DocumentsServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def get(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def roll(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_DocumentsServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'get': grpc.unary_unary_rpc_method_handler(
+                    servicer.get,
+                    request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TypedDocumentRequest.FromString,
+                    response_serializer=nexus_dot_models_dot_proto_dot_typed__document__pb2.TypedDocument.SerializeToString,
+            ),
+            'roll': grpc.unary_unary_rpc_method_handler(
+                    servicer.roll,
+                    request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.FromString,
+                    response_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'nexus.meta_api.proto.Documents', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Documents(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def get(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Documents/get',
+            nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TypedDocumentRequest.SerializeToString,
+            nexus_dot_models_dot_proto_dot_typed__document__pb2.TypedDocument.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def roll(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Documents/roll',
+            nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.SerializeToString,
+            nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

nexus/meta_api/proto/meta_search_service.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+package nexus.meta_api.proto;
+
+import "nexus/models/proto/typed_document.proto";
+
+message ScoredDocument {
+  nexus.models.proto.TypedDocument typed_document = 1;
+  float score = 3;
+  uint32 position = 4;
+}
+
+message SearchResponse {
+  repeated ScoredDocument scored_documents = 1;
+  bool has_next = 2;
+}
+
+message SearchRequest {
+  repeated string schemas = 1;
+  string query = 2;
+  uint32 page = 3;
+  uint32 page_size = 4;
+  string language = 5;
+  int64 user_id = 6;
+  string session_id = 7;
+}
+
+service MetaSearch {
+  rpc search (SearchRequest) returns (SearchResponse) {}
+}

nexus/meta_api/proto/meta_search_service_pb2.py

@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nexus/meta_api/proto/meta_search_service.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from nexus.models.proto import \
+    typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nexus/meta_api/proto/meta_search_service.proto',
+  package='nexus.meta_api.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x03 \x01(\x02\x12\x10\n\x08position\x18\x04 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\xa3\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t\x12\x1a\n\x12skip_query_rewrite\x18\t \x01(\x08\x32\x63\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
+  ,
+  dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
+
+
+
+
+_SCOREDDOCUMENT = _descriptor.Descriptor(
+  name='ScoredDocument',
+  full_name='nexus.meta_api.proto.ScoredDocument',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='typed_document', full_name='nexus.meta_api.proto.ScoredDocument.typed_document', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='score', full_name='nexus.meta_api.proto.ScoredDocument.score', index=1,
+      number=3, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='position', full_name='nexus.meta_api.proto.ScoredDocument.position', index=2,
+      number=4, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=113,
+  serialized_end=221,
+)
+
+
+_SEARCHRESPONSE = _descriptor.Descriptor(
+  name='SearchResponse',
+  full_name='nexus.meta_api.proto.SearchResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='scored_documents', full_name='nexus.meta_api.proto.SearchResponse.scored_documents', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='has_next', full_name='nexus.meta_api.proto.SearchResponse.has_next', index=1,
+      number=2, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=223,
+  serialized_end=321,
+)
+
+
+_SEARCHREQUEST = _descriptor.Descriptor(
+  name='SearchRequest',
+  full_name='nexus.meta_api.proto.SearchRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='schemas', full_name='nexus.meta_api.proto.SearchRequest.schemas', index=0,
+      number=1, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='query', full_name='nexus.meta_api.proto.SearchRequest.query', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='page', full_name='nexus.meta_api.proto.SearchRequest.page', index=2,
+      number=3, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='page_size', full_name='nexus.meta_api.proto.SearchRequest.page_size', index=3,
+      number=4, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='language', full_name='nexus.meta_api.proto.SearchRequest.language', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='user_id', full_name='nexus.meta_api.proto.SearchRequest.user_id', index=5,
+      number=6, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='session_id', full_name='nexus.meta_api.proto.SearchRequest.session_id', index=6,
+      number=7, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='skip_query_rewrite', full_name='nexus.meta_api.proto.SearchRequest.skip_query_rewrite', index=7,
+      number=9, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=324,
+  serialized_end=487,
+)
+
+_SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
+_SEARCHRESPONSE.fields_by_name['scored_documents'].message_type = _SCOREDDOCUMENT
+DESCRIPTOR.message_types_by_name['ScoredDocument'] = _SCOREDDOCUMENT
+DESCRIPTOR.message_types_by_name['SearchResponse'] = _SEARCHRESPONSE
+DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ScoredDocument = _reflection.GeneratedProtocolMessageType('ScoredDocument', (_message.Message,), {
+  'DESCRIPTOR' : _SCOREDDOCUMENT,
+  '__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.ScoredDocument)
+  })
+_sym_db.RegisterMessage(ScoredDocument)
+
+SearchResponse = _reflection.GeneratedProtocolMessageType('SearchResponse', (_message.Message,), {
+  'DESCRIPTOR' : _SEARCHRESPONSE,
+  '__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchResponse)
+  })
+_sym_db.RegisterMessage(SearchResponse)
+
+SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
+  'DESCRIPTOR' : _SEARCHREQUEST,
+  '__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchRequest)
+  })
+_sym_db.RegisterMessage(SearchRequest)
+
+
+
+_METASEARCH = _descriptor.ServiceDescriptor(
+  name='MetaSearch',
+  full_name='nexus.meta_api.proto.MetaSearch',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=489,
+  serialized_end=588,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='search',
+    full_name='nexus.meta_api.proto.MetaSearch.search',
+    index=0,
+    containing_service=None,
+    input_type=_SEARCHREQUEST,
+    output_type=_SEARCHRESPONSE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_METASEARCH)
+
+DESCRIPTOR.services_by_name['MetaSearch'] = _METASEARCH
+
+# @@protoc_insertion_point(module_scope)

nexus/meta_api/proto/meta_search_service_pb2_grpc.py

@@ -0,0 +1,67 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+from nexus.meta_api.proto import \
+    meta_search_service_pb2 as \
+    nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2
+
+
+class MetaSearchStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.search = channel.unary_unary(
+                '/nexus.meta_api.proto.MetaSearch/search',
+                request_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
+                response_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
+                )
+
+
+class MetaSearchServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def search(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_MetaSearchServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'search': grpc.unary_unary_rpc_method_handler(
+                    servicer.search,
+                    request_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.FromString,
+                    response_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'nexus.meta_api.proto.MetaSearch', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class MetaSearch(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def search(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.MetaSearch/search',
+            nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
+            nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

nexus/models/proto/operation.proto

@@ -13,10 +13,17 @@ message CrossReferenceOperation {
 
 message DocumentOperation {
   oneof operation {
+    Vote vote = 1;
     UpdateDocument update_document = 3;
   };
 }
 
+message Vote {
+  int64 document_id = 1;
+  int32 value = 2;
+  int64 voter_id = 3;
+}
+
 message UpdateDocument {
   repeated string fields = 1;
   bool should_fill_from_external_source = 2;

nexus/models/proto/operation_pb2.py

@@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\"nexus/models/proto/operation.proto\x12\x12nexus.models.proto\x1a\'nexus/models/proto/typed_document.proto\"k\n\x17\x43rossReferenceOperation\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\x1b\n\x13last_retry_unixtime\x18\x03 \x01(\r\x12\x13\n\x0bretry_count\x18\x04 \x01(\r\"_\n\x11\x44ocumentOperation\x12=\n\x0fupdate_document\x18\x03 \x01(\x0b\x32\".nexus.models.proto.UpdateDocumentH\x00\x42\x0b\n\toperation\"\xa6\x01\n\x0eUpdateDocument\x12\x0e\n\x06\x66ields\x18\x01 \x03(\t\x12(\n should_fill_from_external_source\x18\x02 \x01(\x08\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\x12\x0f\n\x07reindex\x18\x04 \x01(\x08\x12\x39\n\x0etyped_document\x18\x05 \x01(\x0b\x32!.nexus.models.proto.TypedDocumentb\x06proto3'
+  serialized_pb=b'\n\"nexus/models/proto/operation.proto\x12\x12nexus.models.proto\x1a\'nexus/models/proto/typed_document.proto\"k\n\x17\x43rossReferenceOperation\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\x1b\n\x13last_retry_unixtime\x18\x03 \x01(\r\x12\x13\n\x0bretry_count\x18\x04 \x01(\r\"\x89\x01\n\x11\x44ocumentOperation\x12(\n\x04vote\x18\x01 \x01(\x0b\x32\x18.nexus.models.proto.VoteH\x00\x12=\n\x0fupdate_document\x18\x03 \x01(\x0b\x32\".nexus.models.proto.UpdateDocumentH\x00\x42\x0b\n\toperation\"b\n\x04Vote\x12\x10\n\x08voter_id\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x05\x12\x39\n\x0etyped_document\x18\x03 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\"\xa6\x01\n\x0eUpdateDocument\x12\x0e\n\x06\x66ields\x18\x01 \x03(\t\x12(\n should_fill_from_external_source\x18\x02 \x01(\x08\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\x12\x0f\n\x07reindex\x18\x04 \x01(\x08\x12\x39\n\x0etyped_document\x18\x05 \x01(\x0b\x32!.nexus.models.proto.TypedDocumentb\x06proto3'
   ,
   dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
 
@@ -90,7 +90,14 @@ _DOCUMENTOPERATION = _descriptor.Descriptor(
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='update_document', full_name='nexus.models.proto.DocumentOperation.update_document', index=0,
+      name='vote', full_name='nexus.models.proto.DocumentOperation.vote', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='update_document', full_name='nexus.models.proto.DocumentOperation.update_document', index=1,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
@@ -113,8 +120,54 @@ _DOCUMENTOPERATION = _descriptor.Descriptor(
       create_key=_descriptor._internal_create_key,
     fields=[]),
   ],
-  serialized_start=208,
-  serialized_end=303,
+  serialized_start=209,
+  serialized_end=346,
+)
+
+
+_VOTE = _descriptor.Descriptor(
+  name='Vote',
+  full_name='nexus.models.proto.Vote',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='voter_id', full_name='nexus.models.proto.Vote.voter_id', index=0,
+      number=1, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='nexus.models.proto.Vote.value', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='typed_document', full_name='nexus.models.proto.Vote.typed_document', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=348,
+  serialized_end=446,
 )
 
 
@@ -173,17 +226,23 @@ _UPDATEDOCUMENT = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=306,
-  serialized_end=472,
+  serialized_start=449,
+  serialized_end=615,
 )
 
+_DOCUMENTOPERATION.fields_by_name['vote'].message_type = _VOTE
 _DOCUMENTOPERATION.fields_by_name['update_document'].message_type = _UPDATEDOCUMENT
+_DOCUMENTOPERATION.oneofs_by_name['operation'].fields.append(
+  _DOCUMENTOPERATION.fields_by_name['vote'])
+_DOCUMENTOPERATION.fields_by_name['vote'].containing_oneof = _DOCUMENTOPERATION.oneofs_by_name['operation']
 _DOCUMENTOPERATION.oneofs_by_name['operation'].fields.append(
   _DOCUMENTOPERATION.fields_by_name['update_document'])
 _DOCUMENTOPERATION.fields_by_name['update_document'].containing_oneof = _DOCUMENTOPERATION.oneofs_by_name['operation']
+_VOTE.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
 _UPDATEDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
 DESCRIPTOR.message_types_by_name['CrossReferenceOperation'] = _CROSSREFERENCEOPERATION
 DESCRIPTOR.message_types_by_name['DocumentOperation'] = _DOCUMENTOPERATION
+DESCRIPTOR.message_types_by_name['Vote'] = _VOTE
 DESCRIPTOR.message_types_by_name['UpdateDocument'] = _UPDATEDOCUMENT
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
@@ -201,6 +260,13 @@ DocumentOperation = _reflection.GeneratedProtocolMessageType('DocumentOperation'
   })
 _sym_db.RegisterMessage(DocumentOperation)
 
+Vote = _reflection.GeneratedProtocolMessageType('Vote', (_message.Message,), {
+  'DESCRIPTOR' : _VOTE,
+  '__module__' : 'nexus.models.proto.operation_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.models.proto.Vote)
+  })
+_sym_db.RegisterMessage(Vote)
+
 UpdateDocument = _reflection.GeneratedProtocolMessageType('UpdateDocument', (_message.Message,), {
   'DESCRIPTOR' : _UPDATEDOCUMENT,
   '__module__' : 'nexus.models.proto.operation_pb2'

nexus/models/proto/sharience.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+package nexus.models.proto;
+
+message Sharience {
+  int64 id = 1;
+  int64 parent_id = 2;
+  int64 uploader_id = 3;
+  uint32 updated_at = 4;
+  string md5 = 5;
+  uint32 filesize = 6;
+  repeated string ipfs_multihashes = 7;
+  string telegram_file_id = 8;
+}

nexus/models/proto/sharience_pb2.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nexus/models/proto/sharience.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nexus/models/proto/sharience.proto',
+  package='nexus.models.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\"nexus/models/proto/sharience.proto\x12\x12nexus.models.proto\"\xa6\x01\n\tSharience\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x11\n\tparent_id\x18\x02 \x01(\x03\x12\x13\n\x0buploader_id\x18\x03 \x01(\x03\x12\x12\n\nupdated_at\x18\x04 \x01(\r\x12\x0b\n\x03md5\x18\x05 \x01(\t\x12\x10\n\x08\x66ilesize\x18\x06 \x01(\r\x12\x18\n\x10ipfs_multihashes\x18\x07 \x03(\t\x12\x18\n\x10telegram_file_id\x18\x08 \x01(\tb\x06proto3'
+)
+
+
+
+
+_SHARIENCE = _descriptor.Descriptor(
+  name='Sharience',
+  full_name='nexus.models.proto.Sharience',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='nexus.models.proto.Sharience.id', index=0,
+      number=1, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='parent_id', full_name='nexus.models.proto.Sharience.parent_id', index=1,
+      number=2, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='uploader_id', full_name='nexus.models.proto.Sharience.uploader_id', index=2,
+      number=3, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='updated_at', full_name='nexus.models.proto.Sharience.updated_at', index=3,
+      number=4, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='md5', full_name='nexus.models.proto.Sharience.md5', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='filesize', full_name='nexus.models.proto.Sharience.filesize', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='ipfs_multihashes', full_name='nexus.models.proto.Sharience.ipfs_multihashes', index=6,
+      number=7, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='telegram_file_id', full_name='nexus.models.proto.Sharience.telegram_file_id', index=7,
+      number=8, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=59,
+  serialized_end=225,
+)
+
+DESCRIPTOR.message_types_by_name['Sharience'] = _SHARIENCE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Sharience = _reflection.GeneratedProtocolMessageType('Sharience', (_message.Message,), {
+  'DESCRIPTOR' : _SHARIENCE,
+  '__module__' : 'nexus.models.proto.sharience_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.models.proto.Sharience)
+  })
+_sym_db.RegisterMessage(Sharience)
+
+
+# @@protoc_insertion_point(module_scope)

nexus/models/proto/typed_document.proto

@@ -3,10 +3,12 @@ package nexus.models.proto;
 
 import "nexus/models/proto/scimag.proto";
 import "nexus/models/proto/scitech.proto";
+import "nexus/models/proto/sharience.proto";
 
 message TypedDocument {
   oneof document {
     Scimag scimag = 1;
     Scitech scitech = 2;
+    Sharience sharience = 3;
   }
 }

nexus/models/proto/typed_document_pb2.py

@@ -16,6 +16,8 @@ from nexus.models.proto import \
     scimag_pb2 as nexus_dot_models_dot_proto_dot_scimag__pb2
 from nexus.models.proto import \
     scitech_pb2 as nexus_dot_models_dot_proto_dot_scitech__pb2
+from nexus.models.proto import \
+    sharience_pb2 as nexus_dot_models_dot_proto_dot_sharience__pb2
 
 DESCRIPTOR = _descriptor.FileDescriptor(
   name='nexus/models/proto/typed_document.proto',
@@ -23,9 +25,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\'nexus/models/proto/typed_document.proto\x12\x12nexus.models.proto\x1a\x1fnexus/models/proto/scimag.proto\x1a nexus/models/proto/scitech.proto\"y\n\rTypedDocument\x12,\n\x06scimag\x18\x01 \x01(\x0b\x32\x1a.nexus.models.proto.ScimagH\x00\x12.\n\x07scitech\x18\x02 \x01(\x0b\x32\x1b.nexus.models.proto.ScitechH\x00\x42\n\n\x08\x64ocumentb\x06proto3'
+  serialized_pb=b'\n\'nexus/models/proto/typed_document.proto\x12\x12nexus.models.proto\x1a\x1fnexus/models/proto/scimag.proto\x1a nexus/models/proto/scitech.proto\x1a\"nexus/models/proto/sharience.proto\"\xad\x01\n\rTypedDocument\x12,\n\x06scimag\x18\x01 \x01(\x0b\x32\x1a.nexus.models.proto.ScimagH\x00\x12.\n\x07scitech\x18\x02 \x01(\x0b\x32\x1b.nexus.models.proto.ScitechH\x00\x12\x32\n\tsharience\x18\x03 \x01(\x0b\x32\x1d.nexus.models.proto.SharienceH\x00\x42\n\n\x08\x64ocumentb\x06proto3'
   ,
-  dependencies=[nexus_dot_models_dot_proto_dot_scimag__pb2.DESCRIPTOR,nexus_dot_models_dot_proto_dot_scitech__pb2.DESCRIPTOR,])
+  dependencies=[nexus_dot_models_dot_proto_dot_scimag__pb2.DESCRIPTOR,nexus_dot_models_dot_proto_dot_scitech__pb2.DESCRIPTOR,nexus_dot_models_dot_proto_dot_sharience__pb2.DESCRIPTOR,])
 
 
 
@@ -52,6 +54,13 @@ _TYPEDDOCUMENT = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='sharience', full_name='nexus.models.proto.TypedDocument.sharience', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -69,18 +78,22 @@ _TYPEDDOCUMENT = _descriptor.Descriptor(
       create_key=_descriptor._internal_create_key,
     fields=[]),
   ],
-  serialized_start=130,
-  serialized_end=251,
+  serialized_start=167,
+  serialized_end=340,
 )
 
 _TYPEDDOCUMENT.fields_by_name['scimag'].message_type = nexus_dot_models_dot_proto_dot_scimag__pb2._SCIMAG
 _TYPEDDOCUMENT.fields_by_name['scitech'].message_type = nexus_dot_models_dot_proto_dot_scitech__pb2._SCITECH
+_TYPEDDOCUMENT.fields_by_name['sharience'].message_type = nexus_dot_models_dot_proto_dot_sharience__pb2._SHARIENCE
 _TYPEDDOCUMENT.oneofs_by_name['document'].fields.append(
   _TYPEDDOCUMENT.fields_by_name['scimag'])
 _TYPEDDOCUMENT.fields_by_name['scimag'].containing_oneof = _TYPEDDOCUMENT.oneofs_by_name['document']
 _TYPEDDOCUMENT.oneofs_by_name['document'].fields.append(
   _TYPEDDOCUMENT.fields_by_name['scitech'])
 _TYPEDDOCUMENT.fields_by_name['scitech'].containing_oneof = _TYPEDDOCUMENT.oneofs_by_name['document']
+_TYPEDDOCUMENT.oneofs_by_name['document'].fields.append(
+  _TYPEDDOCUMENT.fields_by_name['sharience'])
+_TYPEDDOCUMENT.fields_by_name['sharience'].containing_oneof = _TYPEDDOCUMENT.oneofs_by_name['document']
 DESCRIPTOR.message_types_by_name['TypedDocument'] = _TYPEDDOCUMENT
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 

nexus/pipe/BUILD.bazel

@@ -15,6 +15,7 @@ py3_image(
         "configs/base.yaml",
         "configs/logging.yaml",
     ],
+    legacy_create_init = False,
     main = "main.py",
     srcs_version = "PY3ONLY",
     visibility = ["//visibility:public"],

nexus/pipe/filters/__init__.py

@@ -4,7 +4,7 @@ from nexus.models.proto.operation_pb2 import \
 
 
 class DocumentOperationFilter(AioThing):
-    def __init__(self, operation, document):
+    def __init__(self, operation, document=None):
         super().__init__()
         self.operation = operation
         self.document = document
@@ -12,5 +12,7 @@ class DocumentOperationFilter(AioThing):
     def filter(self, document_operation_pb: DocumentOperationPb) -> bool:
         if document_operation_pb.WhichOneof('operation') != self.operation:
             return False
-        operation = getattr(document_operation_pb, document_operation_pb.WhichOneof('operation'))
-        return operation.typed_document.HasField(self.document)
+        if self.document is not None:
+            operation = getattr(document_operation_pb, document_operation_pb.WhichOneof('operation'))
+            return operation.typed_document.HasField(self.document)
+        return True

nexus/pipe/processors/telegram_messages_processor.py

@@ -1,108 +0,0 @@
-import time
-from typing import Iterable
-
-from library.nlptools.language_detect import detect_language
-from library.nlptools.utils import (
-    clean_text,
-    replace_telegram_link,
-)
-from pypika import (
-    PostgreSQLQuery,
-    Table,
-)
-
-from .base import BaseDatabaseProcessor
-
-
-class TelegramMessagesProcessor(BaseDatabaseProcessor):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.telegram_messages_table = Table('telegram_messages')
-
-    def generate_delete_sql(self, telegram_message_pb):
-        return (
-            PostgreSQLQuery
-            .from_('telegram_messages')
-            .where(self.telegram_messages_table.id == telegram_message_pb.id)
-            .delete()
-            .get_sql()
-        )
-
-    def generate_insert_sql(self, telegram_message_pb):
-        return (
-            PostgreSQLQuery
-            .into('telegram_messages')
-            .columns(
-                'channel_id',
-                'message_id',
-                'is_deleted',
-                'text',
-                'clean_text',
-                'lang',
-                'message_type',
-                'unixtime',
-                'views'
-            )
-            .insert(
-                telegram_message_pb.channel_id,
-                telegram_message_pb.message_id,
-                telegram_message_pb.is_deleted,
-                telegram_message_pb.text,
-                telegram_message_pb.clean_text,
-                telegram_message_pb.language,
-                telegram_message_pb.message_type,
-                telegram_message_pb.unixtime,
-                telegram_message_pb.views,
-            )
-            .on_conflict(self.telegram_messages_table.channel_id, self.telegram_messages_table.message_id)
-            .do_update(self.telegram_messages_table.is_deleted, telegram_message_pb.is_deleted)
-            .do_update(self.telegram_messages_table.text, telegram_message_pb.text)
-            .do_update(self.telegram_messages_table.clean_text, telegram_message_pb.clean_text)
-            .do_update(self.telegram_messages_table.lang, telegram_message_pb.language)
-            .do_update(self.telegram_messages_table.message_type, telegram_message_pb.message_type)
-            .do_update(self.telegram_messages_table.views, telegram_message_pb.views)
-            .returning(self.telegram_messages_table.id)
-            .get_sql()
-        )
-
-    async def process_bulk(self, messages: Iterable):
-        return
-        insert_sqls = []
-        delete_sqls = []
-        documents_container_pb = DocumentsContainerPb()
-        update_document_operations_pb = []
-
-        current_time = int(time.time())
-
-        for document_operation_pb in messages:
-            if document_operation_pb.WhichOneof('operation') == 'update_document':
-                document_pb = document_operation_pb.update_document.document
-                telegram_message_pb = document_pb.telegram_message
-
-                if telegram_message_pb.is_deleted and telegram_message_pb.id:
-                    delete_sqls.append(self.generate_delete_sql(telegram_message_pb))
-                    documents_container_pb.documents.append(document_pb)
-                else:
-                    telegram_message_pb.clean_text = clean_text(replace_telegram_link(telegram_message_pb.text))
-                    telegram_message_pb.language = detect_language(telegram_message_pb.clean_text) or ''
-
-                    insert_sqls.append(self.generate_insert_sql(telegram_message_pb))
-                    update_document_operations_pb.append(document_operation_pb)
-
-        if insert_sqls:
-            for insert_sql, update_document_operation_pb in zip(insert_sqls, update_document_operations_pb):
-                result = await self.pool_holder.execute(insert_sql, fetch=True)
-
-                document_pb = update_document_operation_pb.update_document.document
-                telegram_message_pb = document_pb.telegram_message
-
-                telegram_message_pb.id = result[0][0]
-                if update_document_operation_pb.update_document.reindex:
-                    if (
-                        len(telegram_message_pb.clean_text) > 120
-                        and telegram_message_pb.unixtime > current_time - 60 * 24 * 60 * 60
-                        and telegram_message_pb.message_type == TelegramMessagePb.Type.TEXTUAL
-                    ):
-                        documents_container_pb.documents.append(document_pb)
-        if delete_sqls:
-            await self.pool_holder.execute(';'.join(delete_sqls))

nexus/pylon/sources/specific/nejm.py

@@ -16,6 +16,6 @@ class NejmSource(DoiSource):
             url = f'{self.base_url}/doi/pdf/{self.doi}'
             async with session.get(
                 url,
-                timeout=self.resolve_timeout
+                timeout=self.resolve_timeout,
             ) as resp:
                 yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))

nexus/translations/translations.yaml

@@ -3,6 +3,7 @@
 
 en:
   ABSTRACT: Abstract
+  ACCEPTED: Accepted!
   ALREADY_DOWNLOADING: |
     I have found this book in the Internet.
     Please, be patient, I will send as it will be ready.
@@ -67,10 +68,10 @@ en:
     Please, try to download `{document}` in a minute.
   EDITION: Edition
   FILE: File
+  FILE_TOO_BIG_ERROR: File is too big, contact @nexus_search
   HELP: |
     Hi! I'm **Nexus** bot!
-    I can search books or science articles by titles, authors or DOIs.
-    Also, I can send recommendations based on what you were looking for before.
+    I can search books or scientific articles by titles, authors or DOIs.
 
     **Just type your request in plain words.**
 
@@ -81,6 +82,8 @@ en:
     Search by author: `authors:Jack authors:London`
     Exact match: `"Fetal Hemoglobin"`
 
+    Also, I can accept new scientific articles from you. Send me PDF and I will process it.
+
     /copyright - make a copyright infringement claim
     /donate - tells how to support us
     /help - shows this help
@@ -123,13 +126,16 @@ en:
   NAMELESS: Without name
   NEXUS_DISCOVERY_DESCRIPTION: |
     **Nexus Discovery** is a recommendation service. It also notifies you about newly arrived publications you were looking for before. Your personal digest will be sent at every Sunday.
+  PROCESSING_PAPER: '`File {filename} is processed...`'
   PROMO: |
     **Nexus Search - Update and News**
     {related_channel}
   READ_ONLY_MODE: ⚠️ Reduced service until midnight (UTC). /settings is not working, search in slow mode.
   REFERENCED_BY: Referenced by
   REPLY_MESSAGE_HAS_BEEN_DELETED: Search message has been (re-)moved. Do search again.
-  SEARCHING: searching...
+  REPORT_BROKEN_FILE: ❌ Broken File
+  REPORT_OK_FILE: ✅ What Is Needed
+  SEARCHING: '`searching...`'
   SEND_YOUR_LOCATION: Send your location (through the left attach button)
   SETTINGS_ROUTER_HELP: >
     Settings could be set up automatically or manually.
@@ -154,6 +160,10 @@ en:
     then create a shortlink for query `NID: <NID>`.
   TOO_MANY_DOWNLOADS: Too many active downloads. Please, wait them to complete.
   TRANSMITTED_FROM: 'transmitted from {source}'
+  UNAVAILABLE_METADATA_ERROR: 'Cannot acquire metadata for {doi} from CrossRef API. Ensure it is resolvable before uploading.'
+  UNKNOWN_FILE_FORMAT_ERROR: Only PDF files are supported
+  UNPARSABLE_DOCUMENT_ERROR: Cannot parse the document
+  UNPARSABLE_DOI_ERROR: DOI cannot be found in the document
   UPGRADE_MAINTENANCE: |
     ```Major update is going on, come back to us tomorrow```[‌]({upgrade_maintenance_picture_url})
   UPLOADED_TO_TELEGRAM: uploaded to Telegram...
@@ -228,7 +238,6 @@ es:
   HELP: |
     ¡Hola! ¡Soy el bot **Nexus**!
     Puedo buscar libros o artículos científicos por títulos, autores o DOI.
-    Además, puedo enviar recomendaciones basadas en lo que estabas buscando antes.
 
     **Simplemente escriba su solicitud en palabras sencillas.**
 
@@ -239,6 +248,8 @@ es:
     Búsqueda por autor: `authors:Jack authors:London`
     Coincidencia exacta: `"Hemoglobina Fetal"`
 
+    Además, puedo aceptar nuevos artículos científicos suyos. Envíame PDF y lo procesaré.
+
     /copyright - hacer un reclamo por infracción de derechos de autor
     /donate - explica cómo apoyarnos
     /help - muestra esta ayuda
@@ -287,7 +298,7 @@ es:
   READ_ONLY_MODE: ⚠️ Servicio reducido hasta la medianoche (UTC). /settings no funciona, busque en modo lento.
   REFERENCED_BY: Referenciado por
   REPLY_MESSAGE_HAS_BEEN_DELETED: El mensaje de búsqueda ha sido (re)movido. Vuelve a buscar.
-  SEARCHING: buscando...
+  SEARCHING: '`buscando...`'
   SEND_YOUR_LOCATION: Envía tu ubicación (a través del botón de adjuntar de la izquierda)
   SETTINGS_ROUTER_HELP: >
     Los ajustes se pueden configurar de forma automática o manual.
@@ -386,7 +397,6 @@ it:
   HELP: |
     Ciao! Sono il bot **Nexus**!
     Posso cercare libri o articoli scientifici per titolo, autore o DOI.
-    Inoltre, posso fornire suggerimenti in base a quello che hai cercato precedentemente.
 
     **Scrivi qui la tua richiesta a parole.**
 
@@ -397,6 +407,8 @@ it:
     Ricerca per autore: `authors:Jack authors:London`
     Risultato esatto: `"Emoglobina fetale"`
 
+    Inoltre, posso accettare nuovi articoli scientifici da te. Inviami PDF e lo elaborerò.
+
     /copyright - segnala una violazione di copyright
     /donate - info su come sostenerci
     /help - mostra questo aiuto
@@ -426,7 +438,7 @@ it:
   READ_ONLY_MODE: ⚠️ Il servizio è limitato fino a mezzanotte (UTC). /settings non è disponibile, la ricerca è più lenta.
   REFERENCED_BY: Riferito da
   REPLY_MESSAGE_HAS_BEEN_DELETED: Il messaggio di ricerca è stato (ri-)mosso. Ripeti la ricerca.
-  SEARCHING: ricerca in corso...
+  SEARCHING: '`ricerca in corso...`'
   SEND_YOUR_LOCATION: Invia la tua posizione (tramite il pulsante allegato a sinistra)
   SETTINGS_ROUTER_HELP: >
     Le impostazioni possono essere configurate automaticamente o manualmente.
@@ -525,7 +537,6 @@ pb:
   HELP: |
     Olá! Eu sou o bot **Nexus**!
     Eu posso procurar livros ou artigos científicos por títulos, autores ou DOIs.
-    Além disso, eu posso enviar recomendações baseadas no que você esteve procurando.
 
     **Basta digitar seu pedido em palavras simples.**
 
@@ -536,6 +547,8 @@ pb:
     Buscar pelo autor: `authors:Jack authors:London`
     Combinação exata: `"Hemoglobina fetal"`
 
+    Além disso, posso aceitar novos artigos científicos seus. Envie-me o PDF e irei processá-lo.
+
     /copyright - fazer uma reivindicação de violação de direitos autorais
     /donate - diz como nos apoiar
     /help - mostra esta ajuda
@@ -584,7 +597,7 @@ pb:
   READ_ONLY_MODE: ⚠️ Serviço reduzido até meia-noite (UTC). /settings não está funcionando, pesquisa está no modo lento.
   REFERENCED_BY: Referenciado por
   REPLY_MESSAGE_HAS_BEEN_DELETED: A mensagem de pesquisa foi (re)movida. Pesquise novamente.
-  SEARCHING: procurando...
+  SEARCHING: '`procurando...`'
   SEND_YOUR_LOCATION: Envie sua localização (através do botão esquerdo anexar)
   SETTINGS_ROUTER_HELP: >
     As configurações podem ser configuradas automática ou manualmente.
@@ -670,7 +683,6 @@ ru:
   HELP: |
     Привет! Я бот **Nexus**!
     Я могу искать книги и научные статьи по их названиям, авторам или DOI.
-    Ещё я могу присылать вам рекомендации на основе вашей поисковой истории.
 
     **Просто наберите запрос как в поисковике.**
 
@@ -681,6 +693,8 @@ ru:
     Поиск по автору: `authors:Jack authors:London`
     Точное совпадение: `"Fetal Hemoglobin"`
 
+    Также я могу принимать от вас новые научные статьи. Отправьте мне PDF-файл, и я его обработаю.
+
     /copyright - пожаловаться на нарушение авторских прав
     /donate - страница поддержки
     /help - помочь разобраться с ботом
@@ -728,7 +742,7 @@ ru:
     {related_channel}
   REFERENCED_BY: Ссылки
   REPLY_MESSAGE_HAS_BEEN_DELETED: Сообщение с запросом было удалено/перемещено, выполните поиск заново.
-  SEARCHING: ищем...
+  SEARCHING: '`ищем...`'
   SEND_YOUR_LOCATION: Отправьте вашу геопозицию (кнопка прикрепления файла слева)
   SETTINGS_ROUTER_HELP: >
     Настройки можно установить автоматически или в ручном режиме.

nexus/views/telegram/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("@pip_modules//:requirements.bzl", "requirement")
+
+py_library(
+    name = "telegram",
+    srcs = glob(
+        ["**/*.py"],
+        exclude = ["tests/**"],
+    ),
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("telethon"),
+        "//nexus/models/proto:models_proto_py",
+        "//nexus/nlptools",
+        "//nexus/translations",
+        requirement("izihawa_utils"),
+    ],
+)

nexus/views/telegram/__init__.py

@@ -0,0 +1,3 @@
+from .registry import parse_typed_document_to_view
+
+__all__ = ['parse_typed_document_to_view']

nexus/views/telegram/base.py

@@ -0,0 +1,133 @@
+import datetime
+from typing import Optional
+from urllib.parse import quote
+
+import numpy as np
+from nexus.nlptools.utils import escape_format
+
+from .common import encode_query_to_deep_link
+
+
+class CustomDatetime:
+    def __init__(self, dt: np.datetime64):
+        self.dt = dt
+
+    @property
+    def year(self) -> int:
+        return self.dt.astype('datetime64[Y]').astype(int) + 1970
+
+    @property
+    def month(self) -> int:
+        return self.dt.astype('datetime64[M]').astype(int) % 12 + 1
+
+    @property
+    def day(self) -> int:
+        return self.dt - self.dt.astype('datetime64[M]') + 1
+
+
+class DoiMixin:
+    def get_doi_link(self) -> str:
+        if not self.doi:
+            return ''
+        escaped_doi = escape_format(self.doi)
+        return f'[{escaped_doi}](https://doi.org/{quote(escaped_doi)})'
+
+
+class AuthorMixin:
+    def get_first_authors(self, et_al=True, first_n_authors=1) -> str:
+        et_al_suffix = (' et al' if et_al else '')
+        if self.authors:
+            if len(self.authors) > first_n_authors:
+                return '; '.join(self.authors[:first_n_authors]) + et_al_suffix
+            elif len(self.authors) == 1:
+                if self.authors[0].count(';') >= 1:
+                    comma_authors = list(map(str.strip, self.authors[0].split(';')))
+                    if len(comma_authors) > first_n_authors:
+                        return '; '.join(comma_authors[:first_n_authors]) + et_al_suffix
+                    else:
+                        return '; '.join(comma_authors)
+                return self.authors[0]
+            else:
+                return '; '.join(self.authors)
+        return ''
+
+
+class IssuedAtMixin:
+    def get_issued_datetime(self) -> Optional[CustomDatetime]:
+        if not self.has_field('issued_at'):
+            return None
+        return CustomDatetime(dt=np.datetime64(self.issued_at, 's'))
+
+    def get_formatted_datetime(self) -> str:
+        if self.has_field('issued_at'):
+            dt = self.get_issued_datetime()
+            try:
+                ct = datetime.date(dt.year, dt.month, 1)
+                if datetime.date.today() - datetime.timedelta(days=365) < ct:
+                    return f'{dt.year}.{dt.month:02d}'
+            except ValueError:
+                pass
+            return str(dt.year)
+
+
+class FileMixin:
+    def get_extension(self) -> str:
+        return 'pdf'
+
+    def get_formatted_filedata(self, show_format=True, show_language=True, show_filesize=False) -> str:
+        parts = []
+        if self.language and show_language:
+            parts.append(self.language.upper())
+        if show_format:
+            parts.append(self.get_extension().upper())
+        if self.filesize and show_filesize:
+            parts.append(self.get_formatted_filesize())
+        return ' | '.join(parts)
+
+    def get_formatted_filesize(self) -> str:
+        if self.filesize:
+            filesize = max(1024, self.filesize)
+            return '{:.1f}Mb'.format(float(filesize) / (1024 * 1024))
+        else:
+            return ''
+
+
+class BaseView:
+    schema = None
+    multihash_ix = 0
+
+    def __getattr__(self, name):
+        return getattr(self.document_pb, name)
+
+    def get_ipfs_gateway_link(self):
+        ipfs_link = (
+            f'https://ipfs.io/ipfs/{self.ipfs_multihashes[self.multihash_ix]}?'
+            f'filename={quote(self.get_filename())}'
+        )
+        return f'[IPFS.io]({ipfs_link})'
+
+    def get_ipfs_link(self):
+        ipfs_link = (
+            f'ipfs://{self.ipfs_multihashes[self.multihash_ix]}?'
+            f'filename={quote(self.get_filename())}'
+        )
+        return f'[IPFS]({ipfs_link})'
+
+    def get_deep_link(self, bot_external_name, text=None):
+        if not text:
+            text = str(self.id)
+        encoded_query = encode_query_to_deep_link(f'NID: {self.id}', bot_external_name)
+        return f'[{text}]({encoded_query})'
+
+    def generate_links(self, bot_external_name):
+        links = [
+            self.get_deep_link(bot_external_name=bot_external_name, text='Nexus Bot')
+        ]
+        if self.ipfs_multihashes:
+            links.append(self.get_ipfs_gateway_link())
+        if self.doi:
+            links.append(self.get_doi_link())
+        return links
+
+    def has_field(self, name):
+        return self.document_pb.HasField(name)

nexus/views/telegram/common.py

@@ -0,0 +1,74 @@
+import base64
+import binascii
+import logging
+
+from izihawa_utils.exceptions import BaseError
+from nexus.translations import t
+from telethon import Button
+
+
+class TooLongQueryError(BaseError):
+    code = 'too_long_query_error'
+    level = logging.WARNING
+
+
+class DecodeDeepQueryError(BaseError):
+    code = 'decode_deep_query_error'
+    level = logging.WARNING
+
+
+def close_button(session_id: str = None):
+    if session_id:
+        return Button.inline(
+            text='✖️',
+            data=f'/close_{session_id}',
+        )
+    else:
+        return Button.inline(
+            text='✖️',
+            data='/close',
+        )
+
+
+def vote_button(language: str, session_id: str, document_id: int, case: str):
+    label = f"REPORT_{case.upper()}_FILE"
+    case = {'broken': 'b', 'ok': 'o'}[case]
+    return Button.inline(
+        text=t(label, language=language),
+        data=f'/vote_{session_id}_{document_id}_{case}',
+    )
+
+
+def encode_query_to_deep_link(query, bot_name):
+    encoded_query = encode_deep_query(query)
+    if len(encoded_query) <= 64:
+        return f'https://t.me/{bot_name}?start={encoded_query}'
+    raise TooLongQueryError()
+
+
+def encode_deep_query(query):
+    return base64.b64encode(query.encode(), altchars=b'-_').decode()
+
+
+def decode_deep_query(query):
+    try:
+        # Padding fix
+        return base64.b64decode(query + "=" * ((4 - len(query) % 4) % 4), altchars=b'-_').decode()
+    except (binascii.Error, ValueError, UnicodeDecodeError) as e:
+        raise DecodeDeepQueryError(nested_error=e)
+
+
+async def remove_button(event, mark, and_empty_too=False):
+    original_message = await event.get_message()
+    if original_message:
+        original_buttons = original_message.buttons
+        buttons = []
+        for original_line in original_buttons:
+            line = []
+            for original_button in original_line:
+                if mark in original_button.text or (and_empty_too and not original_button.text.strip()):
+                    continue
+                line.append(original_button)
+            if line:
+                buttons.append(line)
+        await event.edit(original_message.text, buttons=buttons)

nexus/views/telegram/progress_bar.py

@@ -0,0 +1,120 @@
+import time
+from typing import (
+    Callable,
+    Union,
+)
+
+from izihawa_utils.exceptions import BaseError
+from telethon.errors import MessageIdInvalidError
+
+from .common import close_button
+
+
+class ProgressBarLostMessageError(BaseError):
+    pass
+
+
+bars = {
+    'filled': '█',
+    'empty': ' ',
+}
+
+
+def percent(done, total):
+    return min(float(done) / total, 1.0)
+
+
+class ProgressBar:
+    def __init__(
+        self,
+        telegram_client,
+        request_context,
+        banner,
+        header,
+        tail_text,
+        message=None,
+        source=None,
+        throttle_secs: float = 0,
+    ):
+        self.telegram_client = telegram_client
+        self.request_context = request_context
+        self.banner = banner
+        self.header = header
+        self.tail_text = tail_text
+        self.message = message
+        self.source = source
+        self.done = 0
+        self.total = 1
+        self.throttle_secs = throttle_secs
+
+        self.last_text = None
+        self.last_call = 0
+
+    def share(self):
+        if self.total > 0:
+            return f'{float(percent(self.done, self.total) * 100):.1f}%'
+        else:
+            return f'{float(self.done / (1024 * 1024)):.1f}Mb'
+
+    def _set_progress(self, done, total):
+        self.done = done
+        self.total = total
+
+    def set_source(self, source):
+        self.source = source
+
+    async def render_banner(self):
+        banner = self.banner.format(source=self.source)
+        return f'`{self.header}\n{banner}`'
+
+    async def render_progress(self):
+        total_bars = 20
+        progress_bar = ''
+        if self.total > 0:
+            filled = int(total_bars * percent(self.done, self.total))
+            progress_bar = '|' + filled * bars['filled'] + (total_bars - filled) * bars['empty'] + '| '
+
+        tail_text = self.tail_text.format(source=self.source)
+        return f'`{self.header}\n{progress_bar}{self.share()} {tail_text}`'
+
+    async def send_message(self, text, ignore_last_call=False):
+        now = time.time()
+        if not ignore_last_call and abs(now - self.last_call) < self.throttle_secs:
+            return
+        try:
+            if not self.message:
+                self.message = await self.telegram_client.send_message(
+                    self.request_context.chat.id,
+                    text,
+                    buttons=[close_button()],
+                )
+            elif text != self.last_text:
+                r = await self.message.edit(text, buttons=[close_button()])
+                if not r:
+                    raise ProgressBarLostMessageError()
+        except MessageIdInvalidError:
+            raise ProgressBarLostMessageError()
+        self.last_text = text
+        self.last_call = now
+        return self.message
+
+    async def show_banner(self):
+        return await self.send_message(await self.render_banner(), ignore_last_call=True)
+
+    async def callback(self, done, total):
+        self._set_progress(done, total)
+        return await self.send_message(await self.render_progress())
+
+
+class ThrottlerWrapper:
+    def __init__(self, callback: Callable, throttle_secs: Union[int, float]):
+        self.callback = callback
+        self.last_call = 0
+        self.throttle_secs = throttle_secs
+
+    async def __call__(self, *args, **kwargs):
+        now = time.time()
+        if abs(now - self.last_call) < self.throttle_secs:
+            return
+        self.last_call = now
+        return await self.callback(*args, **kwargs)

nexus/views/telegram/registry.py

@@ -0,0 +1,22 @@
+from typing import Union
+
+from nexus.models.proto.scimag_pb2 import Scimag as ScimagPb
+from nexus.models.proto.scitech_pb2 import Scitech as ScitechPb
+
+from .scimag import ScimagView
+from .scitech import ScitechView
+
+pb_registry = {
+    'scimag': ScimagPb,
+    'scitech': ScitechPb,
+}
+
+views_registry = {
+    'scimag': ScimagView,
+    'scitech': ScitechView,
+}
+
+
+def parse_typed_document_to_view(typed_document_pb) -> Union[ScimagView, ScitechView]:
+    document_pb = getattr(typed_document_pb, typed_document_pb.WhichOneof('document'))
+    return views_registry[typed_document_pb.WhichOneof('document')](document_pb)

nexus/views/telegram/scimag.py

@@ -0,0 +1,235 @@
+import math
+from typing import (
+    List,
+    Optional,
+    Tuple,
+)
+from urllib.parse import quote
+
+from izihawa_utils.common import filter_none
+from nexus.models.proto.scimag_pb2 import Scimag as ScimagPb
+from nexus.nlptools.utils import (
+    cast_string_to_single_string,
+    escape_format,
+)
+from nexus.translations import t
+from nexus.views.telegram.common import close_button
+from telethon import Button
+
+from .base import (
+    AuthorMixin,
+    BaseView,
+    DoiMixin,
+    FileMixin,
+    IssuedAtMixin,
+)
+
+preprints = {'10.1101', '10.21203'}
+
+
+class ScimagView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin):
+    icon = '🔬'
+    schema = 'scimag'
+    multihash_ix = 1
+
+    def __init__(self, document_pb: ScimagPb):
+        self.document_pb = document_pb
+
+    def generate_body(self, language: str, with_referenced_by: bool = False, limit: Optional[int] = None) -> str:
+        head = self.icon
+        title = self.get_robust_title()
+        if title:
+            head += f' **{title}**'
+
+        doi = f'**DOI:** {self.get_doi_link()}'
+
+        referenced_by = None
+        if with_referenced_by:
+            referenced_by = self.get_referenced_by(language=language)
+
+        caption = '\n'.join(filter_none([head, doi, referenced_by, self.get_formatted_locator()]))
+
+        if limit and len(caption) > limit:
+            caption = '\n'.join(filter_none([head, doi, referenced_by]))
+            if len(caption) > limit:
+                subtract = len(self.icon + doi) + 1
+                shorten_title = title[:(limit - subtract)]
+                shorten_head = f'{self.icon} **{shorten_title}**'
+                caption = '\n'.join(filter_none([shorten_head, doi, referenced_by]))
+        return caption
+
+    def get_download_command(self, session_id: str, position: int = 0) -> str:
+        return f'/dla_{session_id}_{self.id}_{position}'
+
+    def get_filename(self) -> str:
+        limit = 55
+
+        processed_author = ''
+        if self.authors:
+            processed_author = self.authors[0]
+
+        processed_author = cast_string_to_single_string(processed_author.lower())
+        processed_title = cast_string_to_single_string((self.title or '').lower())
+
+        parts = []
+        if processed_author:
+            parts.append(processed_author)
+        if processed_title:
+            parts.append(processed_title)
+        filename = '-'.join(parts)
+
+        chars = []
+        size = 0
+        hit_limit = False
+
+        for c in filename:
+            current_size = size + len(c.encode())
+            if current_size > limit:
+                hit_limit = True
+                break
+            chars.append(c)
+            size = current_size
+
+        filename = ''.join(chars)
+        if hit_limit:
+            glyph = filename.rfind('-')
+            if glyph != -1:
+                filename = filename[:glyph]
+
+        if not filename:
+            filename = quote(self.doi, safe='')[:limit]
+
+        dt = self.get_issued_datetime()
+        if dt:
+            filename = f'{filename}-{dt.year}'
+
+        return f'{filename}.pdf'
+
+    def get_formatted_locator(self) -> str:
+        parts = []
+        if self.authors:
+            parts.append(self.get_first_authors(first_n_authors=3))
+        journal = self.get_robust_journal()
+        if journal:
+            parts.extend(['in', journal])
+        dt = self.get_formatted_datetime()
+        if dt:
+            parts.append(f'({dt})')
+        if self.get_robust_volume():
+            parts.append(self.get_robust_volume())
+        if self.get_pages():
+            parts.append(self.get_pages())
+
+        return ' '.join(parts)
+
+    def get_pages(self) -> Optional[str]:
+        if self.first_page:
+            if self.last_page:
+                if self.first_page == self.last_page:
+                    return f'p. {self.first_page}'
+                else:
+                    return f'pp. {self.first_page}-{self.last_page}'
+            else:
+                return f'p. {self.first_page}'
+        elif self.last_page:
+            return f'p. {self.last_page}'
+
+    def get_referenced_by(self, language):
+        if self.ref_by_count:
+            return f'**{t("REFERENCED_BY", language=language)}:** {self.ref_by_count}'
+
+    def get_snippet(
+        self,
+        language: str,
+        view_command: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> str:
+        snippet = self.generate_body(language=language, with_referenced_by=True, limit=limit)
+        if view_command:
+            snippet += f'\n{view_command} | {self.get_formatted_filedata()}'
+        return snippet
+
+    def get_view(
+        self,
+        language: str,
+        session_id: str,
+        bot_external_name: str,
+        position: int = 0,
+        back_command: Optional[str] = None,
+        with_buttons: bool = True,
+    ) -> Tuple[str, Optional[List[List[Button]]]]:
+        parts = [f'**{self.get_robust_title()}**\n']
+
+        if self.authors:
+            parts.append(f'**{t("AUTHORS", language=language)}**: '
+                         f'{escape_format(self.get_first_authors(first_n_authors=3))}')
+        journal = self.get_robust_journal()
+        if journal:
+            parts.append(f'**{t("JOURNAL", language=language)}**: {journal}')
+
+        dt = self.get_formatted_datetime()
+        if dt:
+            parts.append(f'**{t("YEAR", language=language)}**: {dt}')
+
+        if self.downloads_count:
+            parts.append(f'**NRank**: {math.log1p(self.downloads_count):.1f}')
+
+        parts.append(f'**Links**: {" - ".join(self.generate_links(bot_external_name))}')
+
+        if self.abstract:
+            parts.append(
+                f'\n**{t("ABSTRACT", language=language)}**: {escape_format(self.abstract)}',
+            )
+
+        if self.tags:
+            parts.append(f'\n__{escape_format(", ".join(self.tags))}__')
+
+        buttons = None
+        if with_buttons:
+            buttons = [[]]
+            if back_command:
+                buttons[-1].append(
+                    Button.inline(
+                        text='⬅️',
+                        data=back_command
+                    )
+                )
+
+            # ⬇️ is a mark, Find+F over sources before replacing
+            buttons[-1].append(
+                Button.inline(
+                    text=f'⬇️ {self.get_formatted_filedata()}',
+                    data=self.get_download_command(session_id=session_id, position=position),
+                )
+            )
+            if self.ref_by_count:
+                buttons[-1].append(
+                    Button.inline(
+                        text=f'🔗 {self.ref_by_count or ""}',
+                        data=f'/r_{session_id}_{self.id}',
+                    )
+                )
+            buttons[-1].append(close_button(session_id))
+        return '\n'.join(parts).strip()[:4096], buttons
+
+    def get_view_command(self, session_id: str, message_id: int, parent_view_type: str = 's', position: int = 0) -> str:
+        return f'/va{parent_view_type}_{session_id}_{message_id}_{self.id}_{position}'
+
+    def get_robust_journal(self):
+        if self.type != 'chapter' and self.type != 'book-chapter':
+            return escape_format(self.container_title)
+
+    def get_robust_title(self):
+        title = escape_format(self.title or self.doi)
+        if self.doi.split('/')[0] in preprints:
+            title = '`[P]` ' + title
+        if self.type == 'chapter' or self.type == 'book-chapter':
+            title += f' in {self.container_title}'
+        return title
+
+    def get_robust_volume(self) -> str:
+        if self.volume:
+            if self.issue:
+                return f'vol. {self.volume}({self.issue})'
+            else:
+                return self.volume

nexus/views/telegram/scitech.py

@@ -0,0 +1,238 @@
+from typing import (
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+)
+from urllib.parse import quote
+
+from izihawa_utils.common import filter_none
+from nexus.models.proto.scitech_pb2 import Scitech as ScitechPb
+from nexus.nlptools.utils import (
+    cast_string_to_single_string,
+    despace,
+    escape_format,
+)
+from nexus.translations import t
+from nexus.views.telegram.common import close_button
+from telethon import Button
+
+from .base import (
+    AuthorMixin,
+    BaseView,
+    DoiMixin,
+    FileMixin,
+    IssuedAtMixin,
+)
+
+
+class ScitechView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin):
+    icon = '📚'
+    schema = 'scitech'
+
+    def __init__(self, document_pb: ScitechPb, duplicates: Optional[Iterable[ScitechPb]] = tuple()):
+        self.document_pb = document_pb
+        self.duplicates = [ScitechView(
+            document_pb=document_pb,
+        ) for document_pb in duplicates]
+
+    def generate_body(self, language: str, limit: Optional[int] = None) -> str:
+        head = self.icon
+        title = self.get_robust_title()
+        if title:
+            head += f' **{title}**'
+
+        doi = None
+        if self.doi:
+            doi = f'**DOI:** {self.get_doi_link()}'
+
+        locator = self.get_formatted_locator()
+
+        caption = '\n'.join(filter_none([head, doi, locator]))
+        if limit and len(caption) > limit:
+            shorten_title = title[:limit]
+            shorten_title = shorten_title[:max(32, shorten_title.rfind(' '))]
+            caption = f'{self.icon} **{shorten_title}**'
+
+        return caption
+
+    def get_cover_url(self):
+        if self.cu:
+            return self.cu
+        if self.libgen_id or self.fiction_id:
+            if self.libgen_id:
+                bulk_id = (self.libgen_id - (self.libgen_id % 1000))
+                r = f'covers/{bulk_id}/{self.md5}'
+            elif self.fiction_id:
+                bulk_id = (self.fiction_id - (self.fiction_id % 1000))
+                r = f'fictioncovers/{bulk_id}/{self.md5}'
+            else:
+                return None
+            if self.cu_suf:
+                r += f'-{self.cu_suf}'
+            return f'http://gen.lib.rus.ec/{r}.jpg'
+
+    def get_download_command(self, session_id: str, position: int = 0) -> str:
+        return f'/dlb_{session_id}_{self.id}_{position}'
+
+    def get_extension(self):
+        return self.extension
+
+    def get_filename(self) -> str:
+        limit = 55
+
+        processed_author = cast_string_to_single_string(self.get_first_authors(et_al=False).lower())
+        processed_title = cast_string_to_single_string(self.get_robust_title().lower())
+
+        parts = []
+        if processed_author:
+            parts.append(processed_author)
+        if processed_title:
+            parts.append(processed_title)
+        filename = '-'.join(parts)
+
+        chars = []
+        size = 0
+        hit_limit = False
+
+        for c in filename:
+            current_size = size + len(c.encode())
+            if current_size > limit:
+                hit_limit = True
+                break
+            chars.append(c)
+            size = current_size
+
+        filename = ''.join(chars)
+        if hit_limit:
+            glyph = filename.rfind('-')
+            if glyph != -1:
+                filename = filename[:glyph]
+
+        if not filename:
+            if self.doi:
+                filename = quote(self.doi, safe='')
+            else:
+                filename = self.md5
+
+        dt = self.get_issued_datetime()
+        if dt:
+            filename = f'{filename}-{dt.year}'
+
+        return f'{filename}.{self.get_extension()}'
+
+    def get_formatted_locator(self) -> str:
+        parts = []
+        if self.authors:
+            parts.append(self.get_first_authors(first_n_authors=3))
+        dt = self.get_issued_datetime()
+        if dt:
+            parts.append(f'({dt.year})')
+        if self.pages:
+            parts.append(f'pp. {self.pages}')
+
+        return escape_format(' '.join(parts)) or ''
+
+    def get_robust_title(self):
+        title = self.title or ''
+        if self.volume:
+            if title:
+                title += f' ({self.volume})'
+            else:
+                title += self.volume
+        return escape_format(title)
+
+    def get_snippet(
+        self,
+        language: str,
+        view_command: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> str:
+        snippet = self.generate_body(language=language, limit=limit)
+        if view_command:
+            snippet += f'\n{view_command} | {self.get_formatted_filedata(show_format=not self.has_duplicates)}'
+        return snippet
+
+    def get_view(
+        self,
+        language: str,
+        session_id: str,
+        bot_external_name: str,
+        position: int = 0,
+        back_command: Optional[str] = None,
+        with_buttons: bool = True,
+    ) -> Tuple[str, Optional[List[List[Button]]]]:
+        parts = [f'**{self.get_robust_title()}**\n']
+        cover_url = self.get_cover_url()
+        if cover_url:
+            # There is an invisible character inside []!
+            parts[-1] = f'[​]({cover_url})' + parts[-1]
+
+        if self.authors:
+            parts.append(f'**{t("AUTHORS", language=language)}**: '
+                         f'{escape_format(self.get_first_authors(first_n_authors=3))}')
+        dt = self.get_issued_datetime()
+        if dt:
+            parts.append(f'**{t("YEAR", language=language)}**: {dt.year}')
+        if self.edition:
+            parts.append(f'**{t("EDITION", language=language)}**: '
+                         f'{escape_format(self.edition)}')
+
+        parts.append(f'**Links**: {" - ".join(self.generate_links(bot_external_name))}')
+
+        if self.description:
+            parts.append(
+                f'\n**{t("DESCRIPTION", language=language)}**:\n'
+                f'{escape_format(despace(self.description))}',
+            )
+
+        if self.tags:
+            parts.append(f'\n__{escape_format(", ".join(self.tags))}__')
+
+        buttons = None
+        if with_buttons:
+            buttons = [[]]
+            # Plain layout
+            if not self.duplicates:
+                if back_command:
+                    buttons[-1].append(Button.inline(
+                        text='⬅️',
+                        data=back_command
+                    ))
+                buttons[-1].extend([
+                    Button.inline(
+                        text=f'⬇️ {self.get_formatted_filedata(show_language=False)}',
+                        data=self.get_download_command(session_id=session_id, position=position),
+                    ),
+                    close_button(session_id),
+                ])
+            else:
+                buttons = [[]]
+                for view in [self] + self.duplicates:
+                    filedata = view.get_formatted_filedata(show_language=False, show_filesize=True)
+                    if len(buttons[-1]) >= 2:
+                        buttons.append([])
+                    # ⬇️ is a mark, Find+F over sources before replacing
+                    buttons[-1].append(
+                        Button.inline(
+                            text=f'⬇️ {filedata}',
+                            data=view.get_download_command(session_id=session_id, position=position),
+                        )
+                    )
+                if len(buttons[-1]) == 1:
+                    buttons[-1].append(Button.inline(
+                        text=' ',
+                        data='/noop',
+                    ))
+                buttons.append([])
+                if back_command:
+                    buttons[-1].append(Button.inline(
+                        text='⬅️',
+                        data=back_command,
+                    ))
+                buttons[-1].append(close_button(session_id))
+
+        return '\n'.join(parts).strip()[:4096], buttons
+
+    def get_view_command(self, session_id: str, message_id: int, position: int = 0) -> str:
+        return f'/vb_{session_id}_{message_id}_{self.id}_{position}'

rules/python/BUILD.bazel

@@ -2,36 +2,22 @@ load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
 load("@rules_python//python:defs.bzl", "py_runtime")
 
 py_runtime(
-    name = "python2.7",
-    files = [],
-    interpreter_path = select({
-        "//:osx": "/usr/local/bin/python2.7",
-        "//:linux": "/usr/bin/python2.7",
-    }),
-    python_version = "PY2",
-    visibility = ["//visibility:public"],
-)
-
-py_runtime(
-    name = "python3",
-    files = [],
-    interpreter_path = select({
-        "//:osx": "/usr/local/bin/python3.9",
-        "//:linux": "/usr/bin/python3.9",
-    }),
+    name = "python3_runtime",
+    files = ["@python_interpreter//:files"],
+    interpreter = "@python_interpreter//:python_bin",
     python_version = "PY3",
     visibility = ["//visibility:public"],
 )
 
 py_runtime_pair(
-    name = "hyperboria_py_runtime_pair",
-    py2_runtime = ":python2.7",
-    py3_runtime = ":python3",
+    name = "py_runtime_pair",
+    py2_runtime = None,
+    py3_runtime = ":python3_runtime",
 )
 
 toolchain(
-    name = "py_toolchain",
-    toolchain = ":hyperboria_py_runtime_pair",
+    name = "py_3_toolchain",
+    toolchain = ":py_runtime_pair",
     toolchain_type = "@bazel_tools//tools/python:toolchain_type",
 )
 

rules/python/requirements.txt

@@ -12,9 +12,9 @@ aiokit==0.2.1
 aiolibgen==0.2.0
 aiomysql==0.0.21
 aioredis==1.3.1
-aiopg==1.1.0
-aiosumma==0.1.1
-alembic==1.5.7
+aiopg==1.2.1
+aiosumma==0.1.2
+alembic==1.5.8
 aptly-api-client==0.2.3
 argon2-cffi==20.1.0
 attr==0.3.1
@@ -23,25 +23,25 @@ atomicwrites==1.4.0
 babel==2.9.0
 beautifulsoup4==4.9.3
 bencodepy==0.9.5
-bitarray==1.8.0
+bitarray==1.9.1
 bjoern==3.1.0
 blockchain==1.4.4
 brotli==1.0.9
 cachetools==4.2.1
 cchardet==2.1.7
 Celery==5.0.5
-certbot==1.13.0
+certbot==1.14.0
 certbuilder==0.14.2
 certifi==2020.12.5
 cffi==1.14.5
 chardet==4.0.0
 click==7.1.2
 clickhouse-driver==0.2.0
-cloudinary==1.24.0
+cloudinary==1.25.0
 coverage==5.5
-cryptography==3.4.6
+cryptography==3.4.7
 dataclasses-json==0.5.2
-docker==4.4.4
+docker==5.0.0
 ecdsa==0.16.1
 email-validator==1.1.2
 emoji==1.2.0
@@ -53,10 +53,10 @@ Flask-Cors==3.0.10
 Flask-SQLAlchemy==2.5.1
 Flask-WTF==0.14.3
 ftfy==5.9
-geoalchemy2==0.8.4
-grpcio==1.36.1
-grpcio-tools==1.36.1
-gunicorn==20.0.4
+geoalchemy2==0.8.5
+grpcio==1.37.0
+grpcio-tools==1.37.0
+gunicorn==20.1.0
 httpx[http2]==0.17.1
 idna==2.10
 isort==5.8.0
@@ -69,15 +69,15 @@ kazoo==2.8.0
 kubernetes==12.0.1
 langid==1.1.6
 lemminflect==0.2.2
-lightgbm==3.1.1
+lightgbm==3.2.0
 lru-dict==1.1.7
 lxml==4.6.3
 markdown-strings==3.3.0
 MarkupSafe==1.1.1
 mockredispy==2.9.3
 multidict==5.1.0
-networkx==2.5
-numpy==1.20.1
+networkx==2.5.1
+numpy==1.20.2
 orjson==3.5.1
 pandas==1.2.3
 paramiko==2.7.2
@@ -85,7 +85,7 @@ passlib==1.7.4
 pip==21.0.1
 pluggy==0.13.1
 ply==3.11
-protobuf==3.15.6
+protobuf==3.15.7
 psutil==5.8.0
 psycopg2-binary==2.8.6
 pycld3==0.22
@@ -93,13 +93,13 @@ pycountry==20.7.3
 pycparser==2.20
 pyjwt==2.0.1
 pymorphy2[fast]==0.9.1
-pypika==0.48.0
+pypika==0.48.1
 pysocks==1.7.1
-pytest==6.2.2
+pytest==6.2.3
 python-gflags==3.1.2
-python-pdfbox==0.1.8
+python-pdfbox==0.1.8.1
 python-qbittorrent==0.4.2
-python-socks==1.2.2
+python-socks==1.2.4
 pytimeparse==1.1.8
 pyyaml==5.4.1
 rarfile==4.0
@@ -109,8 +109,8 @@ retry==0.9.2
 requests==2.25.1
 ring==0.7.3
 https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web_sm
-SQLAlchemy==1.4.2
-scipy==1.6.1
+SQLAlchemy==1.4.6
+scipy==1.6.2
 sip==6.0.3
 six==1.15.0
 spacy==3.0.5
@@ -118,7 +118,7 @@ tantipy==0.14.0-dev0
 telethon==1.21.1
 tenacity==7.0.0
 timezonefinder==5.2.0
-tqdm==4.59.0
+tqdm==4.60.0
 unrar==0.4
 urllib3==1.26.4
 uvloop==0.15.2
@@ -130,5 +130,5 @@ WTForms==2.3.3
 WTForms-JSON==0.3.3
 www-authenticate==0.9.2
 xmltodict==0.12.0
-yamllint==1.26.0
+yamllint==1.26.1
 zxcvbn-python==4.4.24