Merge pull request #25 from the-superpirate/master

- feat(pylon): Refactor code …

idm/api/BUILD.bazel

@@ -0,0 +1,70 @@
+load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_push")
+
+load("@pip_modules//:requirements.bzl", "requirement")
+
+alias(
+    name = "binary",
+    actual = ":image.binary",
+)
+
+py3_image(
+    name = "image",
+    srcs = glob(
+        [
+            "*.py",
+            "configs/**/*.py",
+            "daemons/**/*.py",
+            "models/**",
+            "proto/**",
+            "services/**",
+        ],
+        exclude = [
+            "**/__pycache__/**",
+            "**/*.pyc",
+            "**/README",
+            "**/*.mako",
+            "proto/**/*.py",
+        ],
+    ),
+    base = "//images/production:base-python-image",
+    data = [
+        "configs/base.yaml",
+        "configs/logging.yaml",
+    ],
+    layers = [
+        requirement("grpcio"),
+        requirement("pypika"),
+        requirement("uvloop"),
+        "//idm/api/proto:idm_grpc_py",
+        "//idm/api/proto:idm_proto_py",
+        "//library/aiogrpctools",
+        requirement("aiokit"),
+        "//library/aiopostgres",
+        "//library/configurator",
+        "//library/telegram",
+        requirement("izihawa_utils"),
+    ],
+    main = "main.py",
+    srcs_version = "PY3ONLY",
+    visibility = ["//visibility:public"],
+)
+
+container_push(
+    name = "push-testing",
+    format = "Docker",
+    image = ":image",
+    registry = "registry.example.com",
+    repository = "idm-api",
+    tag = "testing",
+)
+
+container_push(
+    name = "push-latest",
+    format = "Docker",
+    image = ":image",
+    registry = "registry.example.com",
+    repository = "idm-api",
+    tag = "latest",
+)
+

idm/api/aioclient/BUILD.bazel

@@ -9,7 +9,7 @@ py_library(
         requirement("cachetools"),
         requirement("lru-dict"),
         requirement("tenacity"),
-        "//idm/api2/proto:idm_grpc_py",
+        "//idm/api/proto:idm_grpc_py",
-        "//idm/api2/proto:idm_proto_py",
+        "//idm/api/proto:idm_proto_py",
     ],
 )

idm/api/aioclient/__init__.py

@@ -0,0 +1,3 @@
+from .aioclient import IdmApiGrpcClient
+
+__all__ = ['IdmApiGrpcClient']

idm/api/aioclient/aioclient.py

@@ -4,13 +4,13 @@ from grpc.experimental.aio import (
     AioRpcError,
     insecure_channel,
 )
-from idm.api2.proto.chats_service_pb2 import (
+from idm.api.proto.chat_manager_service_pb2 import (
     CreateChatRequest,
     GetChatRequest,
     ListChatsRequest,
     UpdateChatRequest,
 )
-from idm.api2.proto.chats_service_pb2_grpc import ChatsStub
+from idm.api.proto.chat_manager_service_pb2_grpc import ChatManagerStub
 from lru import LRU
 from tenacity import (
     retry,
@@ -20,7 +20,7 @@ from tenacity import (
 )
 
 
-class IdmApi2GrpcClient(AioThing):
+class IdmApiGrpcClient(AioThing):
     def __init__(
         self,
         base_url,
@@ -33,7 +33,7 @@ class IdmApi2GrpcClient(AioThing):
             ('grpc.min_reconnect_backoff_ms', 1000),
             ('grpc.max_reconnect_backoff_ms', 2000),
         ])
-        self.chats_stub = ChatsStub(self.channel)
+        self.chat_manager_stub = ChatManagerStub(self.channel)
         self.cache = LRU(4096)
 
     async def start(self):
@@ -49,7 +49,7 @@ class IdmApi2GrpcClient(AioThing):
         language,
         request_id: str = None,
     ):
-        response = await self.chats_stub.create_chat(
+        response = await self.chat_manager_stub.create_chat(
             CreateChatRequest(
                 chat_id=chat_id,
                 username=username,
@@ -74,7 +74,7 @@ class IdmApi2GrpcClient(AioThing):
         chat_id,
         request_id: str = None,
     ):
-        response = await self.chats_stub.get_chat(
+        response = await self.chat_manager_stub.get_chat(
             GetChatRequest(chat_id=chat_id),
             metadata=(
                 ('request-id', request_id),
@@ -87,7 +87,7 @@ class IdmApi2GrpcClient(AioThing):
         request_id: str = None,
         banned_at_moment=None,
     ):
-        response = await self.chats_stub.list_chats(
+        response = await self.chat_manager_stub.list_chats(
             ListChatsRequest(banned_at_moment=banned_at_moment),
             metadata=(
                 ('request-id', request_id),
@@ -106,7 +106,7 @@ class IdmApi2GrpcClient(AioThing):
         ban_message=None,
         is_admin=None,
     ):
-        response = await self.chats_stub.update_chat(
+        response = await self.chat_manager_stub.update_chat(
             UpdateChatRequest(
                 chat_id=chat_id,
                 language=language,

idm/api/configs/base.yaml

@@ -0,0 +1,18 @@
+---
+application:
+  debug: true
+  service_name: idm-api
+database:
+  port: 5432
+grpc:
+  address: 0.0.0.0
+  port: 9090
+log_path: '/var/log/idm-api/{{ ENV_TYPE }}'
+telegram:
+  # Telegram App Hash from https://my.telegram.org/
+  app_hash: '{{ APP_HASH }}'
+  # Telegram App ID from https://my.telegram.org/
+  app_id: 00000
+  database:
+    session_id: '/usr/lib/idm-api/{{ ENV_TYPE }}/session.db'
+  related_channel: '@nexus_search'

idm/api/configs/logging.yaml

@@ -0,0 +1,63 @@
+---
+
+logging:
+  disable_existing_loggers: false
+  formatters:
+    base:
+      class: library.logging.formatters.BaseFormatter
+    default:
+      class: library.logging.formatters.DefaultFormatter
+    none:
+      class: logging.Formatter
+    traceback:
+      class: library.logging.formatters.TracebackFormatter
+  handlers:
+    debug:
+      class: logging.StreamHandler
+      formatter: default
+      level: DEBUG
+      stream: 'ext://sys.stderr'
+    error:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/error.log'
+      formatter: default
+      level: ERROR
+    statbox:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/statbox.log'
+      formatter: default
+      level: INFO
+    traceback:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/traceback.log'
+      formatter: traceback
+      level: ERROR
+    warning:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/warning.log'
+      formatter: default
+      level: WARNING
+  loggers:
+    debug:
+      handlers:
+        - debug
+      propagate: false
+    error:
+      handlers:
+        - error
+        - traceback
+        - warning
+      propagate: false
+    statbox:
+      handlers:
+        - statbox
+      propagate: false
+    telethon:
+      handlers:
+        - error
+      propagate: false
+  root:
+    handlers:
+      - debug
+    level: DEBUG
+  version: 1

idm/api/daemons/admin_log_reader.py

@@ -0,0 +1,86 @@
+import asyncio
+import logging
+import string
+
+from aiokit import AioThing
+from library.telegram.base import BaseTelegramClient
+from telethon.tl.functions.channels import GetParticipantsRequest
+from telethon.tl.types import ChannelParticipantsSearch
+
+
+class AdminLogReader(AioThing):
+    def __init__(self, telegram_config):
+        super().__init__()
+        self.subscriptors = set()
+        self.loading = False
+        self.telegram_client = BaseTelegramClient(
+            app_id=telegram_config['app_id'],
+            app_hash=telegram_config['app_hash'],
+            database=telegram_config['database'],
+            flood_sleep_threshold=25,
+        )
+        self.channel_name = telegram_config['related_channel']
+        self.last_max_id = 0
+
+    def statbox(self, **kwargs):
+        logging.getLogger('statbox').info({'mode': 'admin_log_reader', **kwargs})
+
+    async def skip_admin_log(self):
+        async for event in self.telegram_client.iter_admin_log(self.channel_name, limit=1):
+            self.last_max_id = event.id
+        self.statbox(action='skipped_admin_log', max_id=self.last_max_id)
+
+    async def process_admin_log(self, sleep=1.0):
+        self.loading = True
+        try:
+            while 1:
+                async for event in self.telegram_client.iter_admin_log(
+                    self.channel_name, join=True, invite=True, leave=True, min_id=self.last_max_id
+                ):
+                    is_subscribed = event.joined or event.joined_invite
+                    if is_subscribed:
+                        self.subscriptors.add(event.user_id)
+                    elif event.user_id in self.subscriptors:
+                        self.subscriptors.remove(event.user_id)
+                    self.last_max_id = event.id
+                await asyncio.sleep(sleep)
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self.loading = False
+
+    async def _fetch_users(self, query):
+        max_batch_size = 200
+
+        participants = await self.telegram_client(
+            GetParticipantsRequest(
+                channel=self.channel_name,
+                filter=ChannelParticipantsSearch(query),
+                offset=0,
+                limit=max_batch_size,
+                hash=0,
+            )
+        )
+
+        for user in participants.users:
+            self.subscriptors.add(user.id)
+
+        # There is a possibility that more users exist if we hit maximum count of users
+        # So, we are making a recursion to reveal it
+        if len(participants.users) == max_batch_size:
+            for new_letter in string.ascii_lowercase + string.digits:
+                await self._fetch_users(query + new_letter)
+
+    async def load_channel_users(self):
+        self.statbox(action='load_channel_users')
+        await self._fetch_users('')
+        self.statbox(action='loaded_channel_users', subscriptors=len(self.subscriptors))
+
+    async def start(self):
+        await self.telegram_client.start_and_wait()
+        await self.skip_admin_log()
+        await self.load_channel_users()
+        asyncio.create_task(self.process_admin_log())
+
+    def is_subscribed(self, user_id):
+        return not self.loading or user_id in self.subscriptors

idm/api/main.py

@@ -0,0 +1,54 @@
+import asyncio
+
+import uvloop
+from aiopg.sa import create_engine
+from idm.api.configs import get_config
+from idm.api.daemons.admin_log_reader import AdminLogReader
+from idm.api.services.chat_manager import ChatManagerService
+from library.aiogrpctools import AioGrpcServer
+from library.aiopostgres.pool_holder import AioPostgresPoolHolder
+from library.configurator import Configurator
+from library.logging import configure_logging
+
+
+class GrpcServer(AioGrpcServer):
+    def __init__(self, config: Configurator):
+        super().__init__(address=config['grpc']['address'], port=config['grpc']['port'])
+        self.pool_holder = AioPostgresPoolHolder(
+            fn=create_engine,
+            database=config['database']['database'],
+            user=config['database']['username'],
+            password=config['database']['password'],
+            host=config['database']['host'],
+            port=config['database']['port'],
+            timeout=30,
+            pool_recycle=60,
+            maxsize=4,
+        )
+        self.admin_log_reader = AdminLogReader(
+            telegram_config=config['telegram'],
+        )
+        self.chat_manager_service = ChatManagerService(
+            server=self.server,
+            service_name=config['application']['service_name'],
+            pool_holder=self.pool_holder,
+            admin_log_reader=self.admin_log_reader,
+        )
+        self.starts.append(self.admin_log_reader)
+        self.waits.extend([self.chat_manager_service, self.pool_holder])
+
+
+async def create_app(config: Configurator):
+    grpc_server = GrpcServer(config)
+    await grpc_server.start_and_wait()
+
+
+def main():
+    uvloop.install()
+    config = get_config()
+    configure_logging(config)
+    asyncio.get_event_loop().run_until_complete(create_app(config))
+
+
+if __name__ == '__main__':
+    main()

idm/api/proto/BUILD.bazel

@@ -6,8 +6,7 @@ package(default_visibility = ["//visibility:public"])
 proto_library(
     name = "idm_proto",
     srcs = [
-        "chats_service.proto",
+        "chat_manager_service.proto",
-        "location.proto",
     ],
     deps = [
         "@com_google_protobuf//:wrappers_proto",

idm/api/proto/chat_manager_service.proto

@@ -0,0 +1,50 @@
+syntax = "proto3";
+package idm.api.proto;
+
+message Chat {
+    int64 chat_id = 1;
+    string username = 2;
+    string language = 3;
+    bool is_system_messaging_enabled = 4;
+    bool is_discovery_enabled = 5;
+    int32 ban_until = 6;
+    string ban_message = 7;
+    bool is_admin = 8;
+    bool is_subscribed = 9;
+    int64 created_at = 10;
+}
+
+message Chats {
+    repeated Chat chats = 1;
+}
+
+message CreateChatRequest {
+    int64 chat_id = 1;
+    string username = 2;
+    string language = 3;
+}
+
+message GetChatRequest {
+    int64 chat_id = 1;
+}
+
+message ListChatsRequest {
+    optional int32 banned_at_moment = 1;
+}
+
+message UpdateChatRequest {
+    int64 chat_id = 1;
+    optional string language = 2;
+    optional bool is_system_messaging_enabled = 3;
+    optional bool is_discovery_enabled = 4;
+    optional int32 ban_until = 5;
+    optional string ban_message = 6;
+    optional bool is_admin = 7;
+}
+
+service ChatManager {
+    rpc create_chat(CreateChatRequest) returns (Chat) {};
+    rpc get_chat(GetChatRequest) returns (Chat) {};
+    rpc list_chats(ListChatsRequest) returns (Chats) {};
+    rpc update_chat(UpdateChatRequest) returns (Chat) {};
+}

idm/api/proto/chat_manager_service_pb2.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: idm/api2/proto/chats_service.proto
+# source: idm/api/proto/chat_manager_service.proto
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
@@ -12,96 +12,94 @@ from google.protobuf import symbol_database as _symbol_database
 _sym_db = _symbol_database.Default()
 
 
-from idm.api2.proto import \
+
-    location_pb2 as idm_dot_api2_dot_proto_dot_location__pb2
 
 DESCRIPTOR = _descriptor.FileDescriptor(
-  name='idm/api2/proto/chats_service.proto',
+  name='idm/api/proto/chat_manager_service.proto',
-  package='idm.api2.proto',
+  package='idm.api.proto',
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\"idm/api2/proto/chats_service.proto\x12\x0eidm.api2.proto\x1a\x1didm/api2/proto/location.proto\"\xe2\x01\n\x08\x43hatData\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x15\n\ris_subscribed\x18\n \x01(\x08\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\"4\n\tChatsData\x12\'\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x18.idm.api2.proto.ChatData\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"H\n\x10ListChatsRequest\x12\x1a\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x42\x18\n\x16\x62\x61nned_at_moment_oneof\"\xce\x02\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x12\n\x08language\x18\x02 \x01(\tH\x00\x12%\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x12\x1e\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x12\x13\n\tban_until\x18\x05 \x01(\x05H\x03\x12\x15\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x12\x12\n\x08is_admin\x18\x07 \x01(\x08H\x05\x42\x10\n\x0elanguage_oneofB#\n!is_system_messaging_enabled_oneofB\x1c\n\x1ais_discovery_enabled_oneofB\x11\n\x0f\x62\x61n_until_oneofB\x13\n\x11\x62\x61n_message_oneofB\x10\n\x0eis_admin_oneof2\xb8\x02\n\x05\x43hats\x12L\n\x0b\x63reate_chat\x12!.idm.api2.proto.CreateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12\x46\n\x08get_chat\x12\x1e.idm.api2.proto.GetChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12K\n\nlist_chats\x12 .idm.api2.proto.ListChatsRequest\x1a\x19.idm.api2.proto.ChatsData\"\x00\x12L\n\x0bupdate_chat\x12!.idm.api2.proto.UpdateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x62\x06proto3'
+  serialized_pb=b'\n(idm/api/proto/chat_manager_service.proto\x12\ridm.api.proto\"\xe3\x01\n\x04\x43hat\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x15\n\ris_subscribed\x18\t \x01(\x08\x12\x12\n\ncreated_at\x18\n \x01(\x03\"+\n\x05\x43hats\x12\"\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x13.idm.api.proto.Chat\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"F\n\x10ListChatsRequest\x12\x1d\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x88\x01\x01\x42\x13\n\x11_banned_at_moment\"\xc2\x02\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x15\n\x08language\x18\x02 \x01(\tH\x00\x88\x01\x01\x12(\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12!\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x88\x01\x01\x12\x16\n\tban_until\x18\x05 \x01(\x05H\x03\x88\x01\x01\x12\x18\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08is_admin\x18\x07 \x01(\x08H\x05\x88\x01\x01\x42\x0b\n\t_languageB\x1e\n\x1c_is_system_messaging_enabledB\x17\n\x15_is_discovery_enabledB\x0c\n\n_ban_untilB\x0e\n\x0c_ban_messageB\x0b\n\t_is_admin2\xa6\x02\n\x0b\x43hatManager\x12\x46\n\x0b\x63reate_chat\x12 .idm.api.proto.CreateChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x12@\n\x08get_chat\x12\x1d.idm.api.proto.GetChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x12\x45\n\nlist_chats\x12\x1f.idm.api.proto.ListChatsRequest\x1a\x14.idm.api.proto.Chats\"\x00\x12\x46\n\x0bupdate_chat\x12 .idm.api.proto.UpdateChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x62\x06proto3'
-  ,
+)
-  dependencies=[idm_dot_api2_dot_proto_dot_location__pb2.DESCRIPTOR,])
 
 
 
 
-_CHATDATA = _descriptor.Descriptor(
+_CHAT = _descriptor.Descriptor(
-  name='ChatData',
+  name='Chat',
-  full_name='idm.api2.proto.ChatData',
+  full_name='idm.api.proto.Chat',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='id', full_name='idm.api2.proto.ChatData.id', index=0,
+      name='chat_id', full_name='idm.api.proto.Chat.chat_id', index=0,
       number=1, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='username', full_name='idm.api2.proto.ChatData.username', index=1,
+      name='username', full_name='idm.api.proto.Chat.username', index=1,
       number=2, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='language', full_name='idm.api2.proto.ChatData.language', index=2,
+      name='language', full_name='idm.api.proto.Chat.language', index=2,
       number=3, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_system_messaging_enabled', full_name='idm.api2.proto.ChatData.is_system_messaging_enabled', index=3,
+      name='is_system_messaging_enabled', full_name='idm.api.proto.Chat.is_system_messaging_enabled', index=3,
       number=4, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_discovery_enabled', full_name='idm.api2.proto.ChatData.is_discovery_enabled', index=4,
+      name='is_discovery_enabled', full_name='idm.api.proto.Chat.is_discovery_enabled', index=4,
       number=5, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='ban_until', full_name='idm.api2.proto.ChatData.ban_until', index=5,
+      name='ban_until', full_name='idm.api.proto.Chat.ban_until', index=5,
       number=6, type=5, cpp_type=1, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='ban_message', full_name='idm.api2.proto.ChatData.ban_message', index=6,
+      name='ban_message', full_name='idm.api.proto.Chat.ban_message', index=6,
       number=7, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_admin', full_name='idm.api2.proto.ChatData.is_admin', index=7,
+      name='is_admin', full_name='idm.api.proto.Chat.is_admin', index=7,
       number=8, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_subscribed', full_name='idm.api2.proto.ChatData.is_subscribed', index=8,
+      name='is_subscribed', full_name='idm.api.proto.Chat.is_subscribed', index=8,
-      number=10, type=8, cpp_type=7, label=1,
+      number=9, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='created_at', full_name='idm.api2.proto.ChatData.created_at', index=9,
+      name='created_at', full_name='idm.api.proto.Chat.created_at', index=9,
-      number=11, type=3, cpp_type=2, label=1,
+      number=10, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
@@ -118,21 +116,21 @@ _CHATDATA = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=86,
+  serialized_start=60,
-  serialized_end=312,
+  serialized_end=287,
 )
 
 
-_CHATSDATA = _descriptor.Descriptor(
+_CHATS = _descriptor.Descriptor(
-  name='ChatsData',
+  name='Chats',
-  full_name='idm.api2.proto.ChatsData',
+  full_name='idm.api.proto.Chats',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='chats', full_name='idm.api2.proto.ChatsData.chats', index=0,
+      name='chats', full_name='idm.api.proto.Chats.chats', index=0,
       number=1, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
@@ -150,35 +148,35 @@ _CHATSDATA = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=314,
+  serialized_start=289,
-  serialized_end=366,
+  serialized_end=332,
 )
 
 
 _CREATECHATREQUEST = _descriptor.Descriptor(
   name='CreateChatRequest',
-  full_name='idm.api2.proto.CreateChatRequest',
+  full_name='idm.api.proto.CreateChatRequest',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='chat_id', full_name='idm.api2.proto.CreateChatRequest.chat_id', index=0,
+      name='chat_id', full_name='idm.api.proto.CreateChatRequest.chat_id', index=0,
       number=1, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='username', full_name='idm.api2.proto.CreateChatRequest.username', index=1,
+      name='username', full_name='idm.api.proto.CreateChatRequest.username', index=1,
       number=2, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='language', full_name='idm.api2.proto.CreateChatRequest.language', index=2,
+      name='language', full_name='idm.api.proto.CreateChatRequest.language', index=2,
       number=3, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
@@ -196,21 +194,21 @@ _CREATECHATREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=368,
+  serialized_start=334,
-  serialized_end=440,
+  serialized_end=406,
 )
 
 
 _GETCHATREQUEST = _descriptor.Descriptor(
   name='GetChatRequest',
-  full_name='idm.api2.proto.GetChatRequest',
+  full_name='idm.api.proto.GetChatRequest',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='chat_id', full_name='idm.api2.proto.GetChatRequest.chat_id', index=0,
+      name='chat_id', full_name='idm.api.proto.GetChatRequest.chat_id', index=0,
       number=1, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
@@ -228,21 +226,21 @@ _GETCHATREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=442,
+  serialized_start=408,
-  serialized_end=475,
+  serialized_end=441,
 )
 
 
 _LISTCHATSREQUEST = _descriptor.Descriptor(
   name='ListChatsRequest',
-  full_name='idm.api2.proto.ListChatsRequest',
+  full_name='idm.api.proto.ListChatsRequest',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='banned_at_moment', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment', index=0,
+      name='banned_at_moment', full_name='idm.api.proto.ListChatsRequest.banned_at_moment', index=0,
       number=1, type=5, cpp_type=1, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
@@ -260,68 +258,68 @@ _LISTCHATSREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
     _descriptor.OneofDescriptor(
-      name='banned_at_moment_oneof', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment_oneof',
+      name='_banned_at_moment', full_name='idm.api.proto.ListChatsRequest._banned_at_moment',
       index=0, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
   ],
-  serialized_start=477,
+  serialized_start=443,
-  serialized_end=549,
+  serialized_end=513,
 )
 
 
 _UPDATECHATREQUEST = _descriptor.Descriptor(
   name='UpdateChatRequest',
-  full_name='idm.api2.proto.UpdateChatRequest',
+  full_name='idm.api.proto.UpdateChatRequest',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
-      name='chat_id', full_name='idm.api2.proto.UpdateChatRequest.chat_id', index=0,
+      name='chat_id', full_name='idm.api.proto.UpdateChatRequest.chat_id', index=0,
       number=1, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='language', full_name='idm.api2.proto.UpdateChatRequest.language', index=1,
+      name='language', full_name='idm.api.proto.UpdateChatRequest.language', index=1,
       number=2, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_system_messaging_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled', index=2,
+      name='is_system_messaging_enabled', full_name='idm.api.proto.UpdateChatRequest.is_system_messaging_enabled', index=2,
       number=3, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_discovery_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled', index=3,
+      name='is_discovery_enabled', full_name='idm.api.proto.UpdateChatRequest.is_discovery_enabled', index=3,
       number=4, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='ban_until', full_name='idm.api2.proto.UpdateChatRequest.ban_until', index=4,
+      name='ban_until', full_name='idm.api.proto.UpdateChatRequest.ban_until', index=4,
       number=5, type=5, cpp_type=1, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='ban_message', full_name='idm.api2.proto.UpdateChatRequest.ban_message', index=5,
+      name='ban_message', full_name='idm.api.proto.UpdateChatRequest.ban_message', index=5,
       number=6, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='is_admin', full_name='idm.api2.proto.UpdateChatRequest.is_admin', index=6,
+      name='is_admin', full_name='idm.api.proto.UpdateChatRequest.is_admin', index=6,
       number=7, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
@@ -339,167 +337,167 @@ _UPDATECHATREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
     _descriptor.OneofDescriptor(
-      name='language_oneof', full_name='idm.api2.proto.UpdateChatRequest.language_oneof',
+      name='_language', full_name='idm.api.proto.UpdateChatRequest._language',
       index=0, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
     _descriptor.OneofDescriptor(
-      name='is_system_messaging_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled_oneof',
+      name='_is_system_messaging_enabled', full_name='idm.api.proto.UpdateChatRequest._is_system_messaging_enabled',
       index=1, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
     _descriptor.OneofDescriptor(
-      name='is_discovery_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled_oneof',
+      name='_is_discovery_enabled', full_name='idm.api.proto.UpdateChatRequest._is_discovery_enabled',
       index=2, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
     _descriptor.OneofDescriptor(
-      name='ban_until_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_until_oneof',
+      name='_ban_until', full_name='idm.api.proto.UpdateChatRequest._ban_until',
       index=3, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
     _descriptor.OneofDescriptor(
-      name='ban_message_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_message_oneof',
+      name='_ban_message', full_name='idm.api.proto.UpdateChatRequest._ban_message',
       index=4, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
     _descriptor.OneofDescriptor(
-      name='is_admin_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_admin_oneof',
+      name='_is_admin', full_name='idm.api.proto.UpdateChatRequest._is_admin',
       index=5, containing_type=None,
       create_key=_descriptor._internal_create_key,
     fields=[]),
   ],
-  serialized_start=552,
+  serialized_start=516,
-  serialized_end=886,
+  serialized_end=838,
 )
 
-_CHATSDATA.fields_by_name['chats'].message_type = _CHATDATA
+_CHATS.fields_by_name['chats'].message_type = _CHAT
-_LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof'].fields.append(
+_LISTCHATSREQUEST.oneofs_by_name['_banned_at_moment'].fields.append(
   _LISTCHATSREQUEST.fields_by_name['banned_at_moment'])
-_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof']
+_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['_banned_at_moment']
-_UPDATECHATREQUEST.oneofs_by_name['language_oneof'].fields.append(
+_UPDATECHATREQUEST.oneofs_by_name['_language'].fields.append(
   _UPDATECHATREQUEST.fields_by_name['language'])
-_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['language_oneof']
+_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_language']
-_UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof'].fields.append(
+_UPDATECHATREQUEST.oneofs_by_name['_is_system_messaging_enabled'].fields.append(
   _UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'])
-_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof']
+_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_system_messaging_enabled']
-_UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof'].fields.append(
+_UPDATECHATREQUEST.oneofs_by_name['_is_discovery_enabled'].fields.append(
   _UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'])
-_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof']
+_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_discovery_enabled']
-_UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof'].fields.append(
+_UPDATECHATREQUEST.oneofs_by_name['_ban_until'].fields.append(
   _UPDATECHATREQUEST.fields_by_name['ban_until'])
-_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof']
+_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_ban_until']
-_UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof'].fields.append(
+_UPDATECHATREQUEST.oneofs_by_name['_ban_message'].fields.append(
   _UPDATECHATREQUEST.fields_by_name['ban_message'])
-_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof']
+_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_ban_message']
-_UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof'].fields.append(
+_UPDATECHATREQUEST.oneofs_by_name['_is_admin'].fields.append(
   _UPDATECHATREQUEST.fields_by_name['is_admin'])
-_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof']
+_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_admin']
-DESCRIPTOR.message_types_by_name['ChatData'] = _CHATDATA
+DESCRIPTOR.message_types_by_name['Chat'] = _CHAT
-DESCRIPTOR.message_types_by_name['ChatsData'] = _CHATSDATA
+DESCRIPTOR.message_types_by_name['Chats'] = _CHATS
 DESCRIPTOR.message_types_by_name['CreateChatRequest'] = _CREATECHATREQUEST
 DESCRIPTOR.message_types_by_name['GetChatRequest'] = _GETCHATREQUEST
 DESCRIPTOR.message_types_by_name['ListChatsRequest'] = _LISTCHATSREQUEST
 DESCRIPTOR.message_types_by_name['UpdateChatRequest'] = _UPDATECHATREQUEST
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
-ChatData = _reflection.GeneratedProtocolMessageType('ChatData', (_message.Message,), {
+Chat = _reflection.GeneratedProtocolMessageType('Chat', (_message.Message,), {
-  'DESCRIPTOR' : _CHATDATA,
+  'DESCRIPTOR' : _CHAT,
-  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  '__module__' : 'idm.api.proto.chat_manager_service_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.ChatData)
+  # @@protoc_insertion_point(class_scope:idm.api.proto.Chat)
   })
-_sym_db.RegisterMessage(ChatData)
+_sym_db.RegisterMessage(Chat)
 
-ChatsData = _reflection.GeneratedProtocolMessageType('ChatsData', (_message.Message,), {
+Chats = _reflection.GeneratedProtocolMessageType('Chats', (_message.Message,), {
-  'DESCRIPTOR' : _CHATSDATA,
+  'DESCRIPTOR' : _CHATS,
-  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  '__module__' : 'idm.api.proto.chat_manager_service_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.ChatsData)
+  # @@protoc_insertion_point(class_scope:idm.api.proto.Chats)
   })
-_sym_db.RegisterMessage(ChatsData)
+_sym_db.RegisterMessage(Chats)
 
 CreateChatRequest = _reflection.GeneratedProtocolMessageType('CreateChatRequest', (_message.Message,), {
   'DESCRIPTOR' : _CREATECHATREQUEST,
-  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  '__module__' : 'idm.api.proto.chat_manager_service_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.CreateChatRequest)
+  # @@protoc_insertion_point(class_scope:idm.api.proto.CreateChatRequest)
   })
 _sym_db.RegisterMessage(CreateChatRequest)
 
 GetChatRequest = _reflection.GeneratedProtocolMessageType('GetChatRequest', (_message.Message,), {
   'DESCRIPTOR' : _GETCHATREQUEST,
-  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  '__module__' : 'idm.api.proto.chat_manager_service_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.GetChatRequest)
+  # @@protoc_insertion_point(class_scope:idm.api.proto.GetChatRequest)
   })
 _sym_db.RegisterMessage(GetChatRequest)
 
 ListChatsRequest = _reflection.GeneratedProtocolMessageType('ListChatsRequest', (_message.Message,), {
   'DESCRIPTOR' : _LISTCHATSREQUEST,
-  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  '__module__' : 'idm.api.proto.chat_manager_service_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.ListChatsRequest)
+  # @@protoc_insertion_point(class_scope:idm.api.proto.ListChatsRequest)
   })
 _sym_db.RegisterMessage(ListChatsRequest)
 
 UpdateChatRequest = _reflection.GeneratedProtocolMessageType('UpdateChatRequest', (_message.Message,), {
   'DESCRIPTOR' : _UPDATECHATREQUEST,
-  '__module__' : 'idm.api2.proto.chats_service_pb2'
+  '__module__' : 'idm.api.proto.chat_manager_service_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.UpdateChatRequest)
+  # @@protoc_insertion_point(class_scope:idm.api.proto.UpdateChatRequest)
   })
 _sym_db.RegisterMessage(UpdateChatRequest)
 
 
 
-_CHATS = _descriptor.ServiceDescriptor(
+_CHATMANAGER = _descriptor.ServiceDescriptor(
-  name='Chats',
+  name='ChatManager',
-  full_name='idm.api2.proto.Chats',
+  full_name='idm.api.proto.ChatManager',
   file=DESCRIPTOR,
   index=0,
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_start=889,
+  serialized_start=841,
-  serialized_end=1201,
+  serialized_end=1135,
   methods=[
   _descriptor.MethodDescriptor(
     name='create_chat',
-    full_name='idm.api2.proto.Chats.create_chat',
+    full_name='idm.api.proto.ChatManager.create_chat',
     index=0,
     containing_service=None,
     input_type=_CREATECHATREQUEST,
-    output_type=_CHATDATA,
+    output_type=_CHAT,
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
   ),
   _descriptor.MethodDescriptor(
     name='get_chat',
-    full_name='idm.api2.proto.Chats.get_chat',
+    full_name='idm.api.proto.ChatManager.get_chat',
     index=1,
     containing_service=None,
     input_type=_GETCHATREQUEST,
-    output_type=_CHATDATA,
+    output_type=_CHAT,
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
   ),
   _descriptor.MethodDescriptor(
     name='list_chats',
-    full_name='idm.api2.proto.Chats.list_chats',
+    full_name='idm.api.proto.ChatManager.list_chats',
     index=2,
     containing_service=None,
     input_type=_LISTCHATSREQUEST,
-    output_type=_CHATSDATA,
+    output_type=_CHATS,
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
   ),
   _descriptor.MethodDescriptor(
     name='update_chat',
-    full_name='idm.api2.proto.Chats.update_chat',
+    full_name='idm.api.proto.ChatManager.update_chat',
     index=3,
     containing_service=None,
     input_type=_UPDATECHATREQUEST,
-    output_type=_CHATDATA,
+    output_type=_CHAT,
     serialized_options=None,
     create_key=_descriptor._internal_create_key,
   ),
 ])
-_sym_db.RegisterServiceDescriptor(_CHATS)
+_sym_db.RegisterServiceDescriptor(_CHATMANAGER)
 
-DESCRIPTOR.services_by_name['Chats'] = _CHATS
+DESCRIPTOR.services_by_name['ChatManager'] = _CHATMANAGER
 
 # @@protoc_insertion_point(module_scope)

idm/api/proto/chat_manager_service_pb2_grpc.py

@@ -1,11 +1,12 @@
 # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
-from idm.api2.proto import \
+from idm.api.proto import \
-    chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
+    chat_manager_service_pb2 as \
+    idm_dot_api_dot_proto_dot_chat__manager__service__pb2
 
 
-class ChatsStub(object):
+class ChatManagerStub(object):
     """Missing associated documentation comment in .proto file."""
 
     def __init__(self, channel):
@@ -15,28 +16,28 @@ class ChatsStub(object):
             channel: A grpc.Channel.
         """
         self.create_chat = channel.unary_unary(
-                '/idm.api2.proto.Chats/create_chat',
+                '/idm.api.proto.ChatManager/create_chat',
-                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
+                request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.SerializeToString,
-                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+                response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
                 )
         self.get_chat = channel.unary_unary(
-                '/idm.api2.proto.Chats/get_chat',
+                '/idm.api.proto.ChatManager/get_chat',
-                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
+                request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.SerializeToString,
-                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+                response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
                 )
         self.list_chats = channel.unary_unary(
-                '/idm.api2.proto.Chats/list_chats',
+                '/idm.api.proto.ChatManager/list_chats',
-                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
+                request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.SerializeToString,
-                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
+                response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.FromString,
                 )
         self.update_chat = channel.unary_unary(
-                '/idm.api2.proto.Chats/update_chat',
+                '/idm.api.proto.ChatManager/update_chat',
-                request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
+                request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.SerializeToString,
-                response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+                response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
                 )
 
 
-class ChatsServicer(object):
+class ChatManagerServicer(object):
     """Missing associated documentation comment in .proto file."""
 
     def create_chat(self, request, context):
@@ -64,36 +65,36 @@ class ChatsServicer(object):
         raise NotImplementedError('Method not implemented!')
 
 
-def add_ChatsServicer_to_server(servicer, server):
+def add_ChatManagerServicer_to_server(servicer, server):
     rpc_method_handlers = {
             'create_chat': grpc.unary_unary_rpc_method_handler(
                     servicer.create_chat,
-                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.FromString,
+                    request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.FromString,
-                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
+                    response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString,
             ),
             'get_chat': grpc.unary_unary_rpc_method_handler(
                     servicer.get_chat,
-                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.FromString,
+                    request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.FromString,
-                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
+                    response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString,
             ),
             'list_chats': grpc.unary_unary_rpc_method_handler(
                     servicer.list_chats,
-                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.FromString,
+                    request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.FromString,
-                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.SerializeToString,
+                    response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.SerializeToString,
             ),
             'update_chat': grpc.unary_unary_rpc_method_handler(
                     servicer.update_chat,
-                    request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.FromString,
+                    request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.FromString,
-                    response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
+                    response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString,
             ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-            'idm.api2.proto.Chats', rpc_method_handlers)
+            'idm.api.proto.ChatManager', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
 
  # This class is part of an EXPERIMENTAL API.
-class Chats(object):
+class ChatManager(object):
     """Missing associated documentation comment in .proto file."""
 
     @staticmethod
@@ -107,9 +108,9 @@ class Chats(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/create_chat',
+        return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/create_chat',
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.SerializeToString,
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
@@ -124,9 +125,9 @@ class Chats(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/get_chat',
+        return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/get_chat',
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.SerializeToString,
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
@@ -141,9 +142,9 @@ class Chats(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/list_chats',
+        return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/list_chats',
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.SerializeToString,
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
@@ -158,8 +159,8 @@ class Chats(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/update_chat',
+        return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/update_chat',
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.SerializeToString,
-            idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
+            idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

idm/api/services/chat_manager.py

@@ -0,0 +1,109 @@
+from grpc import StatusCode
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
+from idm.api.proto.chat_manager_service_pb2 import Chats as ChatsPb
+from idm.api.proto.chat_manager_service_pb2_grpc import (
+    ChatManagerServicer,
+    add_ChatManagerServicer_to_server,
+)
+from library.aiogrpctools.base import (
+    BaseService,
+    aiogrpc_request_wrapper,
+)
+from pypika import (
+    PostgreSQLQuery,
+    Table,
+)
+
+
+class ChatManagerService(ChatManagerServicer, BaseService):
+    chats_table = Table('chats')
+
+    def __init__(self, server, service_name, pool_holder, admin_log_reader):
+        super().__init__(service_name=service_name)
+        self.server = server
+        self.pool_holder = pool_holder
+        self.admin_log_reader = admin_log_reader
+
+    async def start(self):
+        add_ChatManagerServicer_to_server(self, self.server)
+
+    def enrich_chat(self, chat_pb: ChatPb):
+        chat_pb.is_subscribed = self.admin_log_reader.is_subscribed(chat_pb.chat_id)
+        return chat_pb
+
+    @aiogrpc_request_wrapper()
+    async def create_chat(self, request, context, metadata):
+        chat = ChatPb(
+            chat_id=request.chat_id,
+            language=request.language,
+            username=request.username,
+            is_system_messaging_enabled=True,
+            is_discovery_enabled=True,
+        )
+        query = (
+            PostgreSQLQuery
+            .into(self.chats_table)
+            .columns(
+                self.chats_table.chat_id,
+                self.chats_table.language,
+                self.chats_table.username,
+                self.chats_table.is_system_messaging_enabled,
+                self.chats_table.is_discovery_enabled,
+            )
+            .insert(
+                chat.chat_id,
+                chat.language,
+                chat.username,
+                chat.is_system_messaging_enabled,
+                chat.is_discovery_enabled,
+            )
+            .returning('*')
+        ).get_sql()
+        async with self.pool_holder.pool.acquire() as session:
+            result = await session.execute(query)
+            chat = await result.fetchone()
+        return self.enrich_chat(ChatPb(**chat))
+
+    @aiogrpc_request_wrapper()
+    async def get_chat(self, request, context, metadata):
+        query = (
+            PostgreSQLQuery
+            .from_(self.chats_table)
+            .select('*')
+            .where(self.chats_table.chat_id == request.chat_id)
+        ).get_sql()
+        async with self.pool_holder.pool.acquire() as session:
+            result = await session.execute(query)
+            chat = await result.fetchone()
+            if chat is None:
+                await context.abort(StatusCode.NOT_FOUND, 'not_found')
+            return self.enrich_chat(ChatPb(**chat))
+
+    @aiogrpc_request_wrapper()
+    async def list_chats(self, request, context, metadata):
+        query = (
+            PostgreSQLQuery
+            .from_(self.chats_table)
+            .select('*')
+            .where(self.chats_table.ban_until > request.banned_at_moment)
+            .limit(10)
+        ).get_sql()
+        async with self.pool_holder.pool.acquire() as session:
+            results = await session.execute(query)
+            chats = await results.fetchall()
+            return ChatsPb(
+                chats=list(map(lambda x: self.enrich_chat(ChatPb(**x)), chats))
+            )
+
+    @aiogrpc_request_wrapper()
+    async def update_chat(self, request, context, metadata):
+        query = PostgreSQLQuery.update(self.chats_table)
+        for field in request.DESCRIPTOR.fields:
+            if field.containing_oneof and request.HasField(field.name):
+                field_value = getattr(request, field.name)
+                query = query.set(field.name, field_value)
+        query = query.where(self.chats_table.chat_id == request.chat_id).returning('*').get_sql()
+        async with self.pool_holder.pool.acquire() as session:
+            result = await session.execute(query)
+            chat = await result.fetchone()
+        return self.enrich_chat(ChatPb(**chat))

idm/api2/aioclient/__init__.py

@@ -1,3 +0,0 @@
-from .aioclient import IdmApi2GrpcClient
-
-__all__ = ['IdmApi2GrpcClient']

idm/api2/proto/chats_service.proto

@@ -1,66 +0,0 @@
-syntax = "proto3";
-package idm.api2.proto;
-
-import "idm/api2/proto/location.proto";
-
-message ChatData {
-    int64 id = 1;
-    string username = 2;
-    string language = 3;
-    bool is_system_messaging_enabled = 4;
-    bool is_discovery_enabled = 5;
-    int32 ban_until = 6;
-    string ban_message = 7;
-    bool is_admin = 8;
-    bool is_subscribed = 10;
-    int64 created_at = 11;
-}
-
-message ChatsData {
-    repeated ChatData chats = 1;
-}
-
-message CreateChatRequest {
-    int64 chat_id = 1;
-    string username = 2;
-    string language = 3;
-}
-
-message GetChatRequest {
-    int64 chat_id = 1;
-}
-
-message ListChatsRequest {
-    oneof banned_at_moment_oneof {
-        int32 banned_at_moment = 1;
-    }
-}
-
-message UpdateChatRequest {
-    int64 chat_id = 1;
-    oneof language_oneof {
-        string language = 2;
-    }
-    oneof is_system_messaging_enabled_oneof {
-        bool is_system_messaging_enabled = 3;
-    }
-    oneof is_discovery_enabled_oneof {
-        bool is_discovery_enabled = 4;
-    }
-    oneof ban_until_oneof {
-        int32 ban_until = 5;
-    }
-    oneof ban_message_oneof {
-        string ban_message = 6;
-    }
-    oneof is_admin_oneof {
-        bool is_admin = 7;
-    }
-}
-
-service Chats {
-    rpc create_chat(CreateChatRequest) returns (ChatData) {};
-    rpc get_chat(GetChatRequest) returns (ChatData) {};
-    rpc list_chats(ListChatsRequest) returns (ChatsData) {};
-    rpc update_chat(UpdateChatRequest) returns (ChatData) {};
-}

idm/api2/proto/location.proto

@@ -1,7 +0,0 @@
-syntax = "proto3";
-package idm.api2.proto;
-
-message Location {
-    float lat = 1;
-    float lon = 2;
-}

idm/api2/proto/location_pb2.py

@@ -1,78 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: idm/api2/proto/location.proto
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
-  name='idm/api2/proto/location.proto',
-  package='idm.api2.proto',
-  syntax='proto3',
-  serialized_options=None,
-  create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\x1didm/api2/proto/location.proto\x12\x0eidm.api2.proto\"$\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x02\x12\x0b\n\x03lon\x18\x02 \x01(\x02\x62\x06proto3'
-)
-
-
-
-
-_LOCATION = _descriptor.Descriptor(
-  name='Location',
-  full_name='idm.api2.proto.Location',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='lat', full_name='idm.api2.proto.Location.lat', index=0,
-      number=1, type=2, cpp_type=6, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='lon', full_name='idm.api2.proto.Location.lon', index=1,
-      number=2, type=2, cpp_type=6, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=49,
-  serialized_end=85,
-)
-
-DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), {
-  'DESCRIPTOR' : _LOCATION,
-  '__module__' : 'idm.api2.proto.location_pb2'
-  # @@protoc_insertion_point(class_scope:idm.api2.proto.Location)
-  })
-_sym_db.RegisterMessage(Location)
-
-
-# @@protoc_insertion_point(module_scope)

idm/api2/proto/location_pb2_grpc.py

@@ -1,4 +0,0 @@
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-"""Client and server classes corresponding to protobuf-defined services."""
-import grpc
-

images/install.bzl

@@ -11,14 +11,8 @@ def images_install():
 
     container_pull(
         name = "ubuntu",
-        digest = "sha256:5403064f94b617f7975a19ba4d1a1299fd584397f6ee4393d0e16744ed11aab1",
+        digest = "sha256:45ff0162921e61c004010a67db1bee7d039a677bed0cb294e61f2b47346d42b3",
         registry = "index.docker.io",
         repository = "library/ubuntu",
-        tag = "20.04",
+        tag = "20.10",
-    )
-    container_pull(
-        name = "jupyter",
-        registry = "index.docker.io",
-        repository = "jupyter/tensorflow-notebook",
-        tag = "latest",
     )

images/production/BUILD.bazel

@@ -13,7 +13,7 @@ download_pkgs(
         "libev4",
         "libgomp1",
         "libgoogle-perftools-dev",
-        "libprotobuf17",
+        "libprotobuf23",
         "libssl1.1",
     ],
 )

library/logging/__init__.py

@@ -31,7 +31,7 @@ def error_log(e, level=logging.ERROR, **fields):
         e = e.as_internal_dict()
         e.update(fields)
     elif fields:
-        e = {'error': str(e), **fields}
+        e = {'error': repr(e), **fields}
     logging.getLogger('error').log(
         msg=e,
         level=level

library/telegram/base.py

@@ -136,7 +136,7 @@ class RequestContext:
         self.request_id = request_id or RequestContext.generate_request_id(request_id_length)
         self.default_fields = {
             'bot_name': self.bot_name,
-            'chat_id': self.chat.id,
+            'chat_id': self.chat.chat_id,
             'request_id': self.request_id,
         }
 

nexus/bot/BUILD.bazel

@@ -30,7 +30,7 @@ py3_image(
         requirement("python_socks"),
         requirement("tenacity"),
         requirement("uvloop"),
-        "//idm/api2/aioclient",
+        "//idm/api/aioclient",
         requirement("aiobaseclient"),
         requirement("aiocrossref"),
         requirement("aiokit"),

nexus/bot/application.py

@@ -1,5 +1,5 @@
 from aiokit import AioRootThing
-from idm.api2.aioclient import IdmApi2GrpcClient
+from idm.api.aioclient import IdmApiGrpcClient
 from izihawa_utils.importlib import import_object
 from library.telegram.base import BaseTelegramClient
 from nexus.bot.promotioner import Promotioner
@@ -21,7 +21,7 @@ class TelegramApplication(AioRootThing):
         )
 
         self.hub_client = HubGrpcClient(base_url=self.config['hub']['url'])
-        self.idm_client = IdmApi2GrpcClient(base_url=self.config['idm']['url'])
+        self.idm_client = IdmApiGrpcClient(base_url=self.config['idm']['url'])
         self.meta_api_client = MetaApiGrpcClient(base_url=self.config['meta_api']['url'])
 
         self.promotioner = Promotioner(promotions=self.config['promotions'])

nexus/bot/handlers/base.py

@@ -6,7 +6,7 @@ from typing import Union
 
 from grpc import StatusCode
 from grpc.experimental.aio import AioRpcError
-from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from izihawa_utils.exceptions import BaseError
 from izihawa_utils.random import random_string
 from library.logging import error_log
@@ -40,12 +40,12 @@ def get_language(event: events.ChatAction, chat):
     return chat.lang_code
 
 
-def is_banned(chat: Chat) -> bool:
+def is_banned(chat: ChatPb) -> bool:
     return chat.ban_until is not None and datetime.utcnow().timestamp() < chat.ban_until
 
 
-def is_subscribed(chat: Chat) -> bool:
+def is_subscribed(chat: ChatPb) -> bool:
-    return chat.is_subscribed or chat.id < 0 or chat.created_at > time.time() - 10 * 60
+    return chat.is_subscribed or chat.chat_id < 0 or chat.created_at > time.time() - 10 * 60
 
 
 class ReadOnlyModeError(BaseError):
@@ -97,7 +97,7 @@ class BaseHandler(ABC):
             session_id=session_id,
             position=position,
             request_id=request_context.request_id,
-            user_id=request_context.chat.id,
+            user_id=request_context.chat.chat_id,
         )
 
     async def resolve_scimag(
@@ -136,7 +136,7 @@ class BaseHandler(ABC):
             page_size=16,
             request_id=request_context.request_id,
             session_id=session_id,
-            user_id=request_context.chat.id,
+            user_id=request_context.chat.chat_id,
         )
         duplicates = [
             scored_document.typed_document.scitech
@@ -209,7 +209,7 @@ class BaseHandler(ABC):
             )
             return chat
 
-    async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
+    async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: ChatPb):
         if is_banned(chat):
             if chat.ban_message is not None:
                 async with safe_execution(
@@ -245,7 +245,7 @@ class BaseHandler(ABC):
             )
             raise events.StopPropagation()
 
-    async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
+    async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: ChatPb):
         if (
             self.application.config['application']['is_subscription_required']
             and self.is_subscription_required_for_handler
@@ -261,7 +261,7 @@ class BaseHandler(ABC):
                 ).format(related_channel=self.application.config['telegram']['related_channel']))
             raise events.StopPropagation()
 
-    def _has_access(self, chat: Chat) -> bool:
+    def _has_access(self, chat: ChatPb) -> bool:
         return True
 
     async def _process_chat(self, event: events.ChatAction, request_id: str):
@@ -271,8 +271,8 @@ class BaseHandler(ABC):
             error_log(e)
             event_chat = await event.get_chat()
             username = get_username(event, event_chat)
-            chat = Chat(
+            chat = ChatPb(
-                id=event.chat_id,
+                chat_id=event.chat_id,
                 is_system_messaging_enabled=True,
                 is_discovery_enabled=True,
                 language='en',
@@ -307,7 +307,7 @@ class BaseHandler(ABC):
         await self._check_ban(event=event, request_context=request_context, chat=chat)
 
         if self.should_reset_last_widget:
-            self.reset_last_widget(request_context.chat.id)
+            self.reset_last_widget(request_context.chat.chat_id)
 
         async with safe_execution(
             request_context=request_context,

nexus/bot/handlers/download.py

@@ -19,7 +19,7 @@ class DownloadHandler(BaseCallbackQueryHandler):
         document_id = int(event.pattern_match.group(3))
         position = int(event.pattern_match.group(4).decode())
 
-        self.application.user_manager.last_widget[request_context.chat.id] = None
+        self.application.user_manager.last_widget[request_context.chat.chat_id] = None
 
         request_context.add_default_fields(mode='download', session_id=session_id)
         request_context.statbox(action='get', query=str(document_id), position=position)
@@ -48,4 +48,4 @@ class DownloadHandler(BaseCallbackQueryHandler):
             )
         else:
             await remove_button(event, '⬇️', and_empty_too=True)
-            self.application.user_manager.last_widget[request_context.chat.id] = None
+            self.application.user_manager.last_widget[request_context.chat.chat_id] = None

nexus/bot/handlers/referencing_to.py

@@ -41,7 +41,7 @@ class ReferencingToHandler(BaseCallbackQueryHandler):
 
         serp, buttons = await document_list_widget.render()
         return await self.application.telegram_client.edit_message(
-            request_context.chat.id,
+            request_context.chat.chat_id,
             message_id,
             serp,
             buttons=buttons,

nexus/bot/handlers/roll.py

@@ -20,7 +20,7 @@ class RollHandler(BaseHandler):
             language=request_context.chat.language,
             session_id=session_id,
             request_id=request_context.request_id,
-            user_id=request_context.chat.id,
+            user_id=request_context.chat.chat_id,
         )
         scitech_view = await self.resolve_scitech(
             document_id=roll_response_pb.document_id,
@@ -35,7 +35,7 @@ class RollHandler(BaseHandler):
         )
         actions = [
             self.application.telegram_client.send_message(
-                request_context.chat.id,
+                request_context.chat.chat_id,
                 view,
                 buttons=buttons,
             ),

nexus/bot/handlers/search.py

@@ -52,7 +52,7 @@ class BaseSearchHandler(BaseHandler):
         except AioRpcError as e:
             actions = [
                 self.application.telegram_client.delete_messages(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     [message_id],
                 )
             ]
@@ -118,7 +118,7 @@ class BaseSearchHandler(BaseHandler):
             )
             return await asyncio.gather(
                 self.application.telegram_client.edit_message(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     message_id,
                     view,
                     buttons=buttons,
@@ -127,7 +127,7 @@ class BaseSearchHandler(BaseHandler):
 
         serp, buttons = await search_widget.render()
         return await self.application.telegram_client.edit_message(
-            request_context.chat.id,
+            request_context.chat.chat_id,
             message_id,
             serp,
             buttons=buttons,
@@ -147,7 +147,7 @@ class SearchHandler(BaseSearchHandler):
             'action': 'user_flood_ban',
             'mode': 'search',
             'ban_timeout_seconds': ban_timeout,
-            'chat_id': request_context.chat.id,
+            'chat_id': request_context.chat.chat_id,
         })
         ban_reason = t(
             'BAN_MESSAGE_TOO_MANY_REQUESTS',
@@ -162,10 +162,10 @@ class SearchHandler(BaseSearchHandler):
         ))
 
     async def handler(self, event: events.ChatAction, request_context: RequestContext):
-        ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.id)
+        ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.chat_id)
         if ban_timeout:
             return await self.ban_handler(event, request_context, ban_timeout)
-        self.application.user_manager.add_search_time(user_id=request_context.chat.id, search_time=time.time())
+        self.application.user_manager.add_search_time(user_id=request_context.chat.chat_id, search_time=time.time())
 
         search_prefix = event.pattern_match.group(1)
         query = event.pattern_match.group(2)
@@ -178,7 +178,7 @@ class SearchHandler(BaseSearchHandler):
         prefetch_message = await event.reply(
             t("SEARCHING", language=request_context.chat.language),
         )
-        self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
+        self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
         try:
             await self.do_search(
                 event, request_context, prefetch_message,

nexus/bot/handlers/start.py

@@ -32,7 +32,7 @@ class StartHandler(BaseSearchHandler):
             prefetch_message = await request_message.reply(
                 t("SEARCHING", language=request_context.chat.language),
             )
-            self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
+            self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
             await asyncio.gather(
                 event.delete(),
                 self.do_search(event, request_context, prefetch_message, query=query,

nexus/bot/handlers/view.py

@@ -30,7 +30,7 @@ class ViewHandler(BaseHandler):
 
         request_context.add_default_fields(mode='view', session_id=session_id)
         request_context.statbox(action='view', query=str(document_id), position=position)
-        found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.id)
+        found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id)
 
         try:
             if found_old_widget:
@@ -41,11 +41,11 @@ class ViewHandler(BaseHandler):
                     functions.messages.GetMessagesRequest(id=[old_message_id])
                 )).messages[0]
                 prefetch_message = await self.application.telegram_client.send_message(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     t("SEARCHING", language=request_context.chat.language),
                     reply_to=old_message.reply_to_msg_id,
                 )
-                self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
+                self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
                 message_id = prefetch_message.id
                 link_preview = True
 
@@ -81,7 +81,7 @@ class ViewHandler(BaseHandler):
             )
             actions = [
                 self.application.telegram_client.edit_message(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     message_id,
                     view,
                     buttons=buttons,
@@ -92,7 +92,7 @@ class ViewHandler(BaseHandler):
             if not found_old_widget:
                 actions.append(
                     self.application.telegram_client.delete_messages(
-                        request_context.chat.id,
+                        request_context.chat.chat_id,
                         [old_message_id],
                     )
                 )

nexus/bot/handlers/vote.py

@@ -26,7 +26,7 @@ class VoteHandler(BaseCallbackQueryHandler):
             vote=VotePb(
                 document_id=document_id,
                 value=vote_value,
-                voter_id=request_context.chat.id,
+                voter_id=request_context.chat.chat_id,
             ),
         )
 
@@ -43,7 +43,7 @@ class VoteHandler(BaseCallbackQueryHandler):
         # ToDo: Generalize nexus.views.telegram.common.remove_button and use it here
         return await asyncio.gather(
             self.application.telegram_client.edit_message(
-                request_context.chat.id,
+                request_context.chat.chat_id,
                 message.id,
                 message.text,
                 buttons=None,

nexus/bot/widgets/admin_widget.py

@@ -1,9 +1,9 @@
-from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
 
 
 class AdminWidget:
-    def __init__(self, application: TelegramApplication, chat: Chat):
+    def __init__(self, application: TelegramApplication, chat: ChatPb):
         self.application = application
         self.chat = chat
 

nexus/bot/widgets/banlist_widget.py

@@ -1,25 +1,25 @@
 import time
 
-from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
 
 
 class BanlistWidget:
-    def __init__(self, application: TelegramApplication, chat: Chat):
+    def __init__(self, application: TelegramApplication, chat: ChatPb):
         self.application = application
         self.chat = chat
 
-    async def render(self, chat_list: list[Chat]):
+    async def render(self, chat_list: list[ChatPb]):
         if not chat_list:
             return 'Nobody is banned'
         separator = '------------\n'
         return separator.join(
             map(
                 lambda chat: (
-                    f'```{chat.username} ({chat.id})\n'
+                    f'```{chat.username} ({chat.chat_id})\n'
                     f'Until: {time.ctime(chat.ban_until)}\n'
                     f'Message: {chat.ban_message}```\n'
-                    f'/unban_{chat.id}\n'
+                    f'/unban_{chat.chat_id}\n'
                 ),
                 chat_list
             )

nexus/bot/widgets/document_list_widget.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
 from nexus.meta_api.proto.meta_search_service_pb2 import \
     ScoredDocument as ScoredDocumentPb
@@ -14,7 +14,7 @@ class DocumentListWidget:
     def __init__(
         self,
         application: TelegramApplication,
-        chat: Chat,
+        chat: ChatPb,
         session_id: str,
         message_id: int,
         request_id: str,
@@ -32,7 +32,7 @@ class DocumentListWidget:
     @staticmethod
     async def create(
         application: TelegramApplication,
-        chat: Chat,
+        chat: ChatPb,
         session_id: str,
         message_id: int,
         request_id: str,
@@ -58,7 +58,7 @@ class DocumentListWidget:
             position=0,
             request_id=self.request_id,
             session_id=self.session_id,
-            user_id=self.chat.id,
+            user_id=self.chat.chat_id,
         )
         self._response = await self.application.meta_api_client.search(
             schemas=('scimag',),
@@ -66,7 +66,7 @@ class DocumentListWidget:
             page=self.page,
             request_id=self.request_id,
             session_id=self.session_id,
-            user_id=self.chat.id,
+            user_id=self.chat.chat_id,
         )
 
     @property

nexus/bot/widgets/search_widget.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
 from nexus.meta_api.proto.meta_search_service_pb2 import \
     ScoredDocument as ScoredDocumentPb
@@ -21,7 +21,7 @@ class SearchWidget:
     def __init__(
         self,
         application: TelegramApplication,
-        chat: Chat,
+        chat: ChatPb,
         session_id: str,
         message_id: int,
         request_id: str,
@@ -41,7 +41,7 @@ class SearchWidget:
     @staticmethod
     async def create(
         application: TelegramApplication,
-        chat: Chat,
+        chat: ChatPb,
         session_id: str,
         message_id: int,
         request_id: str,
@@ -70,7 +70,7 @@ class SearchWidget:
             page_size=self.application.config['application']['page_size'],
             request_id=self.request_id,
             session_id=self.session_id,
-            user_id=self.chat.id,
+            user_id=self.chat.chat_id,
             language=self.chat.language,
         )
 

nexus/bot/widgets/settings_widget.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from idm.api2.proto.chats_service_pb2 import ChatData as Chat
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
 from nexus.translations import t
 from telethon import Button
@@ -34,7 +34,7 @@ class SettingsWidget:
     def __init__(
         self,
         application: TelegramApplication,
-        chat: Chat,
+        chat: ChatPb,
         has_language_buttons: Optional[bool] = None,
         is_group_mode: bool = False,
         request_id: Optional[str] = None,
@@ -54,7 +54,7 @@ class SettingsWidget:
 
     async def _switch_language(self, target_language: str):
         self.chat = await self.application.idm_client.update_chat(
-            chat_id=self.chat.id,
+            chat_id=self.chat.chat_id,
             language=target_language,
             request_id=self.request_id,
         )
@@ -62,7 +62,7 @@ class SettingsWidget:
 
     async def _switch_system_messaging(self, is_system_messaging_enabled: str):
         self.chat = await self.application.idm_client.update_chat(
-            chat_id=self.chat.id,
+            chat_id=self.chat.chat_id,
             is_system_messaging_enabled=bool(int(is_system_messaging_enabled)),
             request_id=self.request_id,
         )
@@ -70,7 +70,7 @@ class SettingsWidget:
 
     async def _switch_discovery(self, is_discovery_enabled: str):
         self.chat = await self.application.idm_client.update_chat(
-            chat_id=self.chat.id,
+            chat_id=self.chat.chat_id,
             is_discovery_enabled=bool(int(is_discovery_enabled)),
             request_id=self.request_id,
         )

nexus/hub/BUILD.bazel

@@ -17,10 +17,7 @@ py3_image(
     base = "//images/production:base-python-image",
     data = [
         "configs/base.yaml",
-        "configs/development.yaml",
         "configs/logging.yaml",
-        "configs/production.yaml",
-        "configs/testing.yaml",
     ],
     main = "main.py",
     srcs_version = "PY3ONLY",
@@ -37,7 +34,7 @@ py3_image(
         requirement("python-socks"),
         requirement("tenacity"),
         requirement("uvloop"),
-        "//idm/api2/proto:idm_proto_py",
+        "//idm/api/proto:idm_proto_py",
         requirement("giogrobid"),
         "//library/aiogrpctools",
         requirement("aiokit"),

nexus/hub/README.md

@@ -1,66 +1,9 @@
 # Nexus Search: Hub API
 
-`Hub` is a daemon responsible for retrieving files and sending them to users. This version has cut `configs`
+`Hub` is a daemon responsible for retrieving files and sending them to users.
-subdirectory due to hard reliance of configs on the network infrastructure you are using.
-You have to write your own configs taking example below into account.
 
 The bot requires two other essential parts: 
 - Postgres Database
 - IPFS Daemon
 
 or their substitutions
-
-## Sample `configs/base.yaml`
-
-```yaml
----
-
-application:
-  # Look at the special Postgres `sharience` table to retrieve user-sent files
-  is_sharience_enabled: true
-  maintenance_picture_url:
-  # Used in logging
-  service_name: nexus-hub
-  # Store file hashes into operation log
-  should_store_hashes: true
-database:
-  database: nexus
-  host:
-  password: '{{ DATABASE_PASSWORD }}'
-  username: '{{ DATABASE_USERNAME }}'
-grobid:
-  url:
-grpc:
-  # Listen address
-  address: 0.0.0.0
-  # Listen port
-  port: 9090
-ipfs:
-  address:
-  port: 4001
-log_path: '/var/log/nexus-hub/{{ ENV_TYPE }}'
-meta_api:
-  url:
-pylon:
-  # Proxy used in `pylon` retriever to download files
-  proxy: socks5://127.0.0.1:9050
-  # Proxy used in `pylon` retriever to get metadata
-  resolve_proxy: socks5://127.0.0.1:9050
-telegram:
-  # Telegram App Hash from https://my.telegram.org/
-  app_hash: '{{ APP_HASH }}'
-  # Telegram App ID from https://my.telegram.org/
-  app_id: 00000
-  # External bot name shown in messages to users
-  bot_external_name: libgen_scihub_bot
-  # Internal bot name used in logging
-  bot_name: nexus-bot
-  bot_token: '{{ BOT_TOKEN }}'
-  # Telethon database for keeping cache
-  database:
-    session_id: nexus-hub
-  # Frequency of updating downloading progress
-  progress_throttle_seconds: 5
-  # Send files using stored telegram_file_id
-  should_use_telegram_file_id: true
-```

nexus/hub/aioclient/BUILD.bazel

@@ -6,7 +6,7 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         requirement("grpcio"),
-        "//idm/api2/proto:idm_proto_py",
+        "//idm/api/proto:idm_proto_py",
         requirement("aiokit"),
         "//nexus/hub/proto:hub_grpc_py",
         "//nexus/hub/proto:hub_proto_py",

nexus/hub/aioclient/aioclient.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from aiokit import AioThing
 from grpc.experimental.aio import insecure_channel
-from idm.api2.proto.chats_service_pb2 import ChatData as ChatDataPb
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.hub.proto.delivery_service_pb2 import \
     StartDeliveryRequest as StartDeliveryRequestPb
 from nexus.hub.proto.delivery_service_pb2 import \
@@ -42,7 +42,7 @@ class HubGrpcClient(AioThing):
     async def start_delivery(
         self,
         typed_document_pb: TypedDocumentPb,
-        chat: ChatDataPb,
+        chat: ChatPb,
         request_id: Optional[str],
         session_id: Optional[str],
     ) -> StartDeliveryResponsePb:
@@ -58,7 +58,7 @@ class HubGrpcClient(AioThing):
         self,
         telegram_document: bytes,
         telegram_file_id: str,
-        chat: ChatDataPb,
+        chat: ChatPb,
         request_id: Optional[str] = None,
         session_id: Optional[str] = None,
     ) -> SubmitResponsePb:

nexus/hub/configs/base.yaml

@@ -0,0 +1,51 @@
+---
+
+application:
+  # Enable special Postgres `sharience` table to retrieve user-sent files
+  is_sharience_enabled: true
+  # URL to the picture shown while maintenance
+  maintenance_picture_url:
+  # Used in logging
+  service_name: nexus-hub
+  # Store file hashes into operation log
+  should_store_hashes: true
+database:
+  database: nexus
+  host:
+  password: '{{ DATABASE_PASSWORD }}'
+  username: '{{ DATABASE_USERNAME }}'
+grobid:
+  url:
+grpc:
+  # Listen address
+  address: 0.0.0.0
+  # Listen port
+  port: 9090
+ipfs:
+  address:
+  port: 4001
+log_path: '/var/log/nexus-hub/{{ ENV_TYPE }}'
+meta_api:
+  url:
+pylon:
+  # Proxy used in `pylon` retriever to download files
+  proxy: socks5://127.0.0.1:9050
+  # Proxy used in `pylon` retriever to get metadata
+  resolve_proxy: socks5://127.0.0.1:9050
+telegram:
+  # Telegram App Hash from https://my.telegram.org/
+  app_hash: '{{ APP_HASH }}'
+  # Telegram App ID from https://my.telegram.org/
+  app_id: 00000
+  # External bot name shown in messages to users
+  bot_external_name: libgen_scihub_bot
+  # Internal bot name used in logging
+  bot_name: nexus-hub
+  bot_token: '{{ BOT_TOKEN }}'
+  # Telethon database for keeping cache
+  database:
+    session_id: nexus-hub
+  # Frequency of updating downloading progress
+  progress_throttle_seconds: 5
+  # Send files using stored telegram_file_id
+  should_use_telegram_file_id: true

nexus/hub/configs/logging.yaml

@@ -0,0 +1,76 @@
+---
+
+logging:
+  disable_existing_loggers: false
+  formatters:
+    base:
+      class: library.logging.formatters.BaseFormatter
+    default:
+      class: library.logging.formatters.DefaultFormatter
+    traceback:
+      class: library.logging.formatters.TracebackFormatter
+  handlers:
+    debug:
+      class: logging.StreamHandler
+      formatter: default
+      level: DEBUG
+      stream: 'ext://sys.stderr'
+    error:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/error.log'
+      formatter: default
+      level: ERROR
+    operation:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/operation.log'
+      formatter: base
+      level: DEBUG
+    statbox:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/statbox.log'
+      formatter: default
+      level: INFO
+    traceback:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/traceback.log'
+      formatter: traceback
+      level: ERROR
+    warning:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/warning.log'
+      formatter: default
+      level: WARNING
+  loggers:
+    aiokafka:
+      handlers:
+        - error
+      propagate: false
+    debug:
+      handlers:
+        - debug
+      propagate: false
+    error:
+      handlers:
+        - traceback
+        - error
+        - warning
+      propagate: false
+    operation:
+      handlers:
+        - operation
+      propagate: false
+    statbox:
+      handlers:
+        - statbox
+      propagate: false
+    telethon:
+      handlers:
+        - error
+      propagate: false
+  root:
+    handlers:
+      - debug
+      - error
+      - warning
+    level: DEBUG
+  version: 1

nexus/hub/proto/BUILD.bazel

@@ -9,7 +9,7 @@ proto_library(
         "*.proto",
     ]),
     deps = [
-        "//idm/api2/proto:idm_proto",
+        "//idm/api/proto:idm_proto",
         "//nexus/models/proto:models_proto",
         "@com_google_protobuf//:wrappers_proto",
     ],

nexus/hub/proto/delivery_service.proto

@@ -2,11 +2,11 @@ syntax = "proto3";
 package nexus.hub.proto;
 
 import "nexus/models/proto/typed_document.proto";
-import "idm/api2/proto/chats_service.proto";
+import "idm/api/proto/chat_manager_service.proto";
 
 message StartDeliveryRequest {
   nexus.models.proto.TypedDocument typed_document = 1;
-  idm.api2.proto.ChatData chat = 2;
+  idm.api.proto.Chat chat = 2;
 }
 
 message StartDeliveryResponse {

nexus/hub/proto/delivery_service_pb2.py

@@ -12,8 +12,9 @@ from google.protobuf import symbol_database as _symbol_database
 _sym_db = _symbol_database.Default()
 
 
-from idm.api2.proto import \
+from idm.api.proto import \
-    chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
+    chat_manager_service_pb2 as \
+    idm_dot_api_dot_proto_dot_chat__manager__service__pb2
 from nexus.models.proto import \
     typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
 
@@ -23,9 +24,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a\"idm/api2/proto/chats_service.proto\"y\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x03\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3'
+  serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a(idm/api/proto/chat_manager_service.proto\"t\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12!\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x13.idm.api.proto.Chat\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x02\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3'
   ,
-  dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
+  dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api_dot_proto_dot_chat__manager__service__pb2.DESCRIPTOR,])
 
 
 
@@ -47,15 +48,15 @@ _STARTDELIVERYRESPONSE_STATUS = _descriptor.EnumDescriptor(
       type=None,
       create_key=_descriptor._internal_create_key),
     _descriptor.EnumValueDescriptor(
-      name='ALREADY_DOWNLOADING', index=2, number=3,
+      name='ALREADY_DOWNLOADING', index=2, number=2,
       serialized_options=None,
       type=None,
       create_key=_descriptor._internal_create_key),
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=348,
+  serialized_start=349,
-  serialized_end=413,
+  serialized_end=414,
 )
 _sym_db.RegisterEnumDescriptor(_STARTDELIVERYRESPONSE_STATUS)
 
@@ -94,8 +95,8 @@ _STARTDELIVERYREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=136,
+  serialized_start=142,
-  serialized_end=257,
+  serialized_end=258,
 )
 
 
@@ -127,12 +128,12 @@ _STARTDELIVERYRESPONSE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=260,
+  serialized_start=261,
-  serialized_end=413,
+  serialized_end=414,
 )
 
 _STARTDELIVERYREQUEST.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
-_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
+_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api_dot_proto_dot_chat__manager__service__pb2._CHAT
 _STARTDELIVERYRESPONSE.fields_by_name['status'].enum_type = _STARTDELIVERYRESPONSE_STATUS
 _STARTDELIVERYRESPONSE_STATUS.containing_type = _STARTDELIVERYRESPONSE
 DESCRIPTOR.message_types_by_name['StartDeliveryRequest'] = _STARTDELIVERYREQUEST
@@ -162,8 +163,8 @@ _DELIVERY = _descriptor.ServiceDescriptor(
   index=0,
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_start=415,
+  serialized_start=416,
-  serialized_end=524,
+  serialized_end=525,
   methods=[
   _descriptor.MethodDescriptor(
     name='start_delivery',

nexus/hub/proto/submitter_service.proto

@@ -1,12 +1,12 @@
 syntax = "proto3";
 package nexus.hub.proto;
 
-import "idm/api2/proto/chats_service.proto";
+import "idm/api/proto/chat_manager_service.proto";
 
 message SubmitRequest {
     bytes telegram_document = 1;
     string telegram_file_id = 2;
-    idm.api2.proto.ChatData chat = 3;
+    idm.api.proto.Chat chat = 3;
 }
 message SubmitResponse { }
 

nexus/hub/proto/submitter_service_pb2.py

@@ -12,8 +12,9 @@ from google.protobuf import symbol_database as _symbol_database
 _sym_db = _symbol_database.Default()
 
 
-from idm.api2.proto import \
+from idm.api.proto import \
-    chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
+    chat_manager_service_pb2 as \
+    idm_dot_api_dot_proto_dot_chat__manager__service__pb2
 
 DESCRIPTOR = _descriptor.FileDescriptor(
   name='nexus/hub/proto/submitter_service.proto',
@@ -21,9 +22,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a\"idm/api2/proto/chats_service.proto\"R\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3'
+  serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a(idm/api/proto/chat_manager_service.proto\"g\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12\x18\n\x10telegram_file_id\x18\x02 \x01(\t\x12!\n\x04\x63hat\x18\x03 \x01(\x0b\x32\x13.idm.api.proto.Chat\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3'
   ,
-  dependencies=[idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
+  dependencies=[idm_dot_api_dot_proto_dot_chat__manager__service__pb2.DESCRIPTOR,])
 
 
 
@@ -44,8 +45,15 @@ _SUBMITREQUEST = _descriptor.Descriptor(
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=1,
+      name='telegram_file_id', full_name='nexus.hub.proto.SubmitRequest.telegram_file_id', index=1,
-      number=2, type=11, cpp_type=10, label=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=2,
+      number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
@@ -62,8 +70,8 @@ _SUBMITREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=96,
+  serialized_start=102,
-  serialized_end=178,
+  serialized_end=205,
 )
 
 
@@ -87,11 +95,11 @@ _SUBMITRESPONSE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=180,
+  serialized_start=207,
-  serialized_end=196,
+  serialized_end=223,
 )
 
-_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
+_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api_dot_proto_dot_chat__manager__service__pb2._CHAT
 DESCRIPTOR.message_types_by_name['SubmitRequest'] = _SUBMITREQUEST
 DESCRIPTOR.message_types_by_name['SubmitResponse'] = _SUBMITRESPONSE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -119,8 +127,8 @@ _SUBMITTER = _descriptor.ServiceDescriptor(
   index=0,
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_start=198,
+  serialized_start=225,
-  serialized_end=286,
+  serialized_end=313,
   methods=[
   _descriptor.MethodDescriptor(
     name='submit',

nexus/hub/services/base.py

@@ -70,7 +70,7 @@ class BaseHubService(BaseService):
             buttons=buttons,
             caption=f"{document_view.generate_body(language=request_context.chat.language, limit=512)}\n"
                     f"@{self.bot_external_name}",
-            entity=request_context.chat.id,
+            entity=request_context.chat.chat_id,
             file=file,
             progress_callback=progress_callback
         )

nexus/hub/services/delivery.py

@@ -76,7 +76,7 @@ class DownloadTask:
             )
         )
 
-        self.delivery_service.user_manager.add_task(self.request_context.chat.id, self.document_view.id)
+        self.delivery_service.user_manager.add_task(self.request_context.chat.chat_id, self.document_view.id)
         self.delivery_service.downloadings.add(self)
 
         self.task.add_done_callback(self.done_callback)
@@ -84,7 +84,7 @@ class DownloadTask:
     def done_callback(self, f):
         self.delivery_service.downloadings.remove(self)
         self.delivery_service.user_manager.remove_task(
-            self.request_context.chat.id,
+            self.request_context.chat.chat_id,
             self.document_view.id,
         )
 
@@ -93,7 +93,7 @@ class DownloadTask:
 
         async def _on_fail():
             await self.delivery_service.telegram_client.send_message(
-                request_context.chat.id,
+                request_context.chat.chat_id,
                 t('MAINTENANCE', language=request_context.chat.language).format(
                     maintenance_picture_url=self.delivery_service.maintenance_picture_url
                 ),
@@ -165,7 +165,7 @@ class DownloadTask:
                     progress_callback=progress_bar_upload.callback,
                     request_context=self.request_context,
                     session_id=self.session_id,
-                    voting=not is_group_or_channel(self.request_context.chat.id),
+                    voting=not is_group_or_channel(self.request_context.chat.chat_id),
                 )
                 request_context.statbox(
                     action='uploaded',
@@ -191,7 +191,7 @@ class DownloadTask:
             finally:
                 downloads_gauge.dec()
                 messages = filter_none([progress_bar_download.message])
-                await self.delivery_service.telegram_client.delete_messages(request_context.chat.id, messages)
+                await self.delivery_service.telegram_client.delete_messages(request_context.chat.chat_id, messages)
 
     async def process_resp(self, resp, progress_bar, collected, filesize):
         progress_bar.set_source(get_fancy_name(resp.source))
@@ -206,7 +206,7 @@ class DownloadTask:
 
     async def respond_not_found(self, request_context: RequestContext, document_view):
         return await self.delivery_service.telegram_client.send_message(
-            request_context.chat.id,
+            request_context.chat.chat_id,
             t("SOURCES_UNAVAILABLE", language=request_context.chat.language).format(
                 document=document_view.get_robust_title()
             ),
@@ -244,6 +244,7 @@ class DownloadTask:
                 async for resp in self.delivery_service.pylon_client.by_doi(
                     doi=document_view.doi,
                     md5=document_view.md5,
+                    error_log_func=self.request_context.error_log,
                 ):
                     await self.process_resp(
                         resp=resp,
@@ -252,11 +253,14 @@ class DownloadTask:
                         filesize=document_view.filesize,
                     )
                 return bytes(collected)
-            except DownloadError:
+            except DownloadError as e:
-                pass
+                self.request_context.error_log(e)
         if document_view.md5:
             try:
-                async for resp in self.delivery_service.pylon_client.by_md5(md5=document_view.md5):
+                async for resp in self.delivery_service.pylon_client.by_md5(
+                    md5=document_view.md5,
+                    error_log_func=self.request_context.error_log,
+                ):
                     await self.process_resp(
                         resp=resp,
                         progress_bar=progress_bar,
@@ -264,14 +268,14 @@ class DownloadTask:
                         filesize=document_view.filesize,
                     )
                 return bytes(collected)
-            except DownloadError:
+            except DownloadError as e:
-                pass
+                self.request_context.error_log(e)
 
     async def external_cancel(self):
         self.task.cancel()
         self.request_context.statbox(action='externally_canceled')
         await self.delivery_service.telegram_client.send_message(
-            self.request_context.chat.id,
+            self.request_context.chat.chat_id,
             t("DOWNLOAD_CANCELED", language=self.request_context.chat.language).format(
                 document=self.document_view.get_robust_title()
             ),
@@ -366,15 +370,15 @@ class DeliveryService(DeliveryServicer, BaseHubService):
                     file=document_view.telegram_file_id,
                     session_id=metadata.get('session-id'),
                     request_context=request_context,
-                    voting=not is_group_or_channel(request_context.chat.id),
+                    voting=not is_group_or_channel(request_context.chat.chat_id),
                 )
                 request_context.statbox(action='cache_hit', document_id=document_view.id)
             except ValueError:
                 cache_hit = False
         if not cache_hit:
-            if self.user_manager.has_task(request.chat.id, document_view.id):
+            if self.user_manager.has_task(request.chat.chat_id, document_view.id):
                 return StartDeliveryResponsePb(status=StartDeliveryResponsePb.Status.ALREADY_DOWNLOADING)
-            if self.user_manager.hit_limits(request.chat.id):
+            if self.user_manager.hit_limits(request.chat.chat_id):
                 return StartDeliveryResponsePb(status=StartDeliveryResponsePb.Status.TOO_MANY_DOWNLOADS)
             await DownloadTask(
                 delivery_service=self,

nexus/hub/services/submitter.py

@@ -101,13 +101,13 @@ class SubmitterService(SubmitterServicer, BaseHubService):
         if document.size > 20 * 1024 * 1024:
             request_context.error_log(FileTooBigError(size=document.size))
             await self.telegram_client.send_message(
-                request_context.chat.id,
+                request_context.chat.chat_id,
                 t('FILE_TOO_BIG_ERROR', language=request_context.chat.language),
                 buttons=[close_button()],
             )
             return SubmitResponsePb()
         processing_message = await self.telegram_client.send_message(
-            request_context.chat.id,
+            request_context.chat.chat_id,
             t("PROCESSING_PAPER", language=request_context.chat.language).format(
                 filename=document.attributes[0].file_name,
             ),
@@ -121,7 +121,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
             except BadRequestError as e:
                 request_context.error_log(e)
                 await self.telegram_client.send_message(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     t('UNPARSABLE_DOCUMENT_ERROR', language=request_context.chat.language),
                     buttons=[close_button()],
                 )
@@ -130,7 +130,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
             if not processed_document.get('doi'):
                 request_context.error_log(UnparsableDoiError())
                 await self.telegram_client.send_message(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     t('UNPARSABLE_DOI_ERROR', language=request_context.chat.language),
                     buttons=[close_button()],
                 )
@@ -143,14 +143,14 @@ class SubmitterService(SubmitterServicer, BaseHubService):
                 page_size=1,
                 request_id=request_context.request_id,
                 session_id=session_id,
-                user_id=request_context.chat.id,
+                user_id=request_context.chat.chat_id,
                 language=request_context.chat.language,
             )
 
             if len(search_response_pb.scored_documents) == 0:
                 request_context.error_log(UnavailableMetadataError(doi=processed_document['doi']))
                 await self.telegram_client.send_message(
-                    request_context.chat.id,
+                    request_context.chat.chat_id,
                     t(
                         'UNAVAILABLE_METADATA_ERROR',
                         language=request_context.chat.language
@@ -175,7 +175,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
             update_document=UpdateDocumentPb(
                 typed_document=TypedDocumentPb(sharience=ShariencePb(
                     parent_id=document_view.id,
-                    uploader_id=request_context.chat.id,
+                    uploader_id=request_context.chat.chat_id,
                     updated_at=int(time.time()),
                     md5=hashlib.md5(file).hexdigest(),
                     filesize=document.size,

nexus/pipe/README.md

@@ -1,99 +1,3 @@
 # Nexus Pipe
 
-`Pipe` processes Kafka queue of operations. This version has cut `configs`
+`Pipe` processes Kafka queue of operations.
-subdirectory due to hard reliance of configs on the network infrastructure you are using.
-You have to write your own configs taking example below into account.
-
-## Sample `configs/base.yaml`
-
-```yaml
----
-log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
-pipe:
-  brokers: |
-    kafka-0.example.net
-  schema:
-    - consumers:
-        - class: nexus.pipe.consumers.CrossReferencesBulkConsumer
-          topics:
-            - name: cross_references
-              workers: 4
-      group_id: pipe
-      processors:
-        - class: nexus.pipe.processors.CrossReferencesProcessor
-          kwargs:
-            brokers: |
-              kafka-0.example.net
-            database:
-              database: nexus
-              host: postgres.example.net
-              password: '{{ DATABASE_PASSWORD }}'
-              username: '{{ DATABASE_USERNAME }}'
-    - consumers:
-        - class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
-          topics:
-            - name: operations
-              workers: 2
-        - class: nexus.pipe.consumers.DocumentOperationsConsumer
-          topics:
-            - name: operations_binary_hp
-              workers: 4
-            - name: operations_binary
-              workers: 14
-      group_id: pipe
-      processors:
-        - class: nexus.pipe.processors.ActionProcessor
-          kwargs:
-            actions:
-              - class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
-                kwargs:
-                  crossref:
-                    rps: 50
-                    user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
-              - class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
-              - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
-                kwargs:
-                  database:
-                    database: nexus
-                    host: postgres.example.net
-                    password: '{{ DATABASE_PASSWORD }}'
-                    username: '{{ DATABASE_USERNAME }}'
-              - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
-                kwargs:
-                  brokers: |
-                    kafka-0.example.net
-                  topic: cross_references
-              - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
-                kwargs:
-                  summa:
-                    base_url: http://summa.example.net
-                    timeout: 15
-                    ttl_dns_cache: 30
-            filter:
-              class: nexus.pipe.filters.DocumentOperationFilter
-              kwargs:
-                document: scimag
-                operation: update_document
-        - class: nexus.pipe.processors.ActionProcessor
-          kwargs:
-            actions:
-              - class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
-              - class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
-                kwargs:
-                  database:
-                    database: nexus
-                    host: postgres.example.net
-                    password: '{{ DATABASE_PASSWORD }}'
-                    username: '{{ DATABASE_USERNAME }}'
-              - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
-                kwargs:
-                  summa:
-                    base_url: http://summa.example.net
-                    timeout: 15
-                    ttl_dns_cache: 30
-            filter:
-              class: nexus.pipe.filters.DocumentOperationFilter
-              kwargs:
-                document: scitech
-                operation: update_document
-```

nexus/pipe/configs/base.yaml

@@ -0,0 +1,90 @@
+---
+
+log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
+pipe:
+  brokers: |
+    kafka-0.example.net
+  schema:
+    - consumers:
+        - class: nexus.pipe.consumers.CrossReferencesBulkConsumer
+          topics:
+            - name: cross_references
+              workers: 4
+      group_id: pipe
+      processors:
+        - class: nexus.pipe.processors.CrossReferencesProcessor
+          kwargs:
+            brokers: |
+              kafka-0.example.net
+            database:
+              database: nexus
+              host: postgres.example.net
+              password: '{{ DATABASE_PASSWORD }}'
+              username: '{{ DATABASE_USERNAME }}'
+    - consumers:
+        - class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
+          topics:
+            - name: operations
+              workers: 2
+        - class: nexus.pipe.consumers.DocumentOperationsConsumer
+          topics:
+            - name: operations_binary_hp
+              workers: 4
+            - name: operations_binary
+              workers: 14
+      group_id: pipe
+      processors:
+        - class: nexus.pipe.processors.ActionProcessor
+          kwargs:
+            actions:
+              - class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
+                kwargs:
+                  crossref:
+                    rps: 50
+                    user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
+              - class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
+              - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
+                kwargs:
+                  database:
+                    database: nexus
+                    host: postgres.example.net
+                    password: '{{ DATABASE_PASSWORD }}'
+                    username: '{{ DATABASE_USERNAME }}'
+              - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
+                kwargs:
+                  brokers: |
+                    kafka-0.example.net
+                  topic: cross_references
+              - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
+                kwargs:
+                  summa:
+                    base_url: http://summa.example.net
+                    timeout: 15
+                    ttl_dns_cache: 30
+            filter:
+              class: nexus.pipe.filters.DocumentOperationFilter
+              kwargs:
+                document: scimag
+                operation: update_document
+        - class: nexus.pipe.processors.ActionProcessor
+          kwargs:
+            actions:
+              - class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
+              - class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
+                kwargs:
+                  database:
+                    database: nexus
+                    host: postgres.example.net
+                    password: '{{ DATABASE_PASSWORD }}'
+                    username: '{{ DATABASE_USERNAME }}'
+              - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
+                kwargs:
+                  summa:
+                    base_url: http://summa.example.net
+                    timeout: 15
+                    ttl_dns_cache: 30
+            filter:
+              class: nexus.pipe.filters.DocumentOperationFilter
+              kwargs:
+                document: scitech
+                operation: update_document

nexus/pipe/configs/logging.yaml

@@ -0,0 +1,65 @@
+---
+
+logging:
+  disable_existing_loggers: false
+  formatters:
+    default:
+      class: library.logging.formatters.DefaultFormatter
+    traceback:
+      class: library.logging.formatters.TracebackFormatter
+  handlers:
+    debug:
+      class: logging.StreamHandler
+      formatter: default
+      level: DEBUG
+      stream: 'ext://sys.stderr'
+    error:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/error.log'
+      formatter: default
+      level: ERROR
+    statbox:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/statbox.log'
+      formatter: default
+      level: INFO
+    traceback:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/traceback.log'
+      formatter: traceback
+      level: ERROR
+    warning:
+      class: library.logging.handlers.BaseFileHandler
+      filename: '{{ log_path }}/warning.log'
+      formatter: default
+      level: WARNING
+  loggers:
+    aiokafka:
+      handlers:
+        - error
+      propagate: false
+    asyncio:
+      handlers:
+        - error
+      level: WARNING
+    debug:
+      handlers:
+        - debug
+      propagate: false
+    error:
+      handlers:
+        - error
+        - traceback
+        - warning
+      propagate: false
+    statbox:
+      handlers:
+        - statbox
+      propagate: false
+  root:
+    handlers:
+      - debug
+      - error
+      - warning
+    level: DEBUG
+  version: 1

nexus/pylon/client.py

@@ -1,16 +1,10 @@
-import asyncio
 from typing import (
     AsyncIterable,
+    Callable,
     Iterable,
     Optional,
 )
 
-import aiohttp
-import aiohttp.client_exceptions
-from aiohttp_socks import (
-    ProxyConnectionError,
-    ProxyError,
-)
 from aiokit import AioThing
 from library.logging import error_log
 from nexus.pylon.exceptions import (
@@ -27,7 +21,6 @@ from nexus.pylon.sources import (
     SciHubSeSource,
 )
 from nexus.pylon.sources.specific import get_specific_sources_for_doi
-from python_socks import ProxyTimeoutError
 
 
 class PylonClient(AioThing):
@@ -40,6 +33,7 @@ class PylonClient(AioThing):
         self,
         doi: str,
         md5: Optional[str] = None,
+        error_log_func: Callable = error_log,
     ) -> AsyncIterable[FileResponsePb]:
         sources = []
         sources.extend(get_specific_sources_for_doi(doi, proxy=self.proxy, resolve_proxy=self.resolve_proxy))
@@ -49,49 +43,42 @@ class PylonClient(AioThing):
             LibgenDoiSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
         ])
         sources = filter(lambda x: x.is_enabled, sources)
-        async for resp in self.download(sources=sources):
+        async for resp in self.download(sources=sources, error_log_func=error_log_func):
             yield resp
 
     async def by_md5(
         self,
         md5: str,
+        error_log_func: Callable = error_log,
     ) -> AsyncIterable[FileResponsePb]:
         sources = filter(lambda x: x.is_enabled, [
             LibraryLolSource(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
             LibgenMd5Source(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
         ])
-        async for resp in self.download(sources=sources):
+        async for resp in self.download(sources=sources, error_log_func=error_log_func):
             yield resp
 
-    async def download_source(self, source) -> AsyncIterable[FileResponsePb]:
+    async def download_source(self, source, error_log_func: Callable = error_log) -> AsyncIterable[FileResponsePb]:
-        try:
+        yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url)
-            yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url)
+        async for prepared_file_request in source.resolve(error_log_func=error_log_func):
-            async for prepared_request in source.resolve():
-                async for resp in source.execute_prepared_request(prepared_request=prepared_request):
-                    yield resp
-                return
-            raise DownloadError(error='not_found', source=str(source))
-        except (
-            aiohttp.client_exceptions.ClientConnectionError,
-            aiohttp.client_exceptions.ClientPayloadError,
-            aiohttp.client_exceptions.ClientResponseError,
-            aiohttp.client_exceptions.TooManyRedirects,
-            asyncio.TimeoutError,
-            ProxyConnectionError,
-            ProxyTimeoutError,
-            ProxyError,
-        ) as e:
-            raise DownloadError(nested_error=str(e), nested_error_cls=e.__class__.__name__)
-
-    async def download(self, sources: Iterable[BaseSource]) -> AsyncIterable[FileResponsePb]:
-        for source in sources:
             try:
-                await source.start()
+                async for resp in source.execute_prepared_file_request(prepared_file_request=prepared_file_request):
-                async for resp in self.download_source(source):
                     yield resp
                 return
             except DownloadError as e:
-                error_log(e)
+                error_log_func(e)
+                continue
+        raise DownloadError(error='not_found', source=str(source))
+
+    async def download(self, sources: Iterable[BaseSource], error_log_func: Callable = error_log) -> AsyncIterable[FileResponsePb]:
+        for source in sources:
+            try:
+                await source.start()
+                async for resp in self.download_source(source, error_log_func=error_log_func):
+                    yield resp
+                return
+            except DownloadError as e:
+                error_log_func(e)
                 continue
             finally:
                 await source.stop()

nexus/pylon/sources/base.py

@@ -1,19 +1,27 @@
+import asyncio
 import hashlib
-import random
+import socket
+from contextlib import asynccontextmanager
 from typing import (
     AsyncIterable,
+    Callable,
     Optional,
 )
 
 import aiohttp
 import aiohttp.client_exceptions
+from aiohttp.client_reqrep import ClientRequest
 from aiohttp_socks import (
+    ProxyConnectionError,
     ProxyConnector,
     ProxyError,
 )
 from aiokit import AioThing
+from izihawa_utils.importlib import class_fullname
+from library.logging import error_log
 from nexus.pylon.exceptions import (
     BadResponseError,
+    DownloadError,
     IncorrectMD5Error,
     NotFoundError,
 )
@@ -30,19 +38,32 @@ from tenacity import (
 DEFAULT_USER_AGENT = 'PylonBot/1.0 (Linux x86_64) PylonBot/1.0.0'
 
 
+class KeepAliveClientRequest(ClientRequest):
+    async def send(self, conn):
+        sock = conn.protocol.transport.get_extra_info("socket")
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 60)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 2)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5)
+
+        return await super().send(conn)
+
+
 class PreparedRequest:
     def __init__(
         self,
         method: str,
         url: str,
-        headers: dict = None,
+        headers: Optional[dict] = None,
-        params: dict = None,
+        params: Optional[dict] = None,
-        cookies: dict = None,
+        cookies: Optional[dict] = None,
         ssl: bool = True,
+        timeout: Optional[float] = None
     ):
         self.method = method
         self.url = url
         self.headers = {
+            'Connection': 'keep-alive',
             'User-Agent': DEFAULT_USER_AGENT,
         }
         if headers:
@@ -50,6 +71,7 @@ class PreparedRequest:
         self.params = params
         self.cookies = cookies
         self.ssl = ssl
+        self.timeout = timeout
 
     def __repr__(self):
         return f'{self.method} {self.url} {self.headers} {self.params}'
@@ -57,6 +79,34 @@ class PreparedRequest:
     def __str__(self):
         return repr(self)
 
+    @asynccontextmanager
+    async def execute_with(self, session):
+        async with session.request(
+            method=self.method,
+            url=self.url,
+            timeout=self.timeout,
+            headers=self.headers,
+            cookies=self.cookies,
+            params=self.params,
+            ssl=self.ssl,
+        ) as resp:
+            try:
+                yield resp
+            except BadResponseError as e:
+                e.add('url', self.url)
+                raise e
+            except (
+                aiohttp.client_exceptions.ClientConnectionError,
+                aiohttp.client_exceptions.ClientPayloadError,
+                aiohttp.client_exceptions.ClientResponseError,
+                aiohttp.client_exceptions.TooManyRedirects,
+                asyncio.TimeoutError,
+                ProxyConnectionError,
+                ProxyTimeoutError,
+                ProxyError,
+            ) as e:
+                raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e))
+
 
 class BaseValidator:
     def update(self, chunk: bytes):
@@ -123,12 +173,12 @@ class BaseSource(AioThing):
         return aiohttp.TCPConnector(verify_ssl=self.ssl)
 
     def get_session(self):
-        return aiohttp.ClientSession(connector=self.get_proxy())
+        return aiohttp.ClientSession(request_class=KeepAliveClientRequest, connector=self.get_proxy())
 
     def get_resolve_session(self):
-        return aiohttp.ClientSession(connector=self.get_resolve_proxy())
+        return aiohttp.ClientSession(request_class=KeepAliveClientRequest, connector=self.get_resolve_proxy())
 
-    def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         raise NotImplementedError("`resolve` for BaseSource is not implemented")
 
     def get_validator(self):
@@ -139,19 +189,11 @@ class BaseSource(AioThing):
         stop=stop_after_attempt(3),
         retry=retry_if_exception_type((ProxyError, aiohttp.client_exceptions.ClientPayloadError, ProxyTimeoutError)),
     )
-    async def execute_prepared_request(self, prepared_request: PreparedRequest):
+    async def execute_prepared_file_request(self, prepared_file_request: PreparedRequest):
         async with self.get_session() as session:
-            async with session.request(
+            async with prepared_file_request.execute_with(session=session) as resp:
-                method=prepared_request.method,
-                url=prepared_request.url,
-                timeout=self.timeout,
-                headers=prepared_request.headers,
-                cookies=prepared_request.cookies,
-                params=prepared_request.params,
-                ssl=prepared_request.ssl,
-            ) as resp:
                 if resp.status == 404:
-                    raise NotFoundError(url=prepared_request.url)
+                    raise NotFoundError(url=prepared_file_request.url)
                 elif (
                     resp.status != 200
                     or (
@@ -160,24 +202,17 @@ class BaseSource(AioThing):
                     )
                 ):
                     raise BadResponseError(
-                        request_headers=prepared_request.headers,
+                        request_headers=prepared_file_request.headers,
-                        url=prepared_request.url,
+                        url=prepared_file_request.url,
                         status=resp.status,
                         headers=str(resp.headers),
                     )
                 file_validator = self.get_validator()
-                # Randomness is required due to annoying bug of when separators
+                yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_file_request.url)
-                # (\r\n) are splitted to different chunks
+                async for content, _ in resp.content.iter_chunks():
-                # https://github.com/aio-libs/aiohttp/issues/4677
-                yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_request.url)
-                async for content in resp.content.iter_chunked(1024 * 100 + random.randint(-1024, 1024)):
                     file_validator.update(content)
-                    yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_request.url)
+                    yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_file_request.url)
-                try:
+                file_validator.validate()
-                    file_validator.validate()
-                except BadResponseError as e:
-                    e.add('url', prepared_request.url)
-                    raise e
 
 
 class Md5Source(BaseSource):

nexus/pylon/sources/libgen_doi.py

@@ -1,5 +1,8 @@
 import re
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
 from library.logging import error_log
 from nexus.pylon.exceptions import RegexNotFoundError
@@ -14,13 +17,14 @@ class LibgenDoiSource(DoiSource):
     base_url = 'http://libgen.gs'
     resolve_timeout = 10
 
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/scimag/ads.php?doi={self.doi}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
-                timeout=self.resolve_timeout
+                url=url,
-            ) as resp:
+                timeout=self.resolve_timeout,
+            ).execute_with(session=session) as resp:
                 downloaded_page_bytes = await resp.read()
                 downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace')
             match = re.search(
@@ -29,6 +33,6 @@ class LibgenDoiSource(DoiSource):
                 re.IGNORECASE,
             )
             if match:
-                yield PreparedRequest(method='get', url=match.group())
+                yield PreparedRequest(method='get', url=match.group(), timeout=self.timeout)
             else:
-                error_log(RegexNotFoundError(url=url))
+                error_log_func(RegexNotFoundError(url=url))

nexus/pylon/sources/libgen_md5.py

@@ -1,5 +1,10 @@
 import re
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
+
+from library.logging import error_log
 
 from .base import (
     Md5Source,
@@ -12,10 +17,11 @@ class LibgenMd5Source(Md5Source):
     resolve_timeout = 10
 
     async def resolve_lg(self, session, url):
-        async with session.get(
+        async with PreparedRequest(
-            url,
+            method='get',
+            url=url,
             timeout=self.resolve_timeout
-        ) as resp:
+        ).execute_with(session=session) as resp:
             downloaded_page_fiction = await resp.text()
         match = re.search(
             'https?://.*/get\\.php\\?md5=.*&key=[A-Za-z0-9]+',
@@ -23,9 +29,9 @@ class LibgenMd5Source(Md5Source):
             re.IGNORECASE,
         )
         if match:
-            return PreparedRequest(method='get', url=match.group())
+            return PreparedRequest(method='get', url=match.group(), timeout=self.timeout)
 
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/ads.php?md5={self.md5}'
             result = await self.resolve_lg(session, url)

nexus/pylon/sources/libgen_new.py

@@ -1,5 +1,8 @@
 import re
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
 from library.logging import error_log
 from nexus.pylon.exceptions import RegexNotFoundError
@@ -11,13 +14,14 @@ from .base import (
 
 
 class LibgenNewSource(Md5Source):
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/main/{self.md5.upper()}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
+                url=url,
                 timeout=self.resolve_timeout
-            ) as resp:
+            ).execute_with(session) as resp:
                 downloaded_page = await resp.text()
             match_ipfs = re.search(
                 'https://ipfs.io/ipfs/[A-Za-z0-9]+',
@@ -25,28 +29,28 @@ class LibgenNewSource(Md5Source):
                 re.IGNORECASE,
             )
             if match_ipfs:
-                yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl)
+                yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl, timeout=self.timeout)
             match_cf = re.search(
                 'https://cloudflare-ipfs.com/ipfs/[A-Za-z0-9]+',
                 downloaded_page,
                 re.IGNORECASE,
             )
             if match_cf:
-                yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl)
+                yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl, timeout=self.timeout)
             match_infura = re.search(
                 'https://ipfs.infura.io/ipfs/[A-Za-z0-9]+',
                 downloaded_page,
                 re.IGNORECASE,
             )
             if match_infura:
-                yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl)
+                yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl, timeout=self.timeout)
 
             if not match_cf or not match_infura or not match_ipfs:
-                error_log(RegexNotFoundError(url=url))
+                error_log_func(RegexNotFoundError(url=url))
 
 
 class LibraryLolSource(LibgenNewSource):
     base_url = 'http://library.lol'
-    resolve_timeout = 10
+    resolve_timeout = 20
     ssl = False
-    timeout = 30
+    timeout = 120

nexus/pylon/sources/scihub.py

@@ -1,5 +1,8 @@
 import re
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
 from library.logging import error_log
 from nexus.pylon.exceptions import RegexNotFoundError
@@ -19,16 +22,17 @@ class SciHubSource(DoiSource):
     base_url = None
     ssl = False
 
-    async def resolve(self, timeout=None) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/{self.doi}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
-                timeout=timeout or self.timeout
+                url=url,
-            ) as resp:
+                timeout=self.resolve_timeout
+            ).execute_with(session=session) as resp:
                 # Sometimes sci-hub returns file
                 if resp.headers.get('Content-Type') == 'application/pdf':
-                    yield PreparedRequest(method='get', url=url)
+                    yield PreparedRequest(method='get', url=url, timeout=self.timeout)
                 downloaded_page_bytes = await resp.read()
                 downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace')
             match = re.search('(?:https?:)?//.*\\?download=true', downloaded_page, re.IGNORECASE)
@@ -36,9 +40,9 @@ class SciHubSource(DoiSource):
                 url = match.group()
                 if url.startswith('//'):
                     url = 'http:' + url
-                yield PreparedRequest(method='get', url=url)
+                yield PreparedRequest(method='get', url=url, timeout=self.timeout)
             else:
-                error_log(RegexNotFoundError(url=url))
+                error_log_func(RegexNotFoundError(url=url))
 
 
 class SciHubDoSource(SciHubSource):

nexus/pylon/sources/specific/biorxiv.py

@@ -1,5 +1,9 @@
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
+from library.logging import error_log
 from nexus.pylon.sources.base import (
     DoiSource,
     PreparedRequest,
@@ -9,11 +13,12 @@ from nexus.pylon.sources.base import (
 class BiorxivSource(DoiSource):
     base_url = 'https://dx.doi.org'
 
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/{self.doi}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
-                timeout=self.resolve_timeout
+                url=url,
-            ) as resp:
+                timeout=self.resolve_timeout,
-                yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf')
+            ).execute_with(session=session) as resp:
+                yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf', timeout=self.timeout)

nexus/pylon/sources/specific/lancet.py

@@ -1,5 +1,9 @@
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
+from library.logging import error_log
 from nexus.pylon.sources.base import (
     DoiSource,
     PreparedRequest,
@@ -11,14 +15,20 @@ class LancetSource(DoiSource):
     resolve_timeout = 10
     use_proxy = False
 
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             splitted_doi = self.doi.split("/", maxsplit=1)
             if len(splitted_doi) < 2:
                 return
             url = f'{self.base_url}/action/showPdf?pii={splitted_doi[1].upper()}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
-                timeout=self.resolve_timeout
+                url=url,
-            ) as resp:
+                timeout=self.resolve_timeout,
-                yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))
+            ).execute_with(session=session) as resp:
+                yield PreparedRequest(
+                    method='get',
+                    cookies=resp.cookies,
+                    url=str(resp.url),
+                    timeout=self.resolve_timeout,
+                )

nexus/pylon/sources/specific/nejm.py

@@ -1,5 +1,9 @@
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
+from library.logging import error_log
 from nexus.pylon.sources.base import (
     DoiSource,
     PreparedRequest,
@@ -11,11 +15,12 @@ class NejmSource(DoiSource):
     resolve_timeout = 10
     use_proxy = False
 
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/doi/pdf/{self.doi}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
+                url=url,
                 timeout=self.resolve_timeout,
-            ) as resp:
+            ).execute_with(session=session) as resp:
-                yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))
+                yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url), timeout=self.timeout)

nexus/pylon/sources/specific/research_square.py

@@ -1,8 +1,11 @@
 import re
-from typing import AsyncIterable
+from typing import (
+    AsyncIterable,
+    Callable,
+)
 
+from library.logging import error_log
 from nexus.pylon.exceptions import RegexNotFoundError
-
 from nexus.pylon.sources.base import (
     DoiSource,
     PreparedRequest,
@@ -12,13 +15,14 @@ from nexus.pylon.sources.base import (
 class ResearchSquareSource(DoiSource):
     base_url = 'https://dx.doi.org'
 
-    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+    async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
         async with self.get_resolve_session() as session:
             url = f'{self.base_url}/{self.doi}'
-            async with session.get(
+            async with PreparedRequest(
-                url,
+                method='get',
-                timeout=self.resolve_timeout
+                url=url,
-            ) as resp:
+                timeout=self.resolve_timeout,
+            ).execute_with(session=session) as resp:
                 download_page = await resp.text()
             match = re.search(
                 r'\"(https://www\.researchsquare\.com/article/[^\"]+\.pdf)\"',
@@ -27,4 +31,4 @@ class ResearchSquareSource(DoiSource):
             )
             if not match:
                 raise RegexNotFoundError(url=url)
-            yield PreparedRequest(method='get', url=match.group(1))
+            yield PreparedRequest(method='get', url=match.group(1), timeout=self.timeout)

nexus/views/telegram/progress_bar.py

@@ -84,7 +84,7 @@ class ProgressBar:
         try:
             if not self.message:
                 self.message = await self.telegram_client.send_message(
-                    self.request_context.chat.id,
+                    self.request_context.chat.chat_id,
                     text,
                     buttons=[close_button()],
                 )

nexus/views/telegram/scitech.py

@@ -6,7 +6,10 @@ from typing import (
 )
 from urllib.parse import quote
 
-from izihawa_utils.common import filter_none
+from izihawa_utils.common import (
+    filter_none,
+    is_essential,
+)
 from nexus.models.proto.scitech_pb2 import Scitech as ScitechPb
 from nexus.nlptools.utils import (
     cast_string_to_single_string,
@@ -48,7 +51,7 @@ class ScitechView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin):
 
         locator = self.get_formatted_locator()
 
-        caption = '\n'.join(filter_none([head, doi, locator]))
+        caption = '\n'.join(filter_none([head, doi, locator], predicate=is_essential))
         if limit and len(caption) > limit:
             shorten_title = title[:limit]
             shorten_title = shorten_title[:max(32, shorten_title.rfind(' '))]

rules/python/requirements.txt

@@ -62,7 +62,7 @@ idna==2.10
 isort==5.8.0
 itsdangerous==1.1.0
 izihawa_types==0.1.0
-izihawa_utils==0.3.0
+izihawa_utils==0.3.1
 Jinja2==2.11.3
 jupyter==1.0.0
 kazoo==2.8.0