From 9ce67ec590e7e67db0e6163ed5a0b43d4adddc10 Mon Sep 17 00:00:00 2001 From: the-superpirate Date: Mon, 12 Apr 2021 20:47:21 +0300 Subject: [PATCH] - feat(pylon): Refactor code - feat(idm): Rename IDM-2 to IDM - feat(idm): Open IDM 3 internal commit(s) GitOrigin-RevId: e302e9b5cda18cca1adc4ae8a3d906714d222106 --- idm/api/BUILD.bazel | 70 ++++++ idm/api/__init__.py | 0 idm/{api2 => api}/aioclient/BUILD.bazel | 4 +- idm/api/aioclient/__init__.py | 3 + idm/{api2 => api}/aioclient/aioclient.py | 16 +- idm/api/configs/base.yaml | 17 ++ idm/api/configs/logging.yaml | 63 +++++ idm/api/daemons/__init__.py | 0 idm/api/daemons/admin_log_reader.py | 87 +++++++ idm/api/main.py | 54 +++++ idm/{api2 => api}/proto/BUILD.bazel | 3 +- idm/api/proto/chat_manager_service.proto | 50 ++++ .../proto/chat_manager_service_pb2.py} | 222 +++++++++--------- .../proto/chat_manager_service_pb2_grpc.py} | 79 ++++--- idm/api/services/chat_manager.py | 109 +++++++++ idm/api2/aioclient/__init__.py | 3 - idm/api2/proto/chats_service.proto | 66 ------ idm/api2/proto/location.proto | 7 - idm/api2/proto/location_pb2.py | 78 ------ idm/api2/proto/location_pb2_grpc.py | 4 - images/install.bzl | 10 +- images/production/BUILD.bazel | 2 +- library/logging/__init__.py | 2 +- library/telegram/base.py | 2 +- nexus/bot/BUILD.bazel | 2 +- nexus/bot/application.py | 4 +- nexus/bot/handlers/base.py | 24 +- nexus/bot/handlers/download.py | 4 +- nexus/bot/handlers/referencing_to.py | 2 +- nexus/bot/handlers/roll.py | 4 +- nexus/bot/handlers/search.py | 14 +- nexus/bot/handlers/start.py | 2 +- nexus/bot/handlers/view.py | 10 +- nexus/bot/handlers/vote.py | 4 +- nexus/bot/widgets/admin_widget.py | 4 +- nexus/bot/widgets/banlist_widget.py | 10 +- nexus/bot/widgets/document_list_widget.py | 10 +- nexus/bot/widgets/search_widget.py | 8 +- nexus/bot/widgets/settings_widget.py | 10 +- nexus/hub/BUILD.bazel | 5 +- nexus/hub/README.md | 59 +---- nexus/hub/aioclient/BUILD.bazel | 2 +- nexus/hub/aioclient/aioclient.py | 6 +- nexus/hub/configs/base.yaml | 51 ++++ nexus/hub/configs/logging.yaml | 76 ++++++ nexus/hub/proto/BUILD.bazel | 2 +- nexus/hub/proto/delivery_service.proto | 4 +- nexus/hub/proto/delivery_service_pb2.py | 29 +-- nexus/hub/proto/submitter_service.proto | 4 +- nexus/hub/proto/submitter_service_pb2.py | 34 ++- nexus/hub/services/base.py | 2 +- nexus/hub/services/delivery.py | 34 +-- nexus/hub/services/submitter.py | 14 +- nexus/pipe/README.md | 98 +------- nexus/pipe/configs/base.yaml | 90 +++++++ nexus/pipe/configs/logging.yaml | 65 +++++ nexus/pylon/client.py | 57 ++--- nexus/pylon/sources/base.py | 97 +++++--- nexus/pylon/sources/libgen_doi.py | 20 +- nexus/pylon/sources/libgen_md5.py | 18 +- nexus/pylon/sources/libgen_new.py | 26 +- nexus/pylon/sources/scihub.py | 22 +- nexus/pylon/sources/specific/biorxiv.py | 19 +- nexus/pylon/sources/specific/lancet.py | 24 +- nexus/pylon/sources/specific/nejm.py | 17 +- .../pylon/sources/specific/research_square.py | 20 +- nexus/views/telegram/progress_bar.py | 2 +- nexus/views/telegram/scitech.py | 7 +- 68 files changed, 1230 insertions(+), 737 deletions(-) create mode 100644 idm/api/BUILD.bazel create mode 100644 idm/api/__init__.py rename idm/{api2 => api}/aioclient/BUILD.bazel (80%) create mode 100644 idm/api/aioclient/__init__.py rename idm/{api2 => api}/aioclient/aioclient.py (86%) create mode 100644 idm/api/configs/base.yaml create mode 100644 idm/api/configs/logging.yaml create mode 100644 idm/api/daemons/__init__.py create mode 100644 idm/api/daemons/admin_log_reader.py create mode 100644 idm/api/main.py rename idm/{api2 => api}/proto/BUILD.bazel (89%) create mode 100644 idm/api/proto/chat_manager_service.proto rename idm/{api2/proto/chats_service_pb2.py => api/proto/chat_manager_service_pb2.py} (62%) mode change 100755 => 100644 rename idm/{api2/proto/chats_service_pb2_grpc.py => api/proto/chat_manager_service_pb2_grpc.py} (57%) mode change 100755 => 100644 create mode 100644 idm/api/services/chat_manager.py delete mode 100644 idm/api2/aioclient/__init__.py delete mode 100644 idm/api2/proto/chats_service.proto delete mode 100644 idm/api2/proto/location.proto delete mode 100755 idm/api2/proto/location_pb2.py delete mode 100755 idm/api2/proto/location_pb2_grpc.py create mode 100644 nexus/hub/configs/base.yaml create mode 100644 nexus/hub/configs/logging.yaml create mode 100644 nexus/pipe/configs/base.yaml create mode 100644 nexus/pipe/configs/logging.yaml diff --git a/idm/api/BUILD.bazel b/idm/api/BUILD.bazel new file mode 100644 index 0000000..1b52bcb --- /dev/null +++ b/idm/api/BUILD.bazel @@ -0,0 +1,70 @@ +load("@io_bazel_rules_docker//python3:image.bzl", "py3_image") +load("@io_bazel_rules_docker//container:container.bzl", "container_push") + +load("@pip_modules//:requirements.bzl", "requirement") + +alias( + name = "binary", + actual = ":image.binary", +) + +py3_image( + name = "image", + srcs = glob( + [ + "*.py", + "configs/**/*.py", + "daemons/**/*.py", + "models/**", + "proto/**", + "services/**", + ], + exclude = [ + "**/__pycache__/**", + "**/*.pyc", + "**/README", + "**/*.mako", + "proto/**/*.py", + ], + ), + base = "//images/production:base-python-image", + data = [ + "configs/base.yaml", + "configs/logging.yaml", + ], + layers = [ + requirement("grpcio"), + requirement("pypika"), + requirement("uvloop"), + "//idm/api/proto:idm_grpc_py", + "//idm/api/proto:idm_proto_py", + "//library/aiogrpctools", + requirement("aiokit"), + "//library/aiopostgres", + "//library/configurator", + "//library/telegram", + requirement("izihawa_utils"), + ], + main = "main.py", + srcs_version = "PY3ONLY", + visibility = ["//visibility:public"], +) + +container_push( + name = "push-testing", + format = "Docker", + image = ":image", + registry = "registry.example.com", + repository = "idm-api", + tag = "testing", +) + +container_push( + name = "push-latest", + format = "Docker", + image = ":image", + registry = "registry.example.com", + repository = "idm-api", + tag = "latest", +) + diff --git a/idm/api/__init__.py b/idm/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/idm/api2/aioclient/BUILD.bazel b/idm/api/aioclient/BUILD.bazel similarity index 80% rename from idm/api2/aioclient/BUILD.bazel rename to idm/api/aioclient/BUILD.bazel index eb72b63..cb82832 100644 --- a/idm/api2/aioclient/BUILD.bazel +++ b/idm/api/aioclient/BUILD.bazel @@ -9,7 +9,7 @@ py_library( requirement("cachetools"), requirement("lru-dict"), requirement("tenacity"), - "//idm/api2/proto:idm_grpc_py", - "//idm/api2/proto:idm_proto_py", + "//idm/api/proto:idm_grpc_py", + "//idm/api/proto:idm_proto_py", ], ) diff --git a/idm/api/aioclient/__init__.py b/idm/api/aioclient/__init__.py new file mode 100644 index 0000000..909472f --- /dev/null +++ b/idm/api/aioclient/__init__.py @@ -0,0 +1,3 @@ +from .aioclient import IdmApiGrpcClient + +__all__ = ['IdmApiGrpcClient'] diff --git a/idm/api2/aioclient/aioclient.py b/idm/api/aioclient/aioclient.py similarity index 86% rename from idm/api2/aioclient/aioclient.py rename to idm/api/aioclient/aioclient.py index 16deb2b..9cbbc1f 100644 --- a/idm/api2/aioclient/aioclient.py +++ b/idm/api/aioclient/aioclient.py @@ -4,13 +4,13 @@ from grpc.experimental.aio import ( AioRpcError, insecure_channel, ) -from idm.api2.proto.chats_service_pb2 import ( +from idm.api.proto.chat_manager_service_pb2 import ( CreateChatRequest, GetChatRequest, ListChatsRequest, UpdateChatRequest, ) -from idm.api2.proto.chats_service_pb2_grpc import ChatsStub +from idm.api.proto.chat_manager_service_pb2_grpc import ChatManagerStub from lru import LRU from tenacity import ( retry, @@ -20,7 +20,7 @@ from tenacity import ( ) -class IdmApi2GrpcClient(AioThing): +class IdmApiGrpcClient(AioThing): def __init__( self, base_url, @@ -33,7 +33,7 @@ class IdmApi2GrpcClient(AioThing): ('grpc.min_reconnect_backoff_ms', 1000), ('grpc.max_reconnect_backoff_ms', 2000), ]) - self.chats_stub = ChatsStub(self.channel) + self.chat_manager_stub = ChatManagerStub(self.channel) self.cache = LRU(4096) async def start(self): @@ -49,7 +49,7 @@ class IdmApi2GrpcClient(AioThing): language, request_id: str = None, ): - response = await self.chats_stub.create_chat( + response = await self.chat_manager_stub.create_chat( CreateChatRequest( chat_id=chat_id, username=username, @@ -74,7 +74,7 @@ class IdmApi2GrpcClient(AioThing): chat_id, request_id: str = None, ): - response = await self.chats_stub.get_chat( + response = await self.chat_manager_stub.get_chat( GetChatRequest(chat_id=chat_id), metadata=( ('request-id', request_id), @@ -87,7 +87,7 @@ class IdmApi2GrpcClient(AioThing): request_id: str = None, banned_at_moment=None, ): - response = await self.chats_stub.list_chats( + response = await self.chat_manager_stub.list_chats( ListChatsRequest(banned_at_moment=banned_at_moment), metadata=( ('request-id', request_id), @@ -106,7 +106,7 @@ class IdmApi2GrpcClient(AioThing): ban_message=None, is_admin=None, ): - response = await self.chats_stub.update_chat( + response = await self.chat_manager_stub.update_chat( UpdateChatRequest( chat_id=chat_id, language=language, diff --git a/idm/api/configs/base.yaml b/idm/api/configs/base.yaml new file mode 100644 index 0000000..efb918d --- /dev/null +++ b/idm/api/configs/base.yaml @@ -0,0 +1,17 @@ +--- +application: + debug: true + service_name: idm-api +database: + port: 5432 +grpc: + address: 0.0.0.0 + port: 9090 +log_path: '/var/log/idm-api/{{ ENV_TYPE }}' +telegram: + # Telegram App Hash from https://my.telegram.org/ + app_hash: '{{ APP_HASH }}' + # Telegram App ID from https://my.telegram.org/ + app_id: 00000 + database: + session_id: '/usr/lib/idm-api/{{ ENV_TYPE }}/session.db' diff --git a/idm/api/configs/logging.yaml b/idm/api/configs/logging.yaml new file mode 100644 index 0000000..ee477ba --- /dev/null +++ b/idm/api/configs/logging.yaml @@ -0,0 +1,63 @@ +--- + +logging: + disable_existing_loggers: false + formatters: + base: + class: library.logging.formatters.BaseFormatter + default: + class: library.logging.formatters.DefaultFormatter + none: + class: logging.Formatter + traceback: + class: library.logging.formatters.TracebackFormatter + handlers: + debug: + class: logging.StreamHandler + formatter: default + level: DEBUG + stream: 'ext://sys.stderr' + error: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/error.log' + formatter: default + level: ERROR + statbox: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/statbox.log' + formatter: default + level: INFO + traceback: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/traceback.log' + formatter: traceback + level: ERROR + warning: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/warning.log' + formatter: default + level: WARNING + loggers: + debug: + handlers: + - debug + propagate: false + error: + handlers: + - error + - traceback + - warning + propagate: false + statbox: + handlers: + - statbox + propagate: false + telethon: + handlers: + - error + propagate: false + root: + handlers: + - debug + level: DEBUG + version: 1 diff --git a/idm/api/daemons/__init__.py b/idm/api/daemons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/idm/api/daemons/admin_log_reader.py b/idm/api/daemons/admin_log_reader.py new file mode 100644 index 0000000..74dddaf --- /dev/null +++ b/idm/api/daemons/admin_log_reader.py @@ -0,0 +1,87 @@ +import asyncio +import logging +import string + +from aiokit import AioThing +from library.telegram.base import BaseTelegramClient +from telethon.tl.functions.channels import GetParticipantsRequest +from telethon.tl.types import ChannelParticipantsSearch + + +class AdminLogReader(AioThing): + channel_name = '@nexus_search' + + def __init__(self, telegram_config): + super().__init__() + self.subscriptors = set() + self.loading = False + self.telegram_client = BaseTelegramClient( + app_id=telegram_config['app_id'], + app_hash=telegram_config['app_hash'], + database=telegram_config['database'], + flood_sleep_threshold=25, + ) + self.last_max_id = 0 + + def statbox(self, **kwargs): + logging.getLogger('statbox').info({'mode': 'admin_log_reader', **kwargs}) + + async def skip_admin_log(self): + async for event in self.telegram_client.iter_admin_log(self.channel_name, limit=1): + self.last_max_id = event.id + self.statbox(action='skipped_admin_log', max_id=self.last_max_id) + + async def process_admin_log(self, sleep=1.0): + self.loading = True + try: + while 1: + async for event in self.telegram_client.iter_admin_log( + self.channel_name, join=True, invite=True, leave=True, min_id=self.last_max_id + ): + is_subscribed = event.joined or event.joined_invite + if is_subscribed: + self.subscriptors.add(event.user_id) + elif event.user_id in self.subscriptors: + self.subscriptors.remove(event.user_id) + self.last_max_id = event.id + await asyncio.sleep(sleep) + except asyncio.CancelledError: + pass + finally: + self.loading = False + + async def _fetch_users(self, query): + max_batch_size = 200 + + participants = await self.telegram_client( + GetParticipantsRequest( + channel=self.channel_name, + filter=ChannelParticipantsSearch(query), + offset=0, + limit=max_batch_size, + hash=0, + ) + ) + + for user in participants.users: + self.subscriptors.add(user.id) + + # There is a possibility that more users exist if we hit maximum count of users + # So, we are making a recursion to reveal it + if len(participants.users) == max_batch_size: + for new_letter in string.ascii_lowercase + string.digits: + await self._fetch_users(query + new_letter) + + async def load_channel_users(self): + self.statbox(action='load_channel_users') + await self._fetch_users('') + self.statbox(action='loaded_channel_users', subscriptors=len(self.subscriptors)) + + async def start(self): + await self.telegram_client.start_and_wait() + await self.skip_admin_log() + await self.load_channel_users() + asyncio.create_task(self.process_admin_log()) + + def is_subscribed(self, user_id): + return not self.loading or user_id in self.subscriptors diff --git a/idm/api/main.py b/idm/api/main.py new file mode 100644 index 0000000..b8b9785 --- /dev/null +++ b/idm/api/main.py @@ -0,0 +1,54 @@ +import asyncio + +import uvloop +from aiopg.sa import create_engine +from idm.api.configs import get_config +from idm.api.daemons.admin_log_reader import AdminLogReader +from idm.api.services.chat_manager import ChatManagerService +from library.aiogrpctools import AioGrpcServer +from library.aiopostgres.pool_holder import AioPostgresPoolHolder +from library.configurator import Configurator +from library.logging import configure_logging + + +class GrpcServer(AioGrpcServer): + def __init__(self, config: Configurator): + super().__init__(address=config['grpc']['address'], port=config['grpc']['port']) + self.pool_holder = AioPostgresPoolHolder( + fn=create_engine, + database=config['database']['database'], + user=config['database']['username'], + password=config['database']['password'], + host=config['database']['host'], + port=config['database']['port'], + timeout=30, + pool_recycle=60, + maxsize=4, + ) + self.admin_log_reader = AdminLogReader( + telegram_config=config['telegram'], + ) + self.chat_manager_service = ChatManagerService( + server=self.server, + service_name=config['application']['service_name'], + pool_holder=self.pool_holder, + admin_log_reader=self.admin_log_reader, + ) + self.starts.append(self.admin_log_reader) + self.waits.extend([self.chat_manager_service, self.pool_holder]) + + +async def create_app(config: Configurator): + grpc_server = GrpcServer(config) + await grpc_server.start_and_wait() + + +def main(): + uvloop.install() + config = get_config() + configure_logging(config) + asyncio.get_event_loop().run_until_complete(create_app(config)) + + +if __name__ == '__main__': + main() diff --git a/idm/api2/proto/BUILD.bazel b/idm/api/proto/BUILD.bazel similarity index 89% rename from idm/api2/proto/BUILD.bazel rename to idm/api/proto/BUILD.bazel index cc517c5..977f5c5 100644 --- a/idm/api2/proto/BUILD.bazel +++ b/idm/api/proto/BUILD.bazel @@ -6,8 +6,7 @@ package(default_visibility = ["//visibility:public"]) proto_library( name = "idm_proto", srcs = [ - "chats_service.proto", - "location.proto", + "chat_manager_service.proto", ], deps = [ "@com_google_protobuf//:wrappers_proto", diff --git a/idm/api/proto/chat_manager_service.proto b/idm/api/proto/chat_manager_service.proto new file mode 100644 index 0000000..e801160 --- /dev/null +++ b/idm/api/proto/chat_manager_service.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; +package idm.api.proto; + +message Chat { + int64 chat_id = 1; + string username = 2; + string language = 3; + bool is_system_messaging_enabled = 4; + bool is_discovery_enabled = 5; + int32 ban_until = 6; + string ban_message = 7; + bool is_admin = 8; + bool is_subscribed = 9; + int64 created_at = 10; +} + +message Chats { + repeated Chat chats = 1; +} + +message CreateChatRequest { + int64 chat_id = 1; + string username = 2; + string language = 3; +} + +message GetChatRequest { + int64 chat_id = 1; +} + +message ListChatsRequest { + optional int32 banned_at_moment = 1; +} + +message UpdateChatRequest { + int64 chat_id = 1; + optional string language = 2; + optional bool is_system_messaging_enabled = 3; + optional bool is_discovery_enabled = 4; + optional int32 ban_until = 5; + optional string ban_message = 6; + optional bool is_admin = 7; +} + +service ChatManager { + rpc create_chat(CreateChatRequest) returns (Chat) {}; + rpc get_chat(GetChatRequest) returns (Chat) {}; + rpc list_chats(ListChatsRequest) returns (Chats) {}; + rpc update_chat(UpdateChatRequest) returns (Chat) {}; +} diff --git a/idm/api2/proto/chats_service_pb2.py b/idm/api/proto/chat_manager_service_pb2.py old mode 100755 new mode 100644 similarity index 62% rename from idm/api2/proto/chats_service_pb2.py rename to idm/api/proto/chat_manager_service_pb2.py index d91be68..711355b --- a/idm/api2/proto/chats_service_pb2.py +++ b/idm/api/proto/chat_manager_service_pb2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: idm/api2/proto/chats_service.proto +# source: idm/api/proto/chat_manager_service.proto """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -12,96 +12,94 @@ from google.protobuf import symbol_database as _symbol_database _sym_db = _symbol_database.Default() -from idm.api2.proto import \ - location_pb2 as idm_dot_api2_dot_proto_dot_location__pb2 + DESCRIPTOR = _descriptor.FileDescriptor( - name='idm/api2/proto/chats_service.proto', - package='idm.api2.proto', + name='idm/api/proto/chat_manager_service.proto', + package='idm.api.proto', syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\"idm/api2/proto/chats_service.proto\x12\x0eidm.api2.proto\x1a\x1didm/api2/proto/location.proto\"\xe2\x01\n\x08\x43hatData\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x15\n\ris_subscribed\x18\n \x01(\x08\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\"4\n\tChatsData\x12\'\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x18.idm.api2.proto.ChatData\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"H\n\x10ListChatsRequest\x12\x1a\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x42\x18\n\x16\x62\x61nned_at_moment_oneof\"\xce\x02\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x12\n\x08language\x18\x02 \x01(\tH\x00\x12%\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x12\x1e\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x12\x13\n\tban_until\x18\x05 \x01(\x05H\x03\x12\x15\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x12\x12\n\x08is_admin\x18\x07 \x01(\x08H\x05\x42\x10\n\x0elanguage_oneofB#\n!is_system_messaging_enabled_oneofB\x1c\n\x1ais_discovery_enabled_oneofB\x11\n\x0f\x62\x61n_until_oneofB\x13\n\x11\x62\x61n_message_oneofB\x10\n\x0eis_admin_oneof2\xb8\x02\n\x05\x43hats\x12L\n\x0b\x63reate_chat\x12!.idm.api2.proto.CreateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12\x46\n\x08get_chat\x12\x1e.idm.api2.proto.GetChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12K\n\nlist_chats\x12 .idm.api2.proto.ListChatsRequest\x1a\x19.idm.api2.proto.ChatsData\"\x00\x12L\n\x0bupdate_chat\x12!.idm.api2.proto.UpdateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x62\x06proto3' - , - dependencies=[idm_dot_api2_dot_proto_dot_location__pb2.DESCRIPTOR,]) + serialized_pb=b'\n(idm/api/proto/chat_manager_service.proto\x12\ridm.api.proto\"\xe3\x01\n\x04\x43hat\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x15\n\ris_subscribed\x18\t \x01(\x08\x12\x12\n\ncreated_at\x18\n \x01(\x03\"+\n\x05\x43hats\x12\"\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x13.idm.api.proto.Chat\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"F\n\x10ListChatsRequest\x12\x1d\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x88\x01\x01\x42\x13\n\x11_banned_at_moment\"\xc2\x02\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x15\n\x08language\x18\x02 \x01(\tH\x00\x88\x01\x01\x12(\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12!\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x88\x01\x01\x12\x16\n\tban_until\x18\x05 \x01(\x05H\x03\x88\x01\x01\x12\x18\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08is_admin\x18\x07 \x01(\x08H\x05\x88\x01\x01\x42\x0b\n\t_languageB\x1e\n\x1c_is_system_messaging_enabledB\x17\n\x15_is_discovery_enabledB\x0c\n\n_ban_untilB\x0e\n\x0c_ban_messageB\x0b\n\t_is_admin2\xa6\x02\n\x0b\x43hatManager\x12\x46\n\x0b\x63reate_chat\x12 .idm.api.proto.CreateChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x12@\n\x08get_chat\x12\x1d.idm.api.proto.GetChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x12\x45\n\nlist_chats\x12\x1f.idm.api.proto.ListChatsRequest\x1a\x14.idm.api.proto.Chats\"\x00\x12\x46\n\x0bupdate_chat\x12 .idm.api.proto.UpdateChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x62\x06proto3' +) -_CHATDATA = _descriptor.Descriptor( - name='ChatData', - full_name='idm.api2.proto.ChatData', +_CHAT = _descriptor.Descriptor( + name='Chat', + full_name='idm.api.proto.Chat', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='id', full_name='idm.api2.proto.ChatData.id', index=0, + name='chat_id', full_name='idm.api.proto.Chat.chat_id', index=0, number=1, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='username', full_name='idm.api2.proto.ChatData.username', index=1, + name='username', full_name='idm.api.proto.Chat.username', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='language', full_name='idm.api2.proto.ChatData.language', index=2, + name='language', full_name='idm.api.proto.Chat.language', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_system_messaging_enabled', full_name='idm.api2.proto.ChatData.is_system_messaging_enabled', index=3, + name='is_system_messaging_enabled', full_name='idm.api.proto.Chat.is_system_messaging_enabled', index=3, number=4, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_discovery_enabled', full_name='idm.api2.proto.ChatData.is_discovery_enabled', index=4, + name='is_discovery_enabled', full_name='idm.api.proto.Chat.is_discovery_enabled', index=4, number=5, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='ban_until', full_name='idm.api2.proto.ChatData.ban_until', index=5, + name='ban_until', full_name='idm.api.proto.Chat.ban_until', index=5, number=6, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='ban_message', full_name='idm.api2.proto.ChatData.ban_message', index=6, + name='ban_message', full_name='idm.api.proto.Chat.ban_message', index=6, number=7, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_admin', full_name='idm.api2.proto.ChatData.is_admin', index=7, + name='is_admin', full_name='idm.api.proto.Chat.is_admin', index=7, number=8, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_subscribed', full_name='idm.api2.proto.ChatData.is_subscribed', index=8, - number=10, type=8, cpp_type=7, label=1, + name='is_subscribed', full_name='idm.api.proto.Chat.is_subscribed', index=8, + number=9, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='created_at', full_name='idm.api2.proto.ChatData.created_at', index=9, - number=11, type=3, cpp_type=2, label=1, + name='created_at', full_name='idm.api.proto.Chat.created_at', index=9, + number=10, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -118,21 +116,21 @@ _CHATDATA = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=86, - serialized_end=312, + serialized_start=60, + serialized_end=287, ) -_CHATSDATA = _descriptor.Descriptor( - name='ChatsData', - full_name='idm.api2.proto.ChatsData', +_CHATS = _descriptor.Descriptor( + name='Chats', + full_name='idm.api.proto.Chats', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='chats', full_name='idm.api2.proto.ChatsData.chats', index=0, + name='chats', full_name='idm.api.proto.Chats.chats', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -150,35 +148,35 @@ _CHATSDATA = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=314, - serialized_end=366, + serialized_start=289, + serialized_end=332, ) _CREATECHATREQUEST = _descriptor.Descriptor( name='CreateChatRequest', - full_name='idm.api2.proto.CreateChatRequest', + full_name='idm.api.proto.CreateChatRequest', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='chat_id', full_name='idm.api2.proto.CreateChatRequest.chat_id', index=0, + name='chat_id', full_name='idm.api.proto.CreateChatRequest.chat_id', index=0, number=1, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='username', full_name='idm.api2.proto.CreateChatRequest.username', index=1, + name='username', full_name='idm.api.proto.CreateChatRequest.username', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='language', full_name='idm.api2.proto.CreateChatRequest.language', index=2, + name='language', full_name='idm.api.proto.CreateChatRequest.language', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, @@ -196,21 +194,21 @@ _CREATECHATREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=368, - serialized_end=440, + serialized_start=334, + serialized_end=406, ) _GETCHATREQUEST = _descriptor.Descriptor( name='GetChatRequest', - full_name='idm.api2.proto.GetChatRequest', + full_name='idm.api.proto.GetChatRequest', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='chat_id', full_name='idm.api2.proto.GetChatRequest.chat_id', index=0, + name='chat_id', full_name='idm.api.proto.GetChatRequest.chat_id', index=0, number=1, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, @@ -228,21 +226,21 @@ _GETCHATREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=442, - serialized_end=475, + serialized_start=408, + serialized_end=441, ) _LISTCHATSREQUEST = _descriptor.Descriptor( name='ListChatsRequest', - full_name='idm.api2.proto.ListChatsRequest', + full_name='idm.api.proto.ListChatsRequest', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='banned_at_moment', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment', index=0, + name='banned_at_moment', full_name='idm.api.proto.ListChatsRequest.banned_at_moment', index=0, number=1, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, @@ -260,68 +258,68 @@ _LISTCHATSREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ _descriptor.OneofDescriptor( - name='banned_at_moment_oneof', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment_oneof', + name='_banned_at_moment', full_name='idm.api.proto.ListChatsRequest._banned_at_moment', index=0, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), ], - serialized_start=477, - serialized_end=549, + serialized_start=443, + serialized_end=513, ) _UPDATECHATREQUEST = _descriptor.Descriptor( name='UpdateChatRequest', - full_name='idm.api2.proto.UpdateChatRequest', + full_name='idm.api.proto.UpdateChatRequest', filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='chat_id', full_name='idm.api2.proto.UpdateChatRequest.chat_id', index=0, + name='chat_id', full_name='idm.api.proto.UpdateChatRequest.chat_id', index=0, number=1, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='language', full_name='idm.api2.proto.UpdateChatRequest.language', index=1, + name='language', full_name='idm.api.proto.UpdateChatRequest.language', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_system_messaging_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled', index=2, + name='is_system_messaging_enabled', full_name='idm.api.proto.UpdateChatRequest.is_system_messaging_enabled', index=2, number=3, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_discovery_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled', index=3, + name='is_discovery_enabled', full_name='idm.api.proto.UpdateChatRequest.is_discovery_enabled', index=3, number=4, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='ban_until', full_name='idm.api2.proto.UpdateChatRequest.ban_until', index=4, + name='ban_until', full_name='idm.api.proto.UpdateChatRequest.ban_until', index=4, number=5, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='ban_message', full_name='idm.api2.proto.UpdateChatRequest.ban_message', index=5, + name='ban_message', full_name='idm.api.proto.UpdateChatRequest.ban_message', index=5, number=6, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='is_admin', full_name='idm.api2.proto.UpdateChatRequest.is_admin', index=6, + name='is_admin', full_name='idm.api.proto.UpdateChatRequest.is_admin', index=6, number=7, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, @@ -339,167 +337,167 @@ _UPDATECHATREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ _descriptor.OneofDescriptor( - name='language_oneof', full_name='idm.api2.proto.UpdateChatRequest.language_oneof', + name='_language', full_name='idm.api.proto.UpdateChatRequest._language', index=0, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), _descriptor.OneofDescriptor( - name='is_system_messaging_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled_oneof', + name='_is_system_messaging_enabled', full_name='idm.api.proto.UpdateChatRequest._is_system_messaging_enabled', index=1, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), _descriptor.OneofDescriptor( - name='is_discovery_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled_oneof', + name='_is_discovery_enabled', full_name='idm.api.proto.UpdateChatRequest._is_discovery_enabled', index=2, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), _descriptor.OneofDescriptor( - name='ban_until_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_until_oneof', + name='_ban_until', full_name='idm.api.proto.UpdateChatRequest._ban_until', index=3, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), _descriptor.OneofDescriptor( - name='ban_message_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_message_oneof', + name='_ban_message', full_name='idm.api.proto.UpdateChatRequest._ban_message', index=4, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), _descriptor.OneofDescriptor( - name='is_admin_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_admin_oneof', + name='_is_admin', full_name='idm.api.proto.UpdateChatRequest._is_admin', index=5, containing_type=None, create_key=_descriptor._internal_create_key, fields=[]), ], - serialized_start=552, - serialized_end=886, + serialized_start=516, + serialized_end=838, ) -_CHATSDATA.fields_by_name['chats'].message_type = _CHATDATA -_LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof'].fields.append( +_CHATS.fields_by_name['chats'].message_type = _CHAT +_LISTCHATSREQUEST.oneofs_by_name['_banned_at_moment'].fields.append( _LISTCHATSREQUEST.fields_by_name['banned_at_moment']) -_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof'] -_UPDATECHATREQUEST.oneofs_by_name['language_oneof'].fields.append( +_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['_banned_at_moment'] +_UPDATECHATREQUEST.oneofs_by_name['_language'].fields.append( _UPDATECHATREQUEST.fields_by_name['language']) -_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['language_oneof'] -_UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof'].fields.append( +_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_language'] +_UPDATECHATREQUEST.oneofs_by_name['_is_system_messaging_enabled'].fields.append( _UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled']) -_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof'] -_UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof'].fields.append( +_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_system_messaging_enabled'] +_UPDATECHATREQUEST.oneofs_by_name['_is_discovery_enabled'].fields.append( _UPDATECHATREQUEST.fields_by_name['is_discovery_enabled']) -_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof'] -_UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof'].fields.append( +_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_discovery_enabled'] +_UPDATECHATREQUEST.oneofs_by_name['_ban_until'].fields.append( _UPDATECHATREQUEST.fields_by_name['ban_until']) -_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof'] -_UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof'].fields.append( +_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_ban_until'] +_UPDATECHATREQUEST.oneofs_by_name['_ban_message'].fields.append( _UPDATECHATREQUEST.fields_by_name['ban_message']) -_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof'] -_UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof'].fields.append( +_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_ban_message'] +_UPDATECHATREQUEST.oneofs_by_name['_is_admin'].fields.append( _UPDATECHATREQUEST.fields_by_name['is_admin']) -_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof'] -DESCRIPTOR.message_types_by_name['ChatData'] = _CHATDATA -DESCRIPTOR.message_types_by_name['ChatsData'] = _CHATSDATA +_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_admin'] +DESCRIPTOR.message_types_by_name['Chat'] = _CHAT +DESCRIPTOR.message_types_by_name['Chats'] = _CHATS DESCRIPTOR.message_types_by_name['CreateChatRequest'] = _CREATECHATREQUEST DESCRIPTOR.message_types_by_name['GetChatRequest'] = _GETCHATREQUEST DESCRIPTOR.message_types_by_name['ListChatsRequest'] = _LISTCHATSREQUEST DESCRIPTOR.message_types_by_name['UpdateChatRequest'] = _UPDATECHATREQUEST _sym_db.RegisterFileDescriptor(DESCRIPTOR) -ChatData = _reflection.GeneratedProtocolMessageType('ChatData', (_message.Message,), { - 'DESCRIPTOR' : _CHATDATA, - '__module__' : 'idm.api2.proto.chats_service_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.ChatData) +Chat = _reflection.GeneratedProtocolMessageType('Chat', (_message.Message,), { + 'DESCRIPTOR' : _CHAT, + '__module__' : 'idm.api.proto.chat_manager_service_pb2' + # @@protoc_insertion_point(class_scope:idm.api.proto.Chat) }) -_sym_db.RegisterMessage(ChatData) +_sym_db.RegisterMessage(Chat) -ChatsData = _reflection.GeneratedProtocolMessageType('ChatsData', (_message.Message,), { - 'DESCRIPTOR' : _CHATSDATA, - '__module__' : 'idm.api2.proto.chats_service_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.ChatsData) +Chats = _reflection.GeneratedProtocolMessageType('Chats', (_message.Message,), { + 'DESCRIPTOR' : _CHATS, + '__module__' : 'idm.api.proto.chat_manager_service_pb2' + # @@protoc_insertion_point(class_scope:idm.api.proto.Chats) }) -_sym_db.RegisterMessage(ChatsData) +_sym_db.RegisterMessage(Chats) CreateChatRequest = _reflection.GeneratedProtocolMessageType('CreateChatRequest', (_message.Message,), { 'DESCRIPTOR' : _CREATECHATREQUEST, - '__module__' : 'idm.api2.proto.chats_service_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.CreateChatRequest) + '__module__' : 'idm.api.proto.chat_manager_service_pb2' + # @@protoc_insertion_point(class_scope:idm.api.proto.CreateChatRequest) }) _sym_db.RegisterMessage(CreateChatRequest) GetChatRequest = _reflection.GeneratedProtocolMessageType('GetChatRequest', (_message.Message,), { 'DESCRIPTOR' : _GETCHATREQUEST, - '__module__' : 'idm.api2.proto.chats_service_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.GetChatRequest) + '__module__' : 'idm.api.proto.chat_manager_service_pb2' + # @@protoc_insertion_point(class_scope:idm.api.proto.GetChatRequest) }) _sym_db.RegisterMessage(GetChatRequest) ListChatsRequest = _reflection.GeneratedProtocolMessageType('ListChatsRequest', (_message.Message,), { 'DESCRIPTOR' : _LISTCHATSREQUEST, - '__module__' : 'idm.api2.proto.chats_service_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.ListChatsRequest) + '__module__' : 'idm.api.proto.chat_manager_service_pb2' + # @@protoc_insertion_point(class_scope:idm.api.proto.ListChatsRequest) }) _sym_db.RegisterMessage(ListChatsRequest) UpdateChatRequest = _reflection.GeneratedProtocolMessageType('UpdateChatRequest', (_message.Message,), { 'DESCRIPTOR' : _UPDATECHATREQUEST, - '__module__' : 'idm.api2.proto.chats_service_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.UpdateChatRequest) + '__module__' : 'idm.api.proto.chat_manager_service_pb2' + # @@protoc_insertion_point(class_scope:idm.api.proto.UpdateChatRequest) }) _sym_db.RegisterMessage(UpdateChatRequest) -_CHATS = _descriptor.ServiceDescriptor( - name='Chats', - full_name='idm.api2.proto.Chats', +_CHATMANAGER = _descriptor.ServiceDescriptor( + name='ChatManager', + full_name='idm.api.proto.ChatManager', file=DESCRIPTOR, index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=889, - serialized_end=1201, + serialized_start=841, + serialized_end=1135, methods=[ _descriptor.MethodDescriptor( name='create_chat', - full_name='idm.api2.proto.Chats.create_chat', + full_name='idm.api.proto.ChatManager.create_chat', index=0, containing_service=None, input_type=_CREATECHATREQUEST, - output_type=_CHATDATA, + output_type=_CHAT, serialized_options=None, create_key=_descriptor._internal_create_key, ), _descriptor.MethodDescriptor( name='get_chat', - full_name='idm.api2.proto.Chats.get_chat', + full_name='idm.api.proto.ChatManager.get_chat', index=1, containing_service=None, input_type=_GETCHATREQUEST, - output_type=_CHATDATA, + output_type=_CHAT, serialized_options=None, create_key=_descriptor._internal_create_key, ), _descriptor.MethodDescriptor( name='list_chats', - full_name='idm.api2.proto.Chats.list_chats', + full_name='idm.api.proto.ChatManager.list_chats', index=2, containing_service=None, input_type=_LISTCHATSREQUEST, - output_type=_CHATSDATA, + output_type=_CHATS, serialized_options=None, create_key=_descriptor._internal_create_key, ), _descriptor.MethodDescriptor( name='update_chat', - full_name='idm.api2.proto.Chats.update_chat', + full_name='idm.api.proto.ChatManager.update_chat', index=3, containing_service=None, input_type=_UPDATECHATREQUEST, - output_type=_CHATDATA, + output_type=_CHAT, serialized_options=None, create_key=_descriptor._internal_create_key, ), ]) -_sym_db.RegisterServiceDescriptor(_CHATS) +_sym_db.RegisterServiceDescriptor(_CHATMANAGER) -DESCRIPTOR.services_by_name['Chats'] = _CHATS +DESCRIPTOR.services_by_name['ChatManager'] = _CHATMANAGER # @@protoc_insertion_point(module_scope) diff --git a/idm/api2/proto/chats_service_pb2_grpc.py b/idm/api/proto/chat_manager_service_pb2_grpc.py old mode 100755 new mode 100644 similarity index 57% rename from idm/api2/proto/chats_service_pb2_grpc.py rename to idm/api/proto/chat_manager_service_pb2_grpc.py index df55a5b..b0738c0 --- a/idm/api2/proto/chats_service_pb2_grpc.py +++ b/idm/api/proto/chat_manager_service_pb2_grpc.py @@ -1,11 +1,12 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc -from idm.api2.proto import \ - chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2 +from idm.api.proto import \ + chat_manager_service_pb2 as \ + idm_dot_api_dot_proto_dot_chat__manager__service__pb2 -class ChatsStub(object): +class ChatManagerStub(object): """Missing associated documentation comment in .proto file.""" def __init__(self, channel): @@ -15,28 +16,28 @@ class ChatsStub(object): channel: A grpc.Channel. """ self.create_chat = channel.unary_unary( - '/idm.api2.proto.Chats/create_chat', - request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString, - response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString, + '/idm.api.proto.ChatManager/create_chat', + request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.SerializeToString, + response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString, ) self.get_chat = channel.unary_unary( - '/idm.api2.proto.Chats/get_chat', - request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString, - response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString, + '/idm.api.proto.ChatManager/get_chat', + request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.SerializeToString, + response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString, ) self.list_chats = channel.unary_unary( - '/idm.api2.proto.Chats/list_chats', - request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString, - response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString, + '/idm.api.proto.ChatManager/list_chats', + request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.SerializeToString, + response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.FromString, ) self.update_chat = channel.unary_unary( - '/idm.api2.proto.Chats/update_chat', - request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString, - response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString, + '/idm.api.proto.ChatManager/update_chat', + request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.SerializeToString, + response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString, ) -class ChatsServicer(object): +class ChatManagerServicer(object): """Missing associated documentation comment in .proto file.""" def create_chat(self, request, context): @@ -64,36 +65,36 @@ class ChatsServicer(object): raise NotImplementedError('Method not implemented!') -def add_ChatsServicer_to_server(servicer, server): +def add_ChatManagerServicer_to_server(servicer, server): rpc_method_handlers = { 'create_chat': grpc.unary_unary_rpc_method_handler( servicer.create_chat, - request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.FromString, - response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString, + request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.FromString, + response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString, ), 'get_chat': grpc.unary_unary_rpc_method_handler( servicer.get_chat, - request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.FromString, - response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString, + request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.FromString, + response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString, ), 'list_chats': grpc.unary_unary_rpc_method_handler( servicer.list_chats, - request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.FromString, - response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.SerializeToString, + request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.FromString, + response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.SerializeToString, ), 'update_chat': grpc.unary_unary_rpc_method_handler( servicer.update_chat, - request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.FromString, - response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString, + request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.FromString, + response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( - 'idm.api2.proto.Chats', rpc_method_handlers) + 'idm.api.proto.ChatManager', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) # This class is part of an EXPERIMENTAL API. -class Chats(object): +class ChatManager(object): """Missing associated documentation comment in .proto file.""" @staticmethod @@ -107,9 +108,9 @@ class Chats(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/create_chat', - idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString, - idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString, + return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/create_chat', + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.SerializeToString, + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -124,9 +125,9 @@ class Chats(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/get_chat', - idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString, - idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString, + return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/get_chat', + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.SerializeToString, + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -141,9 +142,9 @@ class Chats(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/list_chats', - idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString, - idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString, + return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/list_chats', + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.SerializeToString, + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -158,8 +159,8 @@ class Chats(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/update_chat', - idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString, - idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString, + return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/update_chat', + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.SerializeToString, + idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/idm/api/services/chat_manager.py b/idm/api/services/chat_manager.py new file mode 100644 index 0000000..a3ac0b3 --- /dev/null +++ b/idm/api/services/chat_manager.py @@ -0,0 +1,109 @@ +from grpc import StatusCode +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb +from idm.api.proto.chat_manager_service_pb2 import Chats as ChatsPb +from idm.api.proto.chat_manager_service_pb2_grpc import ( + ChatManagerServicer, + add_ChatManagerServicer_to_server, +) +from library.aiogrpctools.base import ( + BaseService, + aiogrpc_request_wrapper, +) +from pypika import ( + PostgreSQLQuery, + Table, +) + + +class ChatManagerService(ChatManagerServicer, BaseService): + chats_table = Table('chats') + + def __init__(self, server, service_name, pool_holder, admin_log_reader): + super().__init__(service_name=service_name) + self.server = server + self.pool_holder = pool_holder + self.admin_log_reader = admin_log_reader + + async def start(self): + add_ChatManagerServicer_to_server(self, self.server) + + def enrich_chat(self, chat_pb: ChatPb): + chat_pb.is_subscribed = self.admin_log_reader.is_subscribed(chat_pb.chat_id) + return chat_pb + + @aiogrpc_request_wrapper() + async def create_chat(self, request, context, metadata): + chat = ChatPb( + chat_id=request.chat_id, + language=request.language, + username=request.username, + is_system_messaging_enabled=True, + is_discovery_enabled=True, + ) + query = ( + PostgreSQLQuery + .into(self.chats_table) + .columns( + self.chats_table.chat_id, + self.chats_table.language, + self.chats_table.username, + self.chats_table.is_system_messaging_enabled, + self.chats_table.is_discovery_enabled, + ) + .insert( + chat.chat_id, + chat.language, + chat.username, + chat.is_system_messaging_enabled, + chat.is_discovery_enabled, + ) + .returning('*') + ).get_sql() + async with self.pool_holder.pool.acquire() as session: + result = await session.execute(query) + chat = await result.fetchone() + return self.enrich_chat(ChatPb(**chat)) + + @aiogrpc_request_wrapper() + async def get_chat(self, request, context, metadata): + query = ( + PostgreSQLQuery + .from_(self.chats_table) + .select('*') + .where(self.chats_table.chat_id == request.chat_id) + ).get_sql() + async with self.pool_holder.pool.acquire() as session: + result = await session.execute(query) + chat = await result.fetchone() + if chat is None: + await context.abort(StatusCode.NOT_FOUND, 'not_found') + return self.enrich_chat(ChatPb(**chat)) + + @aiogrpc_request_wrapper() + async def list_chats(self, request, context, metadata): + query = ( + PostgreSQLQuery + .from_(self.chats_table) + .select('*') + .where(self.chats_table.ban_until > request.banned_at_moment) + .limit(10) + ).get_sql() + async with self.pool_holder.pool.acquire() as session: + results = await session.execute(query) + chats = await results.fetchall() + return ChatsPb( + chats=list(map(lambda x: self.enrich_chat(ChatPb(**x)), chats)) + ) + + @aiogrpc_request_wrapper() + async def update_chat(self, request, context, metadata): + query = PostgreSQLQuery.update(self.chats_table) + for field in request.DESCRIPTOR.fields: + if field.containing_oneof and request.HasField(field.name): + field_value = getattr(request, field.name) + query = query.set(field.name, field_value) + query = query.where(self.chats_table.chat_id == request.chat_id).returning('*').get_sql() + async with self.pool_holder.pool.acquire() as session: + result = await session.execute(query) + chat = await result.fetchone() + return self.enrich_chat(ChatPb(**chat)) diff --git a/idm/api2/aioclient/__init__.py b/idm/api2/aioclient/__init__.py deleted file mode 100644 index b4af6b3..0000000 --- a/idm/api2/aioclient/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .aioclient import IdmApi2GrpcClient - -__all__ = ['IdmApi2GrpcClient'] diff --git a/idm/api2/proto/chats_service.proto b/idm/api2/proto/chats_service.proto deleted file mode 100644 index a90010f..0000000 --- a/idm/api2/proto/chats_service.proto +++ /dev/null @@ -1,66 +0,0 @@ -syntax = "proto3"; -package idm.api2.proto; - -import "idm/api2/proto/location.proto"; - -message ChatData { - int64 id = 1; - string username = 2; - string language = 3; - bool is_system_messaging_enabled = 4; - bool is_discovery_enabled = 5; - int32 ban_until = 6; - string ban_message = 7; - bool is_admin = 8; - bool is_subscribed = 10; - int64 created_at = 11; -} - -message ChatsData { - repeated ChatData chats = 1; -} - -message CreateChatRequest { - int64 chat_id = 1; - string username = 2; - string language = 3; -} - -message GetChatRequest { - int64 chat_id = 1; -} - -message ListChatsRequest { - oneof banned_at_moment_oneof { - int32 banned_at_moment = 1; - } -} - -message UpdateChatRequest { - int64 chat_id = 1; - oneof language_oneof { - string language = 2; - } - oneof is_system_messaging_enabled_oneof { - bool is_system_messaging_enabled = 3; - } - oneof is_discovery_enabled_oneof { - bool is_discovery_enabled = 4; - } - oneof ban_until_oneof { - int32 ban_until = 5; - } - oneof ban_message_oneof { - string ban_message = 6; - } - oneof is_admin_oneof { - bool is_admin = 7; - } -} - -service Chats { - rpc create_chat(CreateChatRequest) returns (ChatData) {}; - rpc get_chat(GetChatRequest) returns (ChatData) {}; - rpc list_chats(ListChatsRequest) returns (ChatsData) {}; - rpc update_chat(UpdateChatRequest) returns (ChatData) {}; -} diff --git a/idm/api2/proto/location.proto b/idm/api2/proto/location.proto deleted file mode 100644 index 2b4552e..0000000 --- a/idm/api2/proto/location.proto +++ /dev/null @@ -1,7 +0,0 @@ -syntax = "proto3"; -package idm.api2.proto; - -message Location { - float lat = 1; - float lon = 2; -} diff --git a/idm/api2/proto/location_pb2.py b/idm/api2/proto/location_pb2.py deleted file mode 100755 index 6f97f73..0000000 --- a/idm/api2/proto/location_pb2.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: idm/api2/proto/location.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='idm/api2/proto/location.proto', - package='idm.api2.proto', - syntax='proto3', - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x1didm/api2/proto/location.proto\x12\x0eidm.api2.proto\"$\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x02\x12\x0b\n\x03lon\x18\x02 \x01(\x02\x62\x06proto3' -) - - - - -_LOCATION = _descriptor.Descriptor( - name='Location', - full_name='idm.api2.proto.Location', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='lat', full_name='idm.api2.proto.Location.lat', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='lon', full_name='idm.api2.proto.Location.lon', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=49, - serialized_end=85, -) - -DESCRIPTOR.message_types_by_name['Location'] = _LOCATION -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), { - 'DESCRIPTOR' : _LOCATION, - '__module__' : 'idm.api2.proto.location_pb2' - # @@protoc_insertion_point(class_scope:idm.api2.proto.Location) - }) -_sym_db.RegisterMessage(Location) - - -# @@protoc_insertion_point(module_scope) diff --git a/idm/api2/proto/location_pb2_grpc.py b/idm/api2/proto/location_pb2_grpc.py deleted file mode 100755 index 2daafff..0000000 --- a/idm/api2/proto/location_pb2_grpc.py +++ /dev/null @@ -1,4 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - diff --git a/images/install.bzl b/images/install.bzl index 8233afd..9ec5089 100644 --- a/images/install.bzl +++ b/images/install.bzl @@ -11,14 +11,8 @@ def images_install(): container_pull( name = "ubuntu", - digest = "sha256:5403064f94b617f7975a19ba4d1a1299fd584397f6ee4393d0e16744ed11aab1", + digest = "sha256:45ff0162921e61c004010a67db1bee7d039a677bed0cb294e61f2b47346d42b3", registry = "index.docker.io", repository = "library/ubuntu", - tag = "20.04", - ) - container_pull( - name = "jupyter", - registry = "index.docker.io", - repository = "jupyter/tensorflow-notebook", - tag = "latest", + tag = "20.10", ) diff --git a/images/production/BUILD.bazel b/images/production/BUILD.bazel index f5f664a..ce78233 100644 --- a/images/production/BUILD.bazel +++ b/images/production/BUILD.bazel @@ -13,7 +13,7 @@ download_pkgs( "libev4", "libgomp1", "libgoogle-perftools-dev", - "libprotobuf17", + "libprotobuf23", "libssl1.1", ], ) diff --git a/library/logging/__init__.py b/library/logging/__init__.py index 6b6086a..84b81a6 100644 --- a/library/logging/__init__.py +++ b/library/logging/__init__.py @@ -31,7 +31,7 @@ def error_log(e, level=logging.ERROR, **fields): e = e.as_internal_dict() e.update(fields) elif fields: - e = {'error': str(e), **fields} + e = {'error': repr(e), **fields} logging.getLogger('error').log( msg=e, level=level diff --git a/library/telegram/base.py b/library/telegram/base.py index 008d4c3..9833164 100644 --- a/library/telegram/base.py +++ b/library/telegram/base.py @@ -136,7 +136,7 @@ class RequestContext: self.request_id = request_id or RequestContext.generate_request_id(request_id_length) self.default_fields = { 'bot_name': self.bot_name, - 'chat_id': self.chat.id, + 'chat_id': self.chat.chat_id, 'request_id': self.request_id, } diff --git a/nexus/bot/BUILD.bazel b/nexus/bot/BUILD.bazel index 2509577..f8041ca 100644 --- a/nexus/bot/BUILD.bazel +++ b/nexus/bot/BUILD.bazel @@ -30,7 +30,7 @@ py3_image( requirement("python_socks"), requirement("tenacity"), requirement("uvloop"), - "//idm/api2/aioclient", + "//idm/api/aioclient", requirement("aiobaseclient"), requirement("aiocrossref"), requirement("aiokit"), diff --git a/nexus/bot/application.py b/nexus/bot/application.py index 05be59a..736cbb8 100644 --- a/nexus/bot/application.py +++ b/nexus/bot/application.py @@ -1,5 +1,5 @@ from aiokit import AioRootThing -from idm.api2.aioclient import IdmApi2GrpcClient +from idm.api.aioclient import IdmApiGrpcClient from izihawa_utils.importlib import import_object from library.telegram.base import BaseTelegramClient from nexus.bot.promotioner import Promotioner @@ -21,7 +21,7 @@ class TelegramApplication(AioRootThing): ) self.hub_client = HubGrpcClient(base_url=self.config['hub']['url']) - self.idm_client = IdmApi2GrpcClient(base_url=self.config['idm']['url']) + self.idm_client = IdmApiGrpcClient(base_url=self.config['idm']['url']) self.meta_api_client = MetaApiGrpcClient(base_url=self.config['meta_api']['url']) self.promotioner = Promotioner(promotions=self.config['promotions']) diff --git a/nexus/bot/handlers/base.py b/nexus/bot/handlers/base.py index 4ee5c6f..779dd1b 100644 --- a/nexus/bot/handlers/base.py +++ b/nexus/bot/handlers/base.py @@ -6,7 +6,7 @@ from typing import Union from grpc import StatusCode from grpc.experimental.aio import AioRpcError -from idm.api2.proto.chats_service_pb2 import ChatData as Chat +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from izihawa_utils.exceptions import BaseError from izihawa_utils.random import random_string from library.logging import error_log @@ -40,12 +40,12 @@ def get_language(event: events.ChatAction, chat): return chat.lang_code -def is_banned(chat: Chat) -> bool: +def is_banned(chat: ChatPb) -> bool: return chat.ban_until is not None and datetime.utcnow().timestamp() < chat.ban_until -def is_subscribed(chat: Chat) -> bool: - return chat.is_subscribed or chat.id < 0 or chat.created_at > time.time() - 10 * 60 +def is_subscribed(chat: ChatPb) -> bool: + return chat.is_subscribed or chat.chat_id < 0 or chat.created_at > time.time() - 10 * 60 class ReadOnlyModeError(BaseError): @@ -97,7 +97,7 @@ class BaseHandler(ABC): session_id=session_id, position=position, request_id=request_context.request_id, - user_id=request_context.chat.id, + user_id=request_context.chat.chat_id, ) async def resolve_scimag( @@ -136,7 +136,7 @@ class BaseHandler(ABC): page_size=16, request_id=request_context.request_id, session_id=session_id, - user_id=request_context.chat.id, + user_id=request_context.chat.chat_id, ) duplicates = [ scored_document.typed_document.scitech @@ -209,7 +209,7 @@ class BaseHandler(ABC): ) return chat - async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: Chat): + async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: ChatPb): if is_banned(chat): if chat.ban_message is not None: async with safe_execution( @@ -245,7 +245,7 @@ class BaseHandler(ABC): ) raise events.StopPropagation() - async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: Chat): + async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: ChatPb): if ( self.application.config['application']['is_subscription_required'] and self.is_subscription_required_for_handler @@ -261,7 +261,7 @@ class BaseHandler(ABC): ).format(related_channel=self.application.config['telegram']['related_channel'])) raise events.StopPropagation() - def _has_access(self, chat: Chat) -> bool: + def _has_access(self, chat: ChatPb) -> bool: return True async def _process_chat(self, event: events.ChatAction, request_id: str): @@ -271,8 +271,8 @@ class BaseHandler(ABC): error_log(e) event_chat = await event.get_chat() username = get_username(event, event_chat) - chat = Chat( - id=event.chat_id, + chat = ChatPb( + chat_id=event.chat_id, is_system_messaging_enabled=True, is_discovery_enabled=True, language='en', @@ -307,7 +307,7 @@ class BaseHandler(ABC): await self._check_ban(event=event, request_context=request_context, chat=chat) if self.should_reset_last_widget: - self.reset_last_widget(request_context.chat.id) + self.reset_last_widget(request_context.chat.chat_id) async with safe_execution( request_context=request_context, diff --git a/nexus/bot/handlers/download.py b/nexus/bot/handlers/download.py index 4d183db..12c15bf 100644 --- a/nexus/bot/handlers/download.py +++ b/nexus/bot/handlers/download.py @@ -19,7 +19,7 @@ class DownloadHandler(BaseCallbackQueryHandler): document_id = int(event.pattern_match.group(3)) position = int(event.pattern_match.group(4).decode()) - self.application.user_manager.last_widget[request_context.chat.id] = None + self.application.user_manager.last_widget[request_context.chat.chat_id] = None request_context.add_default_fields(mode='download', session_id=session_id) request_context.statbox(action='get', query=str(document_id), position=position) @@ -48,4 +48,4 @@ class DownloadHandler(BaseCallbackQueryHandler): ) else: await remove_button(event, '⬇️', and_empty_too=True) - self.application.user_manager.last_widget[request_context.chat.id] = None + self.application.user_manager.last_widget[request_context.chat.chat_id] = None diff --git a/nexus/bot/handlers/referencing_to.py b/nexus/bot/handlers/referencing_to.py index 3d7fb12..00ac7d7 100644 --- a/nexus/bot/handlers/referencing_to.py +++ b/nexus/bot/handlers/referencing_to.py @@ -41,7 +41,7 @@ class ReferencingToHandler(BaseCallbackQueryHandler): serp, buttons = await document_list_widget.render() return await self.application.telegram_client.edit_message( - request_context.chat.id, + request_context.chat.chat_id, message_id, serp, buttons=buttons, diff --git a/nexus/bot/handlers/roll.py b/nexus/bot/handlers/roll.py index 6f8ad53..0d4d541 100644 --- a/nexus/bot/handlers/roll.py +++ b/nexus/bot/handlers/roll.py @@ -20,7 +20,7 @@ class RollHandler(BaseHandler): language=request_context.chat.language, session_id=session_id, request_id=request_context.request_id, - user_id=request_context.chat.id, + user_id=request_context.chat.chat_id, ) scitech_view = await self.resolve_scitech( document_id=roll_response_pb.document_id, @@ -35,7 +35,7 @@ class RollHandler(BaseHandler): ) actions = [ self.application.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, view, buttons=buttons, ), diff --git a/nexus/bot/handlers/search.py b/nexus/bot/handlers/search.py index dd5ec40..d229942 100644 --- a/nexus/bot/handlers/search.py +++ b/nexus/bot/handlers/search.py @@ -52,7 +52,7 @@ class BaseSearchHandler(BaseHandler): except AioRpcError as e: actions = [ self.application.telegram_client.delete_messages( - request_context.chat.id, + request_context.chat.chat_id, [message_id], ) ] @@ -118,7 +118,7 @@ class BaseSearchHandler(BaseHandler): ) return await asyncio.gather( self.application.telegram_client.edit_message( - request_context.chat.id, + request_context.chat.chat_id, message_id, view, buttons=buttons, @@ -127,7 +127,7 @@ class BaseSearchHandler(BaseHandler): serp, buttons = await search_widget.render() return await self.application.telegram_client.edit_message( - request_context.chat.id, + request_context.chat.chat_id, message_id, serp, buttons=buttons, @@ -147,7 +147,7 @@ class SearchHandler(BaseSearchHandler): 'action': 'user_flood_ban', 'mode': 'search', 'ban_timeout_seconds': ban_timeout, - 'chat_id': request_context.chat.id, + 'chat_id': request_context.chat.chat_id, }) ban_reason = t( 'BAN_MESSAGE_TOO_MANY_REQUESTS', @@ -162,10 +162,10 @@ class SearchHandler(BaseSearchHandler): )) async def handler(self, event: events.ChatAction, request_context: RequestContext): - ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.id) + ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.chat_id) if ban_timeout: return await self.ban_handler(event, request_context, ban_timeout) - self.application.user_manager.add_search_time(user_id=request_context.chat.id, search_time=time.time()) + self.application.user_manager.add_search_time(user_id=request_context.chat.chat_id, search_time=time.time()) search_prefix = event.pattern_match.group(1) query = event.pattern_match.group(2) @@ -178,7 +178,7 @@ class SearchHandler(BaseSearchHandler): prefetch_message = await event.reply( t("SEARCHING", language=request_context.chat.language), ) - self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id + self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id try: await self.do_search( event, request_context, prefetch_message, diff --git a/nexus/bot/handlers/start.py b/nexus/bot/handlers/start.py index f56fb4c..19287ed 100644 --- a/nexus/bot/handlers/start.py +++ b/nexus/bot/handlers/start.py @@ -32,7 +32,7 @@ class StartHandler(BaseSearchHandler): prefetch_message = await request_message.reply( t("SEARCHING", language=request_context.chat.language), ) - self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id + self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id await asyncio.gather( event.delete(), self.do_search(event, request_context, prefetch_message, query=query, diff --git a/nexus/bot/handlers/view.py b/nexus/bot/handlers/view.py index fb5d2b6..32f8440 100644 --- a/nexus/bot/handlers/view.py +++ b/nexus/bot/handlers/view.py @@ -30,7 +30,7 @@ class ViewHandler(BaseHandler): request_context.add_default_fields(mode='view', session_id=session_id) request_context.statbox(action='view', query=str(document_id), position=position) - found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.id) + found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id) try: if found_old_widget: @@ -41,11 +41,11 @@ class ViewHandler(BaseHandler): functions.messages.GetMessagesRequest(id=[old_message_id]) )).messages[0] prefetch_message = await self.application.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t("SEARCHING", language=request_context.chat.language), reply_to=old_message.reply_to_msg_id, ) - self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id + self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id message_id = prefetch_message.id link_preview = True @@ -81,7 +81,7 @@ class ViewHandler(BaseHandler): ) actions = [ self.application.telegram_client.edit_message( - request_context.chat.id, + request_context.chat.chat_id, message_id, view, buttons=buttons, @@ -92,7 +92,7 @@ class ViewHandler(BaseHandler): if not found_old_widget: actions.append( self.application.telegram_client.delete_messages( - request_context.chat.id, + request_context.chat.chat_id, [old_message_id], ) ) diff --git a/nexus/bot/handlers/vote.py b/nexus/bot/handlers/vote.py index 71a1f3b..4630558 100644 --- a/nexus/bot/handlers/vote.py +++ b/nexus/bot/handlers/vote.py @@ -26,7 +26,7 @@ class VoteHandler(BaseCallbackQueryHandler): vote=VotePb( document_id=document_id, value=vote_value, - voter_id=request_context.chat.id, + voter_id=request_context.chat.chat_id, ), ) @@ -43,7 +43,7 @@ class VoteHandler(BaseCallbackQueryHandler): # ToDo: Generalize nexus.views.telegram.common.remove_button and use it here return await asyncio.gather( self.application.telegram_client.edit_message( - request_context.chat.id, + request_context.chat.chat_id, message.id, message.text, buttons=None, diff --git a/nexus/bot/widgets/admin_widget.py b/nexus/bot/widgets/admin_widget.py index a95bb24..d366560 100644 --- a/nexus/bot/widgets/admin_widget.py +++ b/nexus/bot/widgets/admin_widget.py @@ -1,9 +1,9 @@ -from idm.api2.proto.chats_service_pb2 import ChatData as Chat +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.bot.application import TelegramApplication class AdminWidget: - def __init__(self, application: TelegramApplication, chat: Chat): + def __init__(self, application: TelegramApplication, chat: ChatPb): self.application = application self.chat = chat diff --git a/nexus/bot/widgets/banlist_widget.py b/nexus/bot/widgets/banlist_widget.py index 5d1bc10..66abbf3 100644 --- a/nexus/bot/widgets/banlist_widget.py +++ b/nexus/bot/widgets/banlist_widget.py @@ -1,25 +1,25 @@ import time -from idm.api2.proto.chats_service_pb2 import ChatData as Chat +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.bot.application import TelegramApplication class BanlistWidget: - def __init__(self, application: TelegramApplication, chat: Chat): + def __init__(self, application: TelegramApplication, chat: ChatPb): self.application = application self.chat = chat - async def render(self, chat_list: list[Chat]): + async def render(self, chat_list: list[ChatPb]): if not chat_list: return 'Nobody is banned' separator = '------------\n' return separator.join( map( lambda chat: ( - f'```{chat.username} ({chat.id})\n' + f'```{chat.username} ({chat.chat_id})\n' f'Until: {time.ctime(chat.ban_until)}\n' f'Message: {chat.ban_message}```\n' - f'/unban_{chat.id}\n' + f'/unban_{chat.chat_id}\n' ), chat_list ) diff --git a/nexus/bot/widgets/document_list_widget.py b/nexus/bot/widgets/document_list_widget.py index 35e1e73..8ef1418 100644 --- a/nexus/bot/widgets/document_list_widget.py +++ b/nexus/bot/widgets/document_list_widget.py @@ -1,6 +1,6 @@ from typing import Optional -from idm.api2.proto.chats_service_pb2 import ChatData as Chat +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.bot.application import TelegramApplication from nexus.meta_api.proto.meta_search_service_pb2 import \ ScoredDocument as ScoredDocumentPb @@ -14,7 +14,7 @@ class DocumentListWidget: def __init__( self, application: TelegramApplication, - chat: Chat, + chat: ChatPb, session_id: str, message_id: int, request_id: str, @@ -32,7 +32,7 @@ class DocumentListWidget: @staticmethod async def create( application: TelegramApplication, - chat: Chat, + chat: ChatPb, session_id: str, message_id: int, request_id: str, @@ -58,7 +58,7 @@ class DocumentListWidget: position=0, request_id=self.request_id, session_id=self.session_id, - user_id=self.chat.id, + user_id=self.chat.chat_id, ) self._response = await self.application.meta_api_client.search( schemas=('scimag',), @@ -66,7 +66,7 @@ class DocumentListWidget: page=self.page, request_id=self.request_id, session_id=self.session_id, - user_id=self.chat.id, + user_id=self.chat.chat_id, ) @property diff --git a/nexus/bot/widgets/search_widget.py b/nexus/bot/widgets/search_widget.py index 8f3f338..8791e22 100644 --- a/nexus/bot/widgets/search_widget.py +++ b/nexus/bot/widgets/search_widget.py @@ -1,6 +1,6 @@ from typing import Optional -from idm.api2.proto.chats_service_pb2 import ChatData as Chat +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.bot.application import TelegramApplication from nexus.meta_api.proto.meta_search_service_pb2 import \ ScoredDocument as ScoredDocumentPb @@ -21,7 +21,7 @@ class SearchWidget: def __init__( self, application: TelegramApplication, - chat: Chat, + chat: ChatPb, session_id: str, message_id: int, request_id: str, @@ -41,7 +41,7 @@ class SearchWidget: @staticmethod async def create( application: TelegramApplication, - chat: Chat, + chat: ChatPb, session_id: str, message_id: int, request_id: str, @@ -70,7 +70,7 @@ class SearchWidget: page_size=self.application.config['application']['page_size'], request_id=self.request_id, session_id=self.session_id, - user_id=self.chat.id, + user_id=self.chat.chat_id, language=self.chat.language, ) diff --git a/nexus/bot/widgets/settings_widget.py b/nexus/bot/widgets/settings_widget.py index aa4ba27..1481788 100644 --- a/nexus/bot/widgets/settings_widget.py +++ b/nexus/bot/widgets/settings_widget.py @@ -1,6 +1,6 @@ from typing import Optional -from idm.api2.proto.chats_service_pb2 import ChatData as Chat +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.bot.application import TelegramApplication from nexus.translations import t from telethon import Button @@ -34,7 +34,7 @@ class SettingsWidget: def __init__( self, application: TelegramApplication, - chat: Chat, + chat: ChatPb, has_language_buttons: Optional[bool] = None, is_group_mode: bool = False, request_id: Optional[str] = None, @@ -54,7 +54,7 @@ class SettingsWidget: async def _switch_language(self, target_language: str): self.chat = await self.application.idm_client.update_chat( - chat_id=self.chat.id, + chat_id=self.chat.chat_id, language=target_language, request_id=self.request_id, ) @@ -62,7 +62,7 @@ class SettingsWidget: async def _switch_system_messaging(self, is_system_messaging_enabled: str): self.chat = await self.application.idm_client.update_chat( - chat_id=self.chat.id, + chat_id=self.chat.chat_id, is_system_messaging_enabled=bool(int(is_system_messaging_enabled)), request_id=self.request_id, ) @@ -70,7 +70,7 @@ class SettingsWidget: async def _switch_discovery(self, is_discovery_enabled: str): self.chat = await self.application.idm_client.update_chat( - chat_id=self.chat.id, + chat_id=self.chat.chat_id, is_discovery_enabled=bool(int(is_discovery_enabled)), request_id=self.request_id, ) diff --git a/nexus/hub/BUILD.bazel b/nexus/hub/BUILD.bazel index 3f30db2..aa28850 100644 --- a/nexus/hub/BUILD.bazel +++ b/nexus/hub/BUILD.bazel @@ -17,10 +17,7 @@ py3_image( base = "//images/production:base-python-image", data = [ "configs/base.yaml", - "configs/development.yaml", "configs/logging.yaml", - "configs/production.yaml", - "configs/testing.yaml", ], main = "main.py", srcs_version = "PY3ONLY", @@ -37,7 +34,7 @@ py3_image( requirement("python-socks"), requirement("tenacity"), requirement("uvloop"), - "//idm/api2/proto:idm_proto_py", + "//idm/api/proto:idm_proto_py", requirement("giogrobid"), "//library/aiogrpctools", requirement("aiokit"), diff --git a/nexus/hub/README.md b/nexus/hub/README.md index 971b5fa..b6eee92 100644 --- a/nexus/hub/README.md +++ b/nexus/hub/README.md @@ -1,66 +1,9 @@ # Nexus Search: Hub API -`Hub` is a daemon responsible for retrieving files and sending them to users. This version has cut `configs` -subdirectory due to hard reliance of configs on the network infrastructure you are using. -You have to write your own configs taking example below into account. +`Hub` is a daemon responsible for retrieving files and sending them to users. The bot requires two other essential parts: - Postgres Database - IPFS Daemon or their substitutions - -## Sample `configs/base.yaml` - -```yaml ---- - -application: - # Look at the special Postgres `sharience` table to retrieve user-sent files - is_sharience_enabled: true - maintenance_picture_url: - # Used in logging - service_name: nexus-hub - # Store file hashes into operation log - should_store_hashes: true -database: - database: nexus - host: - password: '{{ DATABASE_PASSWORD }}' - username: '{{ DATABASE_USERNAME }}' -grobid: - url: -grpc: - # Listen address - address: 0.0.0.0 - # Listen port - port: 9090 -ipfs: - address: - port: 4001 -log_path: '/var/log/nexus-hub/{{ ENV_TYPE }}' -meta_api: - url: -pylon: - # Proxy used in `pylon` retriever to download files - proxy: socks5://127.0.0.1:9050 - # Proxy used in `pylon` retriever to get metadata - resolve_proxy: socks5://127.0.0.1:9050 -telegram: - # Telegram App Hash from https://my.telegram.org/ - app_hash: '{{ APP_HASH }}' - # Telegram App ID from https://my.telegram.org/ - app_id: 00000 - # External bot name shown in messages to users - bot_external_name: libgen_scihub_bot - # Internal bot name used in logging - bot_name: nexus-bot - bot_token: '{{ BOT_TOKEN }}' - # Telethon database for keeping cache - database: - session_id: nexus-hub - # Frequency of updating downloading progress - progress_throttle_seconds: 5 - # Send files using stored telegram_file_id - should_use_telegram_file_id: true -``` \ No newline at end of file diff --git a/nexus/hub/aioclient/BUILD.bazel b/nexus/hub/aioclient/BUILD.bazel index c56a59c..d473826 100644 --- a/nexus/hub/aioclient/BUILD.bazel +++ b/nexus/hub/aioclient/BUILD.bazel @@ -6,7 +6,7 @@ py_library( visibility = ["//visibility:public"], deps = [ requirement("grpcio"), - "//idm/api2/proto:idm_proto_py", + "//idm/api/proto:idm_proto_py", requirement("aiokit"), "//nexus/hub/proto:hub_grpc_py", "//nexus/hub/proto:hub_proto_py", diff --git a/nexus/hub/aioclient/aioclient.py b/nexus/hub/aioclient/aioclient.py index 8096ad4..e7062e7 100644 --- a/nexus/hub/aioclient/aioclient.py +++ b/nexus/hub/aioclient/aioclient.py @@ -2,7 +2,7 @@ from typing import Optional from aiokit import AioThing from grpc.experimental.aio import insecure_channel -from idm.api2.proto.chats_service_pb2 import ChatData as ChatDataPb +from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb from nexus.hub.proto.delivery_service_pb2 import \ StartDeliveryRequest as StartDeliveryRequestPb from nexus.hub.proto.delivery_service_pb2 import \ @@ -42,7 +42,7 @@ class HubGrpcClient(AioThing): async def start_delivery( self, typed_document_pb: TypedDocumentPb, - chat: ChatDataPb, + chat: ChatPb, request_id: Optional[str], session_id: Optional[str], ) -> StartDeliveryResponsePb: @@ -58,7 +58,7 @@ class HubGrpcClient(AioThing): self, telegram_document: bytes, telegram_file_id: str, - chat: ChatDataPb, + chat: ChatPb, request_id: Optional[str] = None, session_id: Optional[str] = None, ) -> SubmitResponsePb: diff --git a/nexus/hub/configs/base.yaml b/nexus/hub/configs/base.yaml new file mode 100644 index 0000000..a3044da --- /dev/null +++ b/nexus/hub/configs/base.yaml @@ -0,0 +1,51 @@ +--- + +application: + # Enable special Postgres `sharience` table to retrieve user-sent files + is_sharience_enabled: true + # URL to the picture shown while maintenance + maintenance_picture_url: + # Used in logging + service_name: nexus-hub + # Store file hashes into operation log + should_store_hashes: true +database: + database: nexus + host: + password: '{{ DATABASE_PASSWORD }}' + username: '{{ DATABASE_USERNAME }}' +grobid: + url: +grpc: + # Listen address + address: 0.0.0.0 + # Listen port + port: 9090 +ipfs: + address: + port: 4001 +log_path: '/var/log/nexus-hub/{{ ENV_TYPE }}' +meta_api: + url: +pylon: + # Proxy used in `pylon` retriever to download files + proxy: socks5://127.0.0.1:9050 + # Proxy used in `pylon` retriever to get metadata + resolve_proxy: socks5://127.0.0.1:9050 +telegram: + # Telegram App Hash from https://my.telegram.org/ + app_hash: '{{ APP_HASH }}' + # Telegram App ID from https://my.telegram.org/ + app_id: 00000 + # External bot name shown in messages to users + bot_external_name: libgen_scihub_bot + # Internal bot name used in logging + bot_name: nexus-hub + bot_token: '{{ BOT_TOKEN }}' + # Telethon database for keeping cache + database: + session_id: nexus-hub + # Frequency of updating downloading progress + progress_throttle_seconds: 5 + # Send files using stored telegram_file_id + should_use_telegram_file_id: true diff --git a/nexus/hub/configs/logging.yaml b/nexus/hub/configs/logging.yaml new file mode 100644 index 0000000..5836cd6 --- /dev/null +++ b/nexus/hub/configs/logging.yaml @@ -0,0 +1,76 @@ +--- + +logging: + disable_existing_loggers: false + formatters: + base: + class: library.logging.formatters.BaseFormatter + default: + class: library.logging.formatters.DefaultFormatter + traceback: + class: library.logging.formatters.TracebackFormatter + handlers: + debug: + class: logging.StreamHandler + formatter: default + level: DEBUG + stream: 'ext://sys.stderr' + error: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/error.log' + formatter: default + level: ERROR + operation: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/operation.log' + formatter: base + level: DEBUG + statbox: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/statbox.log' + formatter: default + level: INFO + traceback: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/traceback.log' + formatter: traceback + level: ERROR + warning: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/warning.log' + formatter: default + level: WARNING + loggers: + aiokafka: + handlers: + - error + propagate: false + debug: + handlers: + - debug + propagate: false + error: + handlers: + - traceback + - error + - warning + propagate: false + operation: + handlers: + - operation + propagate: false + statbox: + handlers: + - statbox + propagate: false + telethon: + handlers: + - error + propagate: false + root: + handlers: + - debug + - error + - warning + level: DEBUG + version: 1 diff --git a/nexus/hub/proto/BUILD.bazel b/nexus/hub/proto/BUILD.bazel index 7dcf5e6..fc3e83e 100644 --- a/nexus/hub/proto/BUILD.bazel +++ b/nexus/hub/proto/BUILD.bazel @@ -9,7 +9,7 @@ proto_library( "*.proto", ]), deps = [ - "//idm/api2/proto:idm_proto", + "//idm/api/proto:idm_proto", "//nexus/models/proto:models_proto", "@com_google_protobuf//:wrappers_proto", ], diff --git a/nexus/hub/proto/delivery_service.proto b/nexus/hub/proto/delivery_service.proto index 9a6c116..7f19af8 100644 --- a/nexus/hub/proto/delivery_service.proto +++ b/nexus/hub/proto/delivery_service.proto @@ -2,11 +2,11 @@ syntax = "proto3"; package nexus.hub.proto; import "nexus/models/proto/typed_document.proto"; -import "idm/api2/proto/chats_service.proto"; +import "idm/api/proto/chat_manager_service.proto"; message StartDeliveryRequest { nexus.models.proto.TypedDocument typed_document = 1; - idm.api2.proto.ChatData chat = 2; + idm.api.proto.Chat chat = 2; } message StartDeliveryResponse { diff --git a/nexus/hub/proto/delivery_service_pb2.py b/nexus/hub/proto/delivery_service_pb2.py index acccd3c..407dc80 100644 --- a/nexus/hub/proto/delivery_service_pb2.py +++ b/nexus/hub/proto/delivery_service_pb2.py @@ -12,8 +12,9 @@ from google.protobuf import symbol_database as _symbol_database _sym_db = _symbol_database.Default() -from idm.api2.proto import \ - chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2 +from idm.api.proto import \ + chat_manager_service_pb2 as \ + idm_dot_api_dot_proto_dot_chat__manager__service__pb2 from nexus.models.proto import \ typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2 @@ -23,9 +24,9 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a\"idm/api2/proto/chats_service.proto\"y\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x03\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3' + serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a(idm/api/proto/chat_manager_service.proto\"t\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12!\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x13.idm.api.proto.Chat\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x02\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3' , - dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,]) + dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api_dot_proto_dot_chat__manager__service__pb2.DESCRIPTOR,]) @@ -47,15 +48,15 @@ _STARTDELIVERYRESPONSE_STATUS = _descriptor.EnumDescriptor( type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( - name='ALREADY_DOWNLOADING', index=2, number=3, + name='ALREADY_DOWNLOADING', index=2, number=2, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), ], containing_type=None, serialized_options=None, - serialized_start=348, - serialized_end=413, + serialized_start=349, + serialized_end=414, ) _sym_db.RegisterEnumDescriptor(_STARTDELIVERYRESPONSE_STATUS) @@ -94,8 +95,8 @@ _STARTDELIVERYREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=136, - serialized_end=257, + serialized_start=142, + serialized_end=258, ) @@ -127,12 +128,12 @@ _STARTDELIVERYRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=260, - serialized_end=413, + serialized_start=261, + serialized_end=414, ) _STARTDELIVERYREQUEST.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT -_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA +_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api_dot_proto_dot_chat__manager__service__pb2._CHAT _STARTDELIVERYRESPONSE.fields_by_name['status'].enum_type = _STARTDELIVERYRESPONSE_STATUS _STARTDELIVERYRESPONSE_STATUS.containing_type = _STARTDELIVERYRESPONSE DESCRIPTOR.message_types_by_name['StartDeliveryRequest'] = _STARTDELIVERYREQUEST @@ -162,8 +163,8 @@ _DELIVERY = _descriptor.ServiceDescriptor( index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=415, - serialized_end=524, + serialized_start=416, + serialized_end=525, methods=[ _descriptor.MethodDescriptor( name='start_delivery', diff --git a/nexus/hub/proto/submitter_service.proto b/nexus/hub/proto/submitter_service.proto index 04039cc..fc9f6ca 100644 --- a/nexus/hub/proto/submitter_service.proto +++ b/nexus/hub/proto/submitter_service.proto @@ -1,12 +1,12 @@ syntax = "proto3"; package nexus.hub.proto; -import "idm/api2/proto/chats_service.proto"; +import "idm/api/proto/chat_manager_service.proto"; message SubmitRequest { bytes telegram_document = 1; string telegram_file_id = 2; - idm.api2.proto.ChatData chat = 3; + idm.api.proto.Chat chat = 3; } message SubmitResponse { } diff --git a/nexus/hub/proto/submitter_service_pb2.py b/nexus/hub/proto/submitter_service_pb2.py index 37b53a6..5a48f2f 100644 --- a/nexus/hub/proto/submitter_service_pb2.py +++ b/nexus/hub/proto/submitter_service_pb2.py @@ -12,8 +12,9 @@ from google.protobuf import symbol_database as _symbol_database _sym_db = _symbol_database.Default() -from idm.api2.proto import \ - chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2 +from idm.api.proto import \ + chat_manager_service_pb2 as \ + idm_dot_api_dot_proto_dot_chat__manager__service__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='nexus/hub/proto/submitter_service.proto', @@ -21,9 +22,9 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a\"idm/api2/proto/chats_service.proto\"R\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3' + serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a(idm/api/proto/chat_manager_service.proto\"g\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12\x18\n\x10telegram_file_id\x18\x02 \x01(\t\x12!\n\x04\x63hat\x18\x03 \x01(\x0b\x32\x13.idm.api.proto.Chat\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3' , - dependencies=[idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,]) + dependencies=[idm_dot_api_dot_proto_dot_chat__manager__service__pb2.DESCRIPTOR,]) @@ -44,8 +45,15 @@ _SUBMITREQUEST = _descriptor.Descriptor( is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( - name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=1, - number=2, type=11, cpp_type=10, label=1, + name='telegram_file_id', full_name='nexus.hub.proto.SubmitRequest.telegram_file_id', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=2, + number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -62,8 +70,8 @@ _SUBMITREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=96, - serialized_end=178, + serialized_start=102, + serialized_end=205, ) @@ -87,11 +95,11 @@ _SUBMITRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=180, - serialized_end=196, + serialized_start=207, + serialized_end=223, ) -_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA +_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api_dot_proto_dot_chat__manager__service__pb2._CHAT DESCRIPTOR.message_types_by_name['SubmitRequest'] = _SUBMITREQUEST DESCRIPTOR.message_types_by_name['SubmitResponse'] = _SUBMITRESPONSE _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -119,8 +127,8 @@ _SUBMITTER = _descriptor.ServiceDescriptor( index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=198, - serialized_end=286, + serialized_start=225, + serialized_end=313, methods=[ _descriptor.MethodDescriptor( name='submit', diff --git a/nexus/hub/services/base.py b/nexus/hub/services/base.py index afab074..45c8dcc 100644 --- a/nexus/hub/services/base.py +++ b/nexus/hub/services/base.py @@ -70,7 +70,7 @@ class BaseHubService(BaseService): buttons=buttons, caption=f"{document_view.generate_body(language=request_context.chat.language, limit=512)}\n" f"@{self.bot_external_name}", - entity=request_context.chat.id, + entity=request_context.chat.chat_id, file=file, progress_callback=progress_callback ) diff --git a/nexus/hub/services/delivery.py b/nexus/hub/services/delivery.py index 4c616bf..1d61ad8 100644 --- a/nexus/hub/services/delivery.py +++ b/nexus/hub/services/delivery.py @@ -76,7 +76,7 @@ class DownloadTask: ) ) - self.delivery_service.user_manager.add_task(self.request_context.chat.id, self.document_view.id) + self.delivery_service.user_manager.add_task(self.request_context.chat.chat_id, self.document_view.id) self.delivery_service.downloadings.add(self) self.task.add_done_callback(self.done_callback) @@ -84,7 +84,7 @@ class DownloadTask: def done_callback(self, f): self.delivery_service.downloadings.remove(self) self.delivery_service.user_manager.remove_task( - self.request_context.chat.id, + self.request_context.chat.chat_id, self.document_view.id, ) @@ -93,7 +93,7 @@ class DownloadTask: async def _on_fail(): await self.delivery_service.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t('MAINTENANCE', language=request_context.chat.language).format( maintenance_picture_url=self.delivery_service.maintenance_picture_url ), @@ -165,7 +165,7 @@ class DownloadTask: progress_callback=progress_bar_upload.callback, request_context=self.request_context, session_id=self.session_id, - voting=not is_group_or_channel(self.request_context.chat.id), + voting=not is_group_or_channel(self.request_context.chat.chat_id), ) request_context.statbox( action='uploaded', @@ -191,7 +191,7 @@ class DownloadTask: finally: downloads_gauge.dec() messages = filter_none([progress_bar_download.message]) - await self.delivery_service.telegram_client.delete_messages(request_context.chat.id, messages) + await self.delivery_service.telegram_client.delete_messages(request_context.chat.chat_id, messages) async def process_resp(self, resp, progress_bar, collected, filesize): progress_bar.set_source(get_fancy_name(resp.source)) @@ -206,7 +206,7 @@ class DownloadTask: async def respond_not_found(self, request_context: RequestContext, document_view): return await self.delivery_service.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t("SOURCES_UNAVAILABLE", language=request_context.chat.language).format( document=document_view.get_robust_title() ), @@ -244,6 +244,7 @@ class DownloadTask: async for resp in self.delivery_service.pylon_client.by_doi( doi=document_view.doi, md5=document_view.md5, + error_log_func=self.request_context.error_log, ): await self.process_resp( resp=resp, @@ -252,11 +253,14 @@ class DownloadTask: filesize=document_view.filesize, ) return bytes(collected) - except DownloadError: - pass + except DownloadError as e: + self.request_context.error_log(e) if document_view.md5: try: - async for resp in self.delivery_service.pylon_client.by_md5(md5=document_view.md5): + async for resp in self.delivery_service.pylon_client.by_md5( + md5=document_view.md5, + error_log_func=self.request_context.error_log, + ): await self.process_resp( resp=resp, progress_bar=progress_bar, @@ -264,14 +268,14 @@ class DownloadTask: filesize=document_view.filesize, ) return bytes(collected) - except DownloadError: - pass + except DownloadError as e: + self.request_context.error_log(e) async def external_cancel(self): self.task.cancel() self.request_context.statbox(action='externally_canceled') await self.delivery_service.telegram_client.send_message( - self.request_context.chat.id, + self.request_context.chat.chat_id, t("DOWNLOAD_CANCELED", language=self.request_context.chat.language).format( document=self.document_view.get_robust_title() ), @@ -366,15 +370,15 @@ class DeliveryService(DeliveryServicer, BaseHubService): file=document_view.telegram_file_id, session_id=metadata.get('session-id'), request_context=request_context, - voting=not is_group_or_channel(request_context.chat.id), + voting=not is_group_or_channel(request_context.chat.chat_id), ) request_context.statbox(action='cache_hit', document_id=document_view.id) except ValueError: cache_hit = False if not cache_hit: - if self.user_manager.has_task(request.chat.id, document_view.id): + if self.user_manager.has_task(request.chat.chat_id, document_view.id): return StartDeliveryResponsePb(status=StartDeliveryResponsePb.Status.ALREADY_DOWNLOADING) - if self.user_manager.hit_limits(request.chat.id): + if self.user_manager.hit_limits(request.chat.chat_id): return StartDeliveryResponsePb(status=StartDeliveryResponsePb.Status.TOO_MANY_DOWNLOADS) await DownloadTask( delivery_service=self, diff --git a/nexus/hub/services/submitter.py b/nexus/hub/services/submitter.py index 556fa5c..b7547e7 100644 --- a/nexus/hub/services/submitter.py +++ b/nexus/hub/services/submitter.py @@ -101,13 +101,13 @@ class SubmitterService(SubmitterServicer, BaseHubService): if document.size > 20 * 1024 * 1024: request_context.error_log(FileTooBigError(size=document.size)) await self.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t('FILE_TOO_BIG_ERROR', language=request_context.chat.language), buttons=[close_button()], ) return SubmitResponsePb() processing_message = await self.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t("PROCESSING_PAPER", language=request_context.chat.language).format( filename=document.attributes[0].file_name, ), @@ -121,7 +121,7 @@ class SubmitterService(SubmitterServicer, BaseHubService): except BadRequestError as e: request_context.error_log(e) await self.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t('UNPARSABLE_DOCUMENT_ERROR', language=request_context.chat.language), buttons=[close_button()], ) @@ -130,7 +130,7 @@ class SubmitterService(SubmitterServicer, BaseHubService): if not processed_document.get('doi'): request_context.error_log(UnparsableDoiError()) await self.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t('UNPARSABLE_DOI_ERROR', language=request_context.chat.language), buttons=[close_button()], ) @@ -143,14 +143,14 @@ class SubmitterService(SubmitterServicer, BaseHubService): page_size=1, request_id=request_context.request_id, session_id=session_id, - user_id=request_context.chat.id, + user_id=request_context.chat.chat_id, language=request_context.chat.language, ) if len(search_response_pb.scored_documents) == 0: request_context.error_log(UnavailableMetadataError(doi=processed_document['doi'])) await self.telegram_client.send_message( - request_context.chat.id, + request_context.chat.chat_id, t( 'UNAVAILABLE_METADATA_ERROR', language=request_context.chat.language @@ -175,7 +175,7 @@ class SubmitterService(SubmitterServicer, BaseHubService): update_document=UpdateDocumentPb( typed_document=TypedDocumentPb(sharience=ShariencePb( parent_id=document_view.id, - uploader_id=request_context.chat.id, + uploader_id=request_context.chat.chat_id, updated_at=int(time.time()), md5=hashlib.md5(file).hexdigest(), filesize=document.size, diff --git a/nexus/pipe/README.md b/nexus/pipe/README.md index b90044c..77707f1 100644 --- a/nexus/pipe/README.md +++ b/nexus/pipe/README.md @@ -1,99 +1,3 @@ # Nexus Pipe -`Pipe` processes Kafka queue of operations. This version has cut `configs` -subdirectory due to hard reliance of configs on the network infrastructure you are using. -You have to write your own configs taking example below into account. - -## Sample `configs/base.yaml` - -```yaml ---- -log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}' -pipe: - brokers: | - kafka-0.example.net - schema: - - consumers: - - class: nexus.pipe.consumers.CrossReferencesBulkConsumer - topics: - - name: cross_references - workers: 4 - group_id: pipe - processors: - - class: nexus.pipe.processors.CrossReferencesProcessor - kwargs: - brokers: | - kafka-0.example.net - database: - database: nexus - host: postgres.example.net - password: '{{ DATABASE_PASSWORD }}' - username: '{{ DATABASE_USERNAME }}' - - consumers: - - class: nexus.pipe.consumers.DocumentOperationsJsonConsumer - topics: - - name: operations - workers: 2 - - class: nexus.pipe.consumers.DocumentOperationsConsumer - topics: - - name: operations_binary_hp - workers: 4 - - name: operations_binary - workers: 14 - group_id: pipe - processors: - - class: nexus.pipe.processors.ActionProcessor - kwargs: - actions: - - class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction - kwargs: - crossref: - rps: 50 - user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0' - - class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction - - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction - kwargs: - database: - database: nexus - host: postgres.example.net - password: '{{ DATABASE_PASSWORD }}' - username: '{{ DATABASE_USERNAME }}' - - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction - kwargs: - brokers: | - kafka-0.example.net - topic: cross_references - - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction - kwargs: - summa: - base_url: http://summa.example.net - timeout: 15 - ttl_dns_cache: 30 - filter: - class: nexus.pipe.filters.DocumentOperationFilter - kwargs: - document: scimag - operation: update_document - - class: nexus.pipe.processors.ActionProcessor - kwargs: - actions: - - class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction - - class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction - kwargs: - database: - database: nexus - host: postgres.example.net - password: '{{ DATABASE_PASSWORD }}' - username: '{{ DATABASE_USERNAME }}' - - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction - kwargs: - summa: - base_url: http://summa.example.net - timeout: 15 - ttl_dns_cache: 30 - filter: - class: nexus.pipe.filters.DocumentOperationFilter - kwargs: - document: scitech - operation: update_document -``` \ No newline at end of file +`Pipe` processes Kafka queue of operations. diff --git a/nexus/pipe/configs/base.yaml b/nexus/pipe/configs/base.yaml new file mode 100644 index 0000000..fbc6eb9 --- /dev/null +++ b/nexus/pipe/configs/base.yaml @@ -0,0 +1,90 @@ +--- + +log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}' +pipe: + brokers: | + kafka-0.example.net + schema: + - consumers: + - class: nexus.pipe.consumers.CrossReferencesBulkConsumer + topics: + - name: cross_references + workers: 4 + group_id: pipe + processors: + - class: nexus.pipe.processors.CrossReferencesProcessor + kwargs: + brokers: | + kafka-0.example.net + database: + database: nexus + host: postgres.example.net + password: '{{ DATABASE_PASSWORD }}' + username: '{{ DATABASE_USERNAME }}' + - consumers: + - class: nexus.pipe.consumers.DocumentOperationsJsonConsumer + topics: + - name: operations + workers: 2 + - class: nexus.pipe.consumers.DocumentOperationsConsumer + topics: + - name: operations_binary_hp + workers: 4 + - name: operations_binary + workers: 14 + group_id: pipe + processors: + - class: nexus.pipe.processors.ActionProcessor + kwargs: + actions: + - class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction + kwargs: + crossref: + rps: 50 + user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0' + - class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction + - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction + kwargs: + database: + database: nexus + host: postgres.example.net + password: '{{ DATABASE_PASSWORD }}' + username: '{{ DATABASE_USERNAME }}' + - class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction + kwargs: + brokers: | + kafka-0.example.net + topic: cross_references + - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction + kwargs: + summa: + base_url: http://summa.example.net + timeout: 15 + ttl_dns_cache: 30 + filter: + class: nexus.pipe.filters.DocumentOperationFilter + kwargs: + document: scimag + operation: update_document + - class: nexus.pipe.processors.ActionProcessor + kwargs: + actions: + - class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction + - class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction + kwargs: + database: + database: nexus + host: postgres.example.net + password: '{{ DATABASE_PASSWORD }}' + username: '{{ DATABASE_USERNAME }}' + - class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction + kwargs: + summa: + base_url: http://summa.example.net + timeout: 15 + ttl_dns_cache: 30 + filter: + class: nexus.pipe.filters.DocumentOperationFilter + kwargs: + document: scitech + operation: update_document diff --git a/nexus/pipe/configs/logging.yaml b/nexus/pipe/configs/logging.yaml new file mode 100644 index 0000000..f25c43f --- /dev/null +++ b/nexus/pipe/configs/logging.yaml @@ -0,0 +1,65 @@ +--- + +logging: + disable_existing_loggers: false + formatters: + default: + class: library.logging.formatters.DefaultFormatter + traceback: + class: library.logging.formatters.TracebackFormatter + handlers: + debug: + class: logging.StreamHandler + formatter: default + level: DEBUG + stream: 'ext://sys.stderr' + error: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/error.log' + formatter: default + level: ERROR + statbox: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/statbox.log' + formatter: default + level: INFO + traceback: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/traceback.log' + formatter: traceback + level: ERROR + warning: + class: library.logging.handlers.BaseFileHandler + filename: '{{ log_path }}/warning.log' + formatter: default + level: WARNING + loggers: + aiokafka: + handlers: + - error + propagate: false + asyncio: + handlers: + - error + level: WARNING + debug: + handlers: + - debug + propagate: false + error: + handlers: + - error + - traceback + - warning + propagate: false + statbox: + handlers: + - statbox + propagate: false + root: + handlers: + - debug + - error + - warning + level: DEBUG + version: 1 diff --git a/nexus/pylon/client.py b/nexus/pylon/client.py index f79961b..d7bb363 100644 --- a/nexus/pylon/client.py +++ b/nexus/pylon/client.py @@ -1,16 +1,10 @@ -import asyncio from typing import ( AsyncIterable, + Callable, Iterable, Optional, ) -import aiohttp -import aiohttp.client_exceptions -from aiohttp_socks import ( - ProxyConnectionError, - ProxyError, -) from aiokit import AioThing from library.logging import error_log from nexus.pylon.exceptions import ( @@ -27,7 +21,6 @@ from nexus.pylon.sources import ( SciHubSeSource, ) from nexus.pylon.sources.specific import get_specific_sources_for_doi -from python_socks import ProxyTimeoutError class PylonClient(AioThing): @@ -40,6 +33,7 @@ class PylonClient(AioThing): self, doi: str, md5: Optional[str] = None, + error_log_func: Callable = error_log, ) -> AsyncIterable[FileResponsePb]: sources = [] sources.extend(get_specific_sources_for_doi(doi, proxy=self.proxy, resolve_proxy=self.resolve_proxy)) @@ -49,49 +43,42 @@ class PylonClient(AioThing): LibgenDoiSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), ]) sources = filter(lambda x: x.is_enabled, sources) - async for resp in self.download(sources=sources): + async for resp in self.download(sources=sources, error_log_func=error_log_func): yield resp async def by_md5( self, md5: str, + error_log_func: Callable = error_log, ) -> AsyncIterable[FileResponsePb]: sources = filter(lambda x: x.is_enabled, [ LibraryLolSource(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), LibgenMd5Source(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), ]) - async for resp in self.download(sources=sources): + async for resp in self.download(sources=sources, error_log_func=error_log_func): yield resp - async def download_source(self, source) -> AsyncIterable[FileResponsePb]: - try: - yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url) - async for prepared_request in source.resolve(): - async for resp in source.execute_prepared_request(prepared_request=prepared_request): - yield resp - return - raise DownloadError(error='not_found', source=str(source)) - except ( - aiohttp.client_exceptions.ClientConnectionError, - aiohttp.client_exceptions.ClientPayloadError, - aiohttp.client_exceptions.ClientResponseError, - aiohttp.client_exceptions.TooManyRedirects, - asyncio.TimeoutError, - ProxyConnectionError, - ProxyTimeoutError, - ProxyError, - ) as e: - raise DownloadError(nested_error=str(e), nested_error_cls=e.__class__.__name__) - - async def download(self, sources: Iterable[BaseSource]) -> AsyncIterable[FileResponsePb]: - for source in sources: + async def download_source(self, source, error_log_func: Callable = error_log) -> AsyncIterable[FileResponsePb]: + yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url) + async for prepared_file_request in source.resolve(error_log_func=error_log_func): try: - await source.start() - async for resp in self.download_source(source): + async for resp in source.execute_prepared_file_request(prepared_file_request=prepared_file_request): yield resp return except DownloadError as e: - error_log(e) + error_log_func(e) + continue + raise DownloadError(error='not_found', source=str(source)) + + async def download(self, sources: Iterable[BaseSource], error_log_func: Callable = error_log) -> AsyncIterable[FileResponsePb]: + for source in sources: + try: + await source.start() + async for resp in self.download_source(source, error_log_func=error_log_func): + yield resp + return + except DownloadError as e: + error_log_func(e) continue finally: await source.stop() diff --git a/nexus/pylon/sources/base.py b/nexus/pylon/sources/base.py index 60055f5..837c411 100644 --- a/nexus/pylon/sources/base.py +++ b/nexus/pylon/sources/base.py @@ -1,19 +1,27 @@ +import asyncio import hashlib -import random +import socket +from contextlib import asynccontextmanager from typing import ( AsyncIterable, + Callable, Optional, ) import aiohttp import aiohttp.client_exceptions +from aiohttp.client_reqrep import ClientRequest from aiohttp_socks import ( + ProxyConnectionError, ProxyConnector, ProxyError, ) from aiokit import AioThing +from izihawa_utils.importlib import class_fullname +from library.logging import error_log from nexus.pylon.exceptions import ( BadResponseError, + DownloadError, IncorrectMD5Error, NotFoundError, ) @@ -30,19 +38,32 @@ from tenacity import ( DEFAULT_USER_AGENT = 'PylonBot/1.0 (Linux x86_64) PylonBot/1.0.0' +class KeepAliveClientRequest(ClientRequest): + async def send(self, conn): + sock = conn.protocol.transport.get_extra_info("socket") + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 60) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 2) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5) + + return await super().send(conn) + + class PreparedRequest: def __init__( self, method: str, url: str, - headers: dict = None, - params: dict = None, - cookies: dict = None, + headers: Optional[dict] = None, + params: Optional[dict] = None, + cookies: Optional[dict] = None, ssl: bool = True, + timeout: Optional[float] = None ): self.method = method self.url = url self.headers = { + 'Connection': 'keep-alive', 'User-Agent': DEFAULT_USER_AGENT, } if headers: @@ -50,6 +71,7 @@ class PreparedRequest: self.params = params self.cookies = cookies self.ssl = ssl + self.timeout = timeout def __repr__(self): return f'{self.method} {self.url} {self.headers} {self.params}' @@ -57,6 +79,34 @@ class PreparedRequest: def __str__(self): return repr(self) + @asynccontextmanager + async def execute_with(self, session): + async with session.request( + method=self.method, + url=self.url, + timeout=self.timeout, + headers=self.headers, + cookies=self.cookies, + params=self.params, + ssl=self.ssl, + ) as resp: + try: + yield resp + except BadResponseError as e: + e.add('url', self.url) + raise e + except ( + aiohttp.client_exceptions.ClientConnectionError, + aiohttp.client_exceptions.ClientPayloadError, + aiohttp.client_exceptions.ClientResponseError, + aiohttp.client_exceptions.TooManyRedirects, + asyncio.TimeoutError, + ProxyConnectionError, + ProxyTimeoutError, + ProxyError, + ) as e: + raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e)) + class BaseValidator: def update(self, chunk: bytes): @@ -123,12 +173,12 @@ class BaseSource(AioThing): return aiohttp.TCPConnector(verify_ssl=self.ssl) def get_session(self): - return aiohttp.ClientSession(connector=self.get_proxy()) + return aiohttp.ClientSession(request_class=KeepAliveClientRequest, connector=self.get_proxy()) def get_resolve_session(self): - return aiohttp.ClientSession(connector=self.get_resolve_proxy()) + return aiohttp.ClientSession(request_class=KeepAliveClientRequest, connector=self.get_resolve_proxy()) - def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: raise NotImplementedError("`resolve` for BaseSource is not implemented") def get_validator(self): @@ -139,19 +189,11 @@ class BaseSource(AioThing): stop=stop_after_attempt(3), retry=retry_if_exception_type((ProxyError, aiohttp.client_exceptions.ClientPayloadError, ProxyTimeoutError)), ) - async def execute_prepared_request(self, prepared_request: PreparedRequest): + async def execute_prepared_file_request(self, prepared_file_request: PreparedRequest): async with self.get_session() as session: - async with session.request( - method=prepared_request.method, - url=prepared_request.url, - timeout=self.timeout, - headers=prepared_request.headers, - cookies=prepared_request.cookies, - params=prepared_request.params, - ssl=prepared_request.ssl, - ) as resp: + async with prepared_file_request.execute_with(session=session) as resp: if resp.status == 404: - raise NotFoundError(url=prepared_request.url) + raise NotFoundError(url=prepared_file_request.url) elif ( resp.status != 200 or ( @@ -160,24 +202,17 @@ class BaseSource(AioThing): ) ): raise BadResponseError( - request_headers=prepared_request.headers, - url=prepared_request.url, + request_headers=prepared_file_request.headers, + url=prepared_file_request.url, status=resp.status, headers=str(resp.headers), ) file_validator = self.get_validator() - # Randomness is required due to annoying bug of when separators - # (\r\n) are splitted to different chunks - # https://github.com/aio-libs/aiohttp/issues/4677 - yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_request.url) - async for content in resp.content.iter_chunked(1024 * 100 + random.randint(-1024, 1024)): + yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_file_request.url) + async for content, _ in resp.content.iter_chunks(): file_validator.update(content) - yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_request.url) - try: - file_validator.validate() - except BadResponseError as e: - e.add('url', prepared_request.url) - raise e + yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_file_request.url) + file_validator.validate() class Md5Source(BaseSource): diff --git a/nexus/pylon/sources/libgen_doi.py b/nexus/pylon/sources/libgen_doi.py index 765b167..1d401e0 100644 --- a/nexus/pylon/sources/libgen_doi.py +++ b/nexus/pylon/sources/libgen_doi.py @@ -1,5 +1,8 @@ import re -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) from library.logging import error_log from nexus.pylon.exceptions import RegexNotFoundError @@ -14,13 +17,14 @@ class LibgenDoiSource(DoiSource): base_url = 'http://libgen.gs' resolve_timeout = 10 - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/scimag/ads.php?doi={self.doi}' - async with session.get( - url, - timeout=self.resolve_timeout - ) as resp: + async with PreparedRequest( + method='get', + url=url, + timeout=self.resolve_timeout, + ).execute_with(session=session) as resp: downloaded_page_bytes = await resp.read() downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace') match = re.search( @@ -29,6 +33,6 @@ class LibgenDoiSource(DoiSource): re.IGNORECASE, ) if match: - yield PreparedRequest(method='get', url=match.group()) + yield PreparedRequest(method='get', url=match.group(), timeout=self.timeout) else: - error_log(RegexNotFoundError(url=url)) + error_log_func(RegexNotFoundError(url=url)) diff --git a/nexus/pylon/sources/libgen_md5.py b/nexus/pylon/sources/libgen_md5.py index ececc91..a5cc254 100644 --- a/nexus/pylon/sources/libgen_md5.py +++ b/nexus/pylon/sources/libgen_md5.py @@ -1,5 +1,10 @@ import re -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) + +from library.logging import error_log from .base import ( Md5Source, @@ -12,10 +17,11 @@ class LibgenMd5Source(Md5Source): resolve_timeout = 10 async def resolve_lg(self, session, url): - async with session.get( - url, + async with PreparedRequest( + method='get', + url=url, timeout=self.resolve_timeout - ) as resp: + ).execute_with(session=session) as resp: downloaded_page_fiction = await resp.text() match = re.search( 'https?://.*/get\\.php\\?md5=.*&key=[A-Za-z0-9]+', @@ -23,9 +29,9 @@ class LibgenMd5Source(Md5Source): re.IGNORECASE, ) if match: - return PreparedRequest(method='get', url=match.group()) + return PreparedRequest(method='get', url=match.group(), timeout=self.timeout) - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/ads.php?md5={self.md5}' result = await self.resolve_lg(session, url) diff --git a/nexus/pylon/sources/libgen_new.py b/nexus/pylon/sources/libgen_new.py index 3eabf8f..8b268b9 100644 --- a/nexus/pylon/sources/libgen_new.py +++ b/nexus/pylon/sources/libgen_new.py @@ -1,5 +1,8 @@ import re -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) from library.logging import error_log from nexus.pylon.exceptions import RegexNotFoundError @@ -11,13 +14,14 @@ from .base import ( class LibgenNewSource(Md5Source): - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/main/{self.md5.upper()}' - async with session.get( - url, + async with PreparedRequest( + method='get', + url=url, timeout=self.resolve_timeout - ) as resp: + ).execute_with(session) as resp: downloaded_page = await resp.text() match_ipfs = re.search( 'https://ipfs.io/ipfs/[A-Za-z0-9]+', @@ -25,28 +29,28 @@ class LibgenNewSource(Md5Source): re.IGNORECASE, ) if match_ipfs: - yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl) + yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl, timeout=self.timeout) match_cf = re.search( 'https://cloudflare-ipfs.com/ipfs/[A-Za-z0-9]+', downloaded_page, re.IGNORECASE, ) if match_cf: - yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl) + yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl, timeout=self.timeout) match_infura = re.search( 'https://ipfs.infura.io/ipfs/[A-Za-z0-9]+', downloaded_page, re.IGNORECASE, ) if match_infura: - yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl) + yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl, timeout=self.timeout) if not match_cf or not match_infura or not match_ipfs: - error_log(RegexNotFoundError(url=url)) + error_log_func(RegexNotFoundError(url=url)) class LibraryLolSource(LibgenNewSource): base_url = 'http://library.lol' - resolve_timeout = 10 + resolve_timeout = 20 ssl = False - timeout = 30 + timeout = 120 diff --git a/nexus/pylon/sources/scihub.py b/nexus/pylon/sources/scihub.py index b1a4c79..8e93466 100644 --- a/nexus/pylon/sources/scihub.py +++ b/nexus/pylon/sources/scihub.py @@ -1,5 +1,8 @@ import re -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) from library.logging import error_log from nexus.pylon.exceptions import RegexNotFoundError @@ -19,16 +22,17 @@ class SciHubSource(DoiSource): base_url = None ssl = False - async def resolve(self, timeout=None) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/{self.doi}' - async with session.get( - url, - timeout=timeout or self.timeout - ) as resp: + async with PreparedRequest( + method='get', + url=url, + timeout=self.resolve_timeout + ).execute_with(session=session) as resp: # Sometimes sci-hub returns file if resp.headers.get('Content-Type') == 'application/pdf': - yield PreparedRequest(method='get', url=url) + yield PreparedRequest(method='get', url=url, timeout=self.timeout) downloaded_page_bytes = await resp.read() downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace') match = re.search('(?:https?:)?//.*\\?download=true', downloaded_page, re.IGNORECASE) @@ -36,9 +40,9 @@ class SciHubSource(DoiSource): url = match.group() if url.startswith('//'): url = 'http:' + url - yield PreparedRequest(method='get', url=url) + yield PreparedRequest(method='get', url=url, timeout=self.timeout) else: - error_log(RegexNotFoundError(url=url)) + error_log_func(RegexNotFoundError(url=url)) class SciHubDoSource(SciHubSource): diff --git a/nexus/pylon/sources/specific/biorxiv.py b/nexus/pylon/sources/specific/biorxiv.py index 1190f1b..b652369 100644 --- a/nexus/pylon/sources/specific/biorxiv.py +++ b/nexus/pylon/sources/specific/biorxiv.py @@ -1,5 +1,9 @@ -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) +from library.logging import error_log from nexus.pylon.sources.base import ( DoiSource, PreparedRequest, @@ -9,11 +13,12 @@ from nexus.pylon.sources.base import ( class BiorxivSource(DoiSource): base_url = 'https://dx.doi.org' - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/{self.doi}' - async with session.get( - url, - timeout=self.resolve_timeout - ) as resp: - yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf') + async with PreparedRequest( + method='get', + url=url, + timeout=self.resolve_timeout, + ).execute_with(session=session) as resp: + yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf', timeout=self.timeout) diff --git a/nexus/pylon/sources/specific/lancet.py b/nexus/pylon/sources/specific/lancet.py index 95ef527..7924b99 100644 --- a/nexus/pylon/sources/specific/lancet.py +++ b/nexus/pylon/sources/specific/lancet.py @@ -1,5 +1,9 @@ -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) +from library.logging import error_log from nexus.pylon.sources.base import ( DoiSource, PreparedRequest, @@ -11,14 +15,20 @@ class LancetSource(DoiSource): resolve_timeout = 10 use_proxy = False - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: splitted_doi = self.doi.split("/", maxsplit=1) if len(splitted_doi) < 2: return url = f'{self.base_url}/action/showPdf?pii={splitted_doi[1].upper()}' - async with session.get( - url, - timeout=self.resolve_timeout - ) as resp: - yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url)) + async with PreparedRequest( + method='get', + url=url, + timeout=self.resolve_timeout, + ).execute_with(session=session) as resp: + yield PreparedRequest( + method='get', + cookies=resp.cookies, + url=str(resp.url), + timeout=self.resolve_timeout, + ) diff --git a/nexus/pylon/sources/specific/nejm.py b/nexus/pylon/sources/specific/nejm.py index 682f0a4..ebc3d83 100644 --- a/nexus/pylon/sources/specific/nejm.py +++ b/nexus/pylon/sources/specific/nejm.py @@ -1,5 +1,9 @@ -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) +from library.logging import error_log from nexus.pylon.sources.base import ( DoiSource, PreparedRequest, @@ -11,11 +15,12 @@ class NejmSource(DoiSource): resolve_timeout = 10 use_proxy = False - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/doi/pdf/{self.doi}' - async with session.get( - url, + async with PreparedRequest( + method='get', + url=url, timeout=self.resolve_timeout, - ) as resp: - yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url)) + ).execute_with(session=session) as resp: + yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url), timeout=self.timeout) diff --git a/nexus/pylon/sources/specific/research_square.py b/nexus/pylon/sources/specific/research_square.py index e83d76c..dadc8a6 100644 --- a/nexus/pylon/sources/specific/research_square.py +++ b/nexus/pylon/sources/specific/research_square.py @@ -1,8 +1,11 @@ import re -from typing import AsyncIterable +from typing import ( + AsyncIterable, + Callable, +) +from library.logging import error_log from nexus.pylon.exceptions import RegexNotFoundError - from nexus.pylon.sources.base import ( DoiSource, PreparedRequest, @@ -12,13 +15,14 @@ from nexus.pylon.sources.base import ( class ResearchSquareSource(DoiSource): base_url = 'https://dx.doi.org' - async def resolve(self) -> AsyncIterable[PreparedRequest]: + async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]: async with self.get_resolve_session() as session: url = f'{self.base_url}/{self.doi}' - async with session.get( - url, - timeout=self.resolve_timeout - ) as resp: + async with PreparedRequest( + method='get', + url=url, + timeout=self.resolve_timeout, + ).execute_with(session=session) as resp: download_page = await resp.text() match = re.search( r'\"(https://www\.researchsquare\.com/article/[^\"]+\.pdf)\"', @@ -27,4 +31,4 @@ class ResearchSquareSource(DoiSource): ) if not match: raise RegexNotFoundError(url=url) - yield PreparedRequest(method='get', url=match.group(1)) + yield PreparedRequest(method='get', url=match.group(1), timeout=self.timeout) diff --git a/nexus/views/telegram/progress_bar.py b/nexus/views/telegram/progress_bar.py index 72745ab..fc3bd9c 100644 --- a/nexus/views/telegram/progress_bar.py +++ b/nexus/views/telegram/progress_bar.py @@ -84,7 +84,7 @@ class ProgressBar: try: if not self.message: self.message = await self.telegram_client.send_message( - self.request_context.chat.id, + self.request_context.chat.chat_id, text, buttons=[close_button()], ) diff --git a/nexus/views/telegram/scitech.py b/nexus/views/telegram/scitech.py index f61dd9e..1cae2ae 100644 --- a/nexus/views/telegram/scitech.py +++ b/nexus/views/telegram/scitech.py @@ -6,7 +6,10 @@ from typing import ( ) from urllib.parse import quote -from izihawa_utils.common import filter_none +from izihawa_utils.common import ( + filter_none, + is_essential, +) from nexus.models.proto.scitech_pb2 import Scitech as ScitechPb from nexus.nlptools.utils import ( cast_string_to_single_string, @@ -48,7 +51,7 @@ class ScitechView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin): locator = self.get_formatted_locator() - caption = '\n'.join(filter_none([head, doi, locator])) + caption = '\n'.join(filter_none([head, doi, locator], predicate=is_essential)) if limit and len(caption) > limit: shorten_title = title[:limit] shorten_title = shorten_title[:max(32, shorten_title.rfind(' '))]