MarcoBuster
/
hyperboria
mirror of https://github.com/nexus-stc/hyperboria
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- feat(pylon): Refactor code 

- feat(idm): Rename IDM-2 to IDM
  - feat(idm): Open IDM
3 internal commit(s)

GitOrigin-RevId: e302e9b5cda18cca1adc4ae8a3d906714d222106
This commit is contained in:
the-superpirate 2021-04-12 20:47:21 +03:00
parent a51463c027
commit 9ce67ec590
68 changed files with 1230 additions and 737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
idm/api/BUILD.bazel Normal file
View File

@ -0,0 +1,70 @@
load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
load("@io_bazel_rules_docker//container:container.bzl", "container_push")
load("@pip_modules//:requirements.bzl", "requirement")
alias(
name = "binary",
actual = ":image.binary",
)
py3_image(
name = "image",
srcs = glob(
[
"*.py",
"configs/**/*.py",
"daemons/**/*.py",
"models/**",
"proto/**",
"services/**",
],
exclude = [
"**/__pycache__/**",
"**/*.pyc",
"**/README",
"**/*.mako",
"proto/**/*.py",
],
),
base = "//images/production:base-python-image",
data = [
"configs/base.yaml",
"configs/logging.yaml",
],
layers = [
requirement("grpcio"),
requirement("pypika"),
requirement("uvloop"),
"//idm/api/proto:idm_grpc_py",
"//idm/api/proto:idm_proto_py",
"//library/aiogrpctools",
requirement("aiokit"),
"//library/aiopostgres",
"//library/configurator",
"//library/telegram",
requirement("izihawa_utils"),
],
main = "main.py",
srcs_version = "PY3ONLY",
visibility = ["//visibility:public"],
)
container_push(
name = "push-testing",
format = "Docker",
image = ":image",
registry = "registry.example.com",
repository = "idm-api",
tag = "testing",
)
container_push(
name = "push-latest",
format = "Docker",
image = ":image",
registry = "registry.example.com",
repository = "idm-api",
tag = "latest",
)

0
idm/api/__init__.py Normal file
View File

4
idm/api2/aioclient/BUILD.bazel → idm/api/aioclient/BUILD.bazel
View File

@ -9,7 +9,7 @@ py_library(
requirement("cachetools"),
requirement("lru-dict"),
requirement("tenacity"),
"//idm/api2/proto:idm_grpc_py",
"//idm/api2/proto:idm_proto_py",
"//idm/api/proto:idm_grpc_py",
"//idm/api/proto:idm_proto_py",
],
)

3
idm/api/aioclient/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .aioclient import IdmApiGrpcClient
__all__ = ['IdmApiGrpcClient']

16
idm/api2/aioclient/aioclient.py → idm/api/aioclient/aioclient.py
View File

@ -4,13 +4,13 @@ from grpc.experimental.aio import (
AioRpcError,
insecure_channel,
)
from idm.api2.proto.chats_service_pb2 import (
from idm.api.proto.chat_manager_service_pb2 import (
CreateChatRequest,
GetChatRequest,
ListChatsRequest,
UpdateChatRequest,
)
from idm.api2.proto.chats_service_pb2_grpc import ChatsStub
from idm.api.proto.chat_manager_service_pb2_grpc import ChatManagerStub
from lru import LRU
from tenacity import (
retry,
@ -20,7 +20,7 @@ from tenacity import (
)
class IdmApi2GrpcClient(AioThing):
class IdmApiGrpcClient(AioThing):
def __init__(
self,
base_url,
@ -33,7 +33,7 @@ class IdmApi2GrpcClient(AioThing):
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.chats_stub = ChatsStub(self.channel)
self.chat_manager_stub = ChatManagerStub(self.channel)
self.cache = LRU(4096)
async def start(self):
@ -49,7 +49,7 @@ class IdmApi2GrpcClient(AioThing):
language,
request_id: str = None,
):
response = await self.chats_stub.create_chat(
response = await self.chat_manager_stub.create_chat(
CreateChatRequest(
chat_id=chat_id,
username=username,
@ -74,7 +74,7 @@ class IdmApi2GrpcClient(AioThing):
chat_id,
request_id: str = None,
):
response = await self.chats_stub.get_chat(
response = await self.chat_manager_stub.get_chat(
GetChatRequest(chat_id=chat_id),
metadata=(
('request-id', request_id),
@ -87,7 +87,7 @@ class IdmApi2GrpcClient(AioThing):
request_id: str = None,
banned_at_moment=None,
):
response = await self.chats_stub.list_chats(
response = await self.chat_manager_stub.list_chats(
ListChatsRequest(banned_at_moment=banned_at_moment),
metadata=(
('request-id', request_id),
@ -106,7 +106,7 @@ class IdmApi2GrpcClient(AioThing):
ban_message=None,
is_admin=None,
):
response = await self.chats_stub.update_chat(
response = await self.chat_manager_stub.update_chat(
UpdateChatRequest(
chat_id=chat_id,
language=language,

17
idm/api/configs/base.yaml Normal file
View File

@ -0,0 +1,17 @@
---
application:
debug: true
service_name: idm-api
database:
port: 5432
grpc:
address: 0.0.0.0
port: 9090
log_path: '/var/log/idm-api/{{ ENV_TYPE }}'
telegram:
# Telegram App Hash from https://my.telegram.org/
app_hash: '{{ APP_HASH }}'
# Telegram App ID from https://my.telegram.org/
app_id: 00000
database:
session_id: '/usr/lib/idm-api/{{ ENV_TYPE }}/session.db'

63
idm/api/configs/logging.yaml Normal file
View File

@ -0,0 +1,63 @@
---
logging:
disable_existing_loggers: false
formatters:
base:
class: library.logging.formatters.BaseFormatter
default:
class: library.logging.formatters.DefaultFormatter
none:
class: logging.Formatter
traceback:
class: library.logging.formatters.TracebackFormatter
handlers:
debug:
class: logging.StreamHandler
formatter: default
level: DEBUG
stream: 'ext://sys.stderr'
error:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/error.log'
formatter: default
level: ERROR
statbox:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/statbox.log'
formatter: default
level: INFO
traceback:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/traceback.log'
formatter: traceback
level: ERROR
warning:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/warning.log'
formatter: default
level: WARNING
loggers:
debug:
handlers:
- debug
propagate: false
error:
handlers:
- error
- traceback
- warning
propagate: false
statbox:
handlers:
- statbox
propagate: false
telethon:
handlers:
- error
propagate: false
root:
handlers:
- debug
level: DEBUG
version: 1

0
idm/api/daemons/__init__.py Normal file
View File

87
idm/api/daemons/admin_log_reader.py Normal file
View File

@ -0,0 +1,87 @@
import asyncio
import logging
import string
from aiokit import AioThing
from library.telegram.base import BaseTelegramClient
from telethon.tl.functions.channels import GetParticipantsRequest
from telethon.tl.types import ChannelParticipantsSearch
class AdminLogReader(AioThing):
channel_name = '@nexus_search'
def __init__(self, telegram_config):
super().__init__()
self.subscriptors = set()
self.loading = False
self.telegram_client = BaseTelegramClient(
app_id=telegram_config['app_id'],
app_hash=telegram_config['app_hash'],
database=telegram_config['database'],
flood_sleep_threshold=25,
)
self.last_max_id = 0
def statbox(self, **kwargs):
logging.getLogger('statbox').info({'mode': 'admin_log_reader', **kwargs})
async def skip_admin_log(self):
async for event in self.telegram_client.iter_admin_log(self.channel_name, limit=1):
self.last_max_id = event.id
self.statbox(action='skipped_admin_log', max_id=self.last_max_id)
async def process_admin_log(self, sleep=1.0):
self.loading = True
try:
while 1:
async for event in self.telegram_client.iter_admin_log(
self.channel_name, join=True, invite=True, leave=True, min_id=self.last_max_id
):
is_subscribed = event.joined or event.joined_invite
if is_subscribed:
self.subscriptors.add(event.user_id)
elif event.user_id in self.subscriptors:
self.subscriptors.remove(event.user_id)
self.last_max_id = event.id
await asyncio.sleep(sleep)
except asyncio.CancelledError:
pass
finally:
self.loading = False
async def _fetch_users(self, query):
max_batch_size = 200
participants = await self.telegram_client(
GetParticipantsRequest(
channel=self.channel_name,
filter=ChannelParticipantsSearch(query),
offset=0,
limit=max_batch_size,
hash=0,
)
)
for user in participants.users:
self.subscriptors.add(user.id)
# There is a possibility that more users exist if we hit maximum count of users
# So, we are making a recursion to reveal it
if len(participants.users) == max_batch_size:
for new_letter in string.ascii_lowercase + string.digits:
await self._fetch_users(query + new_letter)
async def load_channel_users(self):
self.statbox(action='load_channel_users')
await self._fetch_users('')
self.statbox(action='loaded_channel_users', subscriptors=len(self.subscriptors))
async def start(self):
await self.telegram_client.start_and_wait()
await self.skip_admin_log()
await self.load_channel_users()
asyncio.create_task(self.process_admin_log())
def is_subscribed(self, user_id):
return not self.loading or user_id in self.subscriptors

54
idm/api/main.py Normal file
View File

@ -0,0 +1,54 @@
import asyncio
import uvloop
from aiopg.sa import create_engine
from idm.api.configs import get_config
from idm.api.daemons.admin_log_reader import AdminLogReader
from idm.api.services.chat_manager import ChatManagerService
from library.aiogrpctools import AioGrpcServer
from library.aiopostgres.pool_holder import AioPostgresPoolHolder
from library.configurator import Configurator
from library.logging import configure_logging
class GrpcServer(AioGrpcServer):
def __init__(self, config: Configurator):
super().__init__(address=config['grpc']['address'], port=config['grpc']['port'])
self.pool_holder = AioPostgresPoolHolder(
fn=create_engine,
database=config['database']['database'],
user=config['database']['username'],
password=config['database']['password'],
host=config['database']['host'],
port=config['database']['port'],
timeout=30,
pool_recycle=60,
maxsize=4,
)
self.admin_log_reader = AdminLogReader(
telegram_config=config['telegram'],
)
self.chat_manager_service = ChatManagerService(
server=self.server,
service_name=config['application']['service_name'],
pool_holder=self.pool_holder,
admin_log_reader=self.admin_log_reader,
)
self.starts.append(self.admin_log_reader)
self.waits.extend([self.chat_manager_service, self.pool_holder])
async def create_app(config: Configurator):
grpc_server = GrpcServer(config)
await grpc_server.start_and_wait()
def main():
uvloop.install()
config = get_config()
configure_logging(config)
asyncio.get_event_loop().run_until_complete(create_app(config))
if __name__ == '__main__':
main()

3
idm/api2/proto/BUILD.bazel → idm/api/proto/BUILD.bazel
View File

@ -6,8 +6,7 @@ package(default_visibility = ["//visibility:public"])
proto_library(
name = "idm_proto",
srcs = [
"chats_service.proto",
"location.proto",
"chat_manager_service.proto",
],
deps = [
"@com_google_protobuf//:wrappers_proto",

50
idm/api/proto/chat_manager_service.proto Normal file
View File

@ -0,0 +1,50 @@
syntax = "proto3";
package idm.api.proto;
message Chat {
int64 chat_id = 1;
string username = 2;
string language = 3;
bool is_system_messaging_enabled = 4;
bool is_discovery_enabled = 5;
int32 ban_until = 6;
string ban_message = 7;
bool is_admin = 8;
bool is_subscribed = 9;
int64 created_at = 10;
}
message Chats {
repeated Chat chats = 1;
}
message CreateChatRequest {
int64 chat_id = 1;
string username = 2;
string language = 3;
}
message GetChatRequest {
int64 chat_id = 1;
}
message ListChatsRequest {
optional int32 banned_at_moment = 1;
}
message UpdateChatRequest {
int64 chat_id = 1;
optional string language = 2;
optional bool is_system_messaging_enabled = 3;
optional bool is_discovery_enabled = 4;
optional int32 ban_until = 5;
optional string ban_message = 6;
optional bool is_admin = 7;
}
service ChatManager {
rpc create_chat(CreateChatRequest) returns (Chat) {};
rpc get_chat(GetChatRequest) returns (Chat) {};
rpc list_chats(ListChatsRequest) returns (Chats) {};
rpc update_chat(UpdateChatRequest) returns (Chat) {};
}

222
idm/api2/proto/chats_service_pb2.py → idm/api/proto/chat_manager_service_pb2.py Executable file → Normal file
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: idm/api2/proto/chats_service.proto
# source: idm/api/proto/chat_manager_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
@ -12,96 +12,94 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from idm.api2.proto import \
location_pb2 as idm_dot_api2_dot_proto_dot_location__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='idm/api2/proto/chats_service.proto',
package='idm.api2.proto',
name='idm/api/proto/chat_manager_service.proto',
package='idm.api.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\"idm/api2/proto/chats_service.proto\x12\x0eidm.api2.proto\x1a\x1didm/api2/proto/location.proto\"\xe2\x01\n\x08\x43hatData\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x15\n\ris_subscribed\x18\n \x01(\x08\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\"4\n\tChatsData\x12\'\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x18.idm.api2.proto.ChatData\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"H\n\x10ListChatsRequest\x12\x1a\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x42\x18\n\x16\x62\x61nned_at_moment_oneof\"\xce\x02\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x12\n\x08language\x18\x02 \x01(\tH\x00\x12%\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x12\x1e\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x12\x13\n\tban_until\x18\x05 \x01(\x05H\x03\x12\x15\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x12\x12\n\x08is_admin\x18\x07 \x01(\x08H\x05\x42\x10\n\x0elanguage_oneofB#\n!is_system_messaging_enabled_oneofB\x1c\n\x1ais_discovery_enabled_oneofB\x11\n\x0f\x62\x61n_until_oneofB\x13\n\x11\x62\x61n_message_oneofB\x10\n\x0eis_admin_oneof2\xb8\x02\n\x05\x43hats\x12L\n\x0b\x63reate_chat\x12!.idm.api2.proto.CreateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12\x46\n\x08get_chat\x12\x1e.idm.api2.proto.GetChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12K\n\nlist_chats\x12 .idm.api2.proto.ListChatsRequest\x1a\x19.idm.api2.proto.ChatsData\"\x00\x12L\n\x0bupdate_chat\x12!.idm.api2.proto.UpdateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x62\x06proto3'
,
dependencies=[idm_dot_api2_dot_proto_dot_location__pb2.DESCRIPTOR,])
serialized_pb=b'\n(idm/api/proto/chat_manager_service.proto\x12\ridm.api.proto\"\xe3\x01\n\x04\x43hat\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x15\n\ris_subscribed\x18\t \x01(\x08\x12\x12\n\ncreated_at\x18\n \x01(\x03\"+\n\x05\x43hats\x12\"\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x13.idm.api.proto.Chat\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"F\n\x10ListChatsRequest\x12\x1d\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x88\x01\x01\x42\x13\n\x11_banned_at_moment\"\xc2\x02\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x15\n\x08language\x18\x02 \x01(\tH\x00\x88\x01\x01\x12(\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12!\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x88\x01\x01\x12\x16\n\tban_until\x18\x05 \x01(\x05H\x03\x88\x01\x01\x12\x18\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08is_admin\x18\x07 \x01(\x08H\x05\x88\x01\x01\x42\x0b\n\t_languageB\x1e\n\x1c_is_system_messaging_enabledB\x17\n\x15_is_discovery_enabledB\x0c\n\n_ban_untilB\x0e\n\x0c_ban_messageB\x0b\n\t_is_admin2\xa6\x02\n\x0b\x43hatManager\x12\x46\n\x0b\x63reate_chat\x12 .idm.api.proto.CreateChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x12@\n\x08get_chat\x12\x1d.idm.api.proto.GetChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x12\x45\n\nlist_chats\x12\x1f.idm.api.proto.ListChatsRequest\x1a\x14.idm.api.proto.Chats\"\x00\x12\x46\n\x0bupdate_chat\x12 .idm.api.proto.UpdateChatRequest\x1a\x13.idm.api.proto.Chat\"\x00\x62\x06proto3'
)
_CHATDATA = _descriptor.Descriptor(
name='ChatData',
full_name='idm.api2.proto.ChatData',
_CHAT = _descriptor.Descriptor(
name='Chat',
full_name='idm.api.proto.Chat',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='idm.api2.proto.ChatData.id', index=0,
name='chat_id', full_name='idm.api.proto.Chat.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='username', full_name='idm.api2.proto.ChatData.username', index=1,
name='username', full_name='idm.api.proto.Chat.username', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='idm.api2.proto.ChatData.language', index=2,
name='language', full_name='idm.api.proto.Chat.language', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_system_messaging_enabled', full_name='idm.api2.proto.ChatData.is_system_messaging_enabled', index=3,
name='is_system_messaging_enabled', full_name='idm.api.proto.Chat.is_system_messaging_enabled', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_discovery_enabled', full_name='idm.api2.proto.ChatData.is_discovery_enabled', index=4,
name='is_discovery_enabled', full_name='idm.api.proto.Chat.is_discovery_enabled', index=4,
number=5, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_until', full_name='idm.api2.proto.ChatData.ban_until', index=5,
name='ban_until', full_name='idm.api.proto.Chat.ban_until', index=5,
number=6, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_message', full_name='idm.api2.proto.ChatData.ban_message', index=6,
name='ban_message', full_name='idm.api.proto.Chat.ban_message', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_admin', full_name='idm.api2.proto.ChatData.is_admin', index=7,
name='is_admin', full_name='idm.api.proto.Chat.is_admin', index=7,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_subscribed', full_name='idm.api2.proto.ChatData.is_subscribed', index=8,
number=10, type=8, cpp_type=7, label=1,
name='is_subscribed', full_name='idm.api.proto.Chat.is_subscribed', index=8,
number=9, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='created_at', full_name='idm.api2.proto.ChatData.created_at', index=9,
number=11, type=3, cpp_type=2, label=1,
name='created_at', full_name='idm.api.proto.Chat.created_at', index=9,
number=10, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
@ -118,21 +116,21 @@ _CHATDATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=86,
serialized_end=312,
serialized_start=60,
serialized_end=287,
)
_CHATSDATA = _descriptor.Descriptor(
name='ChatsData',
full_name='idm.api2.proto.ChatsData',
_CHATS = _descriptor.Descriptor(
name='Chats',
full_name='idm.api.proto.Chats',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chats', full_name='idm.api2.proto.ChatsData.chats', index=0,
name='chats', full_name='idm.api.proto.Chats.chats', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
@ -150,35 +148,35 @@ _CHATSDATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=314,
serialized_end=366,
serialized_start=289,
serialized_end=332,
)
_CREATECHATREQUEST = _descriptor.Descriptor(
name='CreateChatRequest',
full_name='idm.api2.proto.CreateChatRequest',
full_name='idm.api.proto.CreateChatRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chat_id', full_name='idm.api2.proto.CreateChatRequest.chat_id', index=0,
name='chat_id', full_name='idm.api.proto.CreateChatRequest.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='username', full_name='idm.api2.proto.CreateChatRequest.username', index=1,
name='username', full_name='idm.api.proto.CreateChatRequest.username', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='idm.api2.proto.CreateChatRequest.language', index=2,
name='language', full_name='idm.api.proto.CreateChatRequest.language', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@ -196,21 +194,21 @@ _CREATECHATREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=368,
serialized_end=440,
serialized_start=334,
serialized_end=406,
)
_GETCHATREQUEST = _descriptor.Descriptor(
name='GetChatRequest',
full_name='idm.api2.proto.GetChatRequest',
full_name='idm.api.proto.GetChatRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chat_id', full_name='idm.api2.proto.GetChatRequest.chat_id', index=0,
name='chat_id', full_name='idm.api.proto.GetChatRequest.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
@ -228,21 +226,21 @@ _GETCHATREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=442,
serialized_end=475,
serialized_start=408,
serialized_end=441,
)
_LISTCHATSREQUEST = _descriptor.Descriptor(
name='ListChatsRequest',
full_name='idm.api2.proto.ListChatsRequest',
full_name='idm.api.proto.ListChatsRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='banned_at_moment', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment', index=0,
name='banned_at_moment', full_name='idm.api.proto.ListChatsRequest.banned_at_moment', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
@ -260,68 +258,68 @@ _LISTCHATSREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='banned_at_moment_oneof', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment_oneof',
name='_banned_at_moment', full_name='idm.api.proto.ListChatsRequest._banned_at_moment',
index=0, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
],
serialized_start=477,
serialized_end=549,
serialized_start=443,
serialized_end=513,
)
_UPDATECHATREQUEST = _descriptor.Descriptor(
name='UpdateChatRequest',
full_name='idm.api2.proto.UpdateChatRequest',
full_name='idm.api.proto.UpdateChatRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chat_id', full_name='idm.api2.proto.UpdateChatRequest.chat_id', index=0,
name='chat_id', full_name='idm.api.proto.UpdateChatRequest.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='idm.api2.proto.UpdateChatRequest.language', index=1,
name='language', full_name='idm.api.proto.UpdateChatRequest.language', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_system_messaging_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled', index=2,
name='is_system_messaging_enabled', full_name='idm.api.proto.UpdateChatRequest.is_system_messaging_enabled', index=2,
number=3, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_discovery_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled', index=3,
name='is_discovery_enabled', full_name='idm.api.proto.UpdateChatRequest.is_discovery_enabled', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_until', full_name='idm.api2.proto.UpdateChatRequest.ban_until', index=4,
name='ban_until', full_name='idm.api.proto.UpdateChatRequest.ban_until', index=4,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_message', full_name='idm.api2.proto.UpdateChatRequest.ban_message', index=5,
name='ban_message', full_name='idm.api.proto.UpdateChatRequest.ban_message', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_admin', full_name='idm.api2.proto.UpdateChatRequest.is_admin', index=6,
name='is_admin', full_name='idm.api.proto.UpdateChatRequest.is_admin', index=6,
number=7, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
@ -339,167 +337,167 @@ _UPDATECHATREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='language_oneof', full_name='idm.api2.proto.UpdateChatRequest.language_oneof',
name='_language', full_name='idm.api.proto.UpdateChatRequest._language',
index=0, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='is_system_messaging_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled_oneof',
name='_is_system_messaging_enabled', full_name='idm.api.proto.UpdateChatRequest._is_system_messaging_enabled',
index=1, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='is_discovery_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled_oneof',
name='_is_discovery_enabled', full_name='idm.api.proto.UpdateChatRequest._is_discovery_enabled',
index=2, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='ban_until_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_until_oneof',
name='_ban_until', full_name='idm.api.proto.UpdateChatRequest._ban_until',
index=3, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='ban_message_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_message_oneof',
name='_ban_message', full_name='idm.api.proto.UpdateChatRequest._ban_message',
index=4, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='is_admin_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_admin_oneof',
name='_is_admin', full_name='idm.api.proto.UpdateChatRequest._is_admin',
index=5, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
],
serialized_start=552,
serialized_end=886,
serialized_start=516,
serialized_end=838,
)
_CHATSDATA.fields_by_name['chats'].message_type = _CHATDATA
_LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof'].fields.append(
_CHATS.fields_by_name['chats'].message_type = _CHAT
_LISTCHATSREQUEST.oneofs_by_name['_banned_at_moment'].fields.append(
_LISTCHATSREQUEST.fields_by_name['banned_at_moment'])
_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof']
_UPDATECHATREQUEST.oneofs_by_name['language_oneof'].fields.append(
_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['_banned_at_moment']
_UPDATECHATREQUEST.oneofs_by_name['_language'].fields.append(
_UPDATECHATREQUEST.fields_by_name['language'])
_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['language_oneof']
_UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_language']
_UPDATECHATREQUEST.oneofs_by_name['_is_system_messaging_enabled'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'])
_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof']
_UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_system_messaging_enabled']
_UPDATECHATREQUEST.oneofs_by_name['_is_discovery_enabled'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'])
_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof']
_UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_discovery_enabled']
_UPDATECHATREQUEST.oneofs_by_name['_ban_until'].fields.append(
_UPDATECHATREQUEST.fields_by_name['ban_until'])
_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof']
_UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_ban_until']
_UPDATECHATREQUEST.oneofs_by_name['_ban_message'].fields.append(
_UPDATECHATREQUEST.fields_by_name['ban_message'])
_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof']
_UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_ban_message']
_UPDATECHATREQUEST.oneofs_by_name['_is_admin'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_admin'])
_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof']
DESCRIPTOR.message_types_by_name['ChatData'] = _CHATDATA
DESCRIPTOR.message_types_by_name['ChatsData'] = _CHATSDATA
_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['_is_admin']
DESCRIPTOR.message_types_by_name['Chat'] = _CHAT
DESCRIPTOR.message_types_by_name['Chats'] = _CHATS
DESCRIPTOR.message_types_by_name['CreateChatRequest'] = _CREATECHATREQUEST
DESCRIPTOR.message_types_by_name['GetChatRequest'] = _GETCHATREQUEST
DESCRIPTOR.message_types_by_name['ListChatsRequest'] = _LISTCHATSREQUEST
DESCRIPTOR.message_types_by_name['UpdateChatRequest'] = _UPDATECHATREQUEST
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
ChatData = _reflection.GeneratedProtocolMessageType('ChatData', (_message.Message,), {
'DESCRIPTOR' : _CHATDATA,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.ChatData)
Chat = _reflection.GeneratedProtocolMessageType('Chat', (_message.Message,), {
'DESCRIPTOR' : _CHAT,
'__module__' : 'idm.api.proto.chat_manager_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api.proto.Chat)
})
_sym_db.RegisterMessage(ChatData)
_sym_db.RegisterMessage(Chat)
ChatsData = _reflection.GeneratedProtocolMessageType('ChatsData', (_message.Message,), {
'DESCRIPTOR' : _CHATSDATA,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.ChatsData)
Chats = _reflection.GeneratedProtocolMessageType('Chats', (_message.Message,), {
'DESCRIPTOR' : _CHATS,
'__module__' : 'idm.api.proto.chat_manager_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api.proto.Chats)
})
_sym_db.RegisterMessage(ChatsData)
_sym_db.RegisterMessage(Chats)
CreateChatRequest = _reflection.GeneratedProtocolMessageType('CreateChatRequest', (_message.Message,), {
'DESCRIPTOR' : _CREATECHATREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.CreateChatRequest)
'__module__' : 'idm.api.proto.chat_manager_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api.proto.CreateChatRequest)
})
_sym_db.RegisterMessage(CreateChatRequest)
GetChatRequest = _reflection.GeneratedProtocolMessageType('GetChatRequest', (_message.Message,), {
'DESCRIPTOR' : _GETCHATREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.GetChatRequest)
'__module__' : 'idm.api.proto.chat_manager_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api.proto.GetChatRequest)
})
_sym_db.RegisterMessage(GetChatRequest)
ListChatsRequest = _reflection.GeneratedProtocolMessageType('ListChatsRequest', (_message.Message,), {
'DESCRIPTOR' : _LISTCHATSREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.ListChatsRequest)
'__module__' : 'idm.api.proto.chat_manager_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api.proto.ListChatsRequest)
})
_sym_db.RegisterMessage(ListChatsRequest)
UpdateChatRequest = _reflection.GeneratedProtocolMessageType('UpdateChatRequest', (_message.Message,), {
'DESCRIPTOR' : _UPDATECHATREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.UpdateChatRequest)
'__module__' : 'idm.api.proto.chat_manager_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api.proto.UpdateChatRequest)
})
_sym_db.RegisterMessage(UpdateChatRequest)
_CHATS = _descriptor.ServiceDescriptor(
name='Chats',
full_name='idm.api2.proto.Chats',
_CHATMANAGER = _descriptor.ServiceDescriptor(
name='ChatManager',
full_name='idm.api.proto.ChatManager',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=889,
serialized_end=1201,
serialized_start=841,
serialized_end=1135,
methods=[
_descriptor.MethodDescriptor(
name='create_chat',
full_name='idm.api2.proto.Chats.create_chat',
full_name='idm.api.proto.ChatManager.create_chat',
index=0,
containing_service=None,
input_type=_CREATECHATREQUEST,
output_type=_CHATDATA,
output_type=_CHAT,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='get_chat',
full_name='idm.api2.proto.Chats.get_chat',
full_name='idm.api.proto.ChatManager.get_chat',
index=1,
containing_service=None,
input_type=_GETCHATREQUEST,
output_type=_CHATDATA,
output_type=_CHAT,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='list_chats',
full_name='idm.api2.proto.Chats.list_chats',
full_name='idm.api.proto.ChatManager.list_chats',
index=2,
containing_service=None,
input_type=_LISTCHATSREQUEST,
output_type=_CHATSDATA,
output_type=_CHATS,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='update_chat',
full_name='idm.api2.proto.Chats.update_chat',
full_name='idm.api.proto.ChatManager.update_chat',
index=3,
containing_service=None,
input_type=_UPDATECHATREQUEST,
output_type=_CHATDATA,
output_type=_CHAT,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_CHATS)
_sym_db.RegisterServiceDescriptor(_CHATMANAGER)
DESCRIPTOR.services_by_name['Chats'] = _CHATS
DESCRIPTOR.services_by_name['ChatManager'] = _CHATMANAGER
# @@protoc_insertion_point(module_scope)

79
idm/api2/proto/chats_service_pb2_grpc.py → idm/api/proto/chat_manager_service_pb2_grpc.py Executable file → Normal file
View File

@ -1,11 +1,12 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from idm.api2.proto import \
chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
from idm.api.proto import \
chat_manager_service_pb2 as \
idm_dot_api_dot_proto_dot_chat__manager__service__pb2
class ChatsStub(object):
class ChatManagerStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
@ -15,28 +16,28 @@ class ChatsStub(object):
channel: A grpc.Channel.
"""
self.create_chat = channel.unary_unary(
'/idm.api2.proto.Chats/create_chat',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
'/idm.api.proto.ChatManager/create_chat',
request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.SerializeToString,
response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
)
self.get_chat = channel.unary_unary(
'/idm.api2.proto.Chats/get_chat',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
'/idm.api.proto.ChatManager/get_chat',
request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.SerializeToString,
response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
)
self.list_chats = channel.unary_unary(
'/idm.api2.proto.Chats/list_chats',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
'/idm.api.proto.ChatManager/list_chats',
request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.SerializeToString,
response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.FromString,
)
self.update_chat = channel.unary_unary(
'/idm.api2.proto.Chats/update_chat',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
'/idm.api.proto.ChatManager/update_chat',
request_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.SerializeToString,
response_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
)
class ChatsServicer(object):
class ChatManagerServicer(object):
"""Missing associated documentation comment in .proto file."""
def create_chat(self, request, context):
@ -64,36 +65,36 @@ class ChatsServicer(object):
raise NotImplementedError('Method not implemented!')
def add_ChatsServicer_to_server(servicer, server):
def add_ChatManagerServicer_to_server(servicer, server):
rpc_method_handlers = {
'create_chat': grpc.unary_unary_rpc_method_handler(
servicer.create_chat,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.FromString,
response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString,
),
'get_chat': grpc.unary_unary_rpc_method_handler(
servicer.get_chat,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.FromString,
response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString,
),
'list_chats': grpc.unary_unary_rpc_method_handler(
servicer.list_chats,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.SerializeToString,
request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.FromString,
response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.SerializeToString,
),
'update_chat': grpc.unary_unary_rpc_method_handler(
servicer.update_chat,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
request_deserializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.FromString,
response_serializer=idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'idm.api2.proto.Chats', rpc_method_handlers)
'idm.api.proto.ChatManager', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Chats(object):
class ChatManager(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
@ -107,9 +108,9 @@ class Chats(object):
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/create_chat',
idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/create_chat',
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.CreateChatRequest.SerializeToString,
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@ -124,9 +125,9 @@ class Chats(object):
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/get_chat',
idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/get_chat',
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.GetChatRequest.SerializeToString,
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@ -141,9 +142,9 @@ class Chats(object):
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/list_chats',
idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/list_chats',
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.ListChatsRequest.SerializeToString,
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chats.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@ -158,8 +159,8 @@ class Chats(object):
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/update_chat',
idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
return grpc.experimental.unary_unary(request, target, '/idm.api.proto.ChatManager/update_chat',
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.UpdateChatRequest.SerializeToString,
idm_dot_api_dot_proto_dot_chat__manager__service__pb2.Chat.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

109
idm/api/services/chat_manager.py Normal file
View File

@ -0,0 +1,109 @@
from grpc import StatusCode
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from idm.api.proto.chat_manager_service_pb2 import Chats as ChatsPb
from idm.api.proto.chat_manager_service_pb2_grpc import (
ChatManagerServicer,
add_ChatManagerServicer_to_server,
)
from library.aiogrpctools.base import (
BaseService,
aiogrpc_request_wrapper,
)
from pypika import (
PostgreSQLQuery,
Table,
)
class ChatManagerService(ChatManagerServicer, BaseService):
chats_table = Table('chats')
def __init__(self, server, service_name, pool_holder, admin_log_reader):
super().__init__(service_name=service_name)
self.server = server
self.pool_holder = pool_holder
self.admin_log_reader = admin_log_reader
async def start(self):
add_ChatManagerServicer_to_server(self, self.server)
def enrich_chat(self, chat_pb: ChatPb):
chat_pb.is_subscribed = self.admin_log_reader.is_subscribed(chat_pb.chat_id)
return chat_pb
@aiogrpc_request_wrapper()
async def create_chat(self, request, context, metadata):
chat = ChatPb(
chat_id=request.chat_id,
language=request.language,
username=request.username,
is_system_messaging_enabled=True,
is_discovery_enabled=True,
)
query = (
PostgreSQLQuery
.into(self.chats_table)
.columns(
self.chats_table.chat_id,
self.chats_table.language,
self.chats_table.username,
self.chats_table.is_system_messaging_enabled,
self.chats_table.is_discovery_enabled,
)
.insert(
chat.chat_id,
chat.language,
chat.username,
chat.is_system_messaging_enabled,
chat.is_discovery_enabled,
)
.returning('*')
).get_sql()
async with self.pool_holder.pool.acquire() as session:
result = await session.execute(query)
chat = await result.fetchone()
return self.enrich_chat(ChatPb(**chat))
@aiogrpc_request_wrapper()
async def get_chat(self, request, context, metadata):
query = (
PostgreSQLQuery
.from_(self.chats_table)
.select('*')
.where(self.chats_table.chat_id == request.chat_id)
).get_sql()
async with self.pool_holder.pool.acquire() as session:
result = await session.execute(query)
chat = await result.fetchone()
if chat is None:
await context.abort(StatusCode.NOT_FOUND, 'not_found')
return self.enrich_chat(ChatPb(**chat))
@aiogrpc_request_wrapper()
async def list_chats(self, request, context, metadata):
query = (
PostgreSQLQuery
.from_(self.chats_table)
.select('*')
.where(self.chats_table.ban_until > request.banned_at_moment)
.limit(10)
).get_sql()
async with self.pool_holder.pool.acquire() as session:
results = await session.execute(query)
chats = await results.fetchall()
return ChatsPb(
chats=list(map(lambda x: self.enrich_chat(ChatPb(**x)), chats))
)
@aiogrpc_request_wrapper()
async def update_chat(self, request, context, metadata):
query = PostgreSQLQuery.update(self.chats_table)
for field in request.DESCRIPTOR.fields:
if field.containing_oneof and request.HasField(field.name):
field_value = getattr(request, field.name)
query = query.set(field.name, field_value)
query = query.where(self.chats_table.chat_id == request.chat_id).returning('*').get_sql()
async with self.pool_holder.pool.acquire() as session:
result = await session.execute(query)
chat = await result.fetchone()
return self.enrich_chat(ChatPb(**chat))

3
idm/api2/aioclient/__init__.py
View File

@ -1,3 +0,0 @@
from .aioclient import IdmApi2GrpcClient
__all__ = ['IdmApi2GrpcClient']

66
idm/api2/proto/chats_service.proto
View File

@ -1,66 +0,0 @@
syntax = "proto3";
package idm.api2.proto;
import "idm/api2/proto/location.proto";
message ChatData {
int64 id = 1;
string username = 2;
string language = 3;
bool is_system_messaging_enabled = 4;
bool is_discovery_enabled = 5;
int32 ban_until = 6;
string ban_message = 7;
bool is_admin = 8;
bool is_subscribed = 10;
int64 created_at = 11;
}
message ChatsData {
repeated ChatData chats = 1;
}
message CreateChatRequest {
int64 chat_id = 1;
string username = 2;
string language = 3;
}
message GetChatRequest {
int64 chat_id = 1;
}
message ListChatsRequest {
oneof banned_at_moment_oneof {
int32 banned_at_moment = 1;
}
}
message UpdateChatRequest {
int64 chat_id = 1;
oneof language_oneof {
string language = 2;
}
oneof is_system_messaging_enabled_oneof {
bool is_system_messaging_enabled = 3;
}
oneof is_discovery_enabled_oneof {
bool is_discovery_enabled = 4;
}
oneof ban_until_oneof {
int32 ban_until = 5;
}
oneof ban_message_oneof {
string ban_message = 6;
}
oneof is_admin_oneof {
bool is_admin = 7;
}
}
service Chats {
rpc create_chat(CreateChatRequest) returns (ChatData) {};
rpc get_chat(GetChatRequest) returns (ChatData) {};
rpc list_chats(ListChatsRequest) returns (ChatsData) {};
rpc update_chat(UpdateChatRequest) returns (ChatData) {};
}

7
idm/api2/proto/location.proto
View File

@ -1,7 +0,0 @@
syntax = "proto3";
package idm.api2.proto;
message Location {
float lat = 1;
float lon = 2;
}

78
idm/api2/proto/location_pb2.py
View File

@ -1,78 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: idm/api2/proto/location.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='idm/api2/proto/location.proto',
package='idm.api2.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\x1didm/api2/proto/location.proto\x12\x0eidm.api2.proto\"$\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x02\x12\x0b\n\x03lon\x18\x02 \x01(\x02\x62\x06proto3'
)
_LOCATION = _descriptor.Descriptor(
name='Location',
full_name='idm.api2.proto.Location',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='lat', full_name='idm.api2.proto.Location.lat', index=0,
number=1, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='lon', full_name='idm.api2.proto.Location.lon', index=1,
number=2, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=49,
serialized_end=85,
)
DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), {
'DESCRIPTOR' : _LOCATION,
'__module__' : 'idm.api2.proto.location_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.Location)
})
_sym_db.RegisterMessage(Location)
# @@protoc_insertion_point(module_scope)

4
idm/api2/proto/location_pb2_grpc.py
View File

@ -1,4 +0,0 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc

10
images/install.bzl
View File

@ -11,14 +11,8 @@ def images_install():
container_pull(
name = "ubuntu",
digest = "sha256:5403064f94b617f7975a19ba4d1a1299fd584397f6ee4393d0e16744ed11aab1",
digest = "sha256:45ff0162921e61c004010a67db1bee7d039a677bed0cb294e61f2b47346d42b3",
registry = "index.docker.io",
repository = "library/ubuntu",
tag = "20.04",
)
container_pull(
name = "jupyter",
registry = "index.docker.io",
repository = "jupyter/tensorflow-notebook",
tag = "latest",
tag = "20.10",
)

2
images/production/BUILD.bazel
View File

@ -13,7 +13,7 @@ download_pkgs(
"libev4",
"libgomp1",
"libgoogle-perftools-dev",
"libprotobuf17",
"libprotobuf23",
"libssl1.1",
],
)

2
library/logging/__init__.py
View File

@ -31,7 +31,7 @@ def error_log(e, level=logging.ERROR, **fields):
e = e.as_internal_dict()
e.update(fields)
elif fields:
e = {'error': str(e), **fields}
e = {'error': repr(e), **fields}
logging.getLogger('error').log(
msg=e,
level=level

2
library/telegram/base.py
View File

@ -136,7 +136,7 @@ class RequestContext:
self.request_id = request_id or RequestContext.generate_request_id(request_id_length)
self.default_fields = {
'bot_name': self.bot_name,
'chat_id': self.chat.id,
'chat_id': self.chat.chat_id,
'request_id': self.request_id,
}

2
nexus/bot/BUILD.bazel
View File

@ -30,7 +30,7 @@ py3_image(
requirement("python_socks"),
requirement("tenacity"),
requirement("uvloop"),
"//idm/api2/aioclient",
"//idm/api/aioclient",
requirement("aiobaseclient"),
requirement("aiocrossref"),
requirement("aiokit"),

4
nexus/bot/application.py
View File

@ -1,5 +1,5 @@
from aiokit import AioRootThing
from idm.api2.aioclient import IdmApi2GrpcClient
from idm.api.aioclient import IdmApiGrpcClient
from izihawa_utils.importlib import import_object
from library.telegram.base import BaseTelegramClient
from nexus.bot.promotioner import Promotioner
@ -21,7 +21,7 @@ class TelegramApplication(AioRootThing):
)
self.hub_client = HubGrpcClient(base_url=self.config['hub']['url'])
self.idm_client = IdmApi2GrpcClient(base_url=self.config['idm']['url'])
self.idm_client = IdmApiGrpcClient(base_url=self.config['idm']['url'])
self.meta_api_client = MetaApiGrpcClient(base_url=self.config['meta_api']['url'])
self.promotioner = Promotioner(promotions=self.config['promotions'])

24
nexus/bot/handlers/base.py
View File

@ -6,7 +6,7 @@ from typing import Union
from grpc import StatusCode
from grpc.experimental.aio import AioRpcError
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from izihawa_utils.exceptions import BaseError
from izihawa_utils.random import random_string
from library.logging import error_log
@ -40,12 +40,12 @@ def get_language(event: events.ChatAction, chat):
return chat.lang_code
def is_banned(chat: Chat) -> bool:
def is_banned(chat: ChatPb) -> bool:
return chat.ban_until is not None and datetime.utcnow().timestamp() < chat.ban_until
def is_subscribed(chat: Chat) -> bool:
return chat.is_subscribed or chat.id < 0 or chat.created_at > time.time() - 10 * 60
def is_subscribed(chat: ChatPb) -> bool:
return chat.is_subscribed or chat.chat_id < 0 or chat.created_at > time.time() - 10 * 60
class ReadOnlyModeError(BaseError):
@ -97,7 +97,7 @@ class BaseHandler(ABC):
session_id=session_id,
position=position,
request_id=request_context.request_id,
user_id=request_context.chat.id,
user_id=request_context.chat.chat_id,
)
async def resolve_scimag(
@ -136,7 +136,7 @@ class BaseHandler(ABC):
page_size=16,
request_id=request_context.request_id,
session_id=session_id,
user_id=request_context.chat.id,
user_id=request_context.chat.chat_id,
)
duplicates = [
scored_document.typed_document.scitech
@ -209,7 +209,7 @@ class BaseHandler(ABC):
)
return chat
async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: ChatPb):
if is_banned(chat):
if chat.ban_message is not None:
async with safe_execution(
@ -245,7 +245,7 @@ class BaseHandler(ABC):
)
raise events.StopPropagation()
async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: ChatPb):
if (
self.application.config['application']['is_subscription_required']
and self.is_subscription_required_for_handler
@ -261,7 +261,7 @@ class BaseHandler(ABC):
).format(related_channel=self.application.config['telegram']['related_channel']))
raise events.StopPropagation()
def _has_access(self, chat: Chat) -> bool:
def _has_access(self, chat: ChatPb) -> bool:
return True
async def _process_chat(self, event: events.ChatAction, request_id: str):
@ -271,8 +271,8 @@ class BaseHandler(ABC):
error_log(e)
event_chat = await event.get_chat()
username = get_username(event, event_chat)
chat = Chat(
id=event.chat_id,
chat = ChatPb(
chat_id=event.chat_id,
is_system_messaging_enabled=True,
is_discovery_enabled=True,
language='en',
@ -307,7 +307,7 @@ class BaseHandler(ABC):
await self._check_ban(event=event, request_context=request_context, chat=chat)
if self.should_reset_last_widget:
self.reset_last_widget(request_context.chat.id)
self.reset_last_widget(request_context.chat.chat_id)
async with safe_execution(
request_context=request_context,

4
nexus/bot/handlers/download.py
View File

@ -19,7 +19,7 @@ class DownloadHandler(BaseCallbackQueryHandler):
document_id = int(event.pattern_match.group(3))
position = int(event.pattern_match.group(4).decode())
self.application.user_manager.last_widget[request_context.chat.id] = None
self.application.user_manager.last_widget[request_context.chat.chat_id] = None
request_context.add_default_fields(mode='download', session_id=session_id)
request_context.statbox(action='get', query=str(document_id), position=position)
@ -48,4 +48,4 @@ class DownloadHandler(BaseCallbackQueryHandler):
)
else:
await remove_button(event, '⬇️', and_empty_too=True)
self.application.user_manager.last_widget[request_context.chat.id] = None
self.application.user_manager.last_widget[request_context.chat.chat_id] = None

2
nexus/bot/handlers/referencing_to.py
View File

@ -41,7 +41,7 @@ class ReferencingToHandler(BaseCallbackQueryHandler):
serp, buttons = await document_list_widget.render()
return await self.application.telegram_client.edit_message(
request_context.chat.id,
request_context.chat.chat_id,
message_id,
serp,
buttons=buttons,

4
nexus/bot/handlers/roll.py
View File

@ -20,7 +20,7 @@ class RollHandler(BaseHandler):
language=request_context.chat.language,
session_id=session_id,
request_id=request_context.request_id,
user_id=request_context.chat.id,
user_id=request_context.chat.chat_id,
)
scitech_view = await self.resolve_scitech(
document_id=roll_response_pb.document_id,
@ -35,7 +35,7 @@ class RollHandler(BaseHandler):
)
actions = [
self.application.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
view,
buttons=buttons,
),

14
nexus/bot/handlers/search.py
View File

@ -52,7 +52,7 @@ class BaseSearchHandler(BaseHandler):
except AioRpcError as e:
actions = [
self.application.telegram_client.delete_messages(
request_context.chat.id,
request_context.chat.chat_id,
[message_id],
)
]
@ -118,7 +118,7 @@ class BaseSearchHandler(BaseHandler):
)
return await asyncio.gather(
self.application.telegram_client.edit_message(
request_context.chat.id,
request_context.chat.chat_id,
message_id,
view,
buttons=buttons,
@ -127,7 +127,7 @@ class BaseSearchHandler(BaseHandler):
serp, buttons = await search_widget.render()
return await self.application.telegram_client.edit_message(
request_context.chat.id,
request_context.chat.chat_id,
message_id,
serp,
buttons=buttons,
@ -147,7 +147,7 @@ class SearchHandler(BaseSearchHandler):
'action': 'user_flood_ban',
'mode': 'search',
'ban_timeout_seconds': ban_timeout,
'chat_id': request_context.chat.id,
'chat_id': request_context.chat.chat_id,
})
ban_reason = t(
'BAN_MESSAGE_TOO_MANY_REQUESTS',
@ -162,10 +162,10 @@ class SearchHandler(BaseSearchHandler):
))
async def handler(self, event: events.ChatAction, request_context: RequestContext):
ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.id)
ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.chat_id)
if ban_timeout:
return await self.ban_handler(event, request_context, ban_timeout)
self.application.user_manager.add_search_time(user_id=request_context.chat.id, search_time=time.time())
self.application.user_manager.add_search_time(user_id=request_context.chat.chat_id, search_time=time.time())
search_prefix = event.pattern_match.group(1)
query = event.pattern_match.group(2)
@ -178,7 +178,7 @@ class SearchHandler(BaseSearchHandler):
prefetch_message = await event.reply(
t("SEARCHING", language=request_context.chat.language),
)
self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
try:
await self.do_search(
event, request_context, prefetch_message,

2
nexus/bot/handlers/start.py
View File

@ -32,7 +32,7 @@ class StartHandler(BaseSearchHandler):
prefetch_message = await request_message.reply(
t("SEARCHING", language=request_context.chat.language),
)
self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
await asyncio.gather(
event.delete(),
self.do_search(event, request_context, prefetch_message, query=query,

10
nexus/bot/handlers/view.py
View File

@ -30,7 +30,7 @@ class ViewHandler(BaseHandler):
request_context.add_default_fields(mode='view', session_id=session_id)
request_context.statbox(action='view', query=str(document_id), position=position)
found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.id)
found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.chat_id)
try:
if found_old_widget:
@ -41,11 +41,11 @@ class ViewHandler(BaseHandler):
functions.messages.GetMessagesRequest(id=[old_message_id])
)).messages[0]
prefetch_message = await self.application.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t("SEARCHING", language=request_context.chat.language),
reply_to=old_message.reply_to_msg_id,
)
self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
self.application.user_manager.last_widget[request_context.chat.chat_id] = prefetch_message.id
message_id = prefetch_message.id
link_preview = True
@ -81,7 +81,7 @@ class ViewHandler(BaseHandler):
)
actions = [
self.application.telegram_client.edit_message(
request_context.chat.id,
request_context.chat.chat_id,
message_id,
view,
buttons=buttons,
@ -92,7 +92,7 @@ class ViewHandler(BaseHandler):
if not found_old_widget:
actions.append(
self.application.telegram_client.delete_messages(
request_context.chat.id,
request_context.chat.chat_id,
[old_message_id],
)
)

4
nexus/bot/handlers/vote.py
View File

@ -26,7 +26,7 @@ class VoteHandler(BaseCallbackQueryHandler):
vote=VotePb(
document_id=document_id,
value=vote_value,
voter_id=request_context.chat.id,
voter_id=request_context.chat.chat_id,
),
)
@ -43,7 +43,7 @@ class VoteHandler(BaseCallbackQueryHandler):
# ToDo: Generalize nexus.views.telegram.common.remove_button and use it here
return await asyncio.gather(
self.application.telegram_client.edit_message(
request_context.chat.id,
request_context.chat.chat_id,
message.id,
message.text,
buttons=None,

4
nexus/bot/widgets/admin_widget.py
View File

@ -1,9 +1,9 @@
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.bot.application import TelegramApplication
class AdminWidget:
def __init__(self, application: TelegramApplication, chat: Chat):
def __init__(self, application: TelegramApplication, chat: ChatPb):
self.application = application
self.chat = chat

10
nexus/bot/widgets/banlist_widget.py
View File

@ -1,25 +1,25 @@
import time
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.bot.application import TelegramApplication
class BanlistWidget:
def __init__(self, application: TelegramApplication, chat: Chat):
def __init__(self, application: TelegramApplication, chat: ChatPb):
self.application = application
self.chat = chat
async def render(self, chat_list: list[Chat]):
async def render(self, chat_list: list[ChatPb]):
if not chat_list:
return 'Nobody is banned'
separator = '------------\n'
return separator.join(
map(
lambda chat: (
f'```{chat.username} ({chat.id})\n'
f'```{chat.username} ({chat.chat_id})\n'
f'Until: {time.ctime(chat.ban_until)}\n'
f'Message: {chat.ban_message}```\n'
f'/unban_{chat.id}\n'
f'/unban_{chat.chat_id}\n'
),
chat_list
)

10
nexus/bot/widgets/document_list_widget.py
View File

@ -1,6 +1,6 @@
from typing import Optional
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.bot.application import TelegramApplication
from nexus.meta_api.proto.meta_search_service_pb2 import \
ScoredDocument as ScoredDocumentPb
@ -14,7 +14,7 @@ class DocumentListWidget:
def __init__(
self,
application: TelegramApplication,
chat: Chat,
chat: ChatPb,
session_id: str,
message_id: int,
request_id: str,
@ -32,7 +32,7 @@ class DocumentListWidget:
@staticmethod
async def create(
application: TelegramApplication,
chat: Chat,
chat: ChatPb,
session_id: str,
message_id: int,
request_id: str,
@ -58,7 +58,7 @@ class DocumentListWidget:
position=0,
request_id=self.request_id,
session_id=self.session_id,
user_id=self.chat.id,
user_id=self.chat.chat_id,
)
self._response = await self.application.meta_api_client.search(
schemas=('scimag',),
@ -66,7 +66,7 @@ class DocumentListWidget:
page=self.page,
request_id=self.request_id,
session_id=self.session_id,
user_id=self.chat.id,
user_id=self.chat.chat_id,
)
@property

8
nexus/bot/widgets/search_widget.py
View File

@ -1,6 +1,6 @@
from typing import Optional
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.bot.application import TelegramApplication
from nexus.meta_api.proto.meta_search_service_pb2 import \
ScoredDocument as ScoredDocumentPb
@ -21,7 +21,7 @@ class SearchWidget:
def __init__(
self,
application: TelegramApplication,
chat: Chat,
chat: ChatPb,
session_id: str,
message_id: int,
request_id: str,
@ -41,7 +41,7 @@ class SearchWidget:
@staticmethod
async def create(
application: TelegramApplication,
chat: Chat,
chat: ChatPb,
session_id: str,
message_id: int,
request_id: str,
@ -70,7 +70,7 @@ class SearchWidget:
page_size=self.application.config['application']['page_size'],
request_id=self.request_id,
session_id=self.session_id,
user_id=self.chat.id,
user_id=self.chat.chat_id,
language=self.chat.language,
)

10
nexus/bot/widgets/settings_widget.py
View File

@ -1,6 +1,6 @@
from typing import Optional
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.bot.application import TelegramApplication
from nexus.translations import t
from telethon import Button
@ -34,7 +34,7 @@ class SettingsWidget:
def __init__(
self,
application: TelegramApplication,
chat: Chat,
chat: ChatPb,
has_language_buttons: Optional[bool] = None,
is_group_mode: bool = False,
request_id: Optional[str] = None,
@ -54,7 +54,7 @@ class SettingsWidget:
async def _switch_language(self, target_language: str):
self.chat = await self.application.idm_client.update_chat(
chat_id=self.chat.id,
chat_id=self.chat.chat_id,
language=target_language,
request_id=self.request_id,
)
@ -62,7 +62,7 @@ class SettingsWidget:
async def _switch_system_messaging(self, is_system_messaging_enabled: str):
self.chat = await self.application.idm_client.update_chat(
chat_id=self.chat.id,
chat_id=self.chat.chat_id,
is_system_messaging_enabled=bool(int(is_system_messaging_enabled)),
request_id=self.request_id,
)
@ -70,7 +70,7 @@ class SettingsWidget:
async def _switch_discovery(self, is_discovery_enabled: str):
self.chat = await self.application.idm_client.update_chat(
chat_id=self.chat.id,
chat_id=self.chat.chat_id,
is_discovery_enabled=bool(int(is_discovery_enabled)),
request_id=self.request_id,
)

5
nexus/hub/BUILD.bazel
View File

@ -17,10 +17,7 @@ py3_image(
base = "//images/production:base-python-image",
data = [
"configs/base.yaml",
"configs/development.yaml",
"configs/logging.yaml",
"configs/production.yaml",
"configs/testing.yaml",
],
main = "main.py",
srcs_version = "PY3ONLY",
@ -37,7 +34,7 @@ py3_image(
requirement("python-socks"),
requirement("tenacity"),
requirement("uvloop"),
"//idm/api2/proto:idm_proto_py",
"//idm/api/proto:idm_proto_py",
requirement("giogrobid"),
"//library/aiogrpctools",
requirement("aiokit"),

59
nexus/hub/README.md
View File

@ -1,66 +1,9 @@
# Nexus Search: Hub API
`Hub` is a daemon responsible for retrieving files and sending them to users. This version has cut `configs`
subdirectory due to hard reliance of configs on the network infrastructure you are using.
You have to write your own configs taking example below into account.
`Hub` is a daemon responsible for retrieving files and sending them to users.
The bot requires two other essential parts:
- Postgres Database
- IPFS Daemon
or their substitutions
## Sample `configs/base.yaml`
```yaml
---
application:
# Look at the special Postgres `sharience` table to retrieve user-sent files
is_sharience_enabled: true
maintenance_picture_url:
# Used in logging
service_name: nexus-hub
# Store file hashes into operation log
should_store_hashes: true
database:
database: nexus
host:
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
grobid:
url:
grpc:
# Listen address
address: 0.0.0.0
# Listen port
port: 9090
ipfs:
address:
port: 4001
log_path: '/var/log/nexus-hub/{{ ENV_TYPE }}'
meta_api:
url:
pylon:
# Proxy used in `pylon` retriever to download files
proxy: socks5://127.0.0.1:9050
# Proxy used in `pylon` retriever to get metadata
resolve_proxy: socks5://127.0.0.1:9050
telegram:
# Telegram App Hash from https://my.telegram.org/
app_hash: '{{ APP_HASH }}'
# Telegram App ID from https://my.telegram.org/
app_id: 00000
# External bot name shown in messages to users
bot_external_name: libgen_scihub_bot
# Internal bot name used in logging
bot_name: nexus-bot
bot_token: '{{ BOT_TOKEN }}'
# Telethon database for keeping cache
database:
session_id: nexus-hub
# Frequency of updating downloading progress
progress_throttle_seconds: 5
# Send files using stored telegram_file_id
should_use_telegram_file_id: true
```

2
nexus/hub/aioclient/BUILD.bazel
View File

@ -6,7 +6,7 @@ py_library(
visibility = ["//visibility:public"],
deps = [
requirement("grpcio"),
"//idm/api2/proto:idm_proto_py",
"//idm/api/proto:idm_proto_py",
requirement("aiokit"),
"//nexus/hub/proto:hub_grpc_py",
"//nexus/hub/proto:hub_proto_py",

6
nexus/hub/aioclient/aioclient.py
View File

@ -2,7 +2,7 @@ from typing import Optional
from aiokit import AioThing
from grpc.experimental.aio import insecure_channel
from idm.api2.proto.chats_service_pb2 import ChatData as ChatDataPb
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.hub.proto.delivery_service_pb2 import \
StartDeliveryRequest as StartDeliveryRequestPb
from nexus.hub.proto.delivery_service_pb2 import \
@ -42,7 +42,7 @@ class HubGrpcClient(AioThing):
async def start_delivery(
self,
typed_document_pb: TypedDocumentPb,
chat: ChatDataPb,
chat: ChatPb,
request_id: Optional[str],
session_id: Optional[str],
) -> StartDeliveryResponsePb:
@ -58,7 +58,7 @@ class HubGrpcClient(AioThing):
self,
telegram_document: bytes,
telegram_file_id: str,
chat: ChatDataPb,
chat: ChatPb,
request_id: Optional[str] = None,
session_id: Optional[str] = None,
) -> SubmitResponsePb:

51
nexus/hub/configs/base.yaml Normal file
View File

@ -0,0 +1,51 @@
---
application:
# Enable special Postgres `sharience` table to retrieve user-sent files
is_sharience_enabled: true
# URL to the picture shown while maintenance
maintenance_picture_url:
# Used in logging
service_name: nexus-hub
# Store file hashes into operation log
should_store_hashes: true
database:
database: nexus
host:
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
grobid:
url:
grpc:
# Listen address
address: 0.0.0.0
# Listen port
port: 9090
ipfs:
address:
port: 4001
log_path: '/var/log/nexus-hub/{{ ENV_TYPE }}'
meta_api:
url:
pylon:
# Proxy used in `pylon` retriever to download files
proxy: socks5://127.0.0.1:9050
# Proxy used in `pylon` retriever to get metadata
resolve_proxy: socks5://127.0.0.1:9050
telegram:
# Telegram App Hash from https://my.telegram.org/
app_hash: '{{ APP_HASH }}'
# Telegram App ID from https://my.telegram.org/
app_id: 00000
# External bot name shown in messages to users
bot_external_name: libgen_scihub_bot
# Internal bot name used in logging
bot_name: nexus-hub
bot_token: '{{ BOT_TOKEN }}'
# Telethon database for keeping cache
database:
session_id: nexus-hub
# Frequency of updating downloading progress
progress_throttle_seconds: 5
# Send files using stored telegram_file_id
should_use_telegram_file_id: true

76
nexus/hub/configs/logging.yaml Normal file
View File

@ -0,0 +1,76 @@
---
logging:
disable_existing_loggers: false
formatters:
base:
class: library.logging.formatters.BaseFormatter
default:
class: library.logging.formatters.DefaultFormatter
traceback:
class: library.logging.formatters.TracebackFormatter
handlers:
debug:
class: logging.StreamHandler
formatter: default
level: DEBUG
stream: 'ext://sys.stderr'
error:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/error.log'
formatter: default
level: ERROR
operation:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/operation.log'
formatter: base
level: DEBUG
statbox:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/statbox.log'
formatter: default
level: INFO
traceback:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/traceback.log'
formatter: traceback
level: ERROR
warning:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/warning.log'
formatter: default
level: WARNING
loggers:
aiokafka:
handlers:
- error
propagate: false
debug:
handlers:
- debug
propagate: false
error:
handlers:
- traceback
- error
- warning
propagate: false
operation:
handlers:
- operation
propagate: false
statbox:
handlers:
- statbox
propagate: false
telethon:
handlers:
- error
propagate: false
root:
handlers:
- debug
- error
- warning
level: DEBUG
version: 1

2
nexus/hub/proto/BUILD.bazel
View File

@ -9,7 +9,7 @@ proto_library(
"*.proto",
]),
deps = [
"//idm/api2/proto:idm_proto",
"//idm/api/proto:idm_proto",
"//nexus/models/proto:models_proto",
"@com_google_protobuf//:wrappers_proto",
],

4
nexus/hub/proto/delivery_service.proto
View File

@ -2,11 +2,11 @@ syntax = "proto3";
package nexus.hub.proto;
import "nexus/models/proto/typed_document.proto";
import "idm/api2/proto/chats_service.proto";
import "idm/api/proto/chat_manager_service.proto";
message StartDeliveryRequest {
nexus.models.proto.TypedDocument typed_document = 1;
idm.api2.proto.ChatData chat = 2;
idm.api.proto.Chat chat = 2;
}
message StartDeliveryResponse {

29
nexus/hub/proto/delivery_service_pb2.py
View File

@ -12,8 +12,9 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from idm.api2.proto import \
chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
from idm.api.proto import \
chat_manager_service_pb2 as \
idm_dot_api_dot_proto_dot_chat__manager__service__pb2
from nexus.models.proto import \
typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
@ -23,9 +24,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a\"idm/api2/proto/chats_service.proto\"y\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x03\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3'
serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a(idm/api/proto/chat_manager_service.proto\"t\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12!\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x13.idm.api.proto.Chat\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x02\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api_dot_proto_dot_chat__manager__service__pb2.DESCRIPTOR,])
@ -47,15 +48,15 @@ _STARTDELIVERYRESPONSE_STATUS = _descriptor.EnumDescriptor(
type=None,
create_key=_descriptor._internal_create_key),
_descriptor.EnumValueDescriptor(
name='ALREADY_DOWNLOADING', index=2, number=3,
name='ALREADY_DOWNLOADING', index=2, number=2,
serialized_options=None,
type=None,
create_key=_descriptor._internal_create_key),
],
containing_type=None,
serialized_options=None,
serialized_start=348,
serialized_end=413,
serialized_start=349,
serialized_end=414,
)
_sym_db.RegisterEnumDescriptor(_STARTDELIVERYRESPONSE_STATUS)
@ -94,8 +95,8 @@ _STARTDELIVERYREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=136,
serialized_end=257,
serialized_start=142,
serialized_end=258,
)
@ -127,12 +128,12 @@ _STARTDELIVERYRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=260,
serialized_end=413,
serialized_start=261,
serialized_end=414,
)
_STARTDELIVERYREQUEST.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api_dot_proto_dot_chat__manager__service__pb2._CHAT
_STARTDELIVERYRESPONSE.fields_by_name['status'].enum_type = _STARTDELIVERYRESPONSE_STATUS
_STARTDELIVERYRESPONSE_STATUS.containing_type = _STARTDELIVERYRESPONSE
DESCRIPTOR.message_types_by_name['StartDeliveryRequest'] = _STARTDELIVERYREQUEST
@ -162,8 +163,8 @@ _DELIVERY = _descriptor.ServiceDescriptor(
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=415,
serialized_end=524,
serialized_start=416,
serialized_end=525,
methods=[
_descriptor.MethodDescriptor(
name='start_delivery',

4
nexus/hub/proto/submitter_service.proto
View File

@ -1,12 +1,12 @@
syntax = "proto3";
package nexus.hub.proto;
import "idm/api2/proto/chats_service.proto";
import "idm/api/proto/chat_manager_service.proto";
message SubmitRequest {
bytes telegram_document = 1;
string telegram_file_id = 2;
idm.api2.proto.ChatData chat = 3;
idm.api.proto.Chat chat = 3;
}
message SubmitResponse { }

34
nexus/hub/proto/submitter_service_pb2.py
View File

@ -12,8 +12,9 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from idm.api2.proto import \
chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
from idm.api.proto import \
chat_manager_service_pb2 as \
idm_dot_api_dot_proto_dot_chat__manager__service__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/hub/proto/submitter_service.proto',
@ -21,9 +22,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a\"idm/api2/proto/chats_service.proto\"R\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3'
serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a(idm/api/proto/chat_manager_service.proto\"g\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12\x18\n\x10telegram_file_id\x18\x02 \x01(\t\x12!\n\x04\x63hat\x18\x03 \x01(\x0b\x32\x13.idm.api.proto.Chat\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3'
,
dependencies=[idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
dependencies=[idm_dot_api_dot_proto_dot_chat__manager__service__pb2.DESCRIPTOR,])
@ -44,8 +45,15 @@ _SUBMITREQUEST = _descriptor.Descriptor(
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=1,
number=2, type=11, cpp_type=10, label=1,
name='telegram_file_id', full_name='nexus.hub.proto.SubmitRequest.telegram_file_id', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
@ -62,8 +70,8 @@ _SUBMITREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=96,
serialized_end=178,
serialized_start=102,
serialized_end=205,
)
@ -87,11 +95,11 @@ _SUBMITRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=180,
serialized_end=196,
serialized_start=207,
serialized_end=223,
)
_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api_dot_proto_dot_chat__manager__service__pb2._CHAT
DESCRIPTOR.message_types_by_name['SubmitRequest'] = _SUBMITREQUEST
DESCRIPTOR.message_types_by_name['SubmitResponse'] = _SUBMITRESPONSE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@ -119,8 +127,8 @@ _SUBMITTER = _descriptor.ServiceDescriptor(
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=198,
serialized_end=286,
serialized_start=225,
serialized_end=313,
methods=[
_descriptor.MethodDescriptor(
name='submit',

2
nexus/hub/services/base.py
View File

@ -70,7 +70,7 @@ class BaseHubService(BaseService):
buttons=buttons,
caption=f"{document_view.generate_body(language=request_context.chat.language, limit=512)}\n"
f"@{self.bot_external_name}",
entity=request_context.chat.id,
entity=request_context.chat.chat_id,
file=file,
progress_callback=progress_callback
)

34
nexus/hub/services/delivery.py
View File

@ -76,7 +76,7 @@ class DownloadTask:
)
)
self.delivery_service.user_manager.add_task(self.request_context.chat.id, self.document_view.id)
self.delivery_service.user_manager.add_task(self.request_context.chat.chat_id, self.document_view.id)
self.delivery_service.downloadings.add(self)
self.task.add_done_callback(self.done_callback)
@ -84,7 +84,7 @@ class DownloadTask:
def done_callback(self, f):
self.delivery_service.downloadings.remove(self)
self.delivery_service.user_manager.remove_task(
self.request_context.chat.id,
self.request_context.chat.chat_id,
self.document_view.id,
)
@ -93,7 +93,7 @@ class DownloadTask:
async def _on_fail():
await self.delivery_service.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t('MAINTENANCE', language=request_context.chat.language).format(
maintenance_picture_url=self.delivery_service.maintenance_picture_url
),
@ -165,7 +165,7 @@ class DownloadTask:
progress_callback=progress_bar_upload.callback,
request_context=self.request_context,
session_id=self.session_id,
voting=not is_group_or_channel(self.request_context.chat.id),
voting=not is_group_or_channel(self.request_context.chat.chat_id),
)
request_context.statbox(
action='uploaded',
@ -191,7 +191,7 @@ class DownloadTask:
finally:
downloads_gauge.dec()
messages = filter_none([progress_bar_download.message])
await self.delivery_service.telegram_client.delete_messages(request_context.chat.id, messages)
await self.delivery_service.telegram_client.delete_messages(request_context.chat.chat_id, messages)
async def process_resp(self, resp, progress_bar, collected, filesize):
progress_bar.set_source(get_fancy_name(resp.source))
@ -206,7 +206,7 @@ class DownloadTask:
async def respond_not_found(self, request_context: RequestContext, document_view):
return await self.delivery_service.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t("SOURCES_UNAVAILABLE", language=request_context.chat.language).format(
document=document_view.get_robust_title()
),
@ -244,6 +244,7 @@ class DownloadTask:
async for resp in self.delivery_service.pylon_client.by_doi(
doi=document_view.doi,
md5=document_view.md5,
error_log_func=self.request_context.error_log,
):
await self.process_resp(
resp=resp,
@ -252,11 +253,14 @@ class DownloadTask:
filesize=document_view.filesize,
)
return bytes(collected)
except DownloadError:
pass
except DownloadError as e:
self.request_context.error_log(e)
if document_view.md5:
try:
async for resp in self.delivery_service.pylon_client.by_md5(md5=document_view.md5):
async for resp in self.delivery_service.pylon_client.by_md5(
md5=document_view.md5,
error_log_func=self.request_context.error_log,
):
await self.process_resp(
resp=resp,
progress_bar=progress_bar,
@ -264,14 +268,14 @@ class DownloadTask:
filesize=document_view.filesize,
)
return bytes(collected)
except DownloadError:
pass
except DownloadError as e:
self.request_context.error_log(e)
async def external_cancel(self):
self.task.cancel()
self.request_context.statbox(action='externally_canceled')
await self.delivery_service.telegram_client.send_message(
self.request_context.chat.id,
self.request_context.chat.chat_id,
t("DOWNLOAD_CANCELED", language=self.request_context.chat.language).format(
document=self.document_view.get_robust_title()
),
@ -366,15 +370,15 @@ class DeliveryService(DeliveryServicer, BaseHubService):
file=document_view.telegram_file_id,
session_id=metadata.get('session-id'),
request_context=request_context,
voting=not is_group_or_channel(request_context.chat.id),
voting=not is_group_or_channel(request_context.chat.chat_id),
)
request_context.statbox(action='cache_hit', document_id=document_view.id)
except ValueError:
cache_hit = False
if not cache_hit:
if self.user_manager.has_task(request.chat.id, document_view.id):
if self.user_manager.has_task(request.chat.chat_id, document_view.id):
return StartDeliveryResponsePb(status=StartDeliveryResponsePb.Status.ALREADY_DOWNLOADING)
if self.user_manager.hit_limits(request.chat.id):
if self.user_manager.hit_limits(request.chat.chat_id):
return StartDeliveryResponsePb(status=StartDeliveryResponsePb.Status.TOO_MANY_DOWNLOADS)
await DownloadTask(
delivery_service=self,

14
nexus/hub/services/submitter.py
View File

@ -101,13 +101,13 @@ class SubmitterService(SubmitterServicer, BaseHubService):
if document.size > 20 * 1024 * 1024:
request_context.error_log(FileTooBigError(size=document.size))
await self.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t('FILE_TOO_BIG_ERROR', language=request_context.chat.language),
buttons=[close_button()],
)
return SubmitResponsePb()
processing_message = await self.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t("PROCESSING_PAPER", language=request_context.chat.language).format(
filename=document.attributes[0].file_name,
),
@ -121,7 +121,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
except BadRequestError as e:
request_context.error_log(e)
await self.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t('UNPARSABLE_DOCUMENT_ERROR', language=request_context.chat.language),
buttons=[close_button()],
)
@ -130,7 +130,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
if not processed_document.get('doi'):
request_context.error_log(UnparsableDoiError())
await self.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t('UNPARSABLE_DOI_ERROR', language=request_context.chat.language),
buttons=[close_button()],
)
@ -143,14 +143,14 @@ class SubmitterService(SubmitterServicer, BaseHubService):
page_size=1,
request_id=request_context.request_id,
session_id=session_id,
user_id=request_context.chat.id,
user_id=request_context.chat.chat_id,
language=request_context.chat.language,
)
if len(search_response_pb.scored_documents) == 0:
request_context.error_log(UnavailableMetadataError(doi=processed_document['doi']))
await self.telegram_client.send_message(
request_context.chat.id,
request_context.chat.chat_id,
t(
'UNAVAILABLE_METADATA_ERROR',
language=request_context.chat.language
@ -175,7 +175,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
update_document=UpdateDocumentPb(
typed_document=TypedDocumentPb(sharience=ShariencePb(
parent_id=document_view.id,
uploader_id=request_context.chat.id,
uploader_id=request_context.chat.chat_id,
updated_at=int(time.time()),
md5=hashlib.md5(file).hexdigest(),
filesize=document.size,

98
nexus/pipe/README.md
View File

@ -1,99 +1,3 @@
# Nexus Pipe
`Pipe` processes Kafka queue of operations. This version has cut `configs`
subdirectory due to hard reliance of configs on the network infrastructure you are using.
You have to write your own configs taking example below into account.
## Sample `configs/base.yaml`
```yaml
---
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
pipe:
brokers: |
kafka-0.example.net
schema:
- consumers:
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
topics:
- name: cross_references
workers: 4
group_id: pipe
processors:
- class: nexus.pipe.processors.CrossReferencesProcessor
kwargs:
brokers: |
kafka-0.example.net
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- consumers:
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
topics:
- name: operations
workers: 2
- class: nexus.pipe.consumers.DocumentOperationsConsumer
topics:
- name: operations_binary_hp
workers: 4
- name: operations_binary
workers: 14
group_id: pipe
processors:
- class: nexus.pipe.processors.ActionProcessor
kwargs:
actions:
- class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
kwargs:
crossref:
rps: 50
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
kwargs:
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
kwargs:
brokers: |
kafka-0.example.net
topic: cross_references
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:
summa:
base_url: http://summa.example.net
timeout: 15
ttl_dns_cache: 30
filter:
class: nexus.pipe.filters.DocumentOperationFilter
kwargs:
document: scimag
operation: update_document
- class: nexus.pipe.processors.ActionProcessor
kwargs:
actions:
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
- class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
kwargs:
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:
summa:
base_url: http://summa.example.net
timeout: 15
ttl_dns_cache: 30
filter:
class: nexus.pipe.filters.DocumentOperationFilter
kwargs:
document: scitech
operation: update_document
```
`Pipe` processes Kafka queue of operations.

90
nexus/pipe/configs/base.yaml Normal file
View File

@ -0,0 +1,90 @@
---
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
pipe:
brokers: |
kafka-0.example.net
schema:
- consumers:
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
topics:
- name: cross_references
workers: 4
group_id: pipe
processors:
- class: nexus.pipe.processors.CrossReferencesProcessor
kwargs:
brokers: |
kafka-0.example.net
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- consumers:
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
topics:
- name: operations
workers: 2
- class: nexus.pipe.consumers.DocumentOperationsConsumer
topics:
- name: operations_binary_hp
workers: 4
- name: operations_binary
workers: 14
group_id: pipe
processors:
- class: nexus.pipe.processors.ActionProcessor
kwargs:
actions:
- class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
kwargs:
crossref:
rps: 50
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
kwargs:
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
kwargs:
brokers: |
kafka-0.example.net
topic: cross_references
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:
summa:
base_url: http://summa.example.net
timeout: 15
ttl_dns_cache: 30
filter:
class: nexus.pipe.filters.DocumentOperationFilter
kwargs:
document: scimag
operation: update_document
- class: nexus.pipe.processors.ActionProcessor
kwargs:
actions:
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
- class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
kwargs:
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:
summa:
base_url: http://summa.example.net
timeout: 15
ttl_dns_cache: 30
filter:
class: nexus.pipe.filters.DocumentOperationFilter
kwargs:
document: scitech
operation: update_document

65
nexus/pipe/configs/logging.yaml Normal file
View File

@ -0,0 +1,65 @@
---
logging:
disable_existing_loggers: false
formatters:
default:
class: library.logging.formatters.DefaultFormatter
traceback:
class: library.logging.formatters.TracebackFormatter
handlers:
debug:
class: logging.StreamHandler
formatter: default
level: DEBUG
stream: 'ext://sys.stderr'
error:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/error.log'
formatter: default
level: ERROR
statbox:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/statbox.log'
formatter: default
level: INFO
traceback:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/traceback.log'
formatter: traceback
level: ERROR
warning:
class: library.logging.handlers.BaseFileHandler
filename: '{{ log_path }}/warning.log'
formatter: default
level: WARNING
loggers:
aiokafka:
handlers:
- error
propagate: false
asyncio:
handlers:
- error
level: WARNING
debug:
handlers:
- debug
propagate: false
error:
handlers:
- error
- traceback
- warning
propagate: false
statbox:
handlers:
- statbox
propagate: false
root:
handlers:
- debug
- error
- warning
level: DEBUG
version: 1

57
nexus/pylon/client.py
View File

@ -1,16 +1,10 @@
import asyncio
from typing import (
AsyncIterable,
Callable,
Iterable,
Optional,
)
import aiohttp
import aiohttp.client_exceptions
from aiohttp_socks import (
ProxyConnectionError,
ProxyError,
)
from aiokit import AioThing
from library.logging import error_log
from nexus.pylon.exceptions import (
@ -27,7 +21,6 @@ from nexus.pylon.sources import (
SciHubSeSource,
)
from nexus.pylon.sources.specific import get_specific_sources_for_doi
from python_socks import ProxyTimeoutError
class PylonClient(AioThing):
@ -40,6 +33,7 @@ class PylonClient(AioThing):
self,
doi: str,
md5: Optional[str] = None,
error_log_func: Callable = error_log,
) -> AsyncIterable[FileResponsePb]:
sources = []
sources.extend(get_specific_sources_for_doi(doi, proxy=self.proxy, resolve_proxy=self.resolve_proxy))
@ -49,49 +43,42 @@ class PylonClient(AioThing):
LibgenDoiSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
])
sources = filter(lambda x: x.is_enabled, sources)
async for resp in self.download(sources=sources):
async for resp in self.download(sources=sources, error_log_func=error_log_func):
yield resp
async def by_md5(
self,
md5: str,
error_log_func: Callable = error_log,
) -> AsyncIterable[FileResponsePb]:
sources = filter(lambda x: x.is_enabled, [
LibraryLolSource(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
LibgenMd5Source(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
])
async for resp in self.download(sources=sources):
async for resp in self.download(sources=sources, error_log_func=error_log_func):
yield resp
async def download_source(self, source) -> AsyncIterable[FileResponsePb]:
try:
yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url)
async for prepared_request in source.resolve():
async for resp in source.execute_prepared_request(prepared_request=prepared_request):
yield resp
return
raise DownloadError(error='not_found', source=str(source))
except (
aiohttp.client_exceptions.ClientConnectionError,
aiohttp.client_exceptions.ClientPayloadError,
aiohttp.client_exceptions.ClientResponseError,
aiohttp.client_exceptions.TooManyRedirects,
asyncio.TimeoutError,
ProxyConnectionError,
ProxyTimeoutError,
ProxyError,
) as e:
raise DownloadError(nested_error=str(e), nested_error_cls=e.__class__.__name__)
async def download(self, sources: Iterable[BaseSource]) -> AsyncIterable[FileResponsePb]:
for source in sources:
async def download_source(self, source, error_log_func: Callable = error_log) -> AsyncIterable[FileResponsePb]:
yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url)
async for prepared_file_request in source.resolve(error_log_func=error_log_func):
try:
await source.start()
async for resp in self.download_source(source):
async for resp in source.execute_prepared_file_request(prepared_file_request=prepared_file_request):
yield resp
return
except DownloadError as e:
error_log(e)
error_log_func(e)
continue
raise DownloadError(error='not_found', source=str(source))
async def download(self, sources: Iterable[BaseSource], error_log_func: Callable = error_log) -> AsyncIterable[FileResponsePb]:
for source in sources:
try:
await source.start()
async for resp in self.download_source(source, error_log_func=error_log_func):
yield resp
return
except DownloadError as e:
error_log_func(e)
continue
finally:
await source.stop()

97
nexus/pylon/sources/base.py
View File

@ -1,19 +1,27 @@
import asyncio
import hashlib
import random
import socket
from contextlib import asynccontextmanager
from typing import (
AsyncIterable,
Callable,
Optional,
)
import aiohttp
import aiohttp.client_exceptions
from aiohttp.client_reqrep import ClientRequest
from aiohttp_socks import (
ProxyConnectionError,
ProxyConnector,
ProxyError,
)
from aiokit import AioThing
from izihawa_utils.importlib import class_fullname
from library.logging import error_log
from nexus.pylon.exceptions import (
BadResponseError,
DownloadError,
IncorrectMD5Error,
NotFoundError,
)
@ -30,19 +38,32 @@ from tenacity import (
DEFAULT_USER_AGENT = 'PylonBot/1.0 (Linux x86_64) PylonBot/1.0.0'
class KeepAliveClientRequest(ClientRequest):
async def send(self, conn):
sock = conn.protocol.transport.get_extra_info("socket")
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 60)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 2)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5)
return await super().send(conn)
class PreparedRequest:
def __init__(
self,
method: str,
url: str,
headers: dict = None,
params: dict = None,
cookies: dict = None,
headers: Optional[dict] = None,
params: Optional[dict] = None,
cookies: Optional[dict] = None,
ssl: bool = True,
timeout: Optional[float] = None
):
self.method = method
self.url = url
self.headers = {
'Connection': 'keep-alive',
'User-Agent': DEFAULT_USER_AGENT,
}
if headers:
@ -50,6 +71,7 @@ class PreparedRequest:
self.params = params
self.cookies = cookies
self.ssl = ssl
self.timeout = timeout
def __repr__(self):
return f'{self.method} {self.url} {self.headers} {self.params}'
@ -57,6 +79,34 @@ class PreparedRequest:
def __str__(self):
return repr(self)
@asynccontextmanager
async def execute_with(self, session):
async with session.request(
method=self.method,
url=self.url,
timeout=self.timeout,
headers=self.headers,
cookies=self.cookies,
params=self.params,
ssl=self.ssl,
) as resp:
try:
yield resp
except BadResponseError as e:
e.add('url', self.url)
raise e
except (
aiohttp.client_exceptions.ClientConnectionError,
aiohttp.client_exceptions.ClientPayloadError,
aiohttp.client_exceptions.ClientResponseError,
aiohttp.client_exceptions.TooManyRedirects,
asyncio.TimeoutError,
ProxyConnectionError,
ProxyTimeoutError,
ProxyError,
) as e:
raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e))
class BaseValidator:
def update(self, chunk: bytes):
@ -123,12 +173,12 @@ class BaseSource(AioThing):
return aiohttp.TCPConnector(verify_ssl=self.ssl)
def get_session(self):
return aiohttp.ClientSession(connector=self.get_proxy())
return aiohttp.ClientSession(request_class=KeepAliveClientRequest, connector=self.get_proxy())
def get_resolve_session(self):
return aiohttp.ClientSession(connector=self.get_resolve_proxy())
return aiohttp.ClientSession(request_class=KeepAliveClientRequest, connector=self.get_resolve_proxy())
def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
raise NotImplementedError("`resolve` for BaseSource is not implemented")
def get_validator(self):
@ -139,19 +189,11 @@ class BaseSource(AioThing):
stop=stop_after_attempt(3),
retry=retry_if_exception_type((ProxyError, aiohttp.client_exceptions.ClientPayloadError, ProxyTimeoutError)),
)
async def execute_prepared_request(self, prepared_request: PreparedRequest):
async def execute_prepared_file_request(self, prepared_file_request: PreparedRequest):
async with self.get_session() as session:
async with session.request(
method=prepared_request.method,
url=prepared_request.url,
timeout=self.timeout,
headers=prepared_request.headers,
cookies=prepared_request.cookies,
params=prepared_request.params,
ssl=prepared_request.ssl,
) as resp:
async with prepared_file_request.execute_with(session=session) as resp:
if resp.status == 404:
raise NotFoundError(url=prepared_request.url)
raise NotFoundError(url=prepared_file_request.url)
elif (
resp.status != 200
or (
@ -160,24 +202,17 @@ class BaseSource(AioThing):
)
):
raise BadResponseError(
request_headers=prepared_request.headers,
url=prepared_request.url,
request_headers=prepared_file_request.headers,
url=prepared_file_request.url,
status=resp.status,
headers=str(resp.headers),
)
file_validator = self.get_validator()
# Randomness is required due to annoying bug of when separators
# (\r\n) are splitted to different chunks
# https://github.com/aio-libs/aiohttp/issues/4677
yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_request.url)
async for content in resp.content.iter_chunked(1024 * 100 + random.randint(-1024, 1024)):
yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_file_request.url)
async for content, _ in resp.content.iter_chunks():
file_validator.update(content)
yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_request.url)
try:
file_validator.validate()
except BadResponseError as e:
e.add('url', prepared_request.url)
raise e
yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_file_request.url)
file_validator.validate()
class Md5Source(BaseSource):

20
nexus/pylon/sources/libgen_doi.py
View File

@ -1,5 +1,8 @@
import re
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.exceptions import RegexNotFoundError
@ -14,13 +17,14 @@ class LibgenDoiSource(DoiSource):
base_url = 'http://libgen.gs'
resolve_timeout = 10
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/scimag/ads.php?doi={self.doi}'
async with session.get(
url,
timeout=self.resolve_timeout
) as resp:
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout,
).execute_with(session=session) as resp:
downloaded_page_bytes = await resp.read()
downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace')
match = re.search(
@ -29,6 +33,6 @@ class LibgenDoiSource(DoiSource):
re.IGNORECASE,
)
if match:
yield PreparedRequest(method='get', url=match.group())
yield PreparedRequest(method='get', url=match.group(), timeout=self.timeout)
else:
error_log(RegexNotFoundError(url=url))
error_log_func(RegexNotFoundError(url=url))

18
nexus/pylon/sources/libgen_md5.py
View File

@ -1,5 +1,10 @@
import re
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from .base import (
Md5Source,
@ -12,10 +17,11 @@ class LibgenMd5Source(Md5Source):
resolve_timeout = 10
async def resolve_lg(self, session, url):
async with session.get(
url,
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout
) as resp:
).execute_with(session=session) as resp:
downloaded_page_fiction = await resp.text()
match = re.search(
'https?://.*/get\\.php\\?md5=.*&key=[A-Za-z0-9]+',
@ -23,9 +29,9 @@ class LibgenMd5Source(Md5Source):
re.IGNORECASE,
)
if match:
return PreparedRequest(method='get', url=match.group())
return PreparedRequest(method='get', url=match.group(), timeout=self.timeout)
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/ads.php?md5={self.md5}'
result = await self.resolve_lg(session, url)

26
nexus/pylon/sources/libgen_new.py
View File

@ -1,5 +1,8 @@
import re
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.exceptions import RegexNotFoundError
@ -11,13 +14,14 @@ from .base import (
class LibgenNewSource(Md5Source):
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/main/{self.md5.upper()}'
async with session.get(
url,
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout
) as resp:
).execute_with(session) as resp:
downloaded_page = await resp.text()
match_ipfs = re.search(
'https://ipfs.io/ipfs/[A-Za-z0-9]+',
@ -25,28 +29,28 @@ class LibgenNewSource(Md5Source):
re.IGNORECASE,
)
if match_ipfs:
yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl)
yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl, timeout=self.timeout)
match_cf = re.search(
'https://cloudflare-ipfs.com/ipfs/[A-Za-z0-9]+',
downloaded_page,
re.IGNORECASE,
)
if match_cf:
yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl)
yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl, timeout=self.timeout)
match_infura = re.search(
'https://ipfs.infura.io/ipfs/[A-Za-z0-9]+',
downloaded_page,
re.IGNORECASE,
)
if match_infura:
yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl)
yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl, timeout=self.timeout)
if not match_cf or not match_infura or not match_ipfs:
error_log(RegexNotFoundError(url=url))
error_log_func(RegexNotFoundError(url=url))
class LibraryLolSource(LibgenNewSource):
base_url = 'http://library.lol'
resolve_timeout = 10
resolve_timeout = 20
ssl = False
timeout = 30
timeout = 120

22
nexus/pylon/sources/scihub.py
View File

@ -1,5 +1,8 @@
import re
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.exceptions import RegexNotFoundError
@ -19,16 +22,17 @@ class SciHubSource(DoiSource):
base_url = None
ssl = False
async def resolve(self, timeout=None) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/{self.doi}'
async with session.get(
url,
timeout=timeout or self.timeout
) as resp:
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout
).execute_with(session=session) as resp:
# Sometimes sci-hub returns file
if resp.headers.get('Content-Type') == 'application/pdf':
yield PreparedRequest(method='get', url=url)
yield PreparedRequest(method='get', url=url, timeout=self.timeout)
downloaded_page_bytes = await resp.read()
downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace')
match = re.search('(?:https?:)?//.*\\?download=true', downloaded_page, re.IGNORECASE)
@ -36,9 +40,9 @@ class SciHubSource(DoiSource):
url = match.group()
if url.startswith('//'):
url = 'http:' + url
yield PreparedRequest(method='get', url=url)
yield PreparedRequest(method='get', url=url, timeout=self.timeout)
else:
error_log(RegexNotFoundError(url=url))
error_log_func(RegexNotFoundError(url=url))
class SciHubDoSource(SciHubSource):

19
nexus/pylon/sources/specific/biorxiv.py
View File

@ -1,5 +1,9 @@
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.sources.base import (
DoiSource,
PreparedRequest,
@ -9,11 +13,12 @@ from nexus.pylon.sources.base import (
class BiorxivSource(DoiSource):
base_url = 'https://dx.doi.org'
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/{self.doi}'
async with session.get(
url,
timeout=self.resolve_timeout
) as resp:
yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf')
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout,
).execute_with(session=session) as resp:
yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf', timeout=self.timeout)

24
nexus/pylon/sources/specific/lancet.py
View File

@ -1,5 +1,9 @@
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.sources.base import (
DoiSource,
PreparedRequest,
@ -11,14 +15,20 @@ class LancetSource(DoiSource):
resolve_timeout = 10
use_proxy = False
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
splitted_doi = self.doi.split("/", maxsplit=1)
if len(splitted_doi) < 2:
return
url = f'{self.base_url}/action/showPdf?pii={splitted_doi[1].upper()}'
async with session.get(
url,
timeout=self.resolve_timeout
) as resp:
yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout,
).execute_with(session=session) as resp:
yield PreparedRequest(
method='get',
cookies=resp.cookies,
url=str(resp.url),
timeout=self.resolve_timeout,
)

17
nexus/pylon/sources/specific/nejm.py
View File

@ -1,5 +1,9 @@
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.sources.base import (
DoiSource,
PreparedRequest,
@ -11,11 +15,12 @@ class NejmSource(DoiSource):
resolve_timeout = 10
use_proxy = False
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/doi/pdf/{self.doi}'
async with session.get(
url,
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout,
) as resp:
yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))
).execute_with(session=session) as resp:
yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url), timeout=self.timeout)

20
nexus/pylon/sources/specific/research_square.py
View File

@ -1,8 +1,11 @@
import re
from typing import AsyncIterable
from typing import (
AsyncIterable,
Callable,
)
from library.logging import error_log
from nexus.pylon.exceptions import RegexNotFoundError
from nexus.pylon.sources.base import (
DoiSource,
PreparedRequest,
@ -12,13 +15,14 @@ from nexus.pylon.sources.base import (
class ResearchSquareSource(DoiSource):
base_url = 'https://dx.doi.org'
async def resolve(self) -> AsyncIterable[PreparedRequest]:
async def resolve(self, error_log_func: Callable = error_log) -> AsyncIterable[PreparedRequest]:
async with self.get_resolve_session() as session:
url = f'{self.base_url}/{self.doi}'
async with session.get(
url,
timeout=self.resolve_timeout
) as resp:
async with PreparedRequest(
method='get',
url=url,
timeout=self.resolve_timeout,
).execute_with(session=session) as resp:
download_page = await resp.text()
match = re.search(
r'\"(https://www\.researchsquare\.com/article/[^\"]+\.pdf)\"',
@ -27,4 +31,4 @@ class ResearchSquareSource(DoiSource):
)
if not match:
raise RegexNotFoundError(url=url)
yield PreparedRequest(method='get', url=match.group(1))
yield PreparedRequest(method='get', url=match.group(1), timeout=self.timeout)

2
nexus/views/telegram/progress_bar.py
View File

@ -84,7 +84,7 @@ class ProgressBar:
try:
if not self.message:
self.message = await self.telegram_client.send_message(
self.request_context.chat.id,
self.request_context.chat.chat_id,
text,
buttons=[close_button()],
)

7
nexus/views/telegram/scitech.py
View File

@ -6,7 +6,10 @@ from typing import (
)
from urllib.parse import quote
from izihawa_utils.common import filter_none
from izihawa_utils.common import (
filter_none,
is_essential,
)
from nexus.models.proto.scitech_pb2 import Scitech as ScitechPb
from nexus.nlptools.utils import (
cast_string_to_single_string,
@ -48,7 +51,7 @@ class ScitechView(BaseView, AuthorMixin, DoiMixin, FileMixin, IssuedAtMixin):
locator = self.get_formatted_locator()
caption = '\n'.join(filter_none([head, doi, locator]))
caption = '\n'.join(filter_none([head, doi, locator], predicate=is_essential))
if limit and len(caption) > limit:
shorten_title = title[:limit]
shorten_title = shorten_title[:max(32, shorten_title.rfind(' '))]