MarcoBuster/hyperboria
1
0
Fork 0
mirror of https://github.com/nexus-stc/hyperboria synced 2024-11-27 13:36:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15 from the-superpirate/master

Browse Source 
Synchronising downstream
This commit is contained in:
the-superpirate 2021-04-09 13:43:06 +03:00 committed by GitHub
commit 411939dcfc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
534 changed files with 14212 additions and 3396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
BUILD.bazel
View File

@ -14,14 +14,14 @@ config_setting(
platform(
name = "linux_x86",
constraint_values = [
"@io_bazel_rules_rust//rust/platform:linux",
"@rules_rust//rust/platform:linux",
"@bazel_tools//platforms:linux",
"@bazel_tools//platforms:x86_64",
],
)
load("@io_bazel_rules_rust//proto:toolchain.bzl", "rust_proto_toolchain")
load("@rules_rust//proto:toolchain.bzl", "rust_proto_toolchain")
rust_proto_toolchain(
name = "proto-toolchain-impl",
@ -33,5 +33,5 @@ rust_proto_toolchain(
toolchain(
name = "proto-toolchain",
toolchain = ":proto-toolchain-impl",
toolchain_type = "@io_bazel_rules_rust//proto:toolchain",
toolchain_type = "@rules_rust//proto:toolchain",
)

103
WORKSPACE
View File

@ -7,80 +7,55 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "bazel_skylib",
sha256 = "11b0e65ec07113b2ea81be554c7471bb80fc5766aba6239c91d071602c46d50f",
strip_prefix = "bazel-skylib-dc080e95161964a1ff841bfd0b871a1123c027a8",
sha256 = "ebdf850bfef28d923a2cc67ddca86355a449b5e4f38b0a70e584dc24e5984aa6",
strip_prefix = "bazel-skylib-f80bc733d4b9f83d427ce3442be2e07427b2cc8d",
urls = [
"https://github.com/bazelbuild/bazel-skylib/archive/dc080e95161964a1ff841bfd0b871a1123c027a8.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/archive/f80bc733d4b9f83d427ce3442be2e07427b2cc8d.tar.gz",
],
)
http_archive(
name = "build_bazel_rules_nodejs",
sha256 = "6142e9586162b179fdd570a55e50d1332e7d9c030efd853453438d607569721d",
urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.0.0/rules_nodejs-3.0.0.tar.gz"],
)
http_archive(
name = "com_github_grpc_grpc",
sha256 = "f046d4cb4d60d4f2a2087e9d46c7ec0c523cd54ebf68eda6272de4ce65e20ac7",
strip_prefix = "grpc-ae7f520358d7145a7484db693376fdebbd72662d",
sha256 = "dd7ea7efda7655c218ca707f55c3e1b9c68055a70c31a98f264b3445bc8f4cb1",
urls = [
"https://github.com/grpc/grpc/archive/ae7f520358d7145a7484db693376fdebbd72662d.tar.gz",
],
)
http_archive(
name = "com_google_protobuf",
sha256 = "d0f5f605d0d656007ce6c8b5a82df3037e1d8fe8b121ed42e536f569dec16113",
strip_prefix = "protobuf-3.14.0",
urls = [
"https://github.com/protocolbuffers/protobuf/archive/v3.14.0.tar.gz",
],
)
http_archive(
name = "io_bazel_rules_docker",
sha256 = "df3ef4a4b53b0145c9751c1e2a840f900e322e7798612a46257abe285d046dc5",
strip_prefix = "rules_docker-7da0de3d094aae5601c45ae0855b64fb2771cd72",
urls = [
"https://github.com/bazelbuild/rules_docker/archive/7da0de3d094aae5601c45ae0855b64fb2771cd72.zip",
"https://github.com/bazelbuild/rules_nodejs/releases/download/3.2.3/rules_nodejs-3.2.3.tar.gz",
],
)
http_archive(
name = "io_bazel_rules_k8s",
sha256 = "95addfd2b7b07b5a4e75663d15aa57dc271f7b831ec404109322288e1b6bf126",
strip_prefix = "rules_k8s-9f9886c7252d66bb2e2206842b149a6ceebe6fe5",
sha256 = "c1c5a692ec994e99e9e7e77ae693086074d6dedfe72e6930efbcc66d30264032",
strip_prefix = "rules_k8s-f1c6399cdd691b7aca90073398e8f690ec8992c6",
urls = [
"https://github.com/bazelbuild/rules_k8s/archive/9f9886c7252d66bb2e2206842b149a6ceebe6fe5.zip",
"https://github.com/bazelbuild/rules_k8s/archive/f1c6399cdd691b7aca90073398e8f690ec8992c6.tar.gz",
],
)
http_archive(
name = "io_bazel_rules_rust",
sha256 = "50a772198877e21a61823fa292d28539f8bc99d72463e55b5b09942394ec370e",
strip_prefix = "rules_rust-9a8ef691b8e8f682d767189c38339cbee16d0a16",
name = "rules_rust",
sha256 = "d10dd5581f66ee169071ee06d52c52c8c7ca7467ac6266e301c0820d289b0f0b",
strip_prefix = "rules_rust-336e1934b07211fb8736c19749919ef94df4df68",
urls = [
# Master branch as of 2020-10-16
"https://github.com/bazelbuild/rules_rust/archive/9a8ef691b8e8f682d767189c38339cbee16d0a16.tar.gz",
"https://github.com/bazelbuild/rules_rust/archive/336e1934b07211fb8736c19749919ef94df4df68.tar.gz",
],
)
http_archive(
name = "rules_jvm_external",
sha256 = "d85951a92c0908c80bd8551002d66cb23c3434409c814179c0ff026b53544dab",
sha256 = "2a547d8d5e99703de8de54b6188ff0ed470b3bfc88e346972d1c8865e2688391",
strip_prefix = "rules_jvm_external-3.3",
urls = [
"https://github.com/bazelbuild/rules_jvm_external/archive/3.3.zip",
"https://github.com/bazelbuild/rules_jvm_external/archive/3.3.tar.gz",
],
)
http_archive(
name = "rules_pkg",
sha256 = "0a33148c4957e666a29443f75b2c0db1fe3e0baf7256742fc47a35731f7a1d2e",
sha256 = "b9a5bdfe4f8ce0dedf9387eadd9f4844c383118b3f4cc27b586626b7998141c3",
strip_prefix = "rules_pkg-4b0b9f4679484f107f750a60190ff5ec6b164a5f/pkg",
urls = [
"https://github.com/bazelbuild/rules_pkg/archive/4b0b9f4679484f107f750a60190ff5ec6b164a5f.zip",
"https://github.com/bazelbuild/rules_pkg/archive/4b0b9f4679484f107f750a60190ff5ec6b164a5f.tar.gz",
],
)
@ -93,6 +68,36 @@ http_archive(
],
)
_configure_python_based_on_os = """
if [[ "$OSTYPE" == "darwin"* ]]; then
./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
else
./configure --prefix=$(pwd)/bazel_install
fi
"""
http_archive(
name = "python_interpreter",
build_file_content = """
exports_files(["python_bin"])
filegroup(
name = "files",
srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
visibility = ["//visibility:public"],
)
""",
patch_cmds = [
"mkdir $(pwd)/bazel_install",
_configure_python_based_on_os,
"make",
"make install",
"ln -s bazel_install/bin/python3 python_bin",
],
sha256 = "4b0e6644a76f8df864ae24ac500a51bbf68bd098f6a173e27d3b61cdca9aa134",
strip_prefix = "Python-3.9.4",
urls = ["https://www.python.org/ftp/python/3.9.4/Python-3.9.4.tar.xz"],
)
http_archive(
name = "rules_python",
sha256 = "b228318a786d99b665bc83bd6cdb81512cae5f8eb15e8cd19f9956604b8939f5",
@ -104,10 +109,9 @@ http_archive(
http_archive(
name = "subpar",
sha256 = "e6e4332bf9af36c4165ad6cc7b2c76288e9f156eba35dc95b739e58c46f30a50",
strip_prefix = "subpar-9fae6b63cfeace2e0fb93c9c1ebdc28d3991b16f",
urls = [
"https://github.com/google/subpar/archive/9fae6b63cfeace2e0fb93c9c1ebdc28d3991b16f.zip",
"https://github.com/google/subpar/archive/9fae6b63cfeace2e0fb93c9c1ebdc28d3991b16f.tar.gz",
],
)
@ -129,9 +133,9 @@ maven_fetch_remote_artifacts()
# Rust
load("@io_bazel_rules_rust//rust:repositories.bzl", "rust_repository_set")
load("@rules_rust//rust:repositories.bzl", "rust_repository_set")
rust_version = "1.49.0"
rust_version = "1.51.0"
rustfmt_version = "1.4.20"
@ -153,10 +157,6 @@ rust_repository_set(
version = rust_version,
)
load("@io_bazel_rules_rust//:workspace.bzl", "bazel_version")
bazel_version(name = "bazel_version")
load("//rules/rust:crates.bzl", "raze_fetch_remote_crates")
raze_fetch_remote_crates()
@ -219,12 +219,13 @@ py3_image_repos()
rust_image_repos()
# Python
register_toolchains("//rules/python:py_toolchain")
register_toolchains("//rules/python:py_3_toolchain")
load("@rules_python//python:pip.bzl", "pip_install")
pip_install(
name = "pip_modules",
python_interpreter_target = "@python_interpreter//:python_bin",
requirements = "//rules/python:requirements.txt",
)

15
idm/api2/aioclient/BUILD.bazel Normal file
View File

@ -0,0 +1,15 @@
load("@rules_python//python:defs.bzl", "py_library")
load("@pip_modules//:requirements.bzl", "requirement")
py_library(
name = "aioclient",
srcs = glob(["**/*.py"]),
visibility = ["//visibility:public"],
deps = [
requirement("cachetools"),
requirement("lru-dict"),
requirement("tenacity"),
"//idm/api2/proto:idm_grpc_py",
"//idm/api2/proto:idm_proto_py",
],
)

3
idm/api2/aioclient/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .aioclient import IdmApi2GrpcClient
__all__ = ['IdmApi2GrpcClient']

125
idm/api2/aioclient/aioclient.py Normal file
View File

@ -0,0 +1,125 @@
from aiokit import AioThing
from grpc import StatusCode
from grpc.experimental.aio import (
AioRpcError,
insecure_channel,
)
from idm.api2.proto.chats_service_pb2 import (
CreateChatRequest,
GetChatRequest,
ListChatsRequest,
UpdateChatRequest,
)
from idm.api2.proto.chats_service_pb2_grpc import ChatsStub
from lru import LRU
from tenacity import (
retry,
retry_if_exception,
stop_after_attempt,
wait_fixed,
)
class IdmApi2GrpcClient(AioThing):
def __init__(
self,
base_url,
):
super().__init__()
self.channel = insecure_channel(base_url, [
('grpc.dns_min_time_between_resolutions_ms', 1000),
('grpc.initial_reconnect_backoff_ms', 1000),
('grpc.lb_policy_name', 'round_robin'),
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.chats_stub = ChatsStub(self.channel)
self.cache = LRU(4096)
async def start(self):
await self.channel.channel_ready()
async def stop(self):
await self.channel.close()
async def create_chat(
self,
chat_id,
username,
language,
request_id: str = None,
):
response = await self.chats_stub.create_chat(
CreateChatRequest(
chat_id=chat_id,
username=username,
language=language,
),
metadata=(
('request-id', request_id),
),
)
return response
@retry(
retry=retry_if_exception(
lambda e: isinstance(e, AioRpcError) and e.code() == StatusCode.UNAVAILABLE
),
reraise=True,
stop=stop_after_attempt(10),
wait=wait_fixed(5),
)
async def get_chat(
self,
chat_id,
request_id: str = None,
):
response = await self.chats_stub.get_chat(
GetChatRequest(chat_id=chat_id),
metadata=(
('request-id', request_id),
),
)
return response
async def list_chats(
self,
request_id: str = None,
banned_at_moment=None,
):
response = await self.chats_stub.list_chats(
ListChatsRequest(banned_at_moment=banned_at_moment),
metadata=(
('request-id', request_id),
),
)
return response
async def update_chat(
self,
chat_id,
request_id: str = None,
language=None,
is_system_messaging_enabled=None,
is_discovery_enabled=None,
ban_until=None,
ban_message=None,
is_admin=None,
last_location=None,
):
response = await self.chats_stub.update_chat(
UpdateChatRequest(
chat_id=chat_id,
language=language,
is_system_messaging_enabled=is_system_messaging_enabled,
is_discovery_enabled=is_discovery_enabled,
ban_until=ban_until,
ban_message=ban_message,
is_admin=is_admin,
last_location=last_location,
),
metadata=(
('request-id', request_id),
),
)
return response

26
idm/api2/proto/BUILD.bazel Normal file
View File

@ -0,0 +1,26 @@
load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
package(default_visibility = ["//visibility:public"])
proto_library(
name = "idm_proto",
srcs = [
"chats_service.proto",
"location.proto",
],
deps = [
"@com_google_protobuf//:wrappers_proto",
],
)
py_proto_library(
name = "idm_proto_py",
deps = [":idm_proto"],
)
py_grpc_library(
name = "idm_grpc_py",
srcs = [":idm_proto"],
deps = [":idm_proto_py"],
)

70
idm/api2/proto/chats_service.proto Normal file
View File

@ -0,0 +1,70 @@
syntax = "proto3";
package idm.api2.proto;
import "idm/api2/proto/location.proto";
message ChatData {
int64 id = 1;
string username = 2;
string language = 3;
bool is_system_messaging_enabled = 4;
bool is_discovery_enabled = 5;
int32 ban_until = 6;
string ban_message = 7;
bool is_admin = 8;
string tzinfo = 9;
bool is_subscribed = 10;
int64 created_at = 11;
}
message ChatsData {
repeated ChatData chats = 1;
}
message CreateChatRequest {
int64 chat_id = 1;
string username = 2;
string language = 3;
}
message GetChatRequest {
int64 chat_id = 1;
}
message ListChatsRequest {
oneof banned_at_moment_oneof {
int32 banned_at_moment = 1;
}
}
message UpdateChatRequest {
int64 chat_id = 1;
oneof language_oneof {
string language = 2;
}
oneof is_system_messaging_enabled_oneof {
bool is_system_messaging_enabled = 3;
}
oneof is_discovery_enabled_oneof {
bool is_discovery_enabled = 4;
}
oneof ban_until_oneof {
int32 ban_until = 5;
}
oneof ban_message_oneof {
string ban_message = 6;
}
oneof is_admin_oneof {
bool is_admin = 7;
}
oneof last_location_oneof {
idm.api2.proto.Location last_location = 8;
}
}
service Chats {
rpc create_chat(CreateChatRequest) returns (ChatData) {};
rpc get_chat(GetChatRequest) returns (ChatData) {};
rpc list_chats(ListChatsRequest) returns (ChatsData) {};
rpc update_chat(UpdateChatRequest) returns (ChatData) {};
}

528
idm/api2/proto/chats_service_pb2.py Executable file
View File

@ -0,0 +1,528 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: idm/api2/proto/chats_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from idm.api2.proto import \
location_pb2 as idm_dot_api2_dot_proto_dot_location__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='idm/api2/proto/chats_service.proto',
package='idm.api2.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\"idm/api2/proto/chats_service.proto\x12\x0eidm.api2.proto\x1a\x1didm/api2/proto/location.proto\"\xf2\x01\n\x08\x43hatData\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\x12#\n\x1bis_system_messaging_enabled\x18\x04 \x01(\x08\x12\x1c\n\x14is_discovery_enabled\x18\x05 \x01(\x08\x12\x11\n\tban_until\x18\x06 \x01(\x05\x12\x13\n\x0b\x62\x61n_message\x18\x07 \x01(\t\x12\x10\n\x08is_admin\x18\x08 \x01(\x08\x12\x0e\n\x06tzinfo\x18\t \x01(\t\x12\x15\n\ris_subscribed\x18\n \x01(\x08\x12\x12\n\ncreated_at\x18\x0b \x01(\x03\"4\n\tChatsData\x12\'\n\x05\x63hats\x18\x01 \x03(\x0b\x32\x18.idm.api2.proto.ChatData\"H\n\x11\x43reateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x10\n\x08username\x18\x02 \x01(\t\x12\x10\n\x08language\x18\x03 \x01(\t\"!\n\x0eGetChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\"H\n\x10ListChatsRequest\x12\x1a\n\x10\x62\x61nned_at_moment\x18\x01 \x01(\x05H\x00\x42\x18\n\x16\x62\x61nned_at_moment_oneof\"\x98\x03\n\x11UpdateChatRequest\x12\x0f\n\x07\x63hat_id\x18\x01 \x01(\x03\x12\x12\n\x08language\x18\x02 \x01(\tH\x00\x12%\n\x1bis_system_messaging_enabled\x18\x03 \x01(\x08H\x01\x12\x1e\n\x14is_discovery_enabled\x18\x04 \x01(\x08H\x02\x12\x13\n\tban_until\x18\x05 \x01(\x05H\x03\x12\x15\n\x0b\x62\x61n_message\x18\x06 \x01(\tH\x04\x12\x12\n\x08is_admin\x18\x07 \x01(\x08H\x05\x12\x31\n\rlast_location\x18\x08 \x01(\x0b\x32\x18.idm.api2.proto.LocationH\x06\x42\x10\n\x0elanguage_oneofB#\n!is_system_messaging_enabled_oneofB\x1c\n\x1ais_discovery_enabled_oneofB\x11\n\x0f\x62\x61n_until_oneofB\x13\n\x11\x62\x61n_message_oneofB\x10\n\x0eis_admin_oneofB\x15\n\x13last_location_oneof2\xb8\x02\n\x05\x43hats\x12L\n\x0b\x63reate_chat\x12!.idm.api2.proto.CreateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12\x46\n\x08get_chat\x12\x1e.idm.api2.proto.GetChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x12K\n\nlist_chats\x12 .idm.api2.proto.ListChatsRequest\x1a\x19.idm.api2.proto.ChatsData\"\x00\x12L\n\x0bupdate_chat\x12!.idm.api2.proto.UpdateChatRequest\x1a\x18.idm.api2.proto.ChatData\"\x00\x62\x06proto3'
,
dependencies=[idm_dot_api2_dot_proto_dot_location__pb2.DESCRIPTOR,])
_CHATDATA = _descriptor.Descriptor(
name='ChatData',
full_name='idm.api2.proto.ChatData',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='idm.api2.proto.ChatData.id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='username', full_name='idm.api2.proto.ChatData.username', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='idm.api2.proto.ChatData.language', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_system_messaging_enabled', full_name='idm.api2.proto.ChatData.is_system_messaging_enabled', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_discovery_enabled', full_name='idm.api2.proto.ChatData.is_discovery_enabled', index=4,
number=5, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_until', full_name='idm.api2.proto.ChatData.ban_until', index=5,
number=6, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_message', full_name='idm.api2.proto.ChatData.ban_message', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_admin', full_name='idm.api2.proto.ChatData.is_admin', index=7,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='tzinfo', full_name='idm.api2.proto.ChatData.tzinfo', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_subscribed', full_name='idm.api2.proto.ChatData.is_subscribed', index=9,
number=10, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='created_at', full_name='idm.api2.proto.ChatData.created_at', index=10,
number=11, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=86,
serialized_end=328,
)
_CHATSDATA = _descriptor.Descriptor(
name='ChatsData',
full_name='idm.api2.proto.ChatsData',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chats', full_name='idm.api2.proto.ChatsData.chats', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=330,
serialized_end=382,
)
_CREATECHATREQUEST = _descriptor.Descriptor(
name='CreateChatRequest',
full_name='idm.api2.proto.CreateChatRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chat_id', full_name='idm.api2.proto.CreateChatRequest.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='username', full_name='idm.api2.proto.CreateChatRequest.username', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='idm.api2.proto.CreateChatRequest.language', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=384,
serialized_end=456,
)
_GETCHATREQUEST = _descriptor.Descriptor(
name='GetChatRequest',
full_name='idm.api2.proto.GetChatRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chat_id', full_name='idm.api2.proto.GetChatRequest.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=458,
serialized_end=491,
)
_LISTCHATSREQUEST = _descriptor.Descriptor(
name='ListChatsRequest',
full_name='idm.api2.proto.ListChatsRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='banned_at_moment', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='banned_at_moment_oneof', full_name='idm.api2.proto.ListChatsRequest.banned_at_moment_oneof',
index=0, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
],
serialized_start=493,
serialized_end=565,
)
_UPDATECHATREQUEST = _descriptor.Descriptor(
name='UpdateChatRequest',
full_name='idm.api2.proto.UpdateChatRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='chat_id', full_name='idm.api2.proto.UpdateChatRequest.chat_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='idm.api2.proto.UpdateChatRequest.language', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_system_messaging_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled', index=2,
number=3, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_discovery_enabled', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_until', full_name='idm.api2.proto.UpdateChatRequest.ban_until', index=4,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ban_message', full_name='idm.api2.proto.UpdateChatRequest.ban_message', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='is_admin', full_name='idm.api2.proto.UpdateChatRequest.is_admin', index=6,
number=7, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='last_location', full_name='idm.api2.proto.UpdateChatRequest.last_location', index=7,
number=8, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='language_oneof', full_name='idm.api2.proto.UpdateChatRequest.language_oneof',
index=0, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='is_system_messaging_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_system_messaging_enabled_oneof',
index=1, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='is_discovery_enabled_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_discovery_enabled_oneof',
index=2, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='ban_until_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_until_oneof',
index=3, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='ban_message_oneof', full_name='idm.api2.proto.UpdateChatRequest.ban_message_oneof',
index=4, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='is_admin_oneof', full_name='idm.api2.proto.UpdateChatRequest.is_admin_oneof',
index=5, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
_descriptor.OneofDescriptor(
name='last_location_oneof', full_name='idm.api2.proto.UpdateChatRequest.last_location_oneof',
index=6, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
],
serialized_start=568,
serialized_end=976,
)
_CHATSDATA.fields_by_name['chats'].message_type = _CHATDATA
_LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof'].fields.append(
_LISTCHATSREQUEST.fields_by_name['banned_at_moment'])
_LISTCHATSREQUEST.fields_by_name['banned_at_moment'].containing_oneof = _LISTCHATSREQUEST.oneofs_by_name['banned_at_moment_oneof']
_UPDATECHATREQUEST.fields_by_name['last_location'].message_type = idm_dot_api2_dot_proto_dot_location__pb2._LOCATION
_UPDATECHATREQUEST.oneofs_by_name['language_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['language'])
_UPDATECHATREQUEST.fields_by_name['language'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['language_oneof']
_UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'])
_UPDATECHATREQUEST.fields_by_name['is_system_messaging_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_system_messaging_enabled_oneof']
_UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'])
_UPDATECHATREQUEST.fields_by_name['is_discovery_enabled'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_discovery_enabled_oneof']
_UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['ban_until'])
_UPDATECHATREQUEST.fields_by_name['ban_until'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_until_oneof']
_UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['ban_message'])
_UPDATECHATREQUEST.fields_by_name['ban_message'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['ban_message_oneof']
_UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['is_admin'])
_UPDATECHATREQUEST.fields_by_name['is_admin'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['is_admin_oneof']
_UPDATECHATREQUEST.oneofs_by_name['last_location_oneof'].fields.append(
_UPDATECHATREQUEST.fields_by_name['last_location'])
_UPDATECHATREQUEST.fields_by_name['last_location'].containing_oneof = _UPDATECHATREQUEST.oneofs_by_name['last_location_oneof']
DESCRIPTOR.message_types_by_name['ChatData'] = _CHATDATA
DESCRIPTOR.message_types_by_name['ChatsData'] = _CHATSDATA
DESCRIPTOR.message_types_by_name['CreateChatRequest'] = _CREATECHATREQUEST
DESCRIPTOR.message_types_by_name['GetChatRequest'] = _GETCHATREQUEST
DESCRIPTOR.message_types_by_name['ListChatsRequest'] = _LISTCHATSREQUEST
DESCRIPTOR.message_types_by_name['UpdateChatRequest'] = _UPDATECHATREQUEST
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
ChatData = _reflection.GeneratedProtocolMessageType('ChatData', (_message.Message,), {
'DESCRIPTOR' : _CHATDATA,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.ChatData)
})
_sym_db.RegisterMessage(ChatData)
ChatsData = _reflection.GeneratedProtocolMessageType('ChatsData', (_message.Message,), {
'DESCRIPTOR' : _CHATSDATA,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.ChatsData)
})
_sym_db.RegisterMessage(ChatsData)
CreateChatRequest = _reflection.GeneratedProtocolMessageType('CreateChatRequest', (_message.Message,), {
'DESCRIPTOR' : _CREATECHATREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.CreateChatRequest)
})
_sym_db.RegisterMessage(CreateChatRequest)
GetChatRequest = _reflection.GeneratedProtocolMessageType('GetChatRequest', (_message.Message,), {
'DESCRIPTOR' : _GETCHATREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.GetChatRequest)
})
_sym_db.RegisterMessage(GetChatRequest)
ListChatsRequest = _reflection.GeneratedProtocolMessageType('ListChatsRequest', (_message.Message,), {
'DESCRIPTOR' : _LISTCHATSREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.ListChatsRequest)
})
_sym_db.RegisterMessage(ListChatsRequest)
UpdateChatRequest = _reflection.GeneratedProtocolMessageType('UpdateChatRequest', (_message.Message,), {
'DESCRIPTOR' : _UPDATECHATREQUEST,
'__module__' : 'idm.api2.proto.chats_service_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.UpdateChatRequest)
})
_sym_db.RegisterMessage(UpdateChatRequest)
_CHATS = _descriptor.ServiceDescriptor(
name='Chats',
full_name='idm.api2.proto.Chats',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=979,
serialized_end=1291,
methods=[
_descriptor.MethodDescriptor(
name='create_chat',
full_name='idm.api2.proto.Chats.create_chat',
index=0,
containing_service=None,
input_type=_CREATECHATREQUEST,
output_type=_CHATDATA,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='get_chat',
full_name='idm.api2.proto.Chats.get_chat',
index=1,
containing_service=None,
input_type=_GETCHATREQUEST,
output_type=_CHATDATA,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='list_chats',
full_name='idm.api2.proto.Chats.list_chats',
index=2,
containing_service=None,
input_type=_LISTCHATSREQUEST,
output_type=_CHATSDATA,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='update_chat',
full_name='idm.api2.proto.Chats.update_chat',
index=3,
containing_service=None,
input_type=_UPDATECHATREQUEST,
output_type=_CHATDATA,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_CHATS)
DESCRIPTOR.services_by_name['Chats'] = _CHATS
# @@protoc_insertion_point(module_scope)

165
idm/api2/proto/chats_service_pb2_grpc.py Executable file
View File

@ -0,0 +1,165 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from idm.api2.proto import \
chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
class ChatsStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.create_chat = channel.unary_unary(
'/idm.api2.proto.Chats/create_chat',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
)
self.get_chat = channel.unary_unary(
'/idm.api2.proto.Chats/get_chat',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
)
self.list_chats = channel.unary_unary(
'/idm.api2.proto.Chats/list_chats',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
)
self.update_chat = channel.unary_unary(
'/idm.api2.proto.Chats/update_chat',
request_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
response_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
)
class ChatsServicer(object):
"""Missing associated documentation comment in .proto file."""
def create_chat(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def get_chat(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def list_chats(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def update_chat(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_ChatsServicer_to_server(servicer, server):
rpc_method_handlers = {
'create_chat': grpc.unary_unary_rpc_method_handler(
servicer.create_chat,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
),
'get_chat': grpc.unary_unary_rpc_method_handler(
servicer.get_chat,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
),
'list_chats': grpc.unary_unary_rpc_method_handler(
servicer.list_chats,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.SerializeToString,
),
'update_chat': grpc.unary_unary_rpc_method_handler(
servicer.update_chat,
request_deserializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.FromString,
response_serializer=idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'idm.api2.proto.Chats', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Chats(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def create_chat(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/create_chat',
idm_dot_api2_dot_proto_dot_chats__service__pb2.CreateChatRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def get_chat(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/get_chat',
idm_dot_api2_dot_proto_dot_chats__service__pb2.GetChatRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def list_chats(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/list_chats',
idm_dot_api2_dot_proto_dot_chats__service__pb2.ListChatsRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatsData.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def update_chat(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/idm.api2.proto.Chats/update_chat',
idm_dot_api2_dot_proto_dot_chats__service__pb2.UpdateChatRequest.SerializeToString,
idm_dot_api2_dot_proto_dot_chats__service__pb2.ChatData.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

7
idm/api2/proto/location.proto Normal file
View File

@ -0,0 +1,7 @@
syntax = "proto3";
package idm.api2.proto;
message Location {
float lat = 1;
float lon = 2;
}

78
idm/api2/proto/location_pb2.py Executable file
View File

@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: idm/api2/proto/location.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='idm/api2/proto/location.proto',
package='idm.api2.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\x1didm/api2/proto/location.proto\x12\x0eidm.api2.proto\"$\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x02\x12\x0b\n\x03lon\x18\x02 \x01(\x02\x62\x06proto3'
)
_LOCATION = _descriptor.Descriptor(
name='Location',
full_name='idm.api2.proto.Location',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='lat', full_name='idm.api2.proto.Location.lat', index=0,
number=1, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='lon', full_name='idm.api2.proto.Location.lon', index=1,
number=2, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=49,
serialized_end=85,
)
DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), {
'DESCRIPTOR' : _LOCATION,
'__module__' : 'idm.api2.proto.location_pb2'
# @@protoc_insertion_point(class_scope:idm.api2.proto.Location)
})
_sym_db.RegisterMessage(Location)
# @@protoc_insertion_point(module_scope)

4
idm/api2/proto/location_pb2_grpc.py Executable file
View File

@ -0,0 +1,4 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc

4
images/install.bzl
View File

@ -11,8 +11,8 @@ def images_install():
container_pull(
name = "ubuntu",
digest = "sha256:45ff0162921e61c004010a67db1bee7d039a677bed0cb294e61f2b47346d42b3",
registry = "index.docker.io",
repository = "library/ubuntu",
digest = "sha256:4e4bc990609ed865e07afc8427c30ffdddca5153fd4e82c20d8f0783a291e241",
tag = "20.04",
tag = "20.10",
)

44
images/production/BUILD.bazel
View File

@ -1,11 +1,11 @@
load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_push")
load("@io_bazel_rules_docker//container:container.bzl", "container_image")
load("@io_bazel_rules_docker//docker/package_managers:download_pkgs.bzl", "download_pkgs")
load("@io_bazel_rules_docker//docker/package_managers:install_pkgs.bzl", "install_pkgs")
package(default_visibility = ["//visibility:public"])
download_pkgs(
name = "download-base-python-image",
name = "download-base-production-image",
image_tar = "//images:base-image.tar",
packages = [
"bash",
@ -13,15 +13,39 @@ download_pkgs(
"libev4",
"libgomp1",
"libgoogle-perftools-dev",
"libprotobuf17",
"libprotobuf23",
"libssl1.1",
],
)
install_pkgs(
name = "install-base-production-image",
image_tar = "//images:base-image.tar",
installables_tar = ":download-base-production-image.tar",
output_image_name = "base-production-image",
)
container_image(
name = "base-production-image",
base = ":install-base-production-image",
env = {
"LANG": "C.UTF-8",
"LC_CTYPE": "en_US.UTF-8",
},
)
download_pkgs(
name = "download-base-python-image",
image_tar = ":base-production-image.tar",
packages = [
"python3.9",
"python3.9-distutils",
],
)
install_pkgs(
name = "install-base-python-image",
image_tar = "//images:base-image.tar",
image_tar = ":base-production-image.tar",
installables_tar = ":download-base-python-image.tar",
installation_cleanup_commands = "rm -rf /var/lib/apt/lists/*",
output_image_name = "installed-base-python-image",
@ -31,7 +55,6 @@ container_image(
name = "base-python-image",
base = ":install-base-python-image",
entrypoint = ["/usr/bin/python3.9"],
env = {"LANG": "C.UTF-8"},
symlinks = {
"/usr/bin/python": "/usr/bin/python3.9",
"/usr/bin/python3": "/usr/bin/python3.9",
@ -41,20 +64,15 @@ container_image(
download_pkgs(
name = "download-base-nodejs-image",
image_tar = "//images:base-image.tar",
image_tar = ":base-production-image.tar",
packages = [
"bash",
"ca-certificates",
"libgoogle-perftools-dev",
"libprotobuf17",
"libssl1.1",
"nodejs",
],
)
install_pkgs(
name = "install-base-nodejs-image",
image_tar = "//images:base-image.tar",
image_tar = ":base-production-image.tar",
installables_tar = ":download-base-nodejs-image.tar",
installation_cleanup_commands = "rm -rf /var/lib/apt/lists/*",
output_image_name = "installed-base-nodejs-image",
@ -64,7 +82,5 @@ container_image(
name = "base-nodejs-image",
base = ":install-base-nodejs-image",
entrypoint = ["/usr/bin/nodejs"],
env = {"LANG": "C.UTF-8"},
visibility = ["//visibility:public"],
)

3
library/aiopostgres/pool_holder.py
View File

@ -1,3 +1,4 @@
import aiopg
import psycopg2.extras
from aiokit import AioThing
from psycopg2 import OperationalError
@ -10,7 +11,7 @@ from tenacity import (
class AioPostgresPoolHolder(AioThing):
def __init__(self, fn, *args, **kwargs):
def __init__(self, fn=aiopg.create_pool, *args, **kwargs):
super().__init__()
self.fn = fn
self.args = args

1
library/configurator/BUILD.bazel
View File

@ -11,7 +11,6 @@ py_library(
visibility = ["//visibility:public"],
deps = [
requirement("jinja2"),
requirement("orjson"),
requirement("pyyaml"),
requirement("izihawa_utils"),
],

2
library/configurator/__init__.py
View File

@ -1,8 +1,8 @@
import json
import os
import os.path
from types import ModuleType
import orjson as json
import yaml
from izihawa_utils.common import smart_merge_dicts
from jinja2 import Template

1
nexus/actions/BUILD.bazel
View File

@ -18,7 +18,6 @@ py_library(
requirement("aiocrossref"),
requirement("aiolibgen"),
"//library/aiopostgres",
"//nexus/cognitron/schema",
"//nexus/models/proto:models_proto_py",
"//nexus/nlptools",
requirement("aiosumma"),

6
nexus/actions/__init__.py
View File

@ -9,6 +9,10 @@ from .update_document_scitech import (
CleanDocumentOperationUpdateDocumentScitechPbAction,
SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction,
)
from .update_document_sharience import (
SendDocumentOperationUpdateDocumentShariencePbToGoldenPostgresAction,
)
from .vote import SendDocumentOperationVotePbToGoldenPostgresAction
__all__ = [
'CleanDocumentOperationUpdateDocumentScimagPbAction',
@ -18,4 +22,6 @@ __all__ = [
'SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction',
'SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction',
'SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction',
'SendDocumentOperationUpdateDocumentShariencePbToGoldenPostgresAction',
'SendDocumentOperationVotePbToGoldenPostgresAction',
]

11
nexus/actions/update_document.py
View File

@ -1,5 +1,5 @@
from aiosumma import SummaHttpClient
from nexus.cognitron.schema import coders
from izihawa_utils.pb_to_json import MessageToDict
from nexus.models.proto.operation_pb2 import \
DocumentOperation as DocumentOperationPb
@ -19,8 +19,13 @@ class SendDocumentOperationUpdateDocumentPbToSummaAction(BaseAction):
original_id = getattr(document, 'original_id', None)
if not update_document_pb.reindex or original_id:
return document_operation_pb
document_tantivy = coders[schema].encode_document(document)
await self.summa_client.put_document(schema, document_tantivy)
casted_document = MessageToDict(document, preserving_proto_field_name=True)
# ToDo: Required to rework checking for extra fields in document
# ToDo: It is needed to go to actual schema and load real fields and then check against them
casted_document.pop('is_deleted', None)
casted_document.pop('meta_language', None)
casted_document.pop('type', None)
await self.summa_client.put_document(schema, casted_document)
if update_document_pb.commit:
await self.summa_client.commit(schema)
return document_operation_pb

86
nexus/actions/update_document_sharience.py Normal file
View File

@ -0,0 +1,86 @@
from typing import (
Optional,
Set,
)
import aiopg
from library.aiopostgres.pool_holder import AioPostgresPoolHolder
from nexus.models.proto.operation_pb2 import \
DocumentOperation as DocumentOperationPb
from nexus.models.proto.sharience_pb2 import Sharience as ShariencePb
from pypika import (
PostgreSQLQuery,
Table,
)
from pypika.terms import Array
from .base import BaseAction
class SendDocumentOperationUpdateDocumentShariencePbToGoldenPostgresAction(BaseAction):
sharience_table = Table('sharience')
db_multi_fields = {
'ipfs_multihashes',
}
db_single_fields = {
'id',
'parent_id',
'uploader_id',
'filesize',
'md5',
'telegram_file_id',
'updated_at',
}
db_fields = db_single_fields | db_multi_fields
def __init__(self, database):
super().__init__()
self.pool_holder = AioPostgresPoolHolder(
fn=aiopg.create_pool,
dsn=f'dbname={database["database"]} '
f'user={database["username"]} '
f'password={database["password"]} '
f'host={database["host"]}',
timeout=30,
pool_recycle=60,
maxsize=4,
)
self.waits.append(self.pool_holder)
def cast_field_value(self, field_name: str, field_value):
if field_name in self.db_multi_fields:
field_value = Array(*field_value)
return field_name, field_value
def is_field_set(self, sharience_pb: ShariencePb, field_name: str):
field_value = getattr(sharience_pb, field_name)
return field_value
def generate_insert_sql(self, sharience_pb: ShariencePb, fields: Optional[Set[str]] = None):
columns = []
inserts = []
fields = fields or self.db_fields
for field_name in fields:
if self.is_field_set(sharience_pb, field_name):
field_value = getattr(sharience_pb, field_name)
field_name, field_value = self.cast_field_value(field_name, field_value)
columns.append(field_name)
inserts.append(field_value)
query = PostgreSQLQuery.into(self.sharience_table).columns(*columns).insert(*inserts)
return query.returning(self.sharience_table.id).get_sql()
async def do(self, document_operation_pb: DocumentOperationPb) -> DocumentOperationPb:
update_document_pb = document_operation_pb.update_document
sharience_pb = update_document_pb.typed_document.sharience
fields = update_document_pb.fields or self.db_fields
sql = self.generate_insert_sql(
sharience_pb=sharience_pb,
fields=fields,
)
result = await self.pool_holder.execute(sql, fetch=True)
sharience_pb.id = result[0][0]
return document_operation_pb

51
nexus/actions/vote.py Normal file
View File

@ -0,0 +1,51 @@
import aiopg
from library.aiopostgres.pool_holder import AioPostgresPoolHolder
from nexus.models.proto.operation_pb2 import \
DocumentOperation as DocumentOperationPb
from pypika import (
PostgreSQLQuery,
Table,
)
from .base import BaseAction
class SendDocumentOperationVotePbToGoldenPostgresAction(BaseAction):
votes_table = Table('votes')
def __init__(self, database):
super().__init__()
self.pool_holder = AioPostgresPoolHolder(
fn=aiopg.create_pool,
dsn=f'dbname={database["database"]} '
f'user={database["username"]} '
f'password={database["password"]} '
f'host={database["host"]}',
timeout=30,
pool_recycle=60,
maxsize=2,
)
self.waits.append(self.pool_holder)
def generate_insert_sql(self, document_id: int, value: int, voter_id: int):
query = PostgreSQLQuery.into(self.votes_table).columns(
'document_id',
'value',
'voter_id',
).insert(
document_id,
value,
voter_id,
)
query = query.on_conflict('document_id', 'voter_id').do_update('value', value)
return query.get_sql()
async def do(self, document_operation_pb: DocumentOperationPb) -> DocumentOperationPb:
vote_pb = document_operation_pb.vote
sql = self.generate_insert_sql(
document_id=vote_pb.document_id,
value=vote_pb.value,
voter_id=vote_pb.voter_id,
)
await self.pool_holder.execute(sql)
return vote_pb

71
nexus/bot/BUILD.bazel Normal file
View File

@ -0,0 +1,71 @@
load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
load("@io_bazel_rules_docker//container:container.bzl", "container_push")
load("@pip_modules//:requirements.bzl", "requirement")
alias(
name = "binary",
actual = ":image.binary",
visibility = ["//visibility:public"],
)
py3_image(
name = "image",
srcs = glob(["**/*.py"]),
base = "//images/production:base-python-image",
data = [
"configs/base.yaml",
"configs/custom.yaml",
"configs/development.yaml",
"configs/logging.yaml",
"configs/metrics.yaml",
"configs/production.yaml",
"configs/promotions.yaml",
"configs/testing.yaml",
],
main = "main.py",
srcs_version = "PY3ONLY",
visibility = ["//visibility:public"],
deps = [
requirement("aiodns"),
requirement("aiohttp"),
requirement("aiohttp_socks"),
requirement("psycopg2-binary"),
requirement("pytimeparse"),
requirement("python_socks"),
requirement("tenacity"),
requirement("uvloop"),
"//idm/api2/aioclient",
"//library/aiobaseclient",
requirement("aiocrossref"),
requirement("aiokit"),
"//library/configurator",
"//library/logging",
"//library/metrics_server",
"//library/telegram",
"//nexus/hub/aioclient",
"//nexus/meta_api/aioclient",
"//nexus/models/proto:models_proto_py",
"//nexus/nlptools",
"//nexus/views/telegram",
requirement("izihawa_utils"),
],
)
container_push(
name = "push-testing",
format = "Docker",
image = ":image",
registry = "registry.example.com",
repository = "nexus-bot",
tag = "testing",
)
container_push(
name = "push-latest",
format = "Docker",
image = ":image",
registry = "registry.example.com",
repository = "nexus-bot",
tag = "latest",
)

8
nexus/bot/README.md Normal file
View File

@ -0,0 +1,8 @@
# Nexus Search: Telegram Bot
The bot requires three other essential parts:
- IDM API (auth)
- Nexus Hub API (managing files)
- Nexus Meta API (rescoring API for Summa Search server)
or their substitutions

3
nexus/bot/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from . import handlers
__all__ = ['handlers']

44
nexus/bot/application.py Normal file
View File

@ -0,0 +1,44 @@
from aiokit import AioRootThing
from idm.api2.aioclient import IdmApi2GrpcClient
from izihawa_utils.importlib import import_object
from library.telegram.base import BaseTelegramClient
from nexus.bot.promotioner import Promotioner
from nexus.bot.user_manager import UserManager
from nexus.hub.aioclient import HubGrpcClient
from nexus.meta_api.aioclient import MetaApiGrpcClient
class TelegramApplication(AioRootThing):
def __init__(self, config):
super().__init__()
self.config = config
self.telegram_client = BaseTelegramClient(
app_id=self.config['telegram']['app_id'],
app_hash=self.config['telegram']['app_hash'],
bot_token=self.config['telegram']['bot_token'],
database=self.config['telegram'].get('database'),
mtproxy=self.config['telegram'].get('mtproxy'),
)
self.hub_client = HubGrpcClient(base_url=self.config['hub']['url'])
self.idm_client = IdmApi2GrpcClient(base_url=self.config['idm']['url'])
self.meta_api_client = MetaApiGrpcClient(base_url=self.config['meta_api']['url'])
self.promotioner = Promotioner(promotions=self.config['promotions'])
self.user_manager = UserManager()
self._handlers = []
self.starts.extend([self.hub_client, self.idm_client, self.meta_api_client])
def set_handlers(self, telegram_client):
for handler in self.config['telegram']['handlers']:
import_object(handler)(self).register_for(telegram_client)
async def start(self):
self.set_handlers(self.telegram_client)
await self.telegram_client.start_and_wait()
await self.telegram_client.run_until_disconnected()
async def stop(self):
self.telegram_client.remove_event_handlers()
await self.telegram_client.stop()

16
nexus/bot/configs/__init__.py Normal file
View File

@ -0,0 +1,16 @@
from izihawa_utils import env
from library.configurator import Configurator
def get_config():
return Configurator([
'nexus/bot/configs/base.yaml',
'nexus/bot/configs/custom.yaml?',
'nexus/bot/configs/metrics.yaml?',
'nexus/bot/configs/%s.yaml?' % env.type,
'nexus/bot/configs/logging.yaml?',
'nexus/bot/configs/promotions.yaml?',
])
config = get_config()

101
nexus/bot/configs/base.yaml Normal file
View File

@ -0,0 +1,101 @@
---
application:
# Amazon Recipient Email in /donate message
amazon_gift_card_recipient: pirate@ship.space
# Amazon URL for buying card in /donate message
amazon_gift_card_url: https://www.amazon.com/dp/B07TMNGSN4
bot_version: 1.6.0
# Bitcoin Donation address in /donate message
btc_donate_address: 3QbF3zRQVjn3qMJBSbmLC1gb6VUc555xkw
# List of chat IDs that is allowed to bypass maintenance mode
bypass_maintenance: []
# Debugging mode
debug: true
# All users (except `bypass_maintenance` ones) will get UPGRADE_MAINTENANCE message in response
is_maintenance_mode: false
# Disable /settings, uploading new articles (can be used while vacuuming backend Postgres)
# and preventing creation of new users
is_read_only_mode: false
# Require subscription to `related_channel` before allowing to use the bot
is_subscription_required: false
# Libera Pay URL in /donate message
libera_pay_url:
maintenance_picture_url:
nexus_version: InterCom
# Default page size for SERP
page_size: 5
# Length of generated Request-ID used for tracking requests across all backends
request_id_length: 12
# Enabled schemas (passed to Nexus Meta API)
schemas:
- scimag
- scitech
# Length of generated Session-ID used in commands to clue user sessions
session_id_length: 8
too_difficult_picture_url:
upgrade_maintenance_picture_url:
# Configuring behaviour of the bot in some cases
views:
settings:
has_discovery_button: true
has_language_buttons: true
has_location_button: false
has_router: false
has_system_messaging_button: true
hub:
url:
idm:
url:
log_path: '/var/log/nexus-bot/{{ ENV_TYPE }}'
meta_api:
url:
telegram:
# Telegram App Hash from https://my.telegram.org/
app_hash: '{{ APP_HASH }}'
# Telegram App ID from https://my.telegram.org/
app_id: 00000
# External bot name shown in messages to users
bot_external_name: libgen_scihub_bot
# Internal bot name used in logging
bot_name: nexus-bot
bot_token: '{{ BOT_TOKEN }}'
# WARNING! Potentially buggy telethon option. Sometimes it goes mad and overload users with tons of messages
# Collect missed messages at startup time and answer to them
catch_up: false
# Telegram account for forwarding copyright infringements from /copyright command
copyright_infringement_account:
# Telethon database for keeping cache
database:
session_id: '/usr/lib/nexus-bot/{{ ENV_TYPE }}/session.db'
# Enabled handlers
handlers:
- nexus.bot.handlers.admin.AdminHandler
- nexus.bot.handlers.ban.BanHandler
- nexus.bot.handlers.ban.BanlistHandler
- nexus.bot.handlers.ban.UnbanHandler
- nexus.bot.handlers.contact.ContactHandler
- nexus.bot.handlers.copyright.CopyrightHandler
- nexus.bot.handlers.close.CloseHandler
- nexus.bot.handlers.donate.DonateHandler
- nexus.bot.handlers.download.DownloadHandler
- nexus.bot.handlers.emoji.EmojiHandler
- nexus.bot.handlers.help.HelpHandler
- nexus.bot.handlers.referencing_to.ReferencingToHandler
- nexus.bot.handlers.referencing_to.ReferencingToPagingHandler
- nexus.bot.handlers.roll.RollHandler
- nexus.bot.handlers.settings.SettingsButtonsHandler
- nexus.bot.handlers.settings.SettingsRouterHandler
- nexus.bot.handlers.settings.SettingsManualHandler
- nexus.bot.handlers.shortlink.ShortlinkHandler
- nexus.bot.handlers.submit.SubmitHandler
- nexus.bot.handlers.start.StartHandler
- nexus.bot.handlers.stop.StopHandler
- nexus.bot.handlers.view.ViewHandler
- nexus.bot.handlers.vote.VoteHandler
- nexus.bot.handlers.noop.NoopHandler
- nexus.bot.handlers.search.SearchHandler
- nexus.bot.handlers.search.SearchEditHandler
- nexus.bot.handlers.search.SearchPagingHandler
# Channel that will be shown in /help, /donate, /contact and in promotions
related_channel: '@nexus_search'

13
nexus/bot/configs/promotions.yaml Normal file
View File

@ -0,0 +1,13 @@
---
promotions:
- texts:
en: 🎁 Help us at /donate to accelerate knowledge unchaining
weight: 3.0
- texts:
en: ⤴️ Stay tuned with us at {related_channel}
es: ⤴️ Mantente en contacto con nosotros en {related_channel}
it: ⤴️ Resta aggiornato con noi su {related_channel}
pb: ⤴️ Fique ligado conosco em {related_channel}
ru: ⤴️ Оставайся на связи с нами на {related_channel}
weight: 1.0

9
nexus/bot/exceptions.py Normal file
View File

@ -0,0 +1,9 @@
from izihawa_utils.exceptions import BaseError
class UnknownFileFormatError(BaseError):
code = 'unknown_file_format_error'
class UnknownSchemaError(BaseError):
code = 'unknown_schema_error'

27
nexus/bot/handlers/__init__.py Normal file
View File

@ -0,0 +1,27 @@
from . import (
admin,
ban,
close,
contact,
copyright,
donate,
download,
emoji,
help,
legacy,
noop,
referencing_to,
roll,
search,
settings,
shortlink,
start,
stop,
submit,
view,
vote,
)
__all__ = ['admin', 'ban', 'contact', 'copyright', 'close', 'donate', 'download', 'emoji', 'help',
'legacy', 'noop', 'referencing_to', 'roll', 'search', 'settings',
'shortlink', 'start', 'stop', 'submit', 'view', 'vote']

22
nexus/bot/handlers/admin.py Normal file
View File

@ -0,0 +1,22 @@
from abc import ABC
from library.telegram.base import RequestContext
from nexus.bot.widgets.admin_widget import AdminWidget
from telethon import events
from .base import BaseHandler
class BaseAdminHandler(BaseHandler, ABC):
def _has_access(self, chat):
return chat.is_admin
class AdminHandler(BaseAdminHandler):
filter = events.NewMessage(incoming=True, pattern='^/admin$')
async def handler(self, event, request_context: RequestContext):
request_context.statbox(action='show', mode='admin')
admin_widget_view = AdminWidget(application=self.application, chat=request_context.chat)
text = await admin_widget_view.render()
await event.reply(text)

82
nexus/bot/handlers/ban.py Normal file
View File

@ -0,0 +1,82 @@
from datetime import (
datetime,
timedelta,
)
from aiobaseclient.exceptions import ClientError
from library.telegram.base import RequestContext
from nexus.bot.widgets.banlist_widget import BanlistWidget
from pytimeparse.timeparse import timeparse
from telethon import events
from .admin import BaseAdminHandler
class BanHandler(BaseAdminHandler):
filter = events.NewMessage(incoming=True, pattern='^/ban ([0-9]+) ([A-Za-z0-9]+)\\s?(.*)?$')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
chat_id = int(event.pattern_match.group(1))
ban_duration = event.pattern_match.group(2)
ban_message = event.pattern_match.group(3)
ban_end_date = datetime.utcnow() + timedelta(seconds=timeparse(ban_duration))
try:
await self.application.idm_client.update_chat(
chat_id=chat_id,
ban_until=int(ban_end_date.timestamp()),
ban_message=ban_message,
request_id=request_context.request_id,
)
request_context.statbox(
action='banned',
ban_message=ban_message,
ban_until=ban_end_date.timestamp(),
banned_chat_id=chat_id,
)
except ClientError as e:
if e.code == 'nonexistent_entity_error':
await event.reply('Chat not found')
return
raise
await event.reply('User banned until ' + ban_end_date.strftime("%Y-%m-%d %H:%M") + ' UTC')
class UnbanHandler(BaseAdminHandler):
filter = events.NewMessage(incoming=True, pattern='^/unban(?:_|\\s)([0-9]+)$')
async def handler(self, event, request_context: RequestContext):
chat_id = int(event.pattern_match.group(1))
try:
await self.application.idm_client.update_chat(
chat_id=chat_id,
ban_until=0,
request_id=request_context.request_id,
)
request_context.statbox(
action='unbanned',
unbanned_chat_id=chat_id,
)
except ClientError as e:
if e.code == 'nonexistent_entity_error':
await event.reply('Chat not found')
return
raise
await event.reply('User unbanned')
class BanlistHandler(BaseAdminHandler):
filter = events.NewMessage(incoming=True, pattern='^/banlist$')
async def handler(self, event, request_context: RequestContext):
request_context.statbox(action='show', mode='banlist')
chat_list = (await self.application.idm_client.list_chats(
banned_at_moment=int(datetime.utcnow().timestamp()),
request_id=request_context.request_id,
)).chats
banlist_widget_view = BanlistWidget(application=self.application, chat=request_context.chat)
widget_content = await banlist_widget_view.render(chat_list=chat_list)
await event.reply(widget_content)

332
nexus/bot/handlers/base.py Normal file
View File

@ -0,0 +1,332 @@
import logging
import time
from abc import ABC
from datetime import datetime
from typing import Union
from grpc import StatusCode
from grpc.experimental.aio import AioRpcError
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from izihawa_utils.exceptions import BaseError
from izihawa_utils.random import random_string
from library.logging import error_log
from library.telegram.base import RequestContext
from library.telegram.utils import safe_execution
from nexus.bot.application import TelegramApplication
from nexus.bot.exceptions import UnknownSchemaError
from nexus.models.proto.typed_document_pb2 import \
TypedDocument as TypedDocumentPb
from nexus.translations import t
from nexus.views.telegram.common import close_button
from nexus.views.telegram.scimag import ScimagView
from nexus.views.telegram.scitech import ScitechView
from telethon import (
TelegramClient,
events,
)
from telethon.errors import QueryIdInvalidError
def get_username(event: events.ChatAction, chat):
if event.is_group or event.is_channel:
return str(event.chat_id)
else:
return chat.username
def get_language(event: events.ChatAction, chat):
if event.is_group or event.is_channel:
return 'en'
return chat.lang_code
def is_banned(chat: Chat) -> bool:
return chat.ban_until is not None and datetime.utcnow().timestamp() < chat.ban_until
def is_subscribed(chat: Chat) -> bool:
return chat.is_subscribed or chat.id < 0 or chat.created_at > time.time() - 10 * 60
class ReadOnlyModeError(BaseError):
level = logging.WARNING
code = 'read_only_mode_error'
class BaseHandler(ABC):
# Is handler working in the groups
is_group_handler = False
# Is subscription to the `config['telegram']['related_channel'] required to use this handler
is_subscription_required_for_handler = False
# Telethon filter
filter = events.NewMessage(incoming=True)
should_reset_last_widget = True
# Raises StopPropagation in the end of handling. It means this handler would be the last one in chain
stop_propagation = True
# If set to True then read_only mode will disable handler
writing_handler = False
def __init__(self, application: TelegramApplication):
self.application = application
self.schema_to_resolver = {
'scimag': self.resolve_scimag,
'scitech': self.resolve_scitech,
}
self.short_schema_to_schema_dict = {
'a': 'scimag',
'b': 'scitech',
}
def generate_session_id(self) -> str:
return random_string(self.application.config['application']['session_id_length'])
def short_schema_to_schema(self, short_schema: str) -> str:
return self.short_schema_to_schema_dict[short_schema]
async def get_typed_document_pb(
self,
schema: str,
document_id: int,
request_context: RequestContext,
session_id: str,
position: int,
) -> TypedDocumentPb:
return await self.application.meta_api_client.get(
schema=schema,
document_id=document_id,
session_id=session_id,
position=position,
request_id=request_context.request_id,
user_id=request_context.chat.id,
)
async def resolve_scimag(
self,
document_id: int,
position: int,
request_context: RequestContext,
session_id: str,
) -> ScimagView:
typed_document_pb = await self.get_typed_document_pb(
schema='scimag',
document_id=document_id,
position=position,
request_context=request_context,
session_id=session_id,
)
return ScimagView(document_pb=typed_document_pb.scimag)
async def resolve_scitech(
self,
document_id: int,
position: int,
request_context: RequestContext,
session_id: str,
) -> ScitechView:
typed_document_pb = await self.get_typed_document_pb(
schema='scitech',
document_id=document_id,
position=position,
request_context=request_context,
session_id=session_id,
)
search_response_duplicates = await self.application.meta_api_client.search(
schemas=('scitech',),
query=f'original_id:{document_id}',
page_size=16,
request_id=request_context.request_id,
session_id=session_id,
user_id=request_context.chat.id,
)
duplicates = [
scored_document.typed_document.scitech
for scored_document in search_response_duplicates.scored_documents
]
return ScitechView(
document_pb=typed_document_pb.scitech,
duplicates=duplicates,
)
async def resolve_document(
self,
schema: str,
document_id: int,
position: int,
session_id: str,
request_context: RequestContext
) -> Union[ScimagView, ScitechView]:
if schema not in self.schema_to_resolver:
raise UnknownSchemaError()
resolver = self.schema_to_resolver[schema]
return await resolver(
document_id=document_id,
position=position,
request_context=request_context,
session_id=session_id,
)
def reset_last_widget(self, chat_id: int):
self.application.user_manager.last_widget[chat_id] = None
def register_for(self, telegram_client: TelegramClient):
telegram_client.add_event_handler(self._wrapped_handler, self.filter)
return self._wrapped_handler
async def _send_fail_response(self, event: events.ChatAction, request_context: RequestContext):
try:
await event.reply(
t('MAINTENANCE', language=request_context.chat.language).format(
maintenance_picture_url=self.application.config['application']['maintenance_picture_url'],
),
buttons=[close_button()]
)
except (ConnectionError, QueryIdInvalidError) as e:
request_context.error_log(e)
async def _put_chat(self, event: events.ChatAction, request_id: str):
try:
chat = await self.application.idm_client.get_chat(
chat_id=event.chat_id,
request_id=request_id,
)
return chat
except AioRpcError as e:
if e.code() != StatusCode.NOT_FOUND:
raise
if self.application.config['application']['is_read_only_mode']:
raise ReadOnlyModeError()
event_chat = await event.get_chat()
username = get_username(event, event_chat)
language = get_language(event, event_chat)
if language not in {'en', 'ru'}:
language = 'en'
chat = await self.application.idm_client.create_chat(
chat_id=event.chat_id,
username=username,
language=language,
request_id=request_id,
)
return chat
async def _check_ban(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
if is_banned(chat):
if chat.ban_message is not None:
async with safe_execution(
request_context=request_context,
on_fail=lambda: self._send_fail_response(event, request_context),
):
await event.reply(t(
'BANNED',
language=chat.language
).format(
datetime=str(time.ctime(chat.ban_until)),
reason=chat.ban_message,
))
raise events.StopPropagation()
async def _check_maintenance(self, event: events.ChatAction):
if (
self.application.config['application']['is_maintenance_mode']
and event.chat_id not in self.application.config['application']['bypass_maintenance']
):
await event.reply(
t('UPGRADE_MAINTENANCE', language='en').format(
upgrade_maintenance_picture_url=self.application.config['application']
['upgrade_maintenance_picture_url']
),
)
raise events.StopPropagation()
async def _check_read_only(self, event: events.ChatAction):
if self.application.config['application']['is_read_only_mode']:
await event.reply(
t("READ_ONLY_MODE", language='en'),
)
raise events.StopPropagation()
async def _check_subscription(self, event: events.ChatAction, request_context: RequestContext, chat: Chat):
if (
self.application.config['application']['is_subscription_required']
and self.is_subscription_required_for_handler
and not is_subscribed(chat)
):
async with safe_execution(
request_context=request_context,
on_fail=lambda: self._send_fail_response(event, request_context),
):
await event.reply(t(
'SUBSCRIBE_TO_CHANNEL',
language=chat.language
).format(related_channel=self.application.config['telegram']['related_channel']))
raise events.StopPropagation()
def _has_access(self, chat: Chat) -> bool:
return True
async def _process_chat(self, event: events.ChatAction, request_id: str):
try:
chat = await self._put_chat(event, request_id=request_id)
except (AioRpcError, BaseError) as e:
error_log(e)
event_chat = await event.get_chat()
username = get_username(event, event_chat)
chat = Chat(
id=event.chat_id,
is_system_messaging_enabled=True,
is_discovery_enabled=True,
language='en',
username=username,
is_admin=False,
is_subscribed=True,
)
return chat
async def _wrapped_handler(self, event: events.ChatAction) -> None:
# Checking group permissions
if (event.is_group or event.is_channel) and not self.is_group_handler:
return
await self._check_maintenance(event=event)
await self._check_read_only(event=event)
request_id = RequestContext.generate_request_id(self.application.config['application']['request_id_length'])
chat = await self._process_chat(event=event, request_id=request_id)
request_context = RequestContext(
bot_name=self.application.config['telegram']['bot_name'],
chat=chat,
request_id=request_id,
request_id_length=self.application.config['application']['request_id_length'],
)
if not self._has_access(chat):
return
await self._check_subscription(event=event, request_context=request_context, chat=chat)
await self._check_ban(event=event, request_context=request_context, chat=chat)
if self.should_reset_last_widget:
self.reset_last_widget(request_context.chat.id)
async with safe_execution(
request_context=request_context,
on_fail=lambda: self._send_fail_response(event, request_context),
):
await self.handler(
event,
request_context=request_context,
)
if self.stop_propagation:
raise events.StopPropagation()
async def handler(self, event: events.ChatAction, request_context: RequestContext):
raise NotImplementedError()
class BaseCallbackQueryHandler(BaseHandler, ABC):
async def _send_fail_response(self, event, request_context: RequestContext):
try:
await event.answer(t('MAINTENANCE_WO_PIC', language=request_context.chat.language))
except (ConnectionError, QueryIdInvalidError) as e:
request_context.error_log(e)

31
nexus/bot/handlers/close.py Normal file
View File

@ -0,0 +1,31 @@
import asyncio
from library.telegram.base import RequestContext
from telethon import events
from .base import BaseCallbackQueryHandler
class CloseHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern='^/close(?:_([A-Za-z0-9]+))?(?:_([0-9]+))?$')
async def handler(self, event, request_context: RequestContext):
session_id = event.pattern_match.group(1)
if session_id:
session_id = session_id.decode()
request_context.add_default_fields(mode='close')
target_events = [event.answer()]
message = await event.get_message()
if message:
request_context.statbox(
action='close',
message_id=message.id,
session_id=session_id,
)
reply_message = await message.get_reply_message()
if reply_message:
target_events.append(reply_message.delete())
target_events.append(message.delete())
await asyncio.gather(*target_events)

33
nexus/bot/handlers/contact.py Normal file
View File

@ -0,0 +1,33 @@
import re
from library.telegram.base import RequestContext
from nexus.bot.configs import config
from nexus.translations import t
from telethon import events
from .base import BaseHandler
class ContactHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern=re.compile('^/contact\\s?(.*)', re.DOTALL))
is_group_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
query = event.pattern_match.group(1)
if query:
request_context.statbox(action='show', mode='contact', query=query)
await event.reply(
t('THANK_YOU_FOR_CONTACT', language=request_context.chat.language).format(
related_channel=self.application.config['telegram']['related_channel'],
),
)
else:
request_context.statbox(action='show', mode='contact')
await event.reply(
t('CONTACT', language=request_context.chat.language).format(
btc_donate_address=config['application']['btc_donate_address'],
libera_pay_url=config['application']['libera_pay_url'],
related_channel=config['telegram']['related_channel'],
),
link_preview=False,
)

31
nexus/bot/handlers/copyright.py Normal file
View File

@ -0,0 +1,31 @@
import re
from library.telegram.base import RequestContext
from nexus.translations import t
from telethon import events
from .base import BaseHandler
class CopyrightHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern=re.compile('^/copyright\\s?(.*)', re.DOTALL))
async def handler(self, event: events.ChatAction, request_context: RequestContext):
query = event.pattern_match.group(1)
if query:
request_context.statbox(
action='show',
mode='copyright',
query=query,
)
await self.application.telegram_client.forward_messages(
self.application.config['telegram']['copyright_infringement_account'],
event.message,
)
await event.reply(t(
'COPYRIGHT_INFRINGEMENT_ACCEPTED',
language=request_context.chat.language,
))
else:
request_context.statbox(action='show', mode='copyright')
await event.reply(t('COPYRIGHT_DESCRIPTION', language=request_context.chat.language,))

25
nexus/bot/handlers/donate.py Normal file
View File

@ -0,0 +1,25 @@
from library.telegram.base import RequestContext
from nexus.bot.configs import config
from nexus.translations import t
from telethon import events
from .base import BaseHandler
class DonateHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern='^/donate(@[A-Za-z0-9_]+)?$')
is_group_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='show', mode='donate')
await event.reply(
t(
'DONATE',
language=request_context.chat.language
).format(
amazon_gift_card_recipient=config['application'].get('amazon_gift_card_recipient', '🚫'),
amazon_gift_card_url=config['application'].get('amazon_gift_card_url', '🚫'),
btc_donate_address=config['application'].get('btc_donate_address', '🚫'),
libera_pay_url=config['application'].get('libera_pay_url', '🚫'),
related_channel=config['telegram'].get('related_channel', '🚫'),
))

51
nexus/bot/handlers/download.py Normal file
View File

@ -0,0 +1,51 @@
from library.telegram.base import RequestContext
from nexus.hub.proto.delivery_service_pb2 import \
StartDeliveryResponse as StartDeliveryResponsePb
from nexus.translations import t
from nexus.views.telegram.common import remove_button
from telethon import events
from .base import BaseCallbackQueryHandler
class DownloadHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern='^/dl([abcm])_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$')
is_group_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
short_schema = event.pattern_match.group(1).decode()
schema = self.short_schema_to_schema(short_schema)
session_id = event.pattern_match.group(2).decode()
document_id = int(event.pattern_match.group(3))
position = int(event.pattern_match.group(4).decode())
self.application.user_manager.last_widget[request_context.chat.id] = None
request_context.add_default_fields(mode='download', session_id=session_id)
request_context.statbox(action='get', query=str(document_id), position=position)
typed_document_pb = await self.get_typed_document_pb(
schema=schema,
document_id=document_id,
request_context=request_context,
session_id=session_id,
position=position,
)
start_delivery_response_pb = await self.application.hub_client.start_delivery(
typed_document_pb=typed_document_pb,
chat=request_context.chat,
request_id=request_context.request_id,
session_id=session_id,
)
if start_delivery_response_pb.status == StartDeliveryResponsePb.Status.ALREADY_DOWNLOADING:
await event.answer(
f'{t("ALREADY_DOWNLOADING", language=request_context.chat.language)}',
)
await remove_button(event, '⬇️', and_empty_too=True)
elif start_delivery_response_pb.status == StartDeliveryResponsePb.Status.TOO_MANY_DOWNLOADS:
await event.answer(
f'{t("TOO_MANY_DOWNLOADS", language=request_context.chat.language)}',
)
else:
await remove_button(event, '⬇️', and_empty_too=True)
self.application.user_manager.last_widget[request_context.chat.id] = None

17
nexus/bot/handlers/emoji.py Normal file
View File

@ -0,0 +1,17 @@
from library.telegram.base import RequestContext
from nexus.nlptools.regex import STICKER_REGEX
from nexus.translations import t
from telethon import events
from .base import BaseHandler
class EmojiHandler(BaseHandler):
filter = events.NewMessage(
incoming=True,
pattern=STICKER_REGEX,
)
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='show', mode='emoji')
await event.reply(t('TANKS_BRUH', language=request_context.chat.language))

20
nexus/bot/handlers/help.py Normal file
View File

@ -0,0 +1,20 @@
from library.telegram.base import RequestContext
from nexus.translations import t
from telethon import (
Button,
events,
)
from .base import BaseHandler
class HelpHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern='^/help(@[A-Za-z0-9_]+)?$')
is_group_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='show', mode='help')
if event.is_group or event.is_channel:
await event.reply(t('HELP_FOR_GROUPS', language=request_context.chat.language), buttons=Button.clear())
else:
await event.reply(t('HELP', language=request_context.chat.language), buttons=Button.clear())

28
nexus/bot/handlers/legacy.py Normal file
View File

@ -0,0 +1,28 @@
from library.telegram.base import RequestContext
from nexus.translations import t
from telethon import events
from .base import (
BaseCallbackQueryHandler,
BaseHandler,
)
class LegacyHandler(BaseHandler):
filter = events.NewMessage(
incoming=True,
pattern='^/v_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)$')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='show', mode='legacy')
await event.reply(t('LEGACY', language=request_context.chat.language))
class LegacyCallbackHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(
pattern='^/dl_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)$'
)
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='show', mode='legacy')
return await event.answer(t('LEGACY', language=request_context.chat.language))

12
nexus/bot/handlers/noop.py Normal file
View File

@ -0,0 +1,12 @@
from library.telegram.base import RequestContext
from telethon import events
from .base import BaseCallbackQueryHandler
class NoopHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern='^/noop$')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='start', mode='noop')
await event.answer()

94
nexus/bot/handlers/referencing_to.py Normal file
View File

@ -0,0 +1,94 @@
import re
import time
from grpc.experimental.aio import AioRpcError
from library.telegram.base import RequestContext
from nexus.bot.widgets.document_list_widget import DocumentListWidget
from nexus.translations import t
from telethon import events
from .base import BaseCallbackQueryHandler
class ReferencingToHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern=re.compile('^/r_([A-Za-z0-9]+)_([0-9]+)', re.DOTALL))
should_reset_last_widget = False
async def do_request(self, request_context: RequestContext, session_id: str, message_id: int, document_id: int, page: int):
start_time = time.time()
try:
document_list_widget = await DocumentListWidget.create(
application=self.application,
chat=request_context.chat,
session_id=session_id,
message_id=message_id,
request_id=request_context.request_id,
referencing_to=document_id,
page=page,
)
except AioRpcError as e:
raise e
action = 'referencing_to_found'
if len(document_list_widget.scored_documents) == 0:
action = 'referencing_to_not_found'
request_context.statbox(
action=action,
duration=time.time() - start_time,
query=f'{document_id}',
)
serp, buttons = await document_list_widget.render()
return await self.application.telegram_client.edit_message(
request_context.chat.id,
message_id,
serp,
buttons=buttons,
link_preview=False,
)
async def handler(self, event, request_context: RequestContext):
session_id = event.pattern_match.group(1).decode()
document_id = int(event.pattern_match.group(2).decode())
request_context.add_default_fields(
mode='referencing_to',
session_id=session_id,
)
prefetch_message = await event.respond(
t("SEARCHING", language=request_context.chat.language),
)
message_id = prefetch_message.id
return await self.do_request(
request_context=request_context,
session_id=session_id,
message_id=message_id,
document_id=document_id,
page=0,
)
class ReferencingToPagingHandler(ReferencingToHandler):
filter = events.CallbackQuery(pattern=re.compile('^/rp_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)', re.DOTALL))
async def handler(self, event: events.ChatAction, request_context: RequestContext):
session_id = event.pattern_match.group(1).decode()
message_id = int(event.pattern_match.group(2).decode())
document_id = int(event.pattern_match.group(3).decode())
page = int(event.pattern_match.group(4).decode())
request_context.add_default_fields(
mode='referencing_to_paging',
session_id=session_id,
)
return await self.do_request(
request_context=request_context,
session_id=session_id,
message_id=message_id,
document_id=document_id,
page=page,
)

44
nexus/bot/handlers/roll.py Normal file
View File

@ -0,0 +1,44 @@
import asyncio
import re
from library.telegram.base import RequestContext
from telethon import events
from .base import BaseHandler
class RollHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern=re.compile('^/roll(@[A-Za-z0-9_]+)?$', re.DOTALL))
is_group_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
session_id = self.generate_session_id()
request_context.add_default_fields(mode='roll', session_id=session_id)
request_context.statbox(action='show')
roll_response_pb = await self.application.meta_api_client.roll(
language=request_context.chat.language,
session_id=session_id,
request_id=request_context.request_id,
user_id=request_context.chat.id,
)
scitech_view = await self.resolve_scitech(
document_id=roll_response_pb.document_id,
position=0,
request_context=request_context,
session_id=session_id,
)
view, buttons = scitech_view.get_view(
language=request_context.chat.language,
session_id=session_id,
bot_external_name=self.application.config['telegram']['bot_external_name'],
)
actions = [
self.application.telegram_client.send_message(
request_context.chat.id,
view,
buttons=buttons,
),
event.delete(),
]
return await asyncio.gather(*actions)

289
nexus/bot/handlers/search.py Normal file
View File

@ -0,0 +1,289 @@
import asyncio
import logging
import re
import time
from grpc import StatusCode
from grpc.experimental.aio import AioRpcError
from library.telegram.base import RequestContext
from nexus.bot.widgets.search_widget import SearchWidget
from nexus.translations import t
from nexus.views.telegram.common import close_button
from nexus.views.telegram.registry import parse_typed_document_to_view
from telethon import (
events,
functions,
)
from .base import (
BaseCallbackQueryHandler,
BaseHandler,
)
class BaseSearchHandler(BaseHandler):
should_reset_last_widget = False
async def do_search(
self,
event: events.ChatAction,
request_context: RequestContext,
prefetch_message,
query: str,
is_group_mode: bool = False,
is_shortpath_enabled: bool = False,
):
session_id = self.generate_session_id()
message_id = prefetch_message.id
request_context.add_default_fields(is_group_mode=is_group_mode, mode='search', session_id=session_id)
start_time = time.time()
try:
search_widget = await SearchWidget.create(
application=self.application,
chat=request_context.chat,
session_id=session_id,
message_id=message_id,
request_id=request_context.request_id,
query=query,
is_group_mode=is_group_mode,
)
except AioRpcError as e:
actions = [
self.application.telegram_client.delete_messages(
request_context.chat.id,
[message_id],
)
]
if e.code() == StatusCode.INVALID_ARGUMENT:
too_difficult_picture_url = self.application.config['application'].get('too_difficult_picture_url', '')
if e.details() == 'url_query_error':
actions.append(
event.reply(
t('INVALID_QUERY_ERROR', language=request_context.chat.language).format(
too_difficult_picture_url=too_difficult_picture_url,
),
buttons=[close_button()],
)
)
elif e.details() == 'invalid_query_error':
actions.append(
event.reply(
t('INVALID_SYNTAX_ERROR', language=request_context.chat.language).format(
too_difficult_picture_url=too_difficult_picture_url,
),
buttons=[close_button()],
)
)
return await asyncio.gather(*actions)
elif e.code() == StatusCode.CANCELLED:
maintenance_picture_url = self.application.config['application'].get('maintenance_picture_url', '')
request_context.error_log(e)
actions.append(event.reply(
t('MAINTENANCE', language=request_context.chat.language).format(
maintenance_picture_url=maintenance_picture_url,
),
buttons=[close_button()],
))
return await asyncio.gather(*actions)
raise e
action = 'documents_found'
if len(search_widget.scored_documents) == 0:
action = 'documents_not_found'
request_context.statbox(
action=action,
duration=time.time() - start_time,
query=f'page:0 query:{query}',
)
if len(search_widget.scored_documents) == 1 and is_shortpath_enabled:
scored_document = search_widget.scored_documents[0]
document_view = parse_typed_document_to_view(scored_document.typed_document)
# Second (re-)fetching is required to retrieve duplicates
document_view = await self.resolve_document(
schema=scored_document.typed_document.WhichOneof('document'),
document_id=document_view.id,
position=0,
session_id=session_id,
request_context=request_context,
)
view, buttons = document_view.get_view(
language=request_context.chat.language,
session_id=session_id,
bot_external_name=self.application.config['telegram']['bot_external_name'],
with_buttons=not is_group_mode,
)
return await asyncio.gather(
self.application.telegram_client.edit_message(
request_context.chat.id,
message_id,
view,
buttons=buttons,
),
)
serp, buttons = await search_widget.render()
return await self.application.telegram_client.edit_message(
request_context.chat.id,
message_id,
serp,
buttons=buttons,
link_preview=False,
)
class SearchHandler(BaseSearchHandler):
filter = events.NewMessage(incoming=True, pattern=re.compile('^(/search\\s+)?(.*)', flags=re.DOTALL))
is_group_handler = True
should_reset_last_widget = False
is_subscription_required_for_handler = True
async def ban_handler(self, event: events.ChatAction, request_context: RequestContext, ban_timeout: float):
logging.getLogger('statbox').info({
'bot_name': self.application.config['telegram']['bot_name'],
'action': 'user_flood_ban',
'mode': 'search',
'ban_timeout_seconds': ban_timeout,
'chat_id': request_context.chat.id,
})
ban_reason = t(
'BAN_MESSAGE_TOO_MANY_REQUESTS',
language=request_context.chat.language
)
return await event.reply(t(
'BANNED_FOR_SECONDS',
language=request_context.chat.language
).format(
seconds=str(ban_timeout),
reason=ban_reason,
))
async def handler(self, event: events.ChatAction, request_context: RequestContext):
ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=request_context.chat.id)
if ban_timeout:
return await self.ban_handler(event, request_context, ban_timeout)
self.application.user_manager.add_search_time(user_id=request_context.chat.id, search_time=time.time())
search_prefix = event.pattern_match.group(1)
query = event.pattern_match.group(2)
is_group_mode = event.is_group or event.is_channel
if is_group_mode and not search_prefix:
return
if not is_group_mode and search_prefix:
query = event.raw_text
prefetch_message = await event.reply(
t("SEARCHING", language=request_context.chat.language),
)
self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
try:
await self.do_search(
event, request_context, prefetch_message,
query=query,
is_group_mode=is_group_mode,
is_shortpath_enabled=True,
)
except (AioRpcError, asyncio.CancelledError) as e:
await asyncio.gather(
event.delete(),
prefetch_message.delete(),
)
raise e
class SearchEditHandler(BaseSearchHandler):
filter = events.MessageEdited(incoming=True, pattern=re.compile('^(/search\\s+)?(.*)', flags=re.DOTALL))
is_group_handler = True
should_reset_last_widget = False
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.add_default_fields(mode='search_edit')
search_prefix = event.pattern_match.group(1)
query = event.pattern_match.group(2)
is_group_mode = event.is_group or event.is_channel
if is_group_mode and not search_prefix:
return
if not is_group_mode and search_prefix:
query = event.raw_text
result = await self.application.telegram_client(functions.messages.GetMessagesRequest(
id=list(range(event.id + 1, event.id + 10)))
)
if not result:
request_context.statbox(action='failed')
return await event.reply(
t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
)
for next_message in result.messages:
if next_message.is_reply and event.id == next_message.reply_to_msg_id:
request_context.statbox(action='resolved')
await self.do_search(
event,
request_context,
prefetch_message=next_message,
query=query,
is_group_mode=is_group_mode,
)
return
request_context.statbox(action='failed')
return await event.reply(
t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
)
class SearchPagingHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern='^/search_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)$')
should_reset_last_widget = False
async def handler(self, event: events.ChatAction, request_context: RequestContext):
session_id = event.pattern_match.group(1).decode()
message_id = int(event.pattern_match.group(2).decode())
page = int(event.pattern_match.group(3).decode())
request_context.add_default_fields(mode='search_paging', session_id=session_id)
message = await event.get_message()
if not message:
return await event.answer()
reply_message = await message.get_reply_message()
if not reply_message:
return await event.respond(
t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
)
start_time = time.time()
query = reply_message.raw_text
try:
search_widget = await SearchWidget.create(
application=self.application,
chat=request_context.chat,
session_id=session_id,
message_id=message_id,
request_id=request_context.request_id,
query=query,
page=page,
)
except AioRpcError as e:
if e.code() == StatusCode.INVALID_ARGUMENT or e.code() == StatusCode.CANCELLED:
request_context.error_log(e)
return await event.answer(
t('MAINTENANCE_WO_PIC', language=request_context.chat.language),
)
raise e
action = 'documents_found'
if len(search_widget.scored_documents) == 0:
action = 'documents_not_found'
request_context.statbox(
action=action,
duration=time.time() - start_time,
query=f'page:{page} query:{query}',
)
serp, buttons = await search_widget.render()
return await asyncio.gather(
event.answer(),
message.edit(serp, buttons=buttons, link_preview=False)
)

129
nexus/bot/handlers/settings.py Normal file
View File

@ -0,0 +1,129 @@
from izihawa_utils.podolsky_encoding import encode
from library.telegram.base import RequestContext
from nexus.bot.widgets.settings_widget import (
SettingsManualWidget,
SettingsRouterWidget,
)
from nexus.translations import t
from telethon import (
Button,
events,
functions,
)
from .base import (
BaseCallbackQueryHandler,
BaseHandler,
)
class SettingsRouterHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern='^/settings(@[A-Za-z0-9_]+)?$')
is_group_handler = True
writing_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.add_default_fields(mode='settings_router')
request_context.statbox(action='show')
if self.application.config['application']['views']['settings']['has_router']:
settings_router_widget = SettingsRouterWidget(
application=self.application,
chat=request_context.chat,
request_id=request_context.request_id,
)
text, buttons = await settings_router_widget.render()
else:
settings_widget = SettingsManualWidget(
application=self.application,
chat=request_context.chat,
is_group_mode=event.is_group or event.is_channel,
request_id=request_context.request_id,
)
text, buttons = await settings_widget.render()
await event.reply(text, buttons=buttons)
class SettingsAutomaticHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern=f'^🌎{encode("1")}')
async def handler(self, event, request_context: RequestContext):
request_context.add_default_fields(mode='settings_automatic')
request_context.statbox(action='show')
await event.reply(
f'{t("SEND_YOUR_LOCATION", language=request_context.chat.language)}{encode("sg")}',
buttons=Button.clear()
)
class SettingsManualHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern=f'^👇{encode("1")}')
async def handler(self, event, request_context: RequestContext):
request_context.add_default_fields(mode='settings_manual')
request_context.statbox(action='show')
settings_widget = SettingsManualWidget(
application=self.application,
chat=request_context.chat,
is_group_mode=event.is_group or event.is_channel,
request_id=request_context.request_id,
)
text, buttons = await settings_widget.render()
await event.reply(text, buttons=buttons)
class SettingsButtonsHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern='^/settings_([A-Za-z0-9]+)_([A-Za-z0-9]+)$')
is_group_handler = True
async def handler(self, event, request_context: RequestContext):
request_context.add_default_fields(mode='settings')
action_id = event.pattern_match.group(1).decode()
data = event.pattern_match.group(2).decode()
request_context.statbox(action='change', query=f'action_id: {action_id} data: {data}')
settings_manual_widget = SettingsManualWidget(
application=self.application,
chat=request_context.chat,
is_group_mode=event.is_group or event.is_channel,
request_id=request_context.request_id,
)
is_changed = await settings_manual_widget.process_action(action_id=action_id, data=data)
text, buttons = await settings_manual_widget.render()
if not is_changed and not (event.is_group or event.is_channel):
await event.answer()
return
if event.is_group or event.is_channel:
buttons = None
await event.edit(text, buttons=buttons)
class GeoHandler(BaseHandler):
filter = events.NewMessage(incoming=True)
stop_propagation = False
async def handler(self, event, request_context: RequestContext):
request_context.add_default_fields(mode='geo')
if not event.geo:
return
request_context.statbox(action='geo', query=f'lon:{event.geo.long} lat:{event.geo.lat}')
result = await self.application.telegram_client(functions.messages.GetMessagesRequest(id=[event.id - 1]))
if not result.messages:
return
previous_message = result.messages[0]
if previous_message.message.endswith(encode("sg")):
request_context.statbox(action='catched_settings')
settings_manual_widget = SettingsManualWidget(
application=self.application,
chat=request_context.chat,
has_language_buttons=False,
is_group_mode=event.is_group or event.is_channel,
request_id=request_context.request_id,
)
await settings_manual_widget.set_last_location(lon=event.geo.long, lat=event.geo.lat)
text, buttons = await settings_manual_widget.render()
await event.reply(text, buttons=buttons or Button.clear())
raise events.StopPropagation()

25
nexus/bot/handlers/shortlink.py Normal file
View File

@ -0,0 +1,25 @@
from library.telegram.base import RequestContext
from nexus.translations import t
from nexus.views.telegram.common import (
TooLongQueryError,
encode_query_to_deep_link,
)
from telethon import events
from .base import BaseHandler
class ShortlinkHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern='^/shortlink\\s?(.*)?')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
query = event.pattern_match.group(1)
request_context.statbox(action='start', mode='shortlink', query=query)
try:
bot_name = self.application.config["telegram"]["bot_external_name"]
text = encode_query_to_deep_link(query, bot_name)
except TooLongQueryError:
text = t('TOO_LONG_QUERY_FOR_SHORTLINK', language=request_context.chat.language),
return await event.reply(f'`{text}`', link_preview=False)

43
nexus/bot/handlers/start.py Normal file
View File

@ -0,0 +1,43 @@
import asyncio
from library.telegram.base import RequestContext
from nexus.translations import t
from nexus.views.telegram.common import (
DecodeDeepQueryError,
decode_deep_query,
)
from telethon import events
from .search import BaseSearchHandler
class StartHandler(BaseSearchHandler):
should_reset_last_widget = False
filter = events.NewMessage(incoming=True, pattern='^/start\\s?(.*)?')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
raw_query = event.pattern_match.group(1)
query = None
request_context.statbox(action='start', mode='start')
try:
query = decode_deep_query(raw_query)
except DecodeDeepQueryError as e:
request_context.error_log(e, mode='start', raw_query=raw_query)
if query:
request_context.statbox(action='query', mode='start', query=query)
request_message = await self.application.telegram_client.send_message(event.chat, query)
prefetch_message = await request_message.reply(
t("SEARCHING", language=request_context.chat.language),
)
self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
await asyncio.gather(
event.delete(),
self.do_search(event, request_context, prefetch_message, query=query,
is_shortpath_enabled=True),
)
else:
request_context.statbox(action='show', mode='start')
await event.reply(t('HELP', language=request_context.chat.language))

11
nexus/bot/handlers/stop.py Normal file
View File

@ -0,0 +1,11 @@
from library.telegram.base import RequestContext
from telethon import events
from .base import BaseHandler
class StopHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern='^/stop$')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
request_context.statbox(action='show', mode='stop')

42
nexus/bot/handlers/submit.py Normal file
View File

@ -0,0 +1,42 @@
import asyncio
from library.telegram.base import RequestContext
from nexus.bot.exceptions import UnknownFileFormatError
from nexus.translations import t
from nexus.views.telegram.common import close_button
from telethon import events
from .base import BaseHandler
class SubmitHandler(BaseHandler):
filter = events.NewMessage(func=lambda e: e.document, incoming=True)
is_group_handler = False
writing_handler = True
async def handler(self, event: events.ChatAction, request_context: RequestContext):
session_id = self.generate_session_id()
request_context.add_default_fields(session_id=session_id)
request_context.statbox(action='show', mode='submit')
if event.document.mime_type != 'application/pdf':
request_context.error_log(UnknownFileFormatError(format=event.document.mime_type))
return await asyncio.gather(
event.reply(
t('UNKNOWN_FILE_FORMAT_ERROR', language=request_context.chat.language),
buttons=[close_button()],
),
event.delete(),
)
return await asyncio.gather(
self.application.hub_client.submit(
telegram_document=bytes(event.document),
telegram_file_id=event.file.id,
chat=request_context.chat,
request_id=request_context.request_id,
session_id=session_id,
),
event.delete(),
)

101
nexus/bot/handlers/view.py Normal file
View File

@ -0,0 +1,101 @@
import asyncio
import re
from library.telegram.base import RequestContext
from nexus.translations import t
from telethon import (
events,
functions,
)
from telethon.errors import MessageIdInvalidError
from .base import BaseHandler
class ViewHandler(BaseHandler):
filter = events.NewMessage(incoming=True, pattern='^/v([abcm])([sr])?_([A-Za-z0-9]+)_([0-9]+)_([0-9]+)_'
'([0-9]+)')
should_reset_last_widget = False
async def handler(self, event: events.ChatAction, request_context: RequestContext):
short_schema = event.pattern_match.group(1)
parent_view_type = event.pattern_match.group(2) or 's'
schema = self.short_schema_to_schema(short_schema)
session_id = event.pattern_match.group(3)
old_message_id = int(event.pattern_match.group(4))
document_id = int(event.pattern_match.group(5))
position = int(event.pattern_match.group(6))
page = int(position / self.application.config['application']['page_size'])
request_context.add_default_fields(mode='view', session_id=session_id)
request_context.statbox(action='view', query=str(document_id), position=position)
found_old_widget = old_message_id == self.application.user_manager.last_widget.get(request_context.chat.id)
try:
if found_old_widget:
message_id = old_message_id
link_preview = None
else:
old_message = (await self.application.telegram_client(
functions.messages.GetMessagesRequest(id=[old_message_id])
)).messages[0]
prefetch_message = await self.application.telegram_client.send_message(
request_context.chat.id,
t("SEARCHING", language=request_context.chat.language),
reply_to=old_message.reply_to_msg_id,
)
self.application.user_manager.last_widget[request_context.chat.id] = prefetch_message.id
message_id = prefetch_message.id
link_preview = True
document_view = await self.resolve_document(
schema,
document_id,
position,
session_id,
request_context,
)
back_command = None
if parent_view_type == 's':
back_command = f'/search_{session_id}_{message_id}_{page}'
elif parent_view_type == 'r':
messages = (await self.application.telegram_client(
functions.messages.GetMessagesRequest(id=[old_message_id])
)).messages
if not messages:
return await event.respond(
t('REPLY_MESSAGE_HAS_BEEN_DELETED', language=request_context.chat.language),
)
message = messages[0]
referencing_to = re.search(r'Linked to: ([0-9]+)', message.raw_text).group(1)
back_command = f'/rp_{session_id}_{message_id}_{referencing_to}_{page}'
view, buttons = document_view.get_view(
language=request_context.chat.language,
session_id=session_id,
bot_external_name=self.application.config['telegram']['bot_external_name'],
position=position,
back_command=back_command,
)
actions = [
self.application.telegram_client.edit_message(
request_context.chat.id,
message_id,
view,
buttons=buttons,
link_preview=link_preview,
),
event.delete(),
]
if not found_old_widget:
actions.append(
self.application.telegram_client.delete_messages(
request_context.chat.id,
[old_message_id],
)
)
return await asyncio.gather(*actions)
except MessageIdInvalidError:
await event.reply(t("VIEWS_CANNOT_BE_SHARED", language=request_context.chat.language))

52
nexus/bot/handlers/vote.py Normal file
View File

@ -0,0 +1,52 @@
import asyncio
import logging
from izihawa_utils.pb_to_json import MessageToDict
from library.telegram.base import RequestContext
from nexus.models.proto.operation_pb2 import \
DocumentOperation as DocumentOperationPb
from nexus.models.proto.operation_pb2 import Vote as VotePb
from nexus.translations import t
from telethon import events
from .base import BaseCallbackQueryHandler
class VoteHandler(BaseCallbackQueryHandler):
filter = events.CallbackQuery(pattern='^/vote_([A-Za-z0-9]+)_([0-9]+)_([bo])$')
async def handler(self, event: events.ChatAction, request_context: RequestContext):
session_id = event.pattern_match.group(1).decode()
document_id = int(event.pattern_match.group(2).decode())
vote = event.pattern_match.group(3).decode()
vote_value = {'b': -1, 'o': 1}[vote]
request_context.add_default_fields(mode='vote', session_id=session_id)
document_operation_pb = DocumentOperationPb(
vote=VotePb(
document_id=document_id,
value=vote_value,
voter_id=request_context.chat.id,
),
)
request_context.statbox(
action='vote',
document_id=document_id,
)
logging.getLogger('operation').info(
msg=MessageToDict(document_operation_pb),
)
message = await event.get_message()
# ToDo: Generalize nexus.views.telegram.common.remove_button and use it here
return await asyncio.gather(
self.application.telegram_client.edit_message(
request_context.chat.id,
message.id,
message.text,
buttons=None,
),
event.answer(t('TANKS_BRUH')),
)

19
nexus/bot/main.py Normal file
View File

@ -0,0 +1,19 @@
import asyncio
import uvloop
from library.logging import configure_logging
from nexus.bot.application import TelegramApplication
from nexus.bot.configs import get_config
def main(config):
uvloop.install()
configure_logging(config)
if config['metrics']['enabled']:
from library.metrics_server import MetricsServer
MetricsServer(config['metrics']).fork_process()
asyncio.get_event_loop().run_until_complete(TelegramApplication(config=config).start_and_wait())
if __name__ == '__main__':
main(config=get_config())

21
nexus/bot/promotioner.py Normal file
View File

@ -0,0 +1,21 @@
import random
class Promotioner:
"""
Promotioner is used to select promotion randomly based on weights of every promotion.
"""
def __init__(self, promotions: list[dict]):
self.promotions = promotions
self.partial_sums: list = [self.promotions[0]['weight']]
for promotion in self.promotions[1:]:
self.partial_sums.append(promotion['weight'] + self.partial_sums[-1])
def choose_promotion(self, language: str = 'en') -> str:
pivot = random.randrange(self.partial_sums[-1])
for partial_sum, promotion in zip(self.partial_sums, self.promotions):
if partial_sum <= pivot:
continue
if language in promotion['texts']:
return promotion['texts'][language]
return promotion['texts']['en']

3
nexus/bot/user_manager/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .user_manager import UserManager
__all__ = ['UserManager']

41
nexus/bot/user_manager/user_manager.py Normal file
View File

@ -0,0 +1,41 @@
import time
class UserManager:
def __init__(self):
self.last_widget = {}
self.search_times = {}
self.search_ban_times = {}
def add_search_time(self, user_id: int, search_time: float):
current_time = time.time()
search_times = self.search_times.get(user_id, [])
search_times.append(search_time)
counter = 0
for i in reversed(search_times):
if i > current_time - 10:
counter = counter + 1
if counter > 5:
self.search_ban_times[user_id] = current_time + int(60)
del self.search_times[user_id]
return
else:
if counter == 1:
del self.search_times[user_id]
return
if len(search_times) > 20:
self.search_ban_times[user_id] = current_time + int(120)
del self.search_times[user_id]
return
self.search_times[user_id] = search_times
def check_search_ban_timeout(self, user_id: int):
ban_time = self.search_ban_times.get(user_id)
if ban_time:
timeout = int(ban_time - time.time())
if timeout > 0:
return timeout
del self.search_ban_times[user_id]

0
nexus/bot/widgets/__init__.py Normal file
View File

16
nexus/bot/widgets/admin_widget.py Normal file
View File

@ -0,0 +1,16 @@
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from nexus.bot.application import TelegramApplication
class AdminWidget:
def __init__(self, application: TelegramApplication, chat: Chat):
self.application = application
self.chat = chat
async def render(self):
return (
'Ban: `/ban 12345 20d Spam`\n'
'Ban (silent): `/ban 12345 100h`\n'
'List of banned chats: `/banlist`\n'
'Unban chat: `/unban 12345`'
)

26
nexus/bot/widgets/banlist_widget.py Normal file
View File

@ -0,0 +1,26 @@
import time
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from nexus.bot.application import TelegramApplication
class BanlistWidget:
def __init__(self, application: TelegramApplication, chat: Chat):
self.application = application
self.chat = chat
async def render(self, chat_list: list[Chat]):
if not chat_list:
return 'Nobody is banned'
separator = '------------\n'
return separator.join(
map(
lambda chat: (
f'```{chat.username} ({chat.id})\n'
f'Until: {time.ctime(chat.ban_until)}\n'
f'Message: {chat.ban_message}```\n'
f'/unban_{chat.id}\n'
),
chat_list
)
)

130
nexus/bot/widgets/document_list_widget.py Normal file
View File

@ -0,0 +1,130 @@
from typing import Optional
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from nexus.bot.application import TelegramApplication
from nexus.meta_api.proto.meta_search_service_pb2 import \
ScoredDocument as ScoredDocumentPb
from nexus.translations import t
from nexus.views.telegram.common import close_button
from nexus.views.telegram.registry import parse_typed_document_to_view
from telethon import Button
class DocumentListWidget:
def __init__(
self,
application: TelegramApplication,
chat: Chat,
session_id: str,
message_id: int,
request_id: str,
referencing_to: int,
page: int = 0,
):
self.application = application
self.chat = chat
self.session_id = session_id
self.message_id = message_id
self.request_id = request_id
self.referencing_to = referencing_to
self.page = page
@staticmethod
async def create(
application: TelegramApplication,
chat: Chat,
session_id: str,
message_id: int,
request_id: str,
referencing_to: int,
page: int = 0,
) -> 'DocumentListWidget':
document_list_view = DocumentListWidget(
application=application,
chat=chat,
session_id=session_id,
message_id=message_id,
request_id=request_id,
referencing_to=referencing_to,
page=page,
)
await document_list_view._acquire_documents()
return document_list_view
async def _acquire_documents(self):
typed_document_pb = await self.application.meta_api_client.get(
schema='scimag',
document_id=self.referencing_to,
position=0,
request_id=self.request_id,
session_id=self.session_id,
user_id=self.chat.id,
)
self._response = await self.application.meta_api_client.search(
schemas=('scimag',),
query=f'references:"{typed_document_pb.scimag.doi}"',
page=self.page,
request_id=self.request_id,
session_id=self.session_id,
user_id=self.chat.id,
)
@property
def has_next(self) -> bool:
return self._response.has_next
@property
def scored_documents(self) -> list[ScoredDocumentPb]:
return self._response.scored_documents
async def render(self) -> tuple[str, Optional[list]]:
if not len(self.scored_documents):
return t('COULD_NOT_FIND_ANYTHING', language=self.chat.language), [close_button(self.session_id)]
serp_elements = [
f'Linked to: {self.referencing_to}',
]
for scored_document in self.scored_documents:
view = parse_typed_document_to_view(scored_document.typed_document)
view_command = view.get_view_command(
session_id=self.session_id,
message_id=self.message_id,
parent_view_type='r',
position=scored_document.position,
)
serp_elements.append(
view.get_snippet(
language=self.chat.language,
view_command=view_command,
limit=512 + 128,
)
)
promo = self.application.promotioner.choose_promotion(language=self.chat.language).format(
related_channel=self.application.config['telegram']['related_channel'],
)
serp_elements.append(promo)
serp = '\n\n'.join(serp_elements)
buttons = []
if self.has_next or self.page > 0:
buttons = [
Button.inline(
text='<<1' if self.page > 1 else ' ',
data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_0'
if self.page > 1 else '/noop',
),
Button.inline(
text=f'<{self.page}' if self.page > 0 else ' ',
data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page - 1}'
if self.page > 0 else '/noop',
),
Button.inline(
text=f'{self.page + 2}>' if self.has_next else ' ',
data=f'/rp_{self.session_id}_{self.message_id}_{self.referencing_to}_{self.page + 1}'
if self.has_next else '/noop',
)
]
buttons.append(close_button(self.session_id))
return serp, buttons

157
nexus/bot/widgets/search_widget.py Normal file
View File

@ -0,0 +1,157 @@
from typing import Optional
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from nexus.bot.application import TelegramApplication
from nexus.meta_api.proto.meta_search_service_pb2 import \
ScoredDocument as ScoredDocumentPb
from nexus.translations import t
from nexus.views.telegram.common import (
TooLongQueryError,
close_button,
encode_query_to_deep_link,
)
from nexus.views.telegram.registry import parse_typed_document_to_view
from telethon import Button
class SearchWidget:
"""
Presents markup for the SERP.
"""
def __init__(
self,
application: TelegramApplication,
chat: Chat,
session_id: str,
message_id: int,
request_id: str,
query: str,
page: int = 0,
is_group_mode: bool = False,
):
self.application = application
self.chat = chat
self.session_id = session_id
self.message_id = message_id
self.request_id = request_id
self.query = query
self.page = page
self.is_group_mode = is_group_mode
@staticmethod
async def create(
application: TelegramApplication,
chat: Chat,
session_id: str,
message_id: int,
request_id: str,
query: str,
page: int = 0,
is_group_mode: bool = False,
) -> 'SearchWidget':
search_widget_view = SearchWidget(
application=application,
chat=chat,
session_id=session_id,
message_id=message_id,
request_id=request_id,
query=query,
page=page,
is_group_mode=is_group_mode,
)
await search_widget_view._acquire_documents()
return search_widget_view
async def _acquire_documents(self):
self._search_response = await self.application.meta_api_client.search(
schemas=self.application.config['application']['schemas'],
query=self.query,
page=self.page,
page_size=self.application.config['application']['page_size'],
request_id=self.request_id,
session_id=self.session_id,
user_id=self.chat.id,
language=self.chat.language,
)
@property
def has_next(self) -> bool:
return self._search_response.has_next
@property
def scored_documents(self) -> list[ScoredDocumentPb]:
return self._search_response.scored_documents
async def render(self) -> tuple[str, Optional[list]]:
if not len(self.scored_documents):
return t('COULD_NOT_FIND_ANYTHING', language=self.chat.language), [close_button(self.session_id)]
serp_elements = []
bot_external_name = self.application.config['telegram']['bot_external_name']
for scored_document in self.scored_documents:
view = parse_typed_document_to_view(scored_document.typed_document)
if not self.is_group_mode:
view_command = view.get_view_command(
session_id=self.session_id,
message_id=self.message_id,
position=scored_document.position,
)
else:
view_command = view.get_deep_link(bot_external_name, text='⬇️')
serp_elements.append(
view.get_snippet(
language=self.chat.language,
view_command=view_command,
limit=512 + 128,
)
)
serp = '\n\n'.join(serp_elements)
if self.is_group_mode:
try:
encoded_query = encode_query_to_deep_link(
self.query,
bot_external_name,
)
serp = (
f"{serp}\n\n**{t('DOWNLOAD_AND_SEARCH_MORE', language=self.chat.language)}: **"
f'[@{bot_external_name}]'
f'({encoded_query})'
)
except TooLongQueryError:
serp = (
f"{serp}\n\n**{t('DOWNLOAD_AND_SEARCH_MORE', language=self.chat.language)}: **"
f'[@{bot_external_name}]'
f'(https://t.me/{bot_external_name})'
)
if not self.is_group_mode:
promo = self.application.promotioner.choose_promotion(language=self.chat.language).format(
related_channel=self.application.config['telegram']['related_channel'],
)
serp = f'{serp}\n\n{promo}\n'
buttons = None
if not self.is_group_mode:
buttons = []
if self.has_next or self.page > 0:
buttons = [
Button.inline(
text='<<1' if self.page > 1 else ' ',
data=f'/search_{self.session_id}_{self.message_id}_0' if self.page > 1 else '/noop',
),
Button.inline(
text=f'<{self.page}' if self.page > 0 else ' ',
data=f'/search_{self.session_id}_{self.message_id}_{self.page - 1}'
if self.page > 0 else '/noop',
),
Button.inline(
text=f'{self.page + 2}>' if self.has_next else ' ',
data=f'/search_{self.session_id}_{self.message_id}_{self.page + 1}'
if self.has_next else '/noop',
)
]
buttons.append(close_button(self.session_id))
return serp, buttons

158
nexus/bot/widgets/settings_widget.py Normal file
View File

@ -0,0 +1,158 @@
from typing import Optional
from idm.api2.proto.chats_service_pb2 import ChatData as Chat
from idm.api2.proto.location_pb2 import Location
from izihawa_utils.podolsky_encoding import encode
from nexus.bot.application import TelegramApplication
from nexus.translations import t
from telethon import Button
top_languages = {
'am': '🇪🇹',
'ar': '🇦🇪',
'de': '🇩🇪',
'en': '🇬🇧',
'es': '🇪🇸',
'fa': '🇮🇷',
'hi': '🇮🇳',
'id': '🇮🇩',
'it': '🇮🇹',
'ja': '🇯🇵',
'ms': '🇲🇾',
'pb': '🇧🇷',
'ru': '🇷🇺',
'tg': '🇹🇯',
'uk': '🇺🇦',
'uz': '🇺🇿',
}
boolean_emoji = {
False: '❎',
True: '✅️',
}
class SettingsRouterWidget:
def __init__(self, application: TelegramApplication, chat: Chat, request_id: str = None):
self.application = application
self.chat = chat
self.request_id = request_id
async def render(self):
sa = f'🌎{encode("1")}{t("SETUP_AUTOMATICALLY", language=self.chat.language)}'
sm = f'👇{encode("1")}{t("SETUP_MANUALLY", language=self.chat.language)}'
return t("SETTINGS_ROUTER_HELP", language=self.chat.language), [[
Button.text(sa, resize=True, single_use=True),
Button.text(sm, resize=True, single_use=True),
Button.force_reply(),
]]
class SettingsManualWidget:
def __init__(
self,
application: TelegramApplication,
chat: Chat,
has_language_buttons: Optional[bool] = None,
is_group_mode: bool = False,
request_id: Optional[str] = None,
):
self.application = application
self.chat = chat
self.has_language_buttons = has_language_buttons
if self.has_language_buttons is None:
self.has_language_buttons = self.application.config['application']['views']['settings']['has_language_buttons']
self.is_group_mode = is_group_mode
self.request_id = request_id
self._actions = {
'sl': self._switch_language,
'ssm': self._switch_system_messaging,
'sd': self._switch_discovery,
}
async def _switch_language(self, target_language: str):
return await self.application.idm_client.update_chat(
chat_id=self.chat.id,
language=target_language,
request_id=self.request_id,
)
async def _switch_system_messaging(self, is_system_messaging_enabled: str):
self.chat = await self.application.idm_client.update_chat(
chat_id=self.chat.id,
is_system_messaging_enabled=bool(int(is_system_messaging_enabled)),
request_id=self.request_id,
)
return self.chat
async def _switch_discovery(self, is_discovery_enabled: str):
self.chat = await self.application.idm_client.update_chat(
chat_id=self.chat.id,
is_discovery_enabled=bool(int(is_discovery_enabled)),
request_id=self.request_id,
)
return self.chat
async def set_last_location(self, lon: float, lat: float):
self.chat = await self.application.idm_client.update_chat(
chat_id=self.chat.id,
last_location=Location(lon=lon, lat=lat),
request_id=self.request_id,
)
return
async def process_action(self, action_id: str, data: str):
old_chat = self.chat
await self._actions[action_id](data)
return old_chat != self.chat
async def render(self):
text = t('SETTINGS_TEMPLATE', language=self.chat.language).format(
bot_version=self.application.config['application']['bot_version'],
nexus_version=self.application.config['application']['nexus_version'],
language=top_languages.get(self.chat.language, self.chat.language),
tzinfo=self.chat.tzinfo or 'UTC',
)
if not self.is_group_mode and self.application.config['application']['views']['settings']['has_discovery_button']:
text = f"{text}\n\n{t('NEXUS_DISCOVERY_DESCRIPTION', language=self.chat.language)}"
buttons = []
if self.has_language_buttons:
buttons.append([])
for language in sorted(top_languages):
if len(buttons[-1]) >= 4:
buttons.append([])
buttons[-1].append(
Button.inline(
text=top_languages[language],
data=f'/settings_sl_{language}'
)
)
if self.is_group_mode:
return text, buttons
if self.application.config['application']['views']['settings']['has_system_messaging_button']:
buttons.append([
Button.inline(
text=(
f'{t("SYSTEM_MESSAGING_OPTION", language=self.chat.language)}: '
f'{boolean_emoji[self.chat.is_system_messaging_enabled]}'
),
data=f'/settings_ssm_{1 - int(self.chat.is_system_messaging_enabled)}'
)
])
if self.application.config['application']['views']['settings']['has_discovery_button']:
buttons.append([
Button.inline(
text=(
f'{t("DISCOVERY_OPTION", language=self.chat.language)}: '
f'{boolean_emoji[self.chat.is_discovery_enabled]}'
),
data=f'/settings_sd_{1 - int(self.chat.is_discovery_enabled)}'
)
])
if self.application.config['application']['views']['settings']['has_location_button']:
buttons.append([
Button.request_location('Setup preferences automatically', resize=True)
])
return text, buttons

20
nexus/cognitron/schema/BUILD.bazel
View File

@ -1,24 +1,4 @@
load("@pip_modules//:requirements.bzl", "requirement")
load("@rules_python//python:defs.bzl", "py_library")
exports_files([
"scimag.yaml",
"scitech.yaml",
])
py_library(
name = "schema",
srcs = glob([
"**/*.py",
]),
data = [
"scimag.yaml",
"scitech.yaml",
],
srcs_version = "PY3",
visibility = ["//visibility:public"],
deps = [
requirement("tantipy"),
requirement("pyyaml"),
],
)

7
nexus/cognitron/schema/__init__.py
View File

@ -1,7 +0,0 @@
from .scimag import scimag_coder
from .scitech import scitech_coder
coders = {
'scimag': scimag_coder,
'scitech': scitech_coder,
}

5
nexus/cognitron/schema/scimag.py
View File

@ -1,5 +0,0 @@
import yaml
from tantipy import TantivyCoder
with open('nexus/cognitron/schema/scimag.yaml') as file:
scimag_coder = TantivyCoder(yaml.safe_load(file.read()))

5
nexus/cognitron/schema/scitech.py
View File

@ -1,5 +0,0 @@
import yaml
from tantipy import TantivyCoder
with open('nexus/cognitron/schema/scitech.yaml') as file:
scitech_coder = TantivyCoder(yaml.safe_load(file.read()))

15
nexus/hub/aioclient/BUILD.bazel Normal file
View File

@ -0,0 +1,15 @@
load("@pip_modules//:requirements.bzl", "requirement")
py_library(
name = "aioclient",
srcs = glob(["**/*.py"]),
visibility = ["//visibility:public"],
deps = [
requirement("grpcio"),
"//idm/api2/proto:idm_proto_py",
requirement("aiokit"),
"//nexus/hub/proto:hub_grpc_py",
"//nexus/hub/proto:hub_proto_py",
"//nexus/models/proto:models_proto_py",
],
)

3
nexus/hub/aioclient/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .aioclient import HubGrpcClient
__all__ = ['HubGrpcClient']

72
nexus/hub/aioclient/aioclient.py Normal file
View File

@ -0,0 +1,72 @@
from typing import Optional
from aiokit import AioThing
from grpc.experimental.aio import insecure_channel
from idm.api2.proto.chats_service_pb2 import ChatData as ChatDataPb
from nexus.hub.proto.delivery_service_pb2 import \
StartDeliveryRequest as StartDeliveryRequestPb
from nexus.hub.proto.delivery_service_pb2 import \
StartDeliveryResponse as StartDeliveryResponsePb
from nexus.hub.proto.delivery_service_pb2_grpc import DeliveryStub
from nexus.hub.proto.submitter_service_pb2 import \
SubmitRequest as SubmitRequestPb
from nexus.hub.proto.submitter_service_pb2 import \
SubmitResponse as SubmitResponsePb
from nexus.hub.proto.submitter_service_pb2_grpc import SubmitterStub
from nexus.models.proto.typed_document_pb2 import \
TypedDocument as TypedDocumentPb
class HubGrpcClient(AioThing):
def __init__(
self,
base_url: str,
):
super().__init__()
self.channel = insecure_channel(base_url, [
('grpc.dns_min_time_between_resolutions_ms', 1000),
('grpc.initial_reconnect_backoff_ms', 1000),
('grpc.lb_policy_name', 'round_robin'),
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.delivery_stub = DeliveryStub(self.channel)
self.submitter_stub = SubmitterStub(self.channel)
async def start(self):
await self.channel.channel_ready()
async def stop(self):
await self.channel.close()
async def start_delivery(
self,
typed_document_pb: TypedDocumentPb,
chat: ChatDataPb,
request_id: Optional[str],
session_id: Optional[str],
) -> StartDeliveryResponsePb:
return await self.delivery_stub.start_delivery(
StartDeliveryRequestPb(
typed_document=typed_document_pb,
chat=chat,
),
metadata=(('request-id', request_id), ('session-id', session_id))
)
async def submit(
self,
telegram_document: bytes,
telegram_file_id: str,
chat: ChatDataPb,
request_id: Optional[str] = None,
session_id: Optional[str] = None,
) -> SubmitResponsePb:
return await self.submitter_stub.submit(
SubmitRequestPb(
telegram_document=telegram_document,
telegram_file_id=telegram_file_id,
chat=chat,
),
metadata=(('request-id', request_id), ('session-id', session_id))
)

27
nexus/hub/proto/BUILD.bazel Normal file
View File

@ -0,0 +1,27 @@
load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
package(default_visibility = ["//visibility:public"])
proto_library(
name = "hub_proto",
srcs = glob([
"*.proto",
]),
deps = [
"//idm/api2/proto:idm_proto",
"//nexus/models/proto:models_proto",
"@com_google_protobuf//:wrappers_proto",
],
)
py_proto_library(
name = "hub_proto_py",
deps = [":hub_proto"],
)
py_grpc_library(
name = "hub_grpc_py",
srcs = [":hub_proto"],
deps = [":hub_proto_py"],
)

23
nexus/hub/proto/delivery_service.proto Normal file
View File

@ -0,0 +1,23 @@
syntax = "proto3";
package nexus.hub.proto;
import "nexus/models/proto/typed_document.proto";
import "idm/api2/proto/chats_service.proto";
message StartDeliveryRequest {
nexus.models.proto.TypedDocument typed_document = 1;
idm.api2.proto.ChatData chat = 2;
}
message StartDeliveryResponse {
enum Status {
OK = 0;
TOO_MANY_DOWNLOADS = 1;
ALREADY_DOWNLOADING = 2;
}
Status status = 1;
}
service Delivery {
rpc start_delivery(StartDeliveryRequest) returns (StartDeliveryResponse) {};
}

183
nexus/hub/proto/delivery_service_pb2.py Normal file
View File

@ -0,0 +1,183 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nexus/hub/proto/delivery_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from idm.api2.proto import \
chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
from nexus.models.proto import \
typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/hub/proto/delivery_service.proto',
package='nexus.hub.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n&nexus/hub/proto/delivery_service.proto\x12\x0fnexus.hub.proto\x1a\'nexus/models/proto/typed_document.proto\x1a\"idm/api2/proto/chats_service.proto\"y\n\x14StartDeliveryRequest\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x99\x01\n\x15StartDeliveryResponse\x12=\n\x06status\x18\x01 \x01(\x0e\x32-.nexus.hub.proto.StartDeliveryResponse.Status\"A\n\x06Status\x12\x06\n\x02OK\x10\x00\x12\x16\n\x12TOO_MANY_DOWNLOADS\x10\x01\x12\x17\n\x13\x41LREADY_DOWNLOADING\x10\x03\x32m\n\x08\x44\x65livery\x12\x61\n\x0estart_delivery\x12%.nexus.hub.proto.StartDeliveryRequest\x1a&.nexus.hub.proto.StartDeliveryResponse\"\x00\x62\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
_STARTDELIVERYRESPONSE_STATUS = _descriptor.EnumDescriptor(
name='Status',
full_name='nexus.hub.proto.StartDeliveryResponse.Status',
filename=None,
file=DESCRIPTOR,
create_key=_descriptor._internal_create_key,
values=[
_descriptor.EnumValueDescriptor(
name='OK', index=0, number=0,
serialized_options=None,
type=None,
create_key=_descriptor._internal_create_key),
_descriptor.EnumValueDescriptor(
name='TOO_MANY_DOWNLOADS', index=1, number=1,
serialized_options=None,
type=None,
create_key=_descriptor._internal_create_key),
_descriptor.EnumValueDescriptor(
name='ALREADY_DOWNLOADING', index=2, number=3,
serialized_options=None,
type=None,
create_key=_descriptor._internal_create_key),
],
containing_type=None,
serialized_options=None,
serialized_start=348,
serialized_end=413,
)
_sym_db.RegisterEnumDescriptor(_STARTDELIVERYRESPONSE_STATUS)
_STARTDELIVERYREQUEST = _descriptor.Descriptor(
name='StartDeliveryRequest',
full_name='nexus.hub.proto.StartDeliveryRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='typed_document', full_name='nexus.hub.proto.StartDeliveryRequest.typed_document', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='chat', full_name='nexus.hub.proto.StartDeliveryRequest.chat', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=136,
serialized_end=257,
)
_STARTDELIVERYRESPONSE = _descriptor.Descriptor(
name='StartDeliveryResponse',
full_name='nexus.hub.proto.StartDeliveryResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='status', full_name='nexus.hub.proto.StartDeliveryResponse.status', index=0,
number=1, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
_STARTDELIVERYRESPONSE_STATUS,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=260,
serialized_end=413,
)
_STARTDELIVERYREQUEST.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
_STARTDELIVERYREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
_STARTDELIVERYRESPONSE.fields_by_name['status'].enum_type = _STARTDELIVERYRESPONSE_STATUS
_STARTDELIVERYRESPONSE_STATUS.containing_type = _STARTDELIVERYRESPONSE
DESCRIPTOR.message_types_by_name['StartDeliveryRequest'] = _STARTDELIVERYREQUEST
DESCRIPTOR.message_types_by_name['StartDeliveryResponse'] = _STARTDELIVERYRESPONSE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
StartDeliveryRequest = _reflection.GeneratedProtocolMessageType('StartDeliveryRequest', (_message.Message,), {
'DESCRIPTOR' : _STARTDELIVERYREQUEST,
'__module__' : 'nexus.hub.proto.delivery_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.hub.proto.StartDeliveryRequest)
})
_sym_db.RegisterMessage(StartDeliveryRequest)
StartDeliveryResponse = _reflection.GeneratedProtocolMessageType('StartDeliveryResponse', (_message.Message,), {
'DESCRIPTOR' : _STARTDELIVERYRESPONSE,
'__module__' : 'nexus.hub.proto.delivery_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.hub.proto.StartDeliveryResponse)
})
_sym_db.RegisterMessage(StartDeliveryResponse)
_DELIVERY = _descriptor.ServiceDescriptor(
name='Delivery',
full_name='nexus.hub.proto.Delivery',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=415,
serialized_end=524,
methods=[
_descriptor.MethodDescriptor(
name='start_delivery',
full_name='nexus.hub.proto.Delivery.start_delivery',
index=0,
containing_service=None,
input_type=_STARTDELIVERYREQUEST,
output_type=_STARTDELIVERYRESPONSE,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_DELIVERY)
DESCRIPTOR.services_by_name['Delivery'] = _DELIVERY
# @@protoc_insertion_point(module_scope)

66
nexus/hub/proto/delivery_service_pb2_grpc.py Normal file
View File

@ -0,0 +1,66 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from nexus.hub.proto import \
delivery_service_pb2 as nexus_dot_hub_dot_proto_dot_delivery__service__pb2
class DeliveryStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.start_delivery = channel.unary_unary(
'/nexus.hub.proto.Delivery/start_delivery',
request_serializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryRequest.SerializeToString,
response_deserializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryResponse.FromString,
)
class DeliveryServicer(object):
"""Missing associated documentation comment in .proto file."""
def start_delivery(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_DeliveryServicer_to_server(servicer, server):
rpc_method_handlers = {
'start_delivery': grpc.unary_unary_rpc_method_handler(
servicer.start_delivery,
request_deserializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryRequest.FromString,
response_serializer=nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'nexus.hub.proto.Delivery', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Delivery(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def start_delivery(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/nexus.hub.proto.Delivery/start_delivery',
nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryRequest.SerializeToString,
nexus_dot_hub_dot_proto_dot_delivery__service__pb2.StartDeliveryResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

15
nexus/hub/proto/submitter_service.proto Normal file
View File

@ -0,0 +1,15 @@
syntax = "proto3";
package nexus.hub.proto;
import "idm/api2/proto/chats_service.proto";
message SubmitRequest {
bytes telegram_document = 1;
string telegram_file_id = 2;
idm.api2.proto.ChatData chat = 3;
}
message SubmitResponse { }
service Submitter {
rpc submit(SubmitRequest) returns (SubmitResponse) {};
}

140
nexus/hub/proto/submitter_service_pb2.py Normal file
View File

@ -0,0 +1,140 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nexus/hub/proto/submitter_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from idm.api2.proto import \
chats_service_pb2 as idm_dot_api2_dot_proto_dot_chats__service__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/hub/proto/submitter_service.proto',
package='nexus.hub.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\'nexus/hub/proto/submitter_service.proto\x12\x0fnexus.hub.proto\x1a\"idm/api2/proto/chats_service.proto\"R\n\rSubmitRequest\x12\x19\n\x11telegram_document\x18\x01 \x01(\x0c\x12&\n\x04\x63hat\x18\x02 \x01(\x0b\x32\x18.idm.api2.proto.ChatData\"\x10\n\x0eSubmitResponse2X\n\tSubmitter\x12K\n\x06submit\x12\x1e.nexus.hub.proto.SubmitRequest\x1a\x1f.nexus.hub.proto.SubmitResponse\"\x00\x62\x06proto3'
,
dependencies=[idm_dot_api2_dot_proto_dot_chats__service__pb2.DESCRIPTOR,])
_SUBMITREQUEST = _descriptor.Descriptor(
name='SubmitRequest',
full_name='nexus.hub.proto.SubmitRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='telegram_document', full_name='nexus.hub.proto.SubmitRequest.telegram_document', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='chat', full_name='nexus.hub.proto.SubmitRequest.chat', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=96,
serialized_end=178,
)
_SUBMITRESPONSE = _descriptor.Descriptor(
name='SubmitResponse',
full_name='nexus.hub.proto.SubmitResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=180,
serialized_end=196,
)
_SUBMITREQUEST.fields_by_name['chat'].message_type = idm_dot_api2_dot_proto_dot_chats__service__pb2._CHATDATA
DESCRIPTOR.message_types_by_name['SubmitRequest'] = _SUBMITREQUEST
DESCRIPTOR.message_types_by_name['SubmitResponse'] = _SUBMITRESPONSE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
SubmitRequest = _reflection.GeneratedProtocolMessageType('SubmitRequest', (_message.Message,), {
'DESCRIPTOR' : _SUBMITREQUEST,
'__module__' : 'nexus.hub.proto.submitter_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.hub.proto.SubmitRequest)
})
_sym_db.RegisterMessage(SubmitRequest)
SubmitResponse = _reflection.GeneratedProtocolMessageType('SubmitResponse', (_message.Message,), {
'DESCRIPTOR' : _SUBMITRESPONSE,
'__module__' : 'nexus.hub.proto.submitter_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.hub.proto.SubmitResponse)
})
_sym_db.RegisterMessage(SubmitResponse)
_SUBMITTER = _descriptor.ServiceDescriptor(
name='Submitter',
full_name='nexus.hub.proto.Submitter',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=198,
serialized_end=286,
methods=[
_descriptor.MethodDescriptor(
name='submit',
full_name='nexus.hub.proto.Submitter.submit',
index=0,
containing_service=None,
input_type=_SUBMITREQUEST,
output_type=_SUBMITRESPONSE,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_SUBMITTER)
DESCRIPTOR.services_by_name['Submitter'] = _SUBMITTER
# @@protoc_insertion_point(module_scope)

67
nexus/hub/proto/submitter_service_pb2_grpc.py Normal file
View File

@ -0,0 +1,67 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from nexus.hub.proto import \
submitter_service_pb2 as \
nexus_dot_hub_dot_proto_dot_submitter__service__pb2
class SubmitterStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.submit = channel.unary_unary(
'/nexus.hub.proto.Submitter/submit',
request_serializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitRequest.SerializeToString,
response_deserializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitResponse.FromString,
)
class SubmitterServicer(object):
"""Missing associated documentation comment in .proto file."""
def submit(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_SubmitterServicer_to_server(servicer, server):
rpc_method_handlers = {
'submit': grpc.unary_unary_rpc_method_handler(
servicer.submit,
request_deserializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitRequest.FromString,
response_serializer=nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'nexus.hub.proto.Submitter', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Submitter(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def submit(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/nexus.hub.proto.Submitter/submit',
nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitRequest.SerializeToString,
nexus_dot_hub_dot_proto_dot_submitter__service__pb2.SubmitResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

2
nexus/ingest/README.md
View File

@ -23,7 +23,6 @@ jobs:
kwargs:
kafka_hosts:
- kafka-0.example.net
- kafka-1.example.net
topic_name: operations_binary
libgen-api:
class: nexus.ingest.jobs.LibgenApiJob
@ -39,7 +38,6 @@ jobs:
kwargs:
kafka_hosts:
- kafka-0.example.net
- kafka-1.example.net
topic_name: operations_binary
log_path: '/var/log/nexus-ingest/{{ ENV_TYPE }}'
```

17
nexus/meta_api/aioclient/BUILD.bazel Normal file
View File

@ -0,0 +1,17 @@
load("@pip_modules//:requirements.bzl", "requirement")
load("@rules_python//python:defs.bzl", "py_library")
py_library(
name = "aioclient",
srcs = glob(["**/*.py"]),
visibility = ["//visibility:public"],
deps = [
requirement("grpcio"),
requirement("lru-dict"),
requirement("tenacity"),
requirement("aiokit"),
"//nexus/meta_api/proto:meta_api_grpc_py",
"//nexus/meta_api/proto:meta_api_proto_py",
"//nexus/models/proto:models_proto_py",
],
)

3
nexus/meta_api/aioclient/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .aioclient import MetaApiGrpcClient
__all__ = ['MetaApiGrpcClient']

132
nexus/meta_api/aioclient/aioclient.py Normal file
View File

@ -0,0 +1,132 @@
from typing import (
List,
Optional,
Tuple,
Union,
)
from aiokit import AioThing
from grpc import StatusCode
from grpc.experimental.aio import (
AioRpcError,
insecure_channel,
)
from lru import LRU
from nexus.meta_api.proto.documents_service_pb2 import \
RollRequest as RollRequestPb
from nexus.meta_api.proto.documents_service_pb2 import \
RollResponse as RollResponsePb
from nexus.meta_api.proto.documents_service_pb2 import \
TypedDocumentRequest as TypedDocumentRequestPb
from nexus.meta_api.proto.documents_service_pb2_grpc import DocumentsStub
from nexus.meta_api.proto.meta_search_service_pb2 import \
SearchRequest as SearchRequestPb
from nexus.meta_api.proto.meta_search_service_pb2 import \
SearchResponse as SearchResponsePb
from nexus.meta_api.proto.meta_search_service_pb2_grpc import MetaSearchStub
from nexus.models.proto.typed_document_pb2 import \
TypedDocument as TypedDocumentPb
from tenacity import (
retry,
retry_if_exception,
stop_after_attempt,
wait_fixed,
)
class MetaApiGrpcClient(AioThing):
def __init__(self, base_url):
super().__init__()
self.channel = insecure_channel(base_url, [
('grpc.dns_min_time_between_resolutions_ms', 1000),
('grpc.initial_reconnect_backoff_ms', 1000),
('grpc.lb_policy_name', 'round_robin'),
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.meta_search_stub = MetaSearchStub(self.channel)
self.documents_stub = DocumentsStub(self.channel)
self.cache = LRU(4096)
async def start(self):
await self.channel.channel_ready()
async def stop(self):
await self.channel.close()
async def get(
self,
schema: str,
document_id: int,
position: Optional[int] = None,
request_id: Optional[str] = None,
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> TypedDocumentPb:
return await self.documents_stub.get(
TypedDocumentRequestPb(
schema=schema,
document_id=document_id,
position=position,
session_id=session_id,
user_id=user_id,
),
metadata=(
('request-id', request_id),
),
)
async def roll(
self,
language: Optional[str] = None,
request_id: Optional[str] = None,
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> RollResponsePb:
return await self.documents_stub.roll(
RollRequestPb(
language=language,
session_id=session_id,
user_id=user_id,
),
metadata=(
('request-id', request_id),
),
)
@retry(
retry=retry_if_exception(
lambda e: isinstance(e, AioRpcError) and (
e.code() == StatusCode.CANCELLED
or e.code() == StatusCode.UNAVAILABLE
)
),
reraise=True,
stop=stop_after_attempt(5),
wait=wait_fixed(2),
)
async def search(
self,
schemas: Union[List[str], Tuple[str]],
query: str,
page: Optional[int] = None,
page_size: Optional[int] = None,
language: Optional[str] = None,
request_id: Optional[str] = None,
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> SearchResponsePb:
return await self.meta_search_stub.search(
SearchRequestPb(
schemas=schemas,
query=query,
page=page,
page_size=page_size,
language=language,
session_id=session_id,
user_id=user_id,
),
metadata=(
('request-id', request_id),
),
)

34
nexus/meta_api/proto/BUILD.bazel Normal file
View File

@ -0,0 +1,34 @@
load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
load("@rules_rust//proto:proto.bzl", "rust_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
package(default_visibility = ["//visibility:public"])
proto_library(
name = "meta_api_proto",
srcs = glob([
"*.proto",
]),
deps = [
"//nexus/models/proto:models_proto",
"@com_google_protobuf//:wrappers_proto",
],
)
py_proto_library(
name = "meta_api_proto_py",
deps = [":meta_api_proto"],
)
py_grpc_library(
name = "meta_api_grpc_py",
srcs = [":meta_api_proto"],
deps = [":meta_api_proto_py"],
)
rust_proto_library(
name = "meta_api_proto_rust",
rust_deps = ["//rules/rust/cargo:protobuf"],
visibility = ["//visibility:public"],
deps = [":meta_api_proto"],
)

29
nexus/meta_api/proto/documents_service.proto Normal file
View File

@ -0,0 +1,29 @@
syntax = "proto3";
package nexus.meta_api.proto;
import "nexus/models/proto/typed_document.proto";
message RollRequest {
string language = 1;
string session_id = 2;
int64 user_id = 3;
}
message RollResponse {
uint64 document_id = 1;
}
message TypedDocumentRequest {
string schema = 1;
uint64 document_id = 2;
uint32 position = 3;
string session_id = 4;
int64 user_id = 5;
}
message PutTypedDocumentResponse {}
service Documents {
rpc get (TypedDocumentRequest) returns (nexus.models.proto.TypedDocument) {}
rpc roll (RollRequest) returns (RollResponse) {}
}

264
nexus/meta_api/proto/documents_service_pb2.py Executable file
View File

@ -0,0 +1,264 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nexus/meta_api/proto/documents_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from nexus.models.proto import \
typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/meta_api/proto/documents_service.proto',
package='nexus.meta_api.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\"\x1a\n\x18PutTypedDocumentResponse2\xb4\x01\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x62\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
_ROLLREQUEST = _descriptor.Descriptor(
name='RollRequest',
full_name='nexus.meta_api.proto.RollRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='language', full_name='nexus.meta_api.proto.RollRequest.language', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='session_id', full_name='nexus.meta_api.proto.RollRequest.session_id', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='user_id', full_name='nexus.meta_api.proto.RollRequest.user_id', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=111,
serialized_end=179,
)
_ROLLRESPONSE = _descriptor.Descriptor(
name='RollResponse',
full_name='nexus.meta_api.proto.RollResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='document_id', full_name='nexus.meta_api.proto.RollResponse.document_id', index=0,
number=1, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=181,
serialized_end=216,
)
_TYPEDDOCUMENTREQUEST = _descriptor.Descriptor(
name='TypedDocumentRequest',
full_name='nexus.meta_api.proto.TypedDocumentRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='schema', full_name='nexus.meta_api.proto.TypedDocumentRequest.schema', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='document_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.document_id', index=1,
number=2, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='position', full_name='nexus.meta_api.proto.TypedDocumentRequest.position', index=2,
number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='session_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.session_id', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='user_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.user_id', index=4,
number=5, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=218,
serialized_end=332,
)
_PUTTYPEDDOCUMENTRESPONSE = _descriptor.Descriptor(
name='PutTypedDocumentResponse',
full_name='nexus.meta_api.proto.PutTypedDocumentResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=334,
serialized_end=360,
)
DESCRIPTOR.message_types_by_name['RollRequest'] = _ROLLREQUEST
DESCRIPTOR.message_types_by_name['RollResponse'] = _ROLLRESPONSE
DESCRIPTOR.message_types_by_name['TypedDocumentRequest'] = _TYPEDDOCUMENTREQUEST
DESCRIPTOR.message_types_by_name['PutTypedDocumentResponse'] = _PUTTYPEDDOCUMENTRESPONSE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
RollRequest = _reflection.GeneratedProtocolMessageType('RollRequest', (_message.Message,), {
'DESCRIPTOR' : _ROLLREQUEST,
'__module__' : 'nexus.meta_api.proto.documents_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.RollRequest)
})
_sym_db.RegisterMessage(RollRequest)
RollResponse = _reflection.GeneratedProtocolMessageType('RollResponse', (_message.Message,), {
'DESCRIPTOR' : _ROLLRESPONSE,
'__module__' : 'nexus.meta_api.proto.documents_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.RollResponse)
})
_sym_db.RegisterMessage(RollResponse)
TypedDocumentRequest = _reflection.GeneratedProtocolMessageType('TypedDocumentRequest', (_message.Message,), {
'DESCRIPTOR' : _TYPEDDOCUMENTREQUEST,
'__module__' : 'nexus.meta_api.proto.documents_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.TypedDocumentRequest)
})
_sym_db.RegisterMessage(TypedDocumentRequest)
PutTypedDocumentResponse = _reflection.GeneratedProtocolMessageType('PutTypedDocumentResponse', (_message.Message,), {
'DESCRIPTOR' : _PUTTYPEDDOCUMENTRESPONSE,
'__module__' : 'nexus.meta_api.proto.documents_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.PutTypedDocumentResponse)
})
_sym_db.RegisterMessage(PutTypedDocumentResponse)
_DOCUMENTS = _descriptor.ServiceDescriptor(
name='Documents',
full_name='nexus.meta_api.proto.Documents',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=363,
serialized_end=543,
methods=[
_descriptor.MethodDescriptor(
name='get',
full_name='nexus.meta_api.proto.Documents.get',
index=0,
containing_service=None,
input_type=_TYPEDDOCUMENTREQUEST,
output_type=nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
_descriptor.MethodDescriptor(
name='roll',
full_name='nexus.meta_api.proto.Documents.roll',
index=1,
containing_service=None,
input_type=_ROLLREQUEST,
output_type=_ROLLRESPONSE,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_DOCUMENTS)
DESCRIPTOR.services_by_name['Documents'] = _DOCUMENTS
# @@protoc_insertion_point(module_scope)

102
nexus/meta_api/proto/documents_service_pb2_grpc.py Executable file
View File

@ -0,0 +1,102 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from nexus.meta_api.proto import \
documents_service_pb2 as \
nexus_dot_meta__api_dot_proto_dot_documents__service__pb2
from nexus.models.proto import \
typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
class DocumentsStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.get = channel.unary_unary(
'/nexus.meta_api.proto.Documents/get',
request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TypedDocumentRequest.SerializeToString,
response_deserializer=nexus_dot_models_dot_proto_dot_typed__document__pb2.TypedDocument.FromString,
)
self.roll = channel.unary_unary(
'/nexus.meta_api.proto.Documents/roll',
request_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.SerializeToString,
response_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString,
)
class DocumentsServicer(object):
"""Missing associated documentation comment in .proto file."""
def get(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def roll(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_DocumentsServicer_to_server(servicer, server):
rpc_method_handlers = {
'get': grpc.unary_unary_rpc_method_handler(
servicer.get,
request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TypedDocumentRequest.FromString,
response_serializer=nexus_dot_models_dot_proto_dot_typed__document__pb2.TypedDocument.SerializeToString,
),
'roll': grpc.unary_unary_rpc_method_handler(
servicer.roll,
request_deserializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.FromString,
response_serializer=nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'nexus.meta_api.proto.Documents', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Documents(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def get(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Documents/get',
nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.TypedDocumentRequest.SerializeToString,
nexus_dot_models_dot_proto_dot_typed__document__pb2.TypedDocument.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def roll(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Documents/roll',
nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollRequest.SerializeToString,
nexus_dot_meta__api_dot_proto_dot_documents__service__pb2.RollResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

29
nexus/meta_api/proto/meta_search_service.proto Normal file
View File

@ -0,0 +1,29 @@
syntax = "proto3";
package nexus.meta_api.proto;
import "nexus/models/proto/typed_document.proto";
message ScoredDocument {
nexus.models.proto.TypedDocument typed_document = 1;
float score = 3;
uint32 position = 4;
}
message SearchResponse {
repeated ScoredDocument scored_documents = 1;
bool has_next = 2;
}
message SearchRequest {
repeated string schemas = 1;
string query = 2;
uint32 page = 3;
uint32 page_size = 4;
string language = 5;
int64 user_id = 6;
string session_id = 7;
}
service MetaSearch {
rpc search (SearchRequest) returns (SearchResponse) {}
}

251
nexus/meta_api/proto/meta_search_service_pb2.py Executable file
View File

@ -0,0 +1,251 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nexus/meta_api/proto/meta_search_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from nexus.models.proto import \
typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/meta_api/proto/meta_search_service.proto',
package='nexus.meta_api.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x03 \x01(\x02\x12\x10\n\x08position\x18\x04 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\xa3\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t\x12\x1a\n\x12skip_query_rewrite\x18\t \x01(\x08\x32\x63\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
_SCOREDDOCUMENT = _descriptor.Descriptor(
name='ScoredDocument',
full_name='nexus.meta_api.proto.ScoredDocument',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='typed_document', full_name='nexus.meta_api.proto.ScoredDocument.typed_document', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='score', full_name='nexus.meta_api.proto.ScoredDocument.score', index=1,
number=3, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='position', full_name='nexus.meta_api.proto.ScoredDocument.position', index=2,
number=4, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=113,
serialized_end=221,
)
_SEARCHRESPONSE = _descriptor.Descriptor(
name='SearchResponse',
full_name='nexus.meta_api.proto.SearchResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='scored_documents', full_name='nexus.meta_api.proto.SearchResponse.scored_documents', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='has_next', full_name='nexus.meta_api.proto.SearchResponse.has_next', index=1,
number=2, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=223,
serialized_end=321,
)
_SEARCHREQUEST = _descriptor.Descriptor(
name='SearchRequest',
full_name='nexus.meta_api.proto.SearchRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='schemas', full_name='nexus.meta_api.proto.SearchRequest.schemas', index=0,
number=1, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='query', full_name='nexus.meta_api.proto.SearchRequest.query', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='page', full_name='nexus.meta_api.proto.SearchRequest.page', index=2,
number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='page_size', full_name='nexus.meta_api.proto.SearchRequest.page_size', index=3,
number=4, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='language', full_name='nexus.meta_api.proto.SearchRequest.language', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='user_id', full_name='nexus.meta_api.proto.SearchRequest.user_id', index=5,
number=6, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='session_id', full_name='nexus.meta_api.proto.SearchRequest.session_id', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='skip_query_rewrite', full_name='nexus.meta_api.proto.SearchRequest.skip_query_rewrite', index=7,
number=9, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=324,
serialized_end=487,
)
_SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
_SEARCHRESPONSE.fields_by_name['scored_documents'].message_type = _SCOREDDOCUMENT
DESCRIPTOR.message_types_by_name['ScoredDocument'] = _SCOREDDOCUMENT
DESCRIPTOR.message_types_by_name['SearchResponse'] = _SEARCHRESPONSE
DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
ScoredDocument = _reflection.GeneratedProtocolMessageType('ScoredDocument', (_message.Message,), {
'DESCRIPTOR' : _SCOREDDOCUMENT,
'__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.ScoredDocument)
})
_sym_db.RegisterMessage(ScoredDocument)
SearchResponse = _reflection.GeneratedProtocolMessageType('SearchResponse', (_message.Message,), {
'DESCRIPTOR' : _SEARCHRESPONSE,
'__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchResponse)
})
_sym_db.RegisterMessage(SearchResponse)
SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
'DESCRIPTOR' : _SEARCHREQUEST,
'__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchRequest)
})
_sym_db.RegisterMessage(SearchRequest)
_METASEARCH = _descriptor.ServiceDescriptor(
name='MetaSearch',
full_name='nexus.meta_api.proto.MetaSearch',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=489,
serialized_end=588,
methods=[
_descriptor.MethodDescriptor(
name='search',
full_name='nexus.meta_api.proto.MetaSearch.search',
index=0,
containing_service=None,
input_type=_SEARCHREQUEST,
output_type=_SEARCHRESPONSE,
serialized_options=None,
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_METASEARCH)
DESCRIPTOR.services_by_name['MetaSearch'] = _METASEARCH
# @@protoc_insertion_point(module_scope)

67
nexus/meta_api/proto/meta_search_service_pb2_grpc.py Executable file
View File

@ -0,0 +1,67 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from nexus.meta_api.proto import \
meta_search_service_pb2 as \
nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2
class MetaSearchStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.search = channel.unary_unary(
'/nexus.meta_api.proto.MetaSearch/search',
request_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
response_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
)
class MetaSearchServicer(object):
"""Missing associated documentation comment in .proto file."""
def search(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_MetaSearchServicer_to_server(servicer, server):
rpc_method_handlers = {
'search': grpc.unary_unary_rpc_method_handler(
servicer.search,
request_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.FromString,
response_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'nexus.meta_api.proto.MetaSearch', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class MetaSearch(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def search(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.MetaSearch/search',
nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

2
nexus/models/proto/BUILD.bazel
View File

@ -1,5 +1,5 @@
load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
load("@io_bazel_rules_rust//proto:proto.bzl", "rust_proto_library")
load("@rules_rust//proto:proto.bzl", "rust_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
proto_library(

7
nexus/models/proto/operation.proto
View File

@ -13,10 +13,17 @@ message CrossReferenceOperation {
message DocumentOperation {
oneof operation {
Vote vote = 1;
UpdateDocument update_document = 3;
};
}
message Vote {
int64 document_id = 1;
int32 value = 2;
int64 voter_id = 3;
}
message UpdateDocument {
repeated string fields = 1;
bool should_fill_from_external_source = 2;

78
nexus/models/proto/operation_pb2.py
View File

@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\"nexus/models/proto/operation.proto\x12\x12nexus.models.proto\x1a\'nexus/models/proto/typed_document.proto\"k\n\x17\x43rossReferenceOperation\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\x1b\n\x13last_retry_unixtime\x18\x03 \x01(\r\x12\x13\n\x0bretry_count\x18\x04 \x01(\r\"_\n\x11\x44ocumentOperation\x12=\n\x0fupdate_document\x18\x03 \x01(\x0b\x32\".nexus.models.proto.UpdateDocumentH\x00\x42\x0b\n\toperation\"\xa6\x01\n\x0eUpdateDocument\x12\x0e\n\x06\x66ields\x18\x01 \x03(\t\x12(\n should_fill_from_external_source\x18\x02 \x01(\x08\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\x12\x0f\n\x07reindex\x18\x04 \x01(\x08\x12\x39\n\x0etyped_document\x18\x05 \x01(\x0b\x32!.nexus.models.proto.TypedDocumentb\x06proto3'
serialized_pb=b'\n\"nexus/models/proto/operation.proto\x12\x12nexus.models.proto\x1a\'nexus/models/proto/typed_document.proto\"k\n\x17\x43rossReferenceOperation\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\x1b\n\x13last_retry_unixtime\x18\x03 \x01(\r\x12\x13\n\x0bretry_count\x18\x04 \x01(\r\"\x89\x01\n\x11\x44ocumentOperation\x12(\n\x04vote\x18\x01 \x01(\x0b\x32\x18.nexus.models.proto.VoteH\x00\x12=\n\x0fupdate_document\x18\x03 \x01(\x0b\x32\".nexus.models.proto.UpdateDocumentH\x00\x42\x0b\n\toperation\"b\n\x04Vote\x12\x10\n\x08voter_id\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x05\x12\x39\n\x0etyped_document\x18\x03 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\"\xa6\x01\n\x0eUpdateDocument\x12\x0e\n\x06\x66ields\x18\x01 \x03(\t\x12(\n should_fill_from_external_source\x18\x02 \x01(\x08\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\x12\x0f\n\x07reindex\x18\x04 \x01(\x08\x12\x39\n\x0etyped_document\x18\x05 \x01(\x0b\x32!.nexus.models.proto.TypedDocumentb\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
@ -90,7 +90,14 @@ _DOCUMENTOPERATION = _descriptor.Descriptor(
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='update_document', full_name='nexus.models.proto.DocumentOperation.update_document', index=0,
name='vote', full_name='nexus.models.proto.DocumentOperation.vote', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='update_document', full_name='nexus.models.proto.DocumentOperation.update_document', index=1,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@ -113,8 +120,54 @@ _DOCUMENTOPERATION = _descriptor.Descriptor(
create_key=_descriptor._internal_create_key,
fields=[]),
],
serialized_start=208,
serialized_end=303,
serialized_start=209,
serialized_end=346,
)
_VOTE = _descriptor.Descriptor(
name='Vote',
full_name='nexus.models.proto.Vote',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='voter_id', full_name='nexus.models.proto.Vote.voter_id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='value', full_name='nexus.models.proto.Vote.value', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='typed_document', full_name='nexus.models.proto.Vote.typed_document', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=348,
serialized_end=446,
)
@ -173,17 +226,23 @@ _UPDATEDOCUMENT = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=306,
serialized_end=472,
serialized_start=449,
serialized_end=615,
)
_DOCUMENTOPERATION.fields_by_name['vote'].message_type = _VOTE
_DOCUMENTOPERATION.fields_by_name['update_document'].message_type = _UPDATEDOCUMENT
_DOCUMENTOPERATION.oneofs_by_name['operation'].fields.append(
_DOCUMENTOPERATION.fields_by_name['vote'])
_DOCUMENTOPERATION.fields_by_name['vote'].containing_oneof = _DOCUMENTOPERATION.oneofs_by_name['operation']
_DOCUMENTOPERATION.oneofs_by_name['operation'].fields.append(
_DOCUMENTOPERATION.fields_by_name['update_document'])
_DOCUMENTOPERATION.fields_by_name['update_document'].containing_oneof = _DOCUMENTOPERATION.oneofs_by_name['operation']
_VOTE.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
_UPDATEDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
DESCRIPTOR.message_types_by_name['CrossReferenceOperation'] = _CROSSREFERENCEOPERATION
DESCRIPTOR.message_types_by_name['DocumentOperation'] = _DOCUMENTOPERATION
DESCRIPTOR.message_types_by_name['Vote'] = _VOTE
DESCRIPTOR.message_types_by_name['UpdateDocument'] = _UPDATEDOCUMENT
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@ -201,6 +260,13 @@ DocumentOperation = _reflection.GeneratedProtocolMessageType('DocumentOperation'
})
_sym_db.RegisterMessage(DocumentOperation)
Vote = _reflection.GeneratedProtocolMessageType('Vote', (_message.Message,), {
'DESCRIPTOR' : _VOTE,
'__module__' : 'nexus.models.proto.operation_pb2'
# @@protoc_insertion_point(class_scope:nexus.models.proto.Vote)
})
_sym_db.RegisterMessage(Vote)
UpdateDocument = _reflection.GeneratedProtocolMessageType('UpdateDocument', (_message.Message,), {
'DESCRIPTOR' : _UPDATEDOCUMENT,
'__module__' : 'nexus.models.proto.operation_pb2'

13
nexus/models/proto/sharience.proto Normal file
View File

@ -0,0 +1,13 @@
syntax = "proto3";
package nexus.models.proto;
message Sharience {
int64 id = 1;
int64 parent_id = 2;
int64 uploader_id = 3;
uint32 updated_at = 4;
string md5 = 5;
uint32 filesize = 6;
repeated string ipfs_multihashes = 7;
string telegram_file_id = 8;
}

120
nexus/models/proto/sharience_pb2.py Normal file
View File

@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nexus/models/proto/sharience.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/models/proto/sharience.proto',
package='nexus.models.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\"nexus/models/proto/sharience.proto\x12\x12nexus.models.proto\"\xa6\x01\n\tSharience\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x11\n\tparent_id\x18\x02 \x01(\x03\x12\x13\n\x0buploader_id\x18\x03 \x01(\x03\x12\x12\n\nupdated_at\x18\x04 \x01(\r\x12\x0b\n\x03md5\x18\x05 \x01(\t\x12\x10\n\x08\x66ilesize\x18\x06 \x01(\r\x12\x18\n\x10ipfs_multihashes\x18\x07 \x03(\t\x12\x18\n\x10telegram_file_id\x18\x08 \x01(\tb\x06proto3'
)
_SHARIENCE = _descriptor.Descriptor(
name='Sharience',
full_name='nexus.models.proto.Sharience',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='nexus.models.proto.Sharience.id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='parent_id', full_name='nexus.models.proto.Sharience.parent_id', index=1,
number=2, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='uploader_id', full_name='nexus.models.proto.Sharience.uploader_id', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='updated_at', full_name='nexus.models.proto.Sharience.updated_at', index=3,
number=4, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='md5', full_name='nexus.models.proto.Sharience.md5', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='filesize', full_name='nexus.models.proto.Sharience.filesize', index=5,
number=6, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='ipfs_multihashes', full_name='nexus.models.proto.Sharience.ipfs_multihashes', index=6,
number=7, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='telegram_file_id', full_name='nexus.models.proto.Sharience.telegram_file_id', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=59,
serialized_end=225,
)
DESCRIPTOR.message_types_by_name['Sharience'] = _SHARIENCE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Sharience = _reflection.GeneratedProtocolMessageType('Sharience', (_message.Message,), {
'DESCRIPTOR' : _SHARIENCE,
'__module__' : 'nexus.models.proto.sharience_pb2'
# @@protoc_insertion_point(class_scope:nexus.models.proto.Sharience)
})
_sym_db.RegisterMessage(Sharience)
# @@protoc_insertion_point(module_scope)

2
nexus/models/proto/typed_document.proto
View File

@ -3,10 +3,12 @@ package nexus.models.proto;
import "nexus/models/proto/scimag.proto";
import "nexus/models/proto/scitech.proto";
import "nexus/models/proto/sharience.proto";
message TypedDocument {
oneof document {
Scimag scimag = 1;
Scitech scitech = 2;
Sharience sharience = 3;
}
}

21
nexus/models/proto/typed_document_pb2.py
View File

@ -16,6 +16,8 @@ from nexus.models.proto import \
scimag_pb2 as nexus_dot_models_dot_proto_dot_scimag__pb2
from nexus.models.proto import \
scitech_pb2 as nexus_dot_models_dot_proto_dot_scitech__pb2
from nexus.models.proto import \
sharience_pb2 as nexus_dot_models_dot_proto_dot_sharience__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/models/proto/typed_document.proto',
@ -23,9 +25,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\'nexus/models/proto/typed_document.proto\x12\x12nexus.models.proto\x1a\x1fnexus/models/proto/scimag.proto\x1a nexus/models/proto/scitech.proto\"y\n\rTypedDocument\x12,\n\x06scimag\x18\x01 \x01(\x0b\x32\x1a.nexus.models.proto.ScimagH\x00\x12.\n\x07scitech\x18\x02 \x01(\x0b\x32\x1b.nexus.models.proto.ScitechH\x00\x42\n\n\x08\x64ocumentb\x06proto3'
serialized_pb=b'\n\'nexus/models/proto/typed_document.proto\x12\x12nexus.models.proto\x1a\x1fnexus/models/proto/scimag.proto\x1a nexus/models/proto/scitech.proto\x1a\"nexus/models/proto/sharience.proto\"\xad\x01\n\rTypedDocument\x12,\n\x06scimag\x18\x01 \x01(\x0b\x32\x1a.nexus.models.proto.ScimagH\x00\x12.\n\x07scitech\x18\x02 \x01(\x0b\x32\x1b.nexus.models.proto.ScitechH\x00\x12\x32\n\tsharience\x18\x03 \x01(\x0b\x32\x1d.nexus.models.proto.SharienceH\x00\x42\n\n\x08\x64ocumentb\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_scimag__pb2.DESCRIPTOR,nexus_dot_models_dot_proto_dot_scitech__pb2.DESCRIPTOR,])
dependencies=[nexus_dot_models_dot_proto_dot_scimag__pb2.DESCRIPTOR,nexus_dot_models_dot_proto_dot_scitech__pb2.DESCRIPTOR,nexus_dot_models_dot_proto_dot_sharience__pb2.DESCRIPTOR,])
@ -52,6 +54,13 @@ _TYPEDDOCUMENT = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='sharience', full_name='nexus.models.proto.TypedDocument.sharience', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
],
extensions=[
],
@ -69,18 +78,22 @@ _TYPEDDOCUMENT = _descriptor.Descriptor(
create_key=_descriptor._internal_create_key,
fields=[]),
],
serialized_start=130,
serialized_end=251,
serialized_start=167,
serialized_end=340,
)
_TYPEDDOCUMENT.fields_by_name['scimag'].message_type = nexus_dot_models_dot_proto_dot_scimag__pb2._SCIMAG
_TYPEDDOCUMENT.fields_by_name['scitech'].message_type = nexus_dot_models_dot_proto_dot_scitech__pb2._SCITECH
_TYPEDDOCUMENT.fields_by_name['sharience'].message_type = nexus_dot_models_dot_proto_dot_sharience__pb2._SHARIENCE
_TYPEDDOCUMENT.oneofs_by_name['document'].fields.append(
_TYPEDDOCUMENT.fields_by_name['scimag'])
_TYPEDDOCUMENT.fields_by_name['scimag'].containing_oneof = _TYPEDDOCUMENT.oneofs_by_name['document']
_TYPEDDOCUMENT.oneofs_by_name['document'].fields.append(
_TYPEDDOCUMENT.fields_by_name['scitech'])
_TYPEDDOCUMENT.fields_by_name['scitech'].containing_oneof = _TYPEDDOCUMENT.oneofs_by_name['document']
_TYPEDDOCUMENT.oneofs_by_name['document'].fields.append(
_TYPEDDOCUMENT.fields_by_name['sharience'])
_TYPEDDOCUMENT.fields_by_name['sharience'].containing_oneof = _TYPEDDOCUMENT.oneofs_by_name['document']
DESCRIPTOR.message_types_by_name['TypedDocument'] = _TYPEDDOCUMENT
_sym_db.RegisterFileDescriptor(DESCRIPTOR)

2
nexus/pipe/BUILD.bazel
View File

@ -15,6 +15,7 @@ py3_image(
"configs/base.yaml",
"configs/logging.yaml",
],
legacy_create_init = False,
main = "main.py",
srcs_version = "PY3ONLY",
visibility = ["//visibility:public"],
@ -28,7 +29,6 @@ py3_image(
"//library/configurator",
"//library/logging",
"//nexus/actions",
"//nexus/cognitron/schema",
"//nexus/models/proto:models_proto_py",
requirement("aiosumma"),
requirement("izihawa_utils"),

9
nexus/pipe/README.md
View File

@ -11,8 +11,7 @@ You have to write your own configs taking example below into account.
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
pipe:
brokers: |
kafka-0.example.net,
kafka-1.example.net
kafka-0.example.net
schema:
- consumers:
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
@ -24,8 +23,7 @@ pipe:
- class: nexus.pipe.processors.CrossReferencesProcessor
kwargs:
brokers: |
kafka-0.example.net,
kafka-1.example.net
kafka-0.example.net
database:
database: nexus
host: postgres.example.net
@ -63,8 +61,7 @@ pipe:
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
kwargs:
brokers: |
kafka-0.example.net,
kafka-1.example.net
kafka-0.example.net
topic: cross_references
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:

14
nexus/pipe/consumers/base.py
View File

@ -50,12 +50,17 @@ class BaseConsumer(AioRootThing):
async def start(self):
logging.getLogger('statbox').info({
'action': 'started',
'action': 'starting',
'group_id': self.group_id,
'topic_names': self.topic_names,
})
self.consumer = self.create_consumer()
await self.consumer.start()
logging.getLogger('statbox').info({
'action': 'started',
'group_id': self.group_id,
'topic_names': self.topic_names,
})
try:
async for msg in self.consumer:
preprocessed_msg = self.preprocess(msg)
@ -108,12 +113,17 @@ class BaseBulkConsumer(BaseConsumer):
async def start(self):
logging.getLogger('statbox').info({
'action': 'started',
'action': 'starting',
'group_id': self.group_id,
'topic_names': self.topic_names,
})
self.consumer = self.create_consumer()
await self.consumer.start()
logging.getLogger('statbox').info({
'action': 'started',
'group_id': self.group_id,
'topic_names': self.topic_names,
})
while self.started:
try:
result = await self.consumer.getmany(timeout_ms=self.timeout * 1000, max_records=self.bulk_size)

8
nexus/pipe/filters/__init__.py
View File

@ -4,7 +4,7 @@ from nexus.models.proto.operation_pb2 import \
class DocumentOperationFilter(AioThing):
def __init__(self, operation, document):
def __init__(self, operation, document=None):
super().__init__()
self.operation = operation
self.document = document
@ -12,5 +12,7 @@ class DocumentOperationFilter(AioThing):
def filter(self, document_operation_pb: DocumentOperationPb) -> bool:
if document_operation_pb.WhichOneof('operation') != self.operation:
return False
operation = getattr(document_operation_pb, document_operation_pb.WhichOneof('operation'))
return operation.typed_document.HasField(self.document)
if self.document is not None:
operation = getattr(document_operation_pb, document_operation_pb.WhichOneof('operation'))
return operation.typed_document.HasField(self.document)
return True

1
nexus/pylon/BUILD.bazel
View File

@ -12,7 +12,6 @@ py_library(
requirement("brotli"),
requirement("cchardet"),
requirement("orjson"),
requirement("pdfminer.six"),
requirement("python-socks"),
requirement("tenacity"),
requirement("aiokit"),

Some files were not shown because too many files have changed in this diff Show More