MarcoBuster/hyperboria
1
0
Fork 0
mirror of https://github.com/nexus-stc/hyperboria synced 2025-01-11 03:05:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #29 from the-superpirate/master

Browse Source 
Making build working
This commit is contained in:
the-superpirate 2021-04-24 17:30:56 +03:00 committed by GitHub
commit 77b15682d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 113 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
WORKSPACE
View File

@ -229,6 +229,17 @@ container_repositories()
load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
# ToDo: temorary fix as registry was broken at 24.04.2021
load("@bazel_gazelle//:deps.bzl", "go_repository")
go_repository(
name = "com_github_google_go_containerregistry",
importpath = "github.com/google/go-containerregistry",
strip_prefix = "google-go-containerregistry-8a28419",
type = "tar.gz",
urls = ["https://api.github.com/repos/google/go-containerregistry/tarball/8a2841911ffee4f6892ca0083e89752fb46c48dd"], # v0.1.4
)
container_deps()
load("@io_bazel_rules_docker//repositories:py_repositories.bzl", "py_deps")

1
library/aiogrpctools/BUILD.bazel
View File

@ -11,6 +11,7 @@ py_library(
visibility = ["//visibility:public"],
deps = [
requirement("grpcio"),
requirement("pyyaml"),
requirement("aiokit"),
"//library/configurator",
"//library/logging",

6
library/aiogrpctools/base.py
View File

@ -1,6 +1,7 @@
import logging
from functools import wraps
import yaml
from aiokit import (
AioRootThing,
AioThing,
@ -36,6 +37,11 @@ class AioGrpcServer(AioRootThing):
})
await self.server.stop(None)
def log_config(self, config):
logging.getLogger('debug').info(
'\n' + yaml.safe_dump(config.get_files()),
)
class BaseService(AioThing):
error_mapping = {}

3
library/configurator/__init__.py
View File

@ -153,3 +153,6 @@ class Configurator(RichDict):
def has_file(self, basename):
return basename in self._by_basenames
def get_files(self):
return self._by_basenames

20
library/telegram/base.py
View File

@ -1,5 +1,9 @@
import datetime
import logging
from typing import (
Optional,
Union,
)
from aiokit import AioThing
from izihawa_utils.random import random_string
@ -19,8 +23,20 @@ from .session_backend import AlchemySessionContainer
class BaseTelegramClient(AioThing):
def __init__(self, app_id, app_hash, database, bot_token=None, mtproxy=None, flood_sleep_threshold: int = 60):
AioThing.__init__(self)
def __init__(
self,
app_id: Union[int, str],
app_hash: str,
database: dict,
bot_token: Optional[str] = None,
mtproxy: Optional[dict] = None,
flood_sleep_threshold: int = 60,
):
super().__init__()
if not app_id or not app_hash:
raise ValueError(
'Your API ID or Hash cannot be empty or None. Set up telegram.app_id and/or telegram.app_hash'
)
self._telegram_client = TelegramClient(
self._get_session(database),
app_id,

9
nexus/bot/application.py
View File

@ -21,15 +21,18 @@ class TelegramApplication(AioRootThing):
)
self.hub_client = HubGrpcClient(base_url=self.config['hub']['url'])
self.idm_client = IdmApiGrpcClient(base_url=self.config['idm']['url'])
self.starts.append(self.hub_client)
self.idm_client = None
if self.config['idm']['enabled']:
self.idm_client = IdmApiGrpcClient(base_url=self.config['idm']['url'])
self.starts.append(self.idm_client)
self.meta_api_client = MetaApiGrpcClient(base_url=self.config['meta_api']['url'])
self.starts.append(self.meta_api_client)
self.promotioner = Promotioner(promotions=self.config['promotions'])
self.user_manager = UserManager()
self._handlers = []
self.starts.extend([self.hub_client, self.idm_client, self.meta_api_client])
def set_handlers(self, telegram_client):
for handler in self.config['telegram']['handlers']:
import_object(handler)(self).register_for(telegram_client)

6
nexus/bot/configs/base.yaml
View File

@ -29,7 +29,6 @@ application:
request_id_length: 12
# Enabled schemas (passed to Nexus Meta API)
schemas:
- scimag
- scitech
# Length of generated Session-ID used in commands to clue user sessions
session_id_length: 8
@ -44,10 +43,13 @@ application:
hub:
url:
idm:
enabled: false
url:
log_path: '/var/log/nexus-bot'
meta_api:
url:
metrics:
enabled: false
telegram:
# Telegram App Hash from https://my.telegram.org/
app_hash: '{{ APP_HASH }}'
@ -65,7 +67,7 @@ telegram:
copyright_infringement_account:
# Telethon database for keeping cache
database:
session_id: '/usr/lib/nexus-bot/session.db'
session_id: /tmp/nexus-bot.db
# Enabled handlers
handlers:
- nexus.bot.handlers.admin.AdminHandler

12
nexus/bot/handlers/base.py
View File

@ -185,6 +185,15 @@ class BaseHandler(ABC):
request_context.error_log(e)
async def _put_chat(self, event: events.ChatAction, request_id: str):
event_chat = await event.get_chat()
username = get_username(event, event_chat)
language = get_language(event, event_chat)
if not self.application.idm_client:
return ChatPb(
chat_id=event.chat_id,
username=username,
language=language,
)
try:
chat = await self.application.idm_client.get_chat(
chat_id=event.chat_id,
@ -196,9 +205,6 @@ class BaseHandler(ABC):
raise
if self.application.config['application']['is_read_only_mode']:
raise ReadOnlyModeError()
event_chat = await event.get_chat()
username = get_username(event, event_chat)
language = get_language(event, event_chat)
if language not in {'en', 'ru'}:
language = 'en'
chat = await self.application.idm_client.create_chat(

2
nexus/hub/BUILD.bazel
View File

@ -35,7 +35,7 @@ py3_image(
requirement("tenacity"),
requirement("uvloop"),
"//idm/api/proto:idm_proto_py",
requirement("giogrobid"),
requirement("aiogrobid"),
"//library/aiogrpctools",
requirement("aiokit"),
"//library/aiopostgres",

9
nexus/hub/configs/base.yaml
View File

@ -1,8 +1,9 @@
---
application:
debug: true
# Enable special Postgres `sharience` table to retrieve user-sent files
is_sharience_enabled: true
is_sharience_enabled: false
# URL to the picture shown while maintenance
maintenance_picture_url:
# Used in logging
@ -11,10 +12,12 @@ application:
should_store_hashes: true
database:
database: nexus
enabled: false
host:
password:
username:
grobid:
enabled: false
url:
grpc:
# Listen address
@ -22,7 +25,7 @@ grpc:
# Listen port
port: 9090
ipfs:
address:
address: localhost
port: 4001
log_path: '/var/log/nexus-hub'
meta_api:
@ -44,7 +47,7 @@ telegram:
bot_token:
# Telethon database for keeping cache
database:
session_id: nexus-hub
session_id: /tmp/nexus-hub.db
# Frequency of updating downloading progress
progress_throttle_seconds: 5
# Send files using stored telegram_file_id

50
nexus/hub/main.py
View File

@ -13,16 +13,22 @@ from nexus.hub.services.submitter import SubmitterService
class GrpcServer(AioGrpcServer):
def __init__(self, config: Configurator):
self.log_config(config)
super().__init__(address=config['grpc']['address'], port=config['grpc']['port'])
self.pool_holder = AioPostgresPoolHolder(
dsn=f'dbname={config["database"]["database"]} '
f'user={config["database"]["username"]} '
f'password={config["database"]["password"]} '
f'host={config["database"]["host"]}',
timeout=30,
pool_recycle=60,
maxsize=4,
)
self.pool_holder = None
if config['database']['enabled']:
self.pool_holder = AioPostgresPoolHolder(
dsn=f'dbname={config["database"]["database"]} '
f'user={config["database"]["username"]} '
f'password={config["database"]["password"]} '
f'host={config["database"]["host"]}',
timeout=30,
pool_recycle=60,
maxsize=4,
)
self.waits.append(self.pool_holder)
self.telegram_client = BaseTelegramClient(
app_id=config['telegram']['app_id'],
app_hash=config['telegram']['app_hash'],
@ -30,6 +36,8 @@ class GrpcServer(AioGrpcServer):
database=config['telegram'].get('database'),
mtproxy=config['telegram'].get('mtproxy'),
)
self.starts.append(self.telegram_client)
self.delivery_service = DeliveryService(
server=self.server,
service_name=config['application']['service_name'],
@ -43,17 +51,19 @@ class GrpcServer(AioGrpcServer):
should_use_telegram_file_id=config['telegram']['should_use_telegram_file_id'],
telegram_client=self.telegram_client,
)
self.submitter_service = SubmitterService(
server=self.server,
service_name=config['application']['service_name'],
bot_external_name=config['telegram']['bot_external_name'],
grobid_config=config['grobid'],
ipfs_config=config['ipfs'],
meta_api_config=config['meta_api'],
telegram_client=self.telegram_client,
)
self.waits.append(self.pool_holder)
self.starts.extend([self.telegram_client, self.delivery_service, self.submitter_service])
self.starts.append(self.delivery_service)
if config['grobid']['enabled']:
self.submitter_service = SubmitterService(
server=self.server,
service_name=config['application']['service_name'],
bot_external_name=config['telegram']['bot_external_name'],
grobid_config=config['grobid'],
ipfs_config=config['ipfs'],
meta_api_config=config['meta_api'],
telegram_client=self.telegram_client,
)
self.starts.append(self.submitter_service)
def main():

2
nexus/hub/services/delivery.py
View File

@ -324,6 +324,8 @@ class DeliveryService(DeliveryServicer, BaseHubService):
should_use_telegram_file_id: bool,
telegram_client: BaseTelegramClient,
):
if is_sharience_enabled and not pool_holder:
raise ValueError('Sharience can be used only with enabled database')
super().__init__(
service_name=service_name,
bot_external_name=bot_external_name,

21
nexus/meta_api/BUILD.bazel
View File

@ -1,5 +1,6 @@
load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
load("@io_bazel_rules_docker//container:container.bzl", "container_push")
load("@pip_modules//:requirements.bzl", "requirement")
alias(
@ -46,31 +47,13 @@ py3_image(
],
)
py3_image(
name = "learn",
srcs = ["learn.py"],
base = "//images/production:base-python-image",
main = "learn.py",
srcs_version = "PY3ONLY",
visibility = ["//visibility:public"],
deps = [
requirement("fire"),
requirement("lightgbm"),
requirement("numpy"),
requirement("orjson"),
requirement("pandas"),
"//nexus/meta_api/proto:meta_api_proto_py",
requirement("izihawa_utils"),
],
)
py_binary(
name = "cli",
srcs = ["cli.py"],
main = "cli.py",
deps = [
"//nexus/meta_api/aioclient",
requirement("fire"),
"//nexus/meta_api/aioclient",
],
)

6
nexus/meta_api/README.md
View File

@ -1 +1,5 @@
# Nexus Search: Meta API
# Nexus Search: Meta API
```
NEXUS_META_API_summa.url=http://summa bazel run -c opt binary
```

4
nexus/meta_api/configs/base.yaml
View File

@ -12,9 +12,11 @@ grpc:
host: 0.0.0.0
port: 9090
log_path: '/var/log/nexus-meta-api'
metrics:
enabled: false
stat_provider:
enabled: false
summa:
timeout: 15
ttl_dns_cache: 30
url:
url: http://summa

1
nexus/meta_api/main.py
View File

@ -15,6 +15,7 @@ from nexus.meta_api.services.search import SearchService
class GrpcServer(AioGrpcServer):
def __init__(self, config):
self.log_config(config)
super().__init__(address=config['grpc']['host'], port=config['grpc']['port'])
self.config = config

9
rules/python/requirements.txt
View File

@ -1,9 +1,10 @@
setuptools==54.1.2
aiobaseclient==0.2.1
aiobaseclient==0.2.4
aiochclient==2.0.0
aiocrossref==0.2.0
aiogrpcclient==0.1.0
aiocrossref==0.2.1
aiogrobid==0.1.2
aiogrpcclient==0.1.2
aiodns==2.0.0
aioftp==0.18.1
aiohttp[speedups]==3.7.4.post0
@ -64,7 +65,7 @@ idna==2.10
isort==5.8.0
itsdangerous==1.1.0
izihawa_types==0.1.0
izihawa_utils==0.3.1
izihawa_utils==0.3.2
Jinja2==2.11.3
jupyter==1.0.0
kazoo==2.8.0