MarcoBuster/hyperboria
1
0
Fork 0
mirror of https://github.com/nexus-stc/hyperboria synced 2024-11-10 13:29:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #27 from the-superpirate/master

Browse Source 
- feat(nexus): Update dependencies, fix bugs
This commit is contained in:
the-superpirate 2021-04-15 17:24:23 +03:00 committed by GitHub
commit f23418f7e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 2778 additions and 1806 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
WORKSPACE
View File

@ -26,10 +26,8 @@ http_archive(
http_archive(
name = "build_bazel_rules_nodejs",
sha256 = "dd7ea7efda7655c218ca707f55c3e1b9c68055a70c31a98f264b3445bc8f4cb1",
urls = [
"https://github.com/bazelbuild/rules_nodejs/releases/download/3.2.3/rules_nodejs-3.2.3.tar.gz",
],
sha256 = "f533eeefc8fe1ddfe93652ec50f82373d0c431f7faabd5e6323f6903195ef227",
urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.3.0/rules_nodejs-3.3.0.tar.gz"],
)
http_archive(
@ -239,6 +237,19 @@ rules_misc_install_internal()
# Proto / gRPC
http_archive(
name = "rules_proto_grpc",
sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419",
strip_prefix = "rules_proto_grpc-3.1.1",
urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"],
)
load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")
rules_proto_grpc_toolchains()
rules_proto_grpc_repos()
load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
rules_proto_dependencies()

3
idm/api/aioclient/BUILD.bazel
View File

@ -6,10 +6,9 @@ py_library(
srcs = glob(["**/*.py"]),
visibility = ["//visibility:public"],
deps = [
requirement("cachetools"),
requirement("lru-dict"),
requirement("tenacity"),
"//idm/api/proto:idm_grpc_py",
"//idm/api/proto:idm_proto_py",
requirement("aiogrpcclient"),
],
)

85
idm/api/aioclient/aioclient.py
View File

@ -1,9 +1,10 @@
from aiokit import AioThing
from typing import Optional
from aiogrpcclient import BaseGrpcClient
from grpc import StatusCode
from grpc.experimental.aio import (
AioRpcError,
insecure_channel,
)
from grpc.experimental.aio import AioRpcError
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from idm.api.proto.chat_manager_service_pb2 import Chats as ChatsPb
from idm.api.proto.chat_manager_service_pb2 import (
CreateChatRequest,
GetChatRequest,
@ -11,7 +12,6 @@ from idm.api.proto.chat_manager_service_pb2 import (
UpdateChatRequest,
)
from idm.api.proto.chat_manager_service_pb2_grpc import ChatManagerStub
from lru import LRU
from tenacity import (
retry,
retry_if_exception,
@ -20,36 +20,19 @@ from tenacity import (
)
class IdmApiGrpcClient(AioThing):
def __init__(
self,
base_url,
):
super().__init__()
self.channel = insecure_channel(base_url, [
('grpc.dns_min_time_between_resolutions_ms', 1000),
('grpc.initial_reconnect_backoff_ms', 1000),
('grpc.lb_policy_name', 'round_robin'),
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.chat_manager_stub = ChatManagerStub(self.channel)
self.cache = LRU(4096)
async def start(self):
await self.channel.channel_ready()
async def stop(self):
await self.channel.close()
class IdmApiGrpcClient(BaseGrpcClient):
stub_clses = {
'chat_manager': ChatManagerStub,
}
async def create_chat(
self,
chat_id,
username,
language,
request_id: str = None,
):
response = await self.chat_manager_stub.create_chat(
chat_id: int,
username: str,
language: str,
request_id: Optional[str] = None,
) -> ChatPb:
response = await self.stubs['chat_manager'].create_chat(
CreateChatRequest(
chat_id=chat_id,
username=username,
@ -71,10 +54,10 @@ class IdmApiGrpcClient(AioThing):
)
async def get_chat(
self,
chat_id,
request_id: str = None,
):
response = await self.chat_manager_stub.get_chat(
chat_id: int,
request_id: Optional[str] = None,
) -> ChatPb:
response = await self.stubs['chat_manager'].get_chat(
GetChatRequest(chat_id=chat_id),
metadata=(
('request-id', request_id),
@ -84,10 +67,10 @@ class IdmApiGrpcClient(AioThing):
async def list_chats(
self,
request_id: str = None,
banned_at_moment=None,
):
response = await self.chat_manager_stub.list_chats(
request_id: Optional[str] = None,
banned_at_moment: Optional[str] = None,
) -> ChatsPb:
response = await self.stubs['chat_manager'].list_chats(
ListChatsRequest(banned_at_moment=banned_at_moment),
metadata=(
('request-id', request_id),
@ -97,16 +80,16 @@ class IdmApiGrpcClient(AioThing):
async def update_chat(
self,
chat_id,
request_id: str = None,
language=None,
is_system_messaging_enabled=None,
is_discovery_enabled=None,
ban_until=None,
ban_message=None,
is_admin=None,
):
response = await self.chat_manager_stub.update_chat(
chat_id: int,
request_id: Optional[str] = None,
language: Optional[str] = None,
is_system_messaging_enabled: Optional[bool] = None,
is_discovery_enabled: Optional[bool] = None,
ban_until: Optional[int] = None,
ban_message: Optional[str] = None,
is_admin: Optional[bool] = None,
) -> ChatPb:
response = await self.stubs['chat_manager'].update_chat(
UpdateChatRequest(
chat_id=chat_id,
language=language,

5
idm/api/main.py
View File

@ -44,9 +44,12 @@ async def create_app(config: Configurator):
def main():
uvloop.install()
config = get_config()
configure_logging(config)
if config['metrics']['enabled']:
from library.metrics_server import MetricsServer
MetricsServer(config['metrics']).fork_process()
asyncio.set_event_loop(uvloop.new_event_loop())
asyncio.get_event_loop().run_until_complete(create_app(config))

2
nexus/bot/configs/base.yaml
View File

@ -25,7 +25,7 @@ application:
nexus_version: InterCom
# Default page size for SERP
page_size: 5
# Length of generated Request-ID used for tracking requests across all backends
# Length of generated Request-Id used for tracking requests across all backends
request_id_length: 12
# Enabled schemas (passed to Nexus Meta API)
schemas:

1
nexus/bot/exceptions.py
View File

@ -17,6 +17,7 @@ class MessageHasBeenDeletedError(BaseError):
class UnknownFileFormatError(BaseError):
level = logging.WARNING
code = 'unknown_file_format_error'

1
nexus/bot/handlers/search.py
View File

@ -91,6 +91,7 @@ class BaseSearchHandler(BaseHandler, ABC):
buttons=[close_button()],
))
return await asyncio.gather(*actions)
await asyncio.gather(*actions)
raise e
action = 'documents_found'

1
nexus/bot/handlers/submit.py
View File

@ -21,6 +21,7 @@ class SubmitHandler(BaseHandler):
request_context.statbox(action='show', mode='submit')
if event.document.mime_type != 'application/pdf':
request_context.statbox(action='unknown_file_format')
request_context.error_log(UnknownFileFormatError(format=event.document.mime_type))
return await asyncio.gather(
event.reply(

2
nexus/bot/main.py
View File

@ -7,11 +7,11 @@ from nexus.bot.configs import get_config
def main(config):
uvloop.install()
configure_logging(config)
if config['metrics']['enabled']:
from library.metrics_server import MetricsServer
MetricsServer(config['metrics']).fork_process()
asyncio.set_event_loop(uvloop.new_event_loop())
asyncio.get_event_loop().run_until_complete(TelegramApplication(config=config).start_and_wait())

2
nexus/bot/widgets/search_widget.py
View File

@ -2,7 +2,7 @@ from typing import Optional
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.bot.application import TelegramApplication
from nexus.meta_api.proto.meta_search_service_pb2 import \
from nexus.meta_api.proto.search_service_pb2 import \
ScoredDocument as ScoredDocumentPb
from nexus.translations import t
from nexus.views.telegram.common import (

2
nexus/cognitron/web/BUILD.bazel
View File

@ -14,6 +14,8 @@ deps = [
"@npm//sass",
"@npm//sass-loader",
"@npm//vue",
"//nexus/meta_api/proto:meta_api_grpc_js",
"//nexus/meta_api/proto:meta_api_proto_js",
]
js_library(

1
nexus/hub/aioclient/BUILD.bazel
View File

@ -7,6 +7,7 @@ py_library(
deps = [
requirement("grpcio"),
"//idm/api/proto:idm_proto_py",
requirement("aiogrpcclient"),
requirement("aiokit"),
"//nexus/hub/proto:hub_grpc_py",
"//nexus/hub/proto:hub_proto_py",

33
nexus/hub/aioclient/aioclient.py
View File

@ -1,7 +1,6 @@
from typing import Optional
from aiokit import AioThing
from grpc.experimental.aio import insecure_channel
from aiogrpcclient import BaseGrpcClient
from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
from nexus.hub.proto.delivery_service_pb2 import \
StartDeliveryRequest as StartDeliveryRequestPb
@ -17,27 +16,11 @@ from nexus.models.proto.typed_document_pb2 import \
TypedDocument as TypedDocumentPb
class HubGrpcClient(AioThing):
def __init__(
self,
base_url: str,
):
super().__init__()
self.channel = insecure_channel(base_url, [
('grpc.dns_min_time_between_resolutions_ms', 1000),
('grpc.initial_reconnect_backoff_ms', 1000),
('grpc.lb_policy_name', 'round_robin'),
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.delivery_stub = DeliveryStub(self.channel)
self.submitter_stub = SubmitterStub(self.channel)
async def start(self):
await self.channel.channel_ready()
async def stop(self):
await self.channel.close()
class HubGrpcClient(BaseGrpcClient):
stub_clses = {
'delivery': DeliveryStub,
'submitter': SubmitterStub,
}
async def start_delivery(
self,
@ -46,7 +29,7 @@ class HubGrpcClient(AioThing):
request_id: Optional[str],
session_id: Optional[str],
) -> StartDeliveryResponsePb:
return await self.delivery_stub.start_delivery(
return await self.stubs['delivery'].start_delivery(
StartDeliveryRequestPb(
typed_document=typed_document_pb,
chat=chat,
@ -62,7 +45,7 @@ class HubGrpcClient(AioThing):
request_id: Optional[str] = None,
session_id: Optional[str] = None,
) -> SubmitResponsePb:
return await self.submitter_stub.submit(
return await self.stubs['submitter'].submit(
SubmitRequestPb(
telegram_document=telegram_document,
telegram_file_id=telegram_file_id,

2
nexus/hub/services/delivery.py
View File

@ -366,7 +366,7 @@ class DeliveryService(DeliveryServicer, BaseHubService):
request_id=metadata.get('request-id'),
)
request_context.add_default_fields(
mode='start_delivery',
mode='delivery',
session_id=metadata.get('session-id'),
**self.get_default_service_fields(),
)

13
nexus/hub/services/submitter.py
View File

@ -78,7 +78,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
async def stop(self):
await self.ipfs_client.close()
@aiogrpc_request_wrapper()
@aiogrpc_request_wrapper(log=False)
async def submit(
self,
request: SubmitRequestPb,
@ -100,6 +100,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
document = BinaryReader(request.telegram_document).tgread_object()
if document.size > 20 * 1024 * 1024:
request_context.error_log(FileTooBigError(size=document.size))
request_context.statbox(action='file_too_big')
await self.telegram_client.send_message(
request_context.chat.chat_id,
t('FILE_TOO_BIG_ERROR', language=request_context.chat.language),
@ -113,12 +114,11 @@ class SubmitterService(SubmitterServicer, BaseHubService):
),
)
try:
file = await self.telegram_client.download_document(document=document, file=bytes)
try:
processed_document = await self.grobid_client.process_fulltext_document(pdf_file=file)
except BadRequestError as e:
request_context.statbox(action='unparsable_document')
request_context.error_log(e)
await self.telegram_client.send_message(
request_context.chat.chat_id,
@ -128,6 +128,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
return SubmitResponsePb()
if not processed_document.get('doi'):
request_context.statbox(action='unparsable_doi')
request_context.error_log(UnparsableDoiError())
await self.telegram_client.send_message(
request_context.chat.chat_id,
@ -148,6 +149,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
)
if len(search_response_pb.scored_documents) == 0:
request_context.statbox(action='unavailable_metadata')
request_context.error_log(UnavailableMetadataError(doi=processed_document['doi']))
await self.telegram_client.send_message(
request_context.chat.chat_id,
@ -184,5 +186,10 @@ class SubmitterService(SubmitterServicer, BaseHubService):
)),
),
)
request_context.statbox(
action='success',
document_id=document_view.id,
schema='scimag',
)
await operation_log(document_operation_pb)
return SubmitResponsePb()

1
nexus/meta_api/aioclient/BUILD.bazel
View File

@ -9,6 +9,7 @@ py_library(
requirement("grpcio"),
requirement("lru-dict"),
requirement("tenacity"),
requirement("aiogrpcclient"),
requirement("aiokit"),
"//nexus/meta_api/proto:meta_api_grpc_py",
"//nexus/meta_api/proto:meta_api_proto_py",

46
nexus/meta_api/aioclient/aioclient.py
View File

@ -5,13 +5,9 @@ from typing import (
Union,
)
from aiokit import AioThing
from aiogrpcclient import BaseGrpcClient
from grpc import StatusCode
from grpc.experimental.aio import (
AioRpcError,
insecure_channel,
)
from lru import LRU
from grpc.experimental.aio import AioRpcError
from nexus.meta_api.proto.documents_service_pb2 import \
RollRequest as RollRequestPb
from nexus.meta_api.proto.documents_service_pb2 import \
@ -23,11 +19,11 @@ from nexus.meta_api.proto.documents_service_pb2 import \
from nexus.meta_api.proto.documents_service_pb2 import \
TypedDocumentRequest as TypedDocumentRequestPb
from nexus.meta_api.proto.documents_service_pb2_grpc import DocumentsStub
from nexus.meta_api.proto.meta_search_service_pb2 import \
from nexus.meta_api.proto.search_service_pb2 import \
SearchRequest as SearchRequestPb
from nexus.meta_api.proto.meta_search_service_pb2 import \
from nexus.meta_api.proto.search_service_pb2 import \
SearchResponse as SearchResponsePb
from nexus.meta_api.proto.meta_search_service_pb2_grpc import MetaSearchStub
from nexus.meta_api.proto.search_service_pb2_grpc import SearchStub
from nexus.models.proto.typed_document_pb2 import \
TypedDocument as TypedDocumentPb
from tenacity import (
@ -38,25 +34,11 @@ from tenacity import (
)
class MetaApiGrpcClient(AioThing):
def __init__(self, base_url):
super().__init__()
self.channel = insecure_channel(base_url, [
('grpc.dns_min_time_between_resolutions_ms', 1000),
('grpc.initial_reconnect_backoff_ms', 1000),
('grpc.lb_policy_name', 'round_robin'),
('grpc.min_reconnect_backoff_ms', 1000),
('grpc.max_reconnect_backoff_ms', 2000),
])
self.meta_search_stub = MetaSearchStub(self.channel)
self.documents_stub = DocumentsStub(self.channel)
self.cache = LRU(4096)
async def start(self):
await self.channel.channel_ready()
async def stop(self):
await self.channel.close()
class MetaApiGrpcClient(BaseGrpcClient):
stub_clses = {
'documents': DocumentsStub,
'search': SearchStub,
}
async def get(
self,
@ -67,7 +49,7 @@ class MetaApiGrpcClient(AioThing):
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> TypedDocumentPb:
return await self.documents_stub.get(
return await self.stubs['documents'].get(
TypedDocumentRequestPb(
schema=schema,
document_id=document_id,
@ -87,7 +69,7 @@ class MetaApiGrpcClient(AioThing):
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> RollResponsePb:
return await self.documents_stub.roll(
return await self.stubs['documents'].roll(
RollRequestPb(
language=language,
session_id=session_id,
@ -120,7 +102,7 @@ class MetaApiGrpcClient(AioThing):
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> SearchResponsePb:
return await self.meta_search_stub.search(
return await self.stubs['search'].search(
SearchRequestPb(
schemas=schemas,
query=query,
@ -143,7 +125,7 @@ class MetaApiGrpcClient(AioThing):
session_id: Optional[str] = None,
user_id: Optional[int] = None,
) -> TopMissedResponsePb:
return await self.documents_stub.top_missed(
return await self.stubs['documents'].top_missed(
TopMissedRequestPb(
page=page,
page_size=page_size,

12
nexus/meta_api/proto/BUILD.bazel
View File

@ -1,6 +1,7 @@
load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
load("@rules_rust//proto:proto.bzl", "rust_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
load("@rules_proto_grpc//js:defs.bzl", "js_grpc_node_library", "js_proto_library")
package(default_visibility = ["//visibility:public"])
@ -32,3 +33,14 @@ rust_proto_library(
visibility = ["//visibility:public"],
deps = [":meta_api_proto"],
)
js_proto_library(
name = "meta_api_proto_js",
deps = ["meta_api_proto"],
)
js_grpc_node_library(
name = "meta_api_grpc_js",
protos = [":meta_api_proto"],
deps = [":meta_api_proto_js"],
)

2
nexus/meta_api/proto/meta_search_service.proto → nexus/meta_api/proto/search_service.proto
View File

@ -24,6 +24,6 @@ message SearchRequest {
string session_id = 7;
}
service MetaSearch {
service Search {
rpc search (SearchRequest) returns (SearchResponse) {}
}

40
nexus/meta_api/proto/meta_search_service_pb2.py → nexus/meta_api/proto/search_service_pb2.py Executable file → Normal file
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nexus/meta_api/proto/meta_search_service.proto
# source: nexus/meta_api/proto/search_service.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
@ -16,12 +16,12 @@ from nexus.models.proto import \
typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nexus/meta_api/proto/meta_search_service.proto',
name='nexus/meta_api/proto/search_service.proto',
package='nexus.meta_api.proto',
syntax='proto3',
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2c\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
serialized_pb=b'\n)nexus/meta_api/proto/search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2_\n\x06Search\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
,
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
@ -69,8 +69,8 @@ _SCOREDDOCUMENT = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=113,
serialized_end=221,
serialized_start=108,
serialized_end=216,
)
@ -108,8 +108,8 @@ _SEARCHRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=223,
serialized_end=321,
serialized_start=218,
serialized_end=316,
)
@ -182,8 +182,8 @@ _SEARCHREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=324,
serialized_end=459,
serialized_start=319,
serialized_end=454,
)
_SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
@ -195,40 +195,40 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
ScoredDocument = _reflection.GeneratedProtocolMessageType('ScoredDocument', (_message.Message,), {
'DESCRIPTOR' : _SCOREDDOCUMENT,
'__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
'__module__' : 'nexus.meta_api.proto.search_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.ScoredDocument)
})
_sym_db.RegisterMessage(ScoredDocument)
SearchResponse = _reflection.GeneratedProtocolMessageType('SearchResponse', (_message.Message,), {
'DESCRIPTOR' : _SEARCHRESPONSE,
'__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
'__module__' : 'nexus.meta_api.proto.search_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchResponse)
})
_sym_db.RegisterMessage(SearchResponse)
SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
'DESCRIPTOR' : _SEARCHREQUEST,
'__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
'__module__' : 'nexus.meta_api.proto.search_service_pb2'
# @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchRequest)
})
_sym_db.RegisterMessage(SearchRequest)
_METASEARCH = _descriptor.ServiceDescriptor(
name='MetaSearch',
full_name='nexus.meta_api.proto.MetaSearch',
_SEARCH = _descriptor.ServiceDescriptor(
name='Search',
full_name='nexus.meta_api.proto.Search',
file=DESCRIPTOR,
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
serialized_start=461,
serialized_end=560,
serialized_start=456,
serialized_end=551,
methods=[
_descriptor.MethodDescriptor(
name='search',
full_name='nexus.meta_api.proto.MetaSearch.search',
full_name='nexus.meta_api.proto.Search.search',
index=0,
containing_service=None,
input_type=_SEARCHREQUEST,
@ -237,8 +237,8 @@ _METASEARCH = _descriptor.ServiceDescriptor(
create_key=_descriptor._internal_create_key,
),
])
_sym_db.RegisterServiceDescriptor(_METASEARCH)
_sym_db.RegisterServiceDescriptor(_SEARCH)
DESCRIPTOR.services_by_name['MetaSearch'] = _METASEARCH
DESCRIPTOR.services_by_name['Search'] = _SEARCH
# @@protoc_insertion_point(module_scope)

31
nexus/meta_api/proto/meta_search_service_pb2_grpc.py → nexus/meta_api/proto/search_service_pb2_grpc.py Executable file → Normal file
View File

@ -1,12 +1,11 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
from nexus.meta_api.proto import \
meta_search_service_pb2 as \
nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2
from nexus.meta_api.proto import search_service_pb2 as nexus_dot_meta__api_dot_proto_dot_search__service__pb2
class MetaSearchStub(object):
class SearchStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
@ -16,13 +15,13 @@ class MetaSearchStub(object):
channel: A grpc.Channel.
"""
self.search = channel.unary_unary(
'/nexus.meta_api.proto.MetaSearch/search',
request_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
response_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
'/nexus.meta_api.proto.Search/search',
request_serializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchRequest.SerializeToString,
response_deserializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchResponse.FromString,
)
class MetaSearchServicer(object):
class SearchServicer(object):
"""Missing associated documentation comment in .proto file."""
def search(self, request, context):
@ -32,21 +31,21 @@ class MetaSearchServicer(object):
raise NotImplementedError('Method not implemented!')
def add_MetaSearchServicer_to_server(servicer, server):
def add_SearchServicer_to_server(servicer, server):
rpc_method_handlers = {
'search': grpc.unary_unary_rpc_method_handler(
servicer.search,
request_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.FromString,
response_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.SerializeToString,
request_deserializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchRequest.FromString,
response_serializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'nexus.meta_api.proto.MetaSearch', rpc_method_handlers)
'nexus.meta_api.proto.Search', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class MetaSearch(object):
class Search(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
@ -60,8 +59,8 @@ class MetaSearch(object):
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.MetaSearch/search',
nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Search/search',
nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchRequest.SerializeToString,
nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

48
nexus/pylon/sources/base.py
View File

@ -81,31 +81,31 @@ class PreparedRequest:
@asynccontextmanager
async def execute_with(self, session):
async with session.request(
method=self.method,
url=self.url,
timeout=self.timeout,
headers=self.headers,
cookies=self.cookies,
params=self.params,
ssl=self.ssl,
) as resp:
try:
try:
async with session.request(
method=self.method,
url=self.url,
timeout=self.timeout,
headers=self.headers,
cookies=self.cookies,
params=self.params,
ssl=self.ssl,
) as resp:
yield resp
except BadResponseError as e:
e.add('url', self.url)
raise e
except (
aiohttp.client_exceptions.ClientConnectionError,
aiohttp.client_exceptions.ClientPayloadError,
aiohttp.client_exceptions.ClientResponseError,
aiohttp.client_exceptions.TooManyRedirects,
asyncio.TimeoutError,
ProxyConnectionError,
ProxyTimeoutError,
ProxyError,
) as e:
raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e))
except BadResponseError as e:
e.add('url', self.url)
raise e
except (
aiohttp.client_exceptions.ClientConnectionError,
aiohttp.client_exceptions.ClientPayloadError,
aiohttp.client_exceptions.ClientResponseError,
aiohttp.client_exceptions.TooManyRedirects,
asyncio.TimeoutError,
ProxyConnectionError,
ProxyTimeoutError,
ProxyError,
) as e:
raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e))
class BaseValidator:

17
rules/nodejs/package.json
View File

@ -6,6 +6,7 @@
"nuxt": "nuxt"
},
"dependencies": {
"@grpc/grpc-js": "^1.2.12",
"@mdi/font": "^5.8.55",
"@nuxtjs/axios": "^5.12.5",
"@pointotech/pug-plain-loader": "^1.0.1",
@ -15,7 +16,7 @@
"bootstrap": "^4.5.3",
"bootstrap-scss": "^4.5.3",
"bootstrap-vue": "^2.17.3",
"copy-webpack-plugin": "^6.4.1",
"copy-webpack-plugin": "^8.1.1",
"core-js": "^3.6.5",
"css-loader": "^5.0.1",
"dateformat": "^4.4.1",
@ -24,13 +25,16 @@
"eslint-config-standard": "^16.0.2",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-promise": "^5.1.0",
"eslint-plugin-vue": "^7.4.1",
"express": "^4.17.1",
"file-loader": "^6.2.0",
"font-awesome": "^4.7.0",
"html-entities": "^1.4.0",
"html-webpack-plugin": "^4.5.1",
"google-protobuf": "^3.15.8",
"grpc": "^1.24.6",
"grpc-tools": "^1.11.1",
"html-entities": "^2.3.2",
"html-webpack-plugin": "^5.3.1",
"js-cookie": "^2.2.1",
"lodash": "^4.17.20",
"loglevel": "^1.7.1",
@ -42,9 +46,10 @@
"pug-plain-loader": "^1.1.0",
"sass": "^1.32.4",
"sass-lint": "^1.13.1",
"sass-loader": "^10.1.1",
"sass-loader": "^11.0.1",
"sockjs-client": "^1.5.0",
"tempusdominus-bootstrap-4": "^5.39.0",
"ts-protoc-gen": "^0.14.0",
"url": "^0.11.0",
"url-loader": "^4.1.1",
"v8-compile-cache": "^2.2.0",
@ -58,7 +63,7 @@
"vue-template-compiler": "^2.6.12",
"vuejs-dialog": "^1.4.2",
"vuex-persistedstate": "^4.0.0-beta.3",
"webpack": "^4.46.0",
"webpack": "^5.33.2",
"webpack-cli": "^4.3.1",
"webpack-dev-server": "^3.11.1",
"wordwrap": "^1.0.0"

4213
rules/nodejs/yarn.lock
View File

File diff suppressed because it is too large Load Diff

2
rules/python/requirements.txt
View File

@ -1,7 +1,9 @@
setuptools==54.1.2
aiobaseclient==0.2.1
aiochclient==2.0.0
aiocrossref==0.2.0
aiogrpcclient==0.1.0
aiodns==2.0.0
aioftp==0.18.1
aiohttp[speedups]==3.7.4.post0