- feat(nexus): Update dependencies, fix bugs

- feat(idm): Added monitoring
  - feat(nexus): Refactor GRPC base client
  - fix(nexus): Logging fixes
  - feat(hub): Extend logging
  - feat(hub): Extend logging
2 internal commit(s)

GitOrigin-RevId: 01de4153dd9c0edc2e5954563733085249a4551c

WORKSPACE

@@ -26,10 +26,8 @@ http_archive(
 
 http_archive(
     name = "build_bazel_rules_nodejs",
-    sha256 = "dd7ea7efda7655c218ca707f55c3e1b9c68055a70c31a98f264b3445bc8f4cb1",
+    sha256 = "f533eeefc8fe1ddfe93652ec50f82373d0c431f7faabd5e6323f6903195ef227",
-    urls = [
+    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.3.0/rules_nodejs-3.3.0.tar.gz"],
-        "https://github.com/bazelbuild/rules_nodejs/releases/download/3.2.3/rules_nodejs-3.2.3.tar.gz",
-    ],
 )
 
 http_archive(
@@ -239,6 +237,19 @@ rules_misc_install_internal()
 
 # Proto / gRPC
 
+http_archive(
+    name = "rules_proto_grpc",
+    sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419",
+    strip_prefix = "rules_proto_grpc-3.1.1",
+    urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"],
+)
+
+load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")
+
+rules_proto_grpc_toolchains()
+
+rules_proto_grpc_repos()
+
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 
 rules_proto_dependencies()

idm/api/aioclient/BUILD.bazel

@@ -6,10 +6,9 @@ py_library(
     srcs = glob(["**/*.py"]),
     visibility = ["//visibility:public"],
     deps = [
-        requirement("cachetools"),
-        requirement("lru-dict"),
         requirement("tenacity"),
         "//idm/api/proto:idm_grpc_py",
         "//idm/api/proto:idm_proto_py",
+        requirement("aiogrpcclient"),
     ],
 )

idm/api/aioclient/aioclient.py

@@ -1,9 +1,10 @@
-from aiokit import AioThing
+from typing import Optional
+
+from aiogrpcclient import BaseGrpcClient
 from grpc import StatusCode
-from grpc.experimental.aio import (
+from grpc.experimental.aio import AioRpcError
-    AioRpcError,
+from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
-    insecure_channel,
+from idm.api.proto.chat_manager_service_pb2 import Chats as ChatsPb
-)
 from idm.api.proto.chat_manager_service_pb2 import (
     CreateChatRequest,
     GetChatRequest,
@@ -11,7 +12,6 @@ from idm.api.proto.chat_manager_service_pb2 import (
     UpdateChatRequest,
 )
 from idm.api.proto.chat_manager_service_pb2_grpc import ChatManagerStub
-from lru import LRU
 from tenacity import (
     retry,
     retry_if_exception,
@@ -20,36 +20,19 @@ from tenacity import (
 )
 
 
-class IdmApiGrpcClient(AioThing):
+class IdmApiGrpcClient(BaseGrpcClient):
-    def __init__(
+    stub_clses = {
-        self,
+        'chat_manager': ChatManagerStub,
-        base_url,
+    }
-    ):
-        super().__init__()
-        self.channel = insecure_channel(base_url, [
-            ('grpc.dns_min_time_between_resolutions_ms', 1000),
-            ('grpc.initial_reconnect_backoff_ms', 1000),
-            ('grpc.lb_policy_name', 'round_robin'),
-            ('grpc.min_reconnect_backoff_ms', 1000),
-            ('grpc.max_reconnect_backoff_ms', 2000),
-        ])
-        self.chat_manager_stub = ChatManagerStub(self.channel)
-        self.cache = LRU(4096)
-
-    async def start(self):
-        await self.channel.channel_ready()
-
-    async def stop(self):
-        await self.channel.close()
 
     async def create_chat(
         self,
-        chat_id,
+        chat_id: int,
-        username,
+        username: str,
-        language,
+        language: str,
-        request_id: str = None,
+        request_id: Optional[str] = None,
-    ):
+    ) -> ChatPb:
-        response = await self.chat_manager_stub.create_chat(
+        response = await self.stubs['chat_manager'].create_chat(
             CreateChatRequest(
                 chat_id=chat_id,
                 username=username,
@@ -71,10 +54,10 @@ class IdmApiGrpcClient(AioThing):
     )
     async def get_chat(
         self,
-        chat_id,
+        chat_id: int,
-        request_id: str = None,
+        request_id: Optional[str] = None,
-    ):
+    ) -> ChatPb:
-        response = await self.chat_manager_stub.get_chat(
+        response = await self.stubs['chat_manager'].get_chat(
             GetChatRequest(chat_id=chat_id),
             metadata=(
                 ('request-id', request_id),
@@ -84,10 +67,10 @@ class IdmApiGrpcClient(AioThing):
 
     async def list_chats(
         self,
-        request_id: str = None,
+        request_id: Optional[str] = None,
-        banned_at_moment=None,
+        banned_at_moment: Optional[str] = None,
-    ):
+    ) -> ChatsPb:
-        response = await self.chat_manager_stub.list_chats(
+        response = await self.stubs['chat_manager'].list_chats(
             ListChatsRequest(banned_at_moment=banned_at_moment),
             metadata=(
                 ('request-id', request_id),
@@ -97,16 +80,16 @@ class IdmApiGrpcClient(AioThing):
 
     async def update_chat(
         self,
-        chat_id,
+        chat_id: int,
-        request_id: str = None,
+        request_id: Optional[str] = None,
-        language=None,
+        language: Optional[str] = None,
-        is_system_messaging_enabled=None,
+        is_system_messaging_enabled: Optional[bool] = None,
-        is_discovery_enabled=None,
+        is_discovery_enabled: Optional[bool] = None,
-        ban_until=None,
+        ban_until: Optional[int] = None,
-        ban_message=None,
+        ban_message: Optional[str] = None,
-        is_admin=None,
+        is_admin: Optional[bool] = None,
-    ):
+    ) -> ChatPb:
-        response = await self.chat_manager_stub.update_chat(
+        response = await self.stubs['chat_manager'].update_chat(
             UpdateChatRequest(
                 chat_id=chat_id,
                 language=language,

idm/api/main.py

@@ -44,9 +44,12 @@ async def create_app(config: Configurator):
 
 
 def main():
-    uvloop.install()
     config = get_config()
     configure_logging(config)
+    if config['metrics']['enabled']:
+        from library.metrics_server import MetricsServer
+        MetricsServer(config['metrics']).fork_process()
+    asyncio.set_event_loop(uvloop.new_event_loop())
     asyncio.get_event_loop().run_until_complete(create_app(config))
 
 

nexus/bot/configs/base.yaml

@@ -25,7 +25,7 @@ application:
   nexus_version: InterCom
   # Default page size for SERP
   page_size: 5
-  # Length of generated Request-ID used for tracking requests across all backends
+  # Length of generated Request-Id used for tracking requests across all backends
   request_id_length: 12
   # Enabled schemas (passed to Nexus Meta API)
   schemas:

nexus/bot/exceptions.py

@@ -17,6 +17,7 @@ class MessageHasBeenDeletedError(BaseError):
 
 
 class UnknownFileFormatError(BaseError):
+    level = logging.WARNING
     code = 'unknown_file_format_error'
 
 

nexus/bot/handlers/search.py

@@ -91,6 +91,7 @@ class BaseSearchHandler(BaseHandler, ABC):
                     buttons=[close_button()],
                 ))
                 return await asyncio.gather(*actions)
+            await asyncio.gather(*actions)
             raise e
 
         action = 'documents_found'

nexus/bot/handlers/submit.py

@@ -21,6 +21,7 @@ class SubmitHandler(BaseHandler):
         request_context.statbox(action='show', mode='submit')
 
         if event.document.mime_type != 'application/pdf':
+            request_context.statbox(action='unknown_file_format')
             request_context.error_log(UnknownFileFormatError(format=event.document.mime_type))
             return await asyncio.gather(
                 event.reply(

nexus/bot/main.py

@@ -7,11 +7,11 @@ from nexus.bot.configs import get_config
 
 
 def main(config):
-    uvloop.install()
     configure_logging(config)
     if config['metrics']['enabled']:
         from library.metrics_server import MetricsServer
         MetricsServer(config['metrics']).fork_process()
+    asyncio.set_event_loop(uvloop.new_event_loop())
     asyncio.get_event_loop().run_until_complete(TelegramApplication(config=config).start_and_wait())
 
 

nexus/bot/widgets/search_widget.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.bot.application import TelegramApplication
-from nexus.meta_api.proto.meta_search_service_pb2 import \
+from nexus.meta_api.proto.search_service_pb2 import \
     ScoredDocument as ScoredDocumentPb
 from nexus.translations import t
 from nexus.views.telegram.common import (

nexus/cognitron/web/BUILD.bazel

@@ -14,6 +14,8 @@ deps = [
     "@npm//sass",
     "@npm//sass-loader",
     "@npm//vue",
+    "//nexus/meta_api/proto:meta_api_grpc_js",
+    "//nexus/meta_api/proto:meta_api_proto_js",
 ]
 
 js_library(

nexus/hub/aioclient/BUILD.bazel

@@ -7,6 +7,7 @@ py_library(
     deps = [
         requirement("grpcio"),
         "//idm/api/proto:idm_proto_py",
+        requirement("aiogrpcclient"),
         requirement("aiokit"),
         "//nexus/hub/proto:hub_grpc_py",
         "//nexus/hub/proto:hub_proto_py",

nexus/hub/aioclient/aioclient.py

@@ -1,7 +1,6 @@
 from typing import Optional
 
-from aiokit import AioThing
+from aiogrpcclient import BaseGrpcClient
-from grpc.experimental.aio import insecure_channel
 from idm.api.proto.chat_manager_service_pb2 import Chat as ChatPb
 from nexus.hub.proto.delivery_service_pb2 import \
     StartDeliveryRequest as StartDeliveryRequestPb
@@ -17,27 +16,11 @@ from nexus.models.proto.typed_document_pb2 import \
     TypedDocument as TypedDocumentPb
 
 
-class HubGrpcClient(AioThing):
+class HubGrpcClient(BaseGrpcClient):
-    def __init__(
+    stub_clses = {
-        self,
+        'delivery': DeliveryStub,
-        base_url: str,
+        'submitter': SubmitterStub,
-    ):
+    }
-        super().__init__()
-        self.channel = insecure_channel(base_url, [
-            ('grpc.dns_min_time_between_resolutions_ms', 1000),
-            ('grpc.initial_reconnect_backoff_ms', 1000),
-            ('grpc.lb_policy_name', 'round_robin'),
-            ('grpc.min_reconnect_backoff_ms', 1000),
-            ('grpc.max_reconnect_backoff_ms', 2000),
-        ])
-        self.delivery_stub = DeliveryStub(self.channel)
-        self.submitter_stub = SubmitterStub(self.channel)
-
-    async def start(self):
-        await self.channel.channel_ready()
-
-    async def stop(self):
-        await self.channel.close()
 
     async def start_delivery(
         self,
@@ -46,7 +29,7 @@ class HubGrpcClient(AioThing):
         request_id: Optional[str],
         session_id: Optional[str],
     ) -> StartDeliveryResponsePb:
-        return await self.delivery_stub.start_delivery(
+        return await self.stubs['delivery'].start_delivery(
             StartDeliveryRequestPb(
                 typed_document=typed_document_pb,
                 chat=chat,
@@ -62,7 +45,7 @@ class HubGrpcClient(AioThing):
         request_id: Optional[str] = None,
         session_id: Optional[str] = None,
     ) -> SubmitResponsePb:
-        return await self.submitter_stub.submit(
+        return await self.stubs['submitter'].submit(
             SubmitRequestPb(
                 telegram_document=telegram_document,
                 telegram_file_id=telegram_file_id,

nexus/hub/services/delivery.py

@@ -366,7 +366,7 @@ class DeliveryService(DeliveryServicer, BaseHubService):
             request_id=metadata.get('request-id'),
         )
         request_context.add_default_fields(
-            mode='start_delivery',
+            mode='delivery',
             session_id=metadata.get('session-id'),
             **self.get_default_service_fields(),
         )

nexus/hub/services/submitter.py

@@ -78,7 +78,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
     async def stop(self):
         await self.ipfs_client.close()
 
-    @aiogrpc_request_wrapper()
+    @aiogrpc_request_wrapper(log=False)
     async def submit(
         self,
         request: SubmitRequestPb,
@@ -100,6 +100,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
         document = BinaryReader(request.telegram_document).tgread_object()
         if document.size > 20 * 1024 * 1024:
             request_context.error_log(FileTooBigError(size=document.size))
+            request_context.statbox(action='file_too_big')
             await self.telegram_client.send_message(
                 request_context.chat.chat_id,
                 t('FILE_TOO_BIG_ERROR', language=request_context.chat.language),
@@ -113,12 +114,11 @@ class SubmitterService(SubmitterServicer, BaseHubService):
             ),
         )
         try:
-
             file = await self.telegram_client.download_document(document=document, file=bytes)
-
             try:
                 processed_document = await self.grobid_client.process_fulltext_document(pdf_file=file)
             except BadRequestError as e:
+                request_context.statbox(action='unparsable_document')
                 request_context.error_log(e)
                 await self.telegram_client.send_message(
                     request_context.chat.chat_id,
@@ -128,6 +128,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
                 return SubmitResponsePb()
 
             if not processed_document.get('doi'):
+                request_context.statbox(action='unparsable_doi')
                 request_context.error_log(UnparsableDoiError())
                 await self.telegram_client.send_message(
                     request_context.chat.chat_id,
@@ -148,6 +149,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
             )
 
             if len(search_response_pb.scored_documents) == 0:
+                request_context.statbox(action='unavailable_metadata')
                 request_context.error_log(UnavailableMetadataError(doi=processed_document['doi']))
                 await self.telegram_client.send_message(
                     request_context.chat.chat_id,
@@ -184,5 +186,10 @@ class SubmitterService(SubmitterServicer, BaseHubService):
                 )),
             ),
         )
+        request_context.statbox(
+            action='success',
+            document_id=document_view.id,
+            schema='scimag',
+        )
         await operation_log(document_operation_pb)
         return SubmitResponsePb()

nexus/meta_api/aioclient/BUILD.bazel

@@ -9,6 +9,7 @@ py_library(
         requirement("grpcio"),
         requirement("lru-dict"),
         requirement("tenacity"),
+        requirement("aiogrpcclient"),
         requirement("aiokit"),
         "//nexus/meta_api/proto:meta_api_grpc_py",
         "//nexus/meta_api/proto:meta_api_proto_py",

nexus/meta_api/aioclient/aioclient.py

@@ -5,13 +5,9 @@ from typing import (
     Union,
 )
 
-from aiokit import AioThing
+from aiogrpcclient import BaseGrpcClient
 from grpc import StatusCode
-from grpc.experimental.aio import (
+from grpc.experimental.aio import AioRpcError
-    AioRpcError,
-    insecure_channel,
-)
-from lru import LRU
 from nexus.meta_api.proto.documents_service_pb2 import \
     RollRequest as RollRequestPb
 from nexus.meta_api.proto.documents_service_pb2 import \
@@ -23,11 +19,11 @@ from nexus.meta_api.proto.documents_service_pb2 import \
 from nexus.meta_api.proto.documents_service_pb2 import \
     TypedDocumentRequest as TypedDocumentRequestPb
 from nexus.meta_api.proto.documents_service_pb2_grpc import DocumentsStub
-from nexus.meta_api.proto.meta_search_service_pb2 import \
+from nexus.meta_api.proto.search_service_pb2 import \
     SearchRequest as SearchRequestPb
-from nexus.meta_api.proto.meta_search_service_pb2 import \
+from nexus.meta_api.proto.search_service_pb2 import \
     SearchResponse as SearchResponsePb
-from nexus.meta_api.proto.meta_search_service_pb2_grpc import MetaSearchStub
+from nexus.meta_api.proto.search_service_pb2_grpc import SearchStub
 from nexus.models.proto.typed_document_pb2 import \
     TypedDocument as TypedDocumentPb
 from tenacity import (
@@ -38,25 +34,11 @@ from tenacity import (
 )
 
 
-class MetaApiGrpcClient(AioThing):
+class MetaApiGrpcClient(BaseGrpcClient):
-    def __init__(self, base_url):
+    stub_clses = {
-        super().__init__()
+        'documents': DocumentsStub,
-        self.channel = insecure_channel(base_url, [
+        'search': SearchStub,
-            ('grpc.dns_min_time_between_resolutions_ms', 1000),
+    }
-            ('grpc.initial_reconnect_backoff_ms', 1000),
-            ('grpc.lb_policy_name', 'round_robin'),
-            ('grpc.min_reconnect_backoff_ms', 1000),
-            ('grpc.max_reconnect_backoff_ms', 2000),
-        ])
-        self.meta_search_stub = MetaSearchStub(self.channel)
-        self.documents_stub = DocumentsStub(self.channel)
-        self.cache = LRU(4096)
-
-    async def start(self):
-        await self.channel.channel_ready()
-
-    async def stop(self):
-        await self.channel.close()
 
     async def get(
         self,
@@ -67,7 +49,7 @@ class MetaApiGrpcClient(AioThing):
         session_id: Optional[str] = None,
         user_id: Optional[int] = None,
     ) -> TypedDocumentPb:
-        return await self.documents_stub.get(
+        return await self.stubs['documents'].get(
             TypedDocumentRequestPb(
                 schema=schema,
                 document_id=document_id,
@@ -87,7 +69,7 @@ class MetaApiGrpcClient(AioThing):
         session_id: Optional[str] = None,
         user_id: Optional[int] = None,
     ) -> RollResponsePb:
-        return await self.documents_stub.roll(
+        return await self.stubs['documents'].roll(
             RollRequestPb(
                 language=language,
                 session_id=session_id,
@@ -120,7 +102,7 @@ class MetaApiGrpcClient(AioThing):
         session_id: Optional[str] = None,
         user_id: Optional[int] = None,
     ) -> SearchResponsePb:
-        return await self.meta_search_stub.search(
+        return await self.stubs['search'].search(
             SearchRequestPb(
                 schemas=schemas,
                 query=query,
@@ -143,7 +125,7 @@ class MetaApiGrpcClient(AioThing):
         session_id: Optional[str] = None,
         user_id: Optional[int] = None,
     ) -> TopMissedResponsePb:
-        return await self.documents_stub.top_missed(
+        return await self.stubs['documents'].top_missed(
             TopMissedRequestPb(
                 page=page,
                 page_size=page_size,

nexus/meta_api/proto/BUILD.bazel

@@ -1,6 +1,7 @@
 load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
 load("@rules_rust//proto:proto.bzl", "rust_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@rules_proto_grpc//js:defs.bzl", "js_grpc_node_library", "js_proto_library")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -32,3 +33,14 @@ rust_proto_library(
     visibility = ["//visibility:public"],
     deps = [":meta_api_proto"],
 )
+
+js_proto_library(
+    name = "meta_api_proto_js",
+    deps = ["meta_api_proto"],
+)
+
+js_grpc_node_library(
+    name = "meta_api_grpc_js",
+    protos = [":meta_api_proto"],
+    deps = [":meta_api_proto_js"],
+)

nexus/meta_api/proto/search_service.proto

@@ -24,6 +24,6 @@ message SearchRequest {
   string session_id = 7;
 }
 
-service MetaSearch {
+service Search {
   rpc search (SearchRequest) returns (SearchResponse) {}
 }

nexus/meta_api/proto/search_service_pb2.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: nexus/meta_api/proto/meta_search_service.proto
+# source: nexus/meta_api/proto/search_service.proto
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
@@ -16,12 +16,12 @@ from nexus.models.proto import \
     typed_document_pb2 as nexus_dot_models_dot_proto_dot_typed__document__pb2
 
 DESCRIPTOR = _descriptor.FileDescriptor(
-  name='nexus/meta_api/proto/meta_search_service.proto',
+  name='nexus/meta_api/proto/search_service.proto',
   package='nexus.meta_api.proto',
   syntax='proto3',
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n.nexus/meta_api/proto/meta_search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2c\n\nMetaSearch\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
+  serialized_pb=b'\n)nexus/meta_api/proto/search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2_\n\x06Search\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
   ,
   dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
 
@@ -69,8 +69,8 @@ _SCOREDDOCUMENT = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=113,
+  serialized_start=108,
-  serialized_end=221,
+  serialized_end=216,
 )
 
 
@@ -108,8 +108,8 @@ _SEARCHRESPONSE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=223,
+  serialized_start=218,
-  serialized_end=321,
+  serialized_end=316,
 )
 
 
@@ -182,8 +182,8 @@ _SEARCHREQUEST = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=324,
+  serialized_start=319,
-  serialized_end=459,
+  serialized_end=454,
 )
 
 _SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
@@ -195,40 +195,40 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 ScoredDocument = _reflection.GeneratedProtocolMessageType('ScoredDocument', (_message.Message,), {
   'DESCRIPTOR' : _SCOREDDOCUMENT,
-  '__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
+  '__module__' : 'nexus.meta_api.proto.search_service_pb2'
   # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.ScoredDocument)
   })
 _sym_db.RegisterMessage(ScoredDocument)
 
 SearchResponse = _reflection.GeneratedProtocolMessageType('SearchResponse', (_message.Message,), {
   'DESCRIPTOR' : _SEARCHRESPONSE,
-  '__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
+  '__module__' : 'nexus.meta_api.proto.search_service_pb2'
   # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchResponse)
   })
 _sym_db.RegisterMessage(SearchResponse)
 
 SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
   'DESCRIPTOR' : _SEARCHREQUEST,
-  '__module__' : 'nexus.meta_api.proto.meta_search_service_pb2'
+  '__module__' : 'nexus.meta_api.proto.search_service_pb2'
   # @@protoc_insertion_point(class_scope:nexus.meta_api.proto.SearchRequest)
   })
 _sym_db.RegisterMessage(SearchRequest)
 
 
 
-_METASEARCH = _descriptor.ServiceDescriptor(
+_SEARCH = _descriptor.ServiceDescriptor(
-  name='MetaSearch',
+  name='Search',
-  full_name='nexus.meta_api.proto.MetaSearch',
+  full_name='nexus.meta_api.proto.Search',
   file=DESCRIPTOR,
   index=0,
   serialized_options=None,
   create_key=_descriptor._internal_create_key,
-  serialized_start=461,
+  serialized_start=456,
-  serialized_end=560,
+  serialized_end=551,
   methods=[
   _descriptor.MethodDescriptor(
     name='search',
-    full_name='nexus.meta_api.proto.MetaSearch.search',
+    full_name='nexus.meta_api.proto.Search.search',
     index=0,
     containing_service=None,
     input_type=_SEARCHREQUEST,
@@ -237,8 +237,8 @@ _METASEARCH = _descriptor.ServiceDescriptor(
     create_key=_descriptor._internal_create_key,
   ),
 ])
-_sym_db.RegisterServiceDescriptor(_METASEARCH)
+_sym_db.RegisterServiceDescriptor(_SEARCH)
 
-DESCRIPTOR.services_by_name['MetaSearch'] = _METASEARCH
+DESCRIPTOR.services_by_name['Search'] = _SEARCH
 
 # @@protoc_insertion_point(module_scope)

nexus/meta_api/proto/search_service_pb2_grpc.py

@@ -1,12 +1,11 @@
 # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
-from nexus.meta_api.proto import \
+
-    meta_search_service_pb2 as \
+from nexus.meta_api.proto import search_service_pb2 as nexus_dot_meta__api_dot_proto_dot_search__service__pb2
-    nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2
 
 
-class MetaSearchStub(object):
+class SearchStub(object):
     """Missing associated documentation comment in .proto file."""
 
     def __init__(self, channel):
@@ -16,13 +15,13 @@ class MetaSearchStub(object):
             channel: A grpc.Channel.
         """
         self.search = channel.unary_unary(
-                '/nexus.meta_api.proto.MetaSearch/search',
+                '/nexus.meta_api.proto.Search/search',
-                request_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
+                request_serializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchRequest.SerializeToString,
-                response_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
+                response_deserializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchResponse.FromString,
                 )
 
 
-class MetaSearchServicer(object):
+class SearchServicer(object):
     """Missing associated documentation comment in .proto file."""
 
     def search(self, request, context):
@@ -32,21 +31,21 @@ class MetaSearchServicer(object):
         raise NotImplementedError('Method not implemented!')
 
 
-def add_MetaSearchServicer_to_server(servicer, server):
+def add_SearchServicer_to_server(servicer, server):
     rpc_method_handlers = {
             'search': grpc.unary_unary_rpc_method_handler(
                     servicer.search,
-                    request_deserializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.FromString,
+                    request_deserializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchRequest.FromString,
-                    response_serializer=nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.SerializeToString,
+                    response_serializer=nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchResponse.SerializeToString,
             ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-            'nexus.meta_api.proto.MetaSearch', rpc_method_handlers)
+            'nexus.meta_api.proto.Search', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
 
  # This class is part of an EXPERIMENTAL API.
-class MetaSearch(object):
+class Search(object):
     """Missing associated documentation comment in .proto file."""
 
     @staticmethod
@@ -60,8 +59,8 @@ class MetaSearch(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.MetaSearch/search',
+        return grpc.experimental.unary_unary(request, target, '/nexus.meta_api.proto.Search/search',
-            nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchRequest.SerializeToString,
+            nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchRequest.SerializeToString,
-            nexus_dot_meta__api_dot_proto_dot_meta__search__service__pb2.SearchResponse.FromString,
+            nexus_dot_meta__api_dot_proto_dot_search__service__pb2.SearchResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

nexus/pylon/sources/base.py

@@ -81,31 +81,31 @@ class PreparedRequest:
 
     @asynccontextmanager
     async def execute_with(self, session):
-        async with session.request(
+        try:
-            method=self.method,
+            async with session.request(
-            url=self.url,
+                method=self.method,
-            timeout=self.timeout,
+                url=self.url,
-            headers=self.headers,
+                timeout=self.timeout,
-            cookies=self.cookies,
+                headers=self.headers,
-            params=self.params,
+                cookies=self.cookies,
-            ssl=self.ssl,
+                params=self.params,
-        ) as resp:
+                ssl=self.ssl,
-            try:
+            ) as resp:
                 yield resp
-            except BadResponseError as e:
+        except BadResponseError as e:
-                e.add('url', self.url)
+            e.add('url', self.url)
-                raise e
+            raise e
-            except (
+        except (
-                aiohttp.client_exceptions.ClientConnectionError,
+            aiohttp.client_exceptions.ClientConnectionError,
-                aiohttp.client_exceptions.ClientPayloadError,
+            aiohttp.client_exceptions.ClientPayloadError,
-                aiohttp.client_exceptions.ClientResponseError,
+            aiohttp.client_exceptions.ClientResponseError,
-                aiohttp.client_exceptions.TooManyRedirects,
+            aiohttp.client_exceptions.TooManyRedirects,
-                asyncio.TimeoutError,
+            asyncio.TimeoutError,
-                ProxyConnectionError,
+            ProxyConnectionError,
-                ProxyTimeoutError,
+            ProxyTimeoutError,
-                ProxyError,
+            ProxyError,
-            ) as e:
+        ) as e:
-                raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e))
+            raise DownloadError(nested_error=repr(e), nested_error_cls=class_fullname(e))
 
 
 class BaseValidator:

rules/nodejs/package.json

@@ -6,6 +6,7 @@
         "nuxt": "nuxt"
     },
     "dependencies": {
+        "@grpc/grpc-js": "^1.2.12",
         "@mdi/font": "^5.8.55",
         "@nuxtjs/axios": "^5.12.5",
         "@pointotech/pug-plain-loader": "^1.0.1",
@@ -15,7 +16,7 @@
         "bootstrap": "^4.5.3",
         "bootstrap-scss": "^4.5.3",
         "bootstrap-vue": "^2.17.3",
-        "copy-webpack-plugin": "^6.4.1",
+        "copy-webpack-plugin": "^8.1.1",
         "core-js": "^3.6.5",
         "css-loader": "^5.0.1",
         "dateformat": "^4.4.1",
@@ -24,13 +25,16 @@
         "eslint-config-standard": "^16.0.2",
         "eslint-plugin-import": "^2.22.1",
         "eslint-plugin-node": "^11.1.0",
-        "eslint-plugin-promise": "^4.2.1",
+        "eslint-plugin-promise": "^5.1.0",
         "eslint-plugin-vue": "^7.4.1",
         "express": "^4.17.1",
         "file-loader": "^6.2.0",
         "font-awesome": "^4.7.0",
-        "html-entities": "^1.4.0",
+        "google-protobuf": "^3.15.8",
-        "html-webpack-plugin": "^4.5.1",
+        "grpc": "^1.24.6",
+        "grpc-tools": "^1.11.1",
+        "html-entities": "^2.3.2",
+        "html-webpack-plugin": "^5.3.1",
         "js-cookie": "^2.2.1",
         "lodash": "^4.17.20",
         "loglevel": "^1.7.1",
@@ -42,9 +46,10 @@
         "pug-plain-loader": "^1.1.0",
         "sass": "^1.32.4",
         "sass-lint": "^1.13.1",
-        "sass-loader": "^10.1.1",
+        "sass-loader": "^11.0.1",
         "sockjs-client": "^1.5.0",
         "tempusdominus-bootstrap-4": "^5.39.0",
+        "ts-protoc-gen": "^0.14.0",
         "url": "^0.11.0",
         "url-loader": "^4.1.1",
         "v8-compile-cache": "^2.2.0",
@@ -58,7 +63,7 @@
         "vue-template-compiler": "^2.6.12",
         "vuejs-dialog": "^1.4.2",
         "vuex-persistedstate": "^4.0.0-beta.3",
-        "webpack": "^4.46.0",
+        "webpack": "^5.33.2",
         "webpack-cli": "^4.3.1",
         "webpack-dev-server": "^3.11.1",
         "wordwrap": "^1.0.0"

rules/python/requirements.txt

@@ -1,7 +1,9 @@
 setuptools==54.1.2
 
+aiobaseclient==0.2.1
 aiochclient==2.0.0
 aiocrossref==0.2.0
+aiogrpcclient==0.1.0
 aiodns==2.0.0
 aioftp==0.18.1
 aiohttp[speedups]==3.7.4.post0