mirror of
https://github.com/nexus-stc/hyperboria
synced 2024-11-11 22:09:24 +01:00
Merge pull request #35 from the-superpirate/master
- feat(nexus): Change user_id to str type due to needs of e...
This commit is contained in:
commit
86c66bf789
@ -4,9 +4,9 @@
|
|||||||
|
|
||||||
## Introduction
|
## Introduction
|
||||||
|
|
||||||
Hyperboria is a monorepository of tools aimed to enhance availability of science.
|
Hyperboria is a monorepository of tools aimed to enhance an availability of the science.
|
||||||
|
|
||||||
Here you find applications for accessing and searching in the biggest libraries of the Earth and other supportive tools.
|
Here you find Docker and desktop application for accessing and searching in the biggest libraries of the Earth.
|
||||||
|
|
||||||
All sources are under [The Unlicense](https://unlicense.org). They are literally yours.
|
All sources are under [The Unlicense](https://unlicense.org). They are literally yours.
|
||||||
|
|
||||||
|
@ -1,15 +1,16 @@
|
|||||||
# Packages
|
|
||||||
|
|
||||||
All packages requires data dumps. Older base dumps can be found in the end of this page.
|
|
||||||
|
|
||||||
## Packages
|
## Packages
|
||||||
|
|
||||||
- [`Telegram Bot`](nexus-bot)
|
### Desktop Applications
|
||||||
- [`Headless Search Server`](nexus-cognitron)
|
|
||||||
- [`Web Application`](nexus-cognitron-web)
|
To Be Done
|
||||||
|
|
||||||
|
### Docker-compose sets
|
||||||
|
- [`Nexus Telegram Bot `](nexus-bot)
|
||||||
|
- [`Nexus Cognitron (headless)`](nexus-cognitron)
|
||||||
|
- [`Nexus Cognitron Web`](nexus-cognitron-web)
|
||||||
|
|
||||||
## Data Dumps
|
## Data Dumps
|
||||||
|
|
||||||
| Date | IPFS Hash |
|
| Date | Database | IPFS Hash |
|
||||||
| --- | ----------- |
|
| ---- | -------- | --------- |
|
||||||
| 2021-03-01 | `bafykbzacebzohi352bddfunaub5rgqv5b324nejk5v6fltjh45be5ykw5jsjg` |
|
| 2021-03-01 | SciTech | `bafykbzacebzohi352bddfunaub5rgqv5b324nejk5v6fltjh45be5ykw5jsjg` |
|
@ -97,7 +97,7 @@ class BaseHandler(ABC):
|
|||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
position=position,
|
position=position,
|
||||||
request_id=request_context.request_id,
|
request_id=request_context.request_id,
|
||||||
user_id=request_context.chat.chat_id,
|
user_id=str(request_context.chat.chat_id),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def resolve_scimag(
|
async def resolve_scimag(
|
||||||
@ -136,7 +136,7 @@ class BaseHandler(ABC):
|
|||||||
page_size=16,
|
page_size=16,
|
||||||
request_id=request_context.request_id,
|
request_id=request_context.request_id,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
user_id=request_context.chat.chat_id,
|
user_id=str(request_context.chat.chat_id),
|
||||||
)
|
)
|
||||||
duplicates = [
|
duplicates = [
|
||||||
scored_document.typed_document.scitech
|
scored_document.typed_document.scitech
|
||||||
|
@ -20,7 +20,7 @@ class RollHandler(BaseHandler):
|
|||||||
language=request_context.chat.language,
|
language=request_context.chat.language,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
request_id=request_context.request_id,
|
request_id=request_context.request_id,
|
||||||
user_id=request_context.chat.chat_id,
|
user_id=str(request_context.chat.chat_id),
|
||||||
)
|
)
|
||||||
scitech_view = await self.resolve_scitech(
|
scitech_view = await self.resolve_scitech(
|
||||||
document_id=roll_response_pb.document_id,
|
document_id=roll_response_pb.document_id,
|
||||||
|
@ -146,11 +146,11 @@ class SearchHandler(BaseSearchHandler):
|
|||||||
should_reset_last_widget = False
|
should_reset_last_widget = False
|
||||||
is_subscription_required_for_handler = True
|
is_subscription_required_for_handler = True
|
||||||
|
|
||||||
def check_search_ban_timeout(self, chat_id: int):
|
def check_search_ban_timeout(self, user_id: str):
|
||||||
ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=chat_id)
|
ban_timeout = self.application.user_manager.check_search_ban_timeout(user_id=user_id)
|
||||||
if ban_timeout:
|
if ban_timeout:
|
||||||
raise BannedUserError(ban_timeout=ban_timeout)
|
raise BannedUserError(ban_timeout=ban_timeout)
|
||||||
self.application.user_manager.add_search_time(user_id=chat_id, search_time=time.time())
|
self.application.user_manager.add_search_time(user_id=user_id, search_time=time.time())
|
||||||
|
|
||||||
def parse_pattern(self, event: events.ChatAction):
|
def parse_pattern(self, event: events.ChatAction):
|
||||||
search_prefix = event.pattern_match.group(1)
|
search_prefix = event.pattern_match.group(1)
|
||||||
@ -161,7 +161,7 @@ class SearchHandler(BaseSearchHandler):
|
|||||||
|
|
||||||
async def handler(self, event: events.ChatAction, request_context: RequestContext):
|
async def handler(self, event: events.ChatAction, request_context: RequestContext):
|
||||||
try:
|
try:
|
||||||
self.check_search_ban_timeout(chat_id=request_context.chat.chat_id)
|
self.check_search_ban_timeout(user_id=str(request_context.chat.chat_id))
|
||||||
except BannedUserError as e:
|
except BannedUserError as e:
|
||||||
request_context.error_log(e)
|
request_context.error_log(e)
|
||||||
return await event.reply(t(
|
return await event.reply(t(
|
||||||
|
@ -7,7 +7,7 @@ class UserManager:
|
|||||||
self.search_times = {}
|
self.search_times = {}
|
||||||
self.search_ban_times = {}
|
self.search_ban_times = {}
|
||||||
|
|
||||||
def add_search_time(self, user_id: int, search_time: float):
|
def add_search_time(self, user_id: str, search_time: float):
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
search_times = self.search_times.get(user_id, [])
|
search_times = self.search_times.get(user_id, [])
|
||||||
search_times.append(search_time)
|
search_times.append(search_time)
|
||||||
@ -32,7 +32,7 @@ class UserManager:
|
|||||||
|
|
||||||
self.search_times[user_id] = search_times
|
self.search_times[user_id] = search_times
|
||||||
|
|
||||||
def check_search_ban_timeout(self, user_id: int):
|
def check_search_ban_timeout(self, user_id: str):
|
||||||
ban_time = self.search_ban_times.get(user_id)
|
ban_time = self.search_ban_times.get(user_id)
|
||||||
if ban_time:
|
if ban_time:
|
||||||
timeout = int(ban_time - time.time())
|
timeout = int(ban_time - time.time())
|
||||||
|
@ -70,7 +70,7 @@ class SearchWidget:
|
|||||||
page_size=self.application.config['application']['page_size'],
|
page_size=self.application.config['application']['page_size'],
|
||||||
request_id=self.request_id,
|
request_id=self.request_id,
|
||||||
session_id=self.session_id,
|
session_id=self.session_id,
|
||||||
user_id=self.chat.chat_id,
|
user_id=str(self.chat.chat_id),
|
||||||
language=self.chat.language,
|
language=self.chat.language,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -148,7 +148,7 @@ class SubmitterService(SubmitterServicer, BaseHubService):
|
|||||||
page_size=1,
|
page_size=1,
|
||||||
request_id=request_context.request_id,
|
request_id=request_context.request_id,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
user_id=request_context.chat.chat_id,
|
user_id=str(request_context.chat.chat_id),
|
||||||
language=request_context.chat.language,
|
language=request_context.chat.language,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -44,39 +44,39 @@ class MetaApiGrpcClient(BaseGrpcClient):
|
|||||||
self,
|
self,
|
||||||
schema: str,
|
schema: str,
|
||||||
document_id: int,
|
document_id: int,
|
||||||
|
user_id: str,
|
||||||
position: Optional[int] = None,
|
position: Optional[int] = None,
|
||||||
request_id: Optional[str] = None,
|
request_id: Optional[str] = None,
|
||||||
session_id: Optional[str] = None,
|
session_id: Optional[str] = None,
|
||||||
user_id: Optional[int] = None,
|
|
||||||
) -> TypedDocumentPb:
|
) -> TypedDocumentPb:
|
||||||
return await self.stubs['documents'].get(
|
return await self.stubs['documents'].get(
|
||||||
TypedDocumentRequestPb(
|
TypedDocumentRequestPb(
|
||||||
schema=schema,
|
schema=schema,
|
||||||
document_id=document_id,
|
document_id=document_id,
|
||||||
position=position,
|
position=position,
|
||||||
session_id=session_id,
|
|
||||||
user_id=user_id,
|
|
||||||
),
|
),
|
||||||
metadata=(
|
metadata=(
|
||||||
('request-id', request_id),
|
('request-id', request_id),
|
||||||
|
('session-id', session_id),
|
||||||
|
('user-id', user_id),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def roll(
|
async def roll(
|
||||||
self,
|
self,
|
||||||
|
user_id: str,
|
||||||
language: Optional[str] = None,
|
language: Optional[str] = None,
|
||||||
request_id: Optional[str] = None,
|
request_id: Optional[str] = None,
|
||||||
session_id: Optional[str] = None,
|
session_id: Optional[str] = None,
|
||||||
user_id: Optional[int] = None,
|
|
||||||
) -> RollResponsePb:
|
) -> RollResponsePb:
|
||||||
return await self.stubs['documents'].roll(
|
return await self.stubs['documents'].roll(
|
||||||
RollRequestPb(
|
RollRequestPb(
|
||||||
language=language,
|
language=language,
|
||||||
session_id=session_id,
|
|
||||||
user_id=user_id,
|
|
||||||
),
|
),
|
||||||
metadata=(
|
metadata=(
|
||||||
('request-id', request_id),
|
('request-id', request_id),
|
||||||
|
('session-id', session_id),
|
||||||
|
('user-id', user_id),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -95,12 +95,12 @@ class MetaApiGrpcClient(BaseGrpcClient):
|
|||||||
self,
|
self,
|
||||||
schemas: Union[List[str], Tuple[str]],
|
schemas: Union[List[str], Tuple[str]],
|
||||||
query: str,
|
query: str,
|
||||||
|
user_id: str,
|
||||||
page: Optional[int] = None,
|
page: Optional[int] = None,
|
||||||
page_size: Optional[int] = None,
|
page_size: Optional[int] = None,
|
||||||
language: Optional[str] = None,
|
language: Optional[str] = None,
|
||||||
request_id: Optional[str] = None,
|
request_id: Optional[str] = None,
|
||||||
session_id: Optional[str] = None,
|
session_id: Optional[str] = None,
|
||||||
user_id: Optional[int] = None,
|
|
||||||
) -> SearchResponsePb:
|
) -> SearchResponsePb:
|
||||||
return await self.stubs['search'].search(
|
return await self.stubs['search'].search(
|
||||||
SearchRequestPb(
|
SearchRequestPb(
|
||||||
@ -109,11 +109,11 @@ class MetaApiGrpcClient(BaseGrpcClient):
|
|||||||
page=page,
|
page=page,
|
||||||
page_size=page_size,
|
page_size=page_size,
|
||||||
language=language,
|
language=language,
|
||||||
session_id=session_id,
|
|
||||||
user_id=user_id,
|
|
||||||
),
|
),
|
||||||
metadata=(
|
metadata=(
|
||||||
('request-id', request_id),
|
('request-id', request_id),
|
||||||
|
('session-id', session_id),
|
||||||
|
('user-id', user_id),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -121,18 +121,18 @@ class MetaApiGrpcClient(BaseGrpcClient):
|
|||||||
self,
|
self,
|
||||||
page: int,
|
page: int,
|
||||||
page_size: int,
|
page_size: int,
|
||||||
|
user_id: str,
|
||||||
request_id: Optional[str] = None,
|
request_id: Optional[str] = None,
|
||||||
session_id: Optional[str] = None,
|
session_id: Optional[str] = None,
|
||||||
user_id: Optional[int] = None,
|
|
||||||
) -> TopMissedResponsePb:
|
) -> TopMissedResponsePb:
|
||||||
return await self.stubs['documents'].top_missed(
|
return await self.stubs['documents'].top_missed(
|
||||||
TopMissedRequestPb(
|
TopMissedRequestPb(
|
||||||
page=page,
|
page=page,
|
||||||
page_size=page_size,
|
page_size=page_size,
|
||||||
session_id=session_id,
|
|
||||||
user_id=user_id,
|
|
||||||
),
|
),
|
||||||
metadata=(
|
metadata=(
|
||||||
('request-id', request_id),
|
('request-id', request_id),
|
||||||
|
('session-id', session_id),
|
||||||
|
('user-id', user_id),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
@ -22,14 +22,16 @@ export default class MetaApi {
|
|||||||
}
|
}
|
||||||
|
|
||||||
prepareMetadata () {
|
prepareMetadata () {
|
||||||
return Object.assign({ 'request-id': this.generateId(12) }, this.metadata)
|
return Object.assign({
|
||||||
|
'request-id': this.generateId(12),
|
||||||
|
'session-id': this.generateId(8)
|
||||||
|
}, this.metadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
async get (schema, documentId) {
|
async get (schema, documentId) {
|
||||||
const request = new documentsProto.TypedDocumentRequest()
|
const request = new documentsProto.TypedDocumentRequest()
|
||||||
request.setSchema(schema)
|
request.setSchema(schema)
|
||||||
request.setDocumentId(documentId)
|
request.setDocumentId(documentId)
|
||||||
request.setSessionId(this.generateId(8))
|
|
||||||
const response = await this.documentsClient.get(request, this.prepareMetadata())
|
const response = await this.documentsClient.get(request, this.prepareMetadata())
|
||||||
return response.toObject()
|
return response.toObject()
|
||||||
}
|
}
|
||||||
@ -40,7 +42,6 @@ export default class MetaApi {
|
|||||||
request.setPageSize(pageSize)
|
request.setPageSize(pageSize)
|
||||||
schemas.forEach((schema) => request.addSchemas(schema))
|
schemas.forEach((schema) => request.addSchemas(schema))
|
||||||
request.setQuery(query)
|
request.setQuery(query)
|
||||||
request.setSessionId(this.generateId(8))
|
|
||||||
const response = await this.searchClient.search(request, this.prepareMetadata())
|
const response = await this.searchClient.search(request, this.prepareMetadata())
|
||||||
return response.toObject()
|
return response.toObject()
|
||||||
}
|
}
|
||||||
|
@ -5,8 +5,6 @@ import "nexus/models/proto/typed_document.proto";
|
|||||||
|
|
||||||
message RollRequest {
|
message RollRequest {
|
||||||
string language = 1;
|
string language = 1;
|
||||||
string session_id = 2;
|
|
||||||
int64 user_id = 3;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
message RollResponse {
|
message RollResponse {
|
||||||
@ -16,8 +14,6 @@ message RollResponse {
|
|||||||
message TopMissedRequest {
|
message TopMissedRequest {
|
||||||
uint32 page = 1;
|
uint32 page = 1;
|
||||||
uint32 page_size = 2;
|
uint32 page_size = 2;
|
||||||
string session_id = 3;
|
|
||||||
int64 user_id = 4;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
message TopMissedResponse {
|
message TopMissedResponse {
|
||||||
@ -29,8 +25,6 @@ message TypedDocumentRequest {
|
|||||||
string schema = 1;
|
string schema = 1;
|
||||||
uint64 document_id = 2;
|
uint64 document_id = 2;
|
||||||
uint32 position = 3;
|
uint32 position = 3;
|
||||||
string session_id = 4;
|
|
||||||
int64 user_id = 5;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
service Documents {
|
service Documents {
|
||||||
|
66
nexus/meta_api/proto/documents_service_pb2.py
Executable file → Normal file
66
nexus/meta_api/proto/documents_service_pb2.py
Executable file → Normal file
@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
|||||||
syntax='proto3',
|
syntax='proto3',
|
||||||
serialized_options=None,
|
serialized_options=None,
|
||||||
create_key=_descriptor._internal_create_key,
|
create_key=_descriptor._internal_create_key,
|
||||||
serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"D\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nsession_id\x18\x02 \x01(\t\x12\x0f\n\x07user_id\x18\x03 \x01(\x03\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"X\n\x10TopMissedRequest\x12\x0c\n\x04page\x18\x01 \x01(\r\x12\x11\n\tpage_size\x18\x02 \x01(\r\x12\x12\n\nsession_id\x18\x03 \x01(\t\x12\x0f\n\x07user_id\x18\x04 \x01(\x03\"a\n\x11TopMissedResponse\x12:\n\x0ftyped_documents\x18\x01 \x03(\x0b\x32!.nexus.models.proto.TypedDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"r\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r\x12\x12\n\nsession_id\x18\x04 \x01(\t\x12\x0f\n\x07user_id\x18\x05 \x01(\x03\x32\x95\x02\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x12_\n\ntop_missed\x12&.nexus.meta_api.proto.TopMissedRequest\x1a\'.nexus.meta_api.proto.TopMissedResponse\"\x00\x62\x06proto3'
|
serialized_pb=b'\n,nexus/meta_api/proto/documents_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"\x1f\n\x0bRollRequest\x12\x10\n\x08language\x18\x01 \x01(\t\"#\n\x0cRollResponse\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x04\"3\n\x10TopMissedRequest\x12\x0c\n\x04page\x18\x01 \x01(\r\x12\x11\n\tpage_size\x18\x02 \x01(\r\"a\n\x11TopMissedResponse\x12:\n\x0ftyped_documents\x18\x01 \x03(\x0b\x32!.nexus.models.proto.TypedDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"M\n\x14TypedDocumentRequest\x12\x0e\n\x06schema\x18\x01 \x01(\t\x12\x13\n\x0b\x64ocument_id\x18\x02 \x01(\x04\x12\x10\n\x08position\x18\x03 \x01(\r2\x95\x02\n\tDocuments\x12V\n\x03get\x12*.nexus.meta_api.proto.TypedDocumentRequest\x1a!.nexus.models.proto.TypedDocument\"\x00\x12O\n\x04roll\x12!.nexus.meta_api.proto.RollRequest\x1a\".nexus.meta_api.proto.RollResponse\"\x00\x12_\n\ntop_missed\x12&.nexus.meta_api.proto.TopMissedRequest\x1a\'.nexus.meta_api.proto.TopMissedResponse\"\x00\x62\x06proto3'
|
||||||
,
|
,
|
||||||
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
|
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
|
||||||
|
|
||||||
@ -43,20 +43,6 @@ _ROLLREQUEST = _descriptor.Descriptor(
|
|||||||
message_type=None, enum_type=None, containing_type=None,
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
is_extension=False, extension_scope=None,
|
is_extension=False, extension_scope=None,
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='session_id', full_name='nexus.meta_api.proto.RollRequest.session_id', index=1,
|
|
||||||
number=2, type=9, cpp_type=9, label=1,
|
|
||||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='user_id', full_name='nexus.meta_api.proto.RollRequest.user_id', index=2,
|
|
||||||
number=3, type=3, cpp_type=2, label=1,
|
|
||||||
has_default_value=False, default_value=0,
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
],
|
],
|
||||||
extensions=[
|
extensions=[
|
||||||
],
|
],
|
||||||
@ -70,7 +56,7 @@ _ROLLREQUEST = _descriptor.Descriptor(
|
|||||||
oneofs=[
|
oneofs=[
|
||||||
],
|
],
|
||||||
serialized_start=111,
|
serialized_start=111,
|
||||||
serialized_end=179,
|
serialized_end=142,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -101,8 +87,8 @@ _ROLLRESPONSE = _descriptor.Descriptor(
|
|||||||
extension_ranges=[],
|
extension_ranges=[],
|
||||||
oneofs=[
|
oneofs=[
|
||||||
],
|
],
|
||||||
serialized_start=181,
|
serialized_start=144,
|
||||||
serialized_end=216,
|
serialized_end=179,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -128,20 +114,6 @@ _TOPMISSEDREQUEST = _descriptor.Descriptor(
|
|||||||
message_type=None, enum_type=None, containing_type=None,
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
is_extension=False, extension_scope=None,
|
is_extension=False, extension_scope=None,
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='session_id', full_name='nexus.meta_api.proto.TopMissedRequest.session_id', index=2,
|
|
||||||
number=3, type=9, cpp_type=9, label=1,
|
|
||||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='user_id', full_name='nexus.meta_api.proto.TopMissedRequest.user_id', index=3,
|
|
||||||
number=4, type=3, cpp_type=2, label=1,
|
|
||||||
has_default_value=False, default_value=0,
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
],
|
],
|
||||||
extensions=[
|
extensions=[
|
||||||
],
|
],
|
||||||
@ -154,8 +126,8 @@ _TOPMISSEDREQUEST = _descriptor.Descriptor(
|
|||||||
extension_ranges=[],
|
extension_ranges=[],
|
||||||
oneofs=[
|
oneofs=[
|
||||||
],
|
],
|
||||||
serialized_start=218,
|
serialized_start=181,
|
||||||
serialized_end=306,
|
serialized_end=232,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -193,8 +165,8 @@ _TOPMISSEDRESPONSE = _descriptor.Descriptor(
|
|||||||
extension_ranges=[],
|
extension_ranges=[],
|
||||||
oneofs=[
|
oneofs=[
|
||||||
],
|
],
|
||||||
serialized_start=308,
|
serialized_start=234,
|
||||||
serialized_end=405,
|
serialized_end=331,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -227,20 +199,6 @@ _TYPEDDOCUMENTREQUEST = _descriptor.Descriptor(
|
|||||||
message_type=None, enum_type=None, containing_type=None,
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
is_extension=False, extension_scope=None,
|
is_extension=False, extension_scope=None,
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='session_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.session_id', index=3,
|
|
||||||
number=4, type=9, cpp_type=9, label=1,
|
|
||||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='user_id', full_name='nexus.meta_api.proto.TypedDocumentRequest.user_id', index=4,
|
|
||||||
number=5, type=3, cpp_type=2, label=1,
|
|
||||||
has_default_value=False, default_value=0,
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
],
|
],
|
||||||
extensions=[
|
extensions=[
|
||||||
],
|
],
|
||||||
@ -253,8 +211,8 @@ _TYPEDDOCUMENTREQUEST = _descriptor.Descriptor(
|
|||||||
extension_ranges=[],
|
extension_ranges=[],
|
||||||
oneofs=[
|
oneofs=[
|
||||||
],
|
],
|
||||||
serialized_start=407,
|
serialized_start=333,
|
||||||
serialized_end=521,
|
serialized_end=410,
|
||||||
)
|
)
|
||||||
|
|
||||||
_TOPMISSEDRESPONSE.fields_by_name['typed_documents'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
|
_TOPMISSEDRESPONSE.fields_by_name['typed_documents'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
|
||||||
@ -309,8 +267,8 @@ _DOCUMENTS = _descriptor.ServiceDescriptor(
|
|||||||
index=0,
|
index=0,
|
||||||
serialized_options=None,
|
serialized_options=None,
|
||||||
create_key=_descriptor._internal_create_key,
|
create_key=_descriptor._internal_create_key,
|
||||||
serialized_start=524,
|
serialized_start=413,
|
||||||
serialized_end=801,
|
serialized_end=690,
|
||||||
methods=[
|
methods=[
|
||||||
_descriptor.MethodDescriptor(
|
_descriptor.MethodDescriptor(
|
||||||
name='get',
|
name='get',
|
||||||
|
0
nexus/meta_api/proto/documents_service_pb2_grpc.py
Executable file → Normal file
0
nexus/meta_api/proto/documents_service_pb2_grpc.py
Executable file → Normal file
@ -20,8 +20,6 @@ message SearchRequest {
|
|||||||
uint32 page = 3;
|
uint32 page = 3;
|
||||||
uint32 page_size = 4;
|
uint32 page_size = 4;
|
||||||
string language = 5;
|
string language = 5;
|
||||||
int64 user_id = 6;
|
|
||||||
string session_id = 7;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
service Search {
|
service Search {
|
||||||
|
@ -21,7 +21,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
|||||||
syntax='proto3',
|
syntax='proto3',
|
||||||
serialized_options=None,
|
serialized_options=None,
|
||||||
create_key=_descriptor._internal_create_key,
|
create_key=_descriptor._internal_create_key,
|
||||||
serialized_pb=b'\n)nexus/meta_api/proto/search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"\x87\x01\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t\x12\x0f\n\x07user_id\x18\x06 \x01(\x03\x12\x12\n\nsession_id\x18\x07 \x01(\t2_\n\x06Search\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
|
serialized_pb=b'\n)nexus/meta_api/proto/search_service.proto\x12\x14nexus.meta_api.proto\x1a\'nexus/models/proto/typed_document.proto\"l\n\x0eScoredDocument\x12\x39\n\x0etyped_document\x18\x01 \x01(\x0b\x32!.nexus.models.proto.TypedDocument\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x10\n\x08position\x18\x03 \x01(\r\"b\n\x0eSearchResponse\x12>\n\x10scored_documents\x18\x01 \x03(\x0b\x32$.nexus.meta_api.proto.ScoredDocument\x12\x10\n\x08has_next\x18\x02 \x01(\x08\"b\n\rSearchRequest\x12\x0f\n\x07schemas\x18\x01 \x03(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04page\x18\x03 \x01(\r\x12\x11\n\tpage_size\x18\x04 \x01(\r\x12\x10\n\x08language\x18\x05 \x01(\t2_\n\x06Search\x12U\n\x06search\x12#.nexus.meta_api.proto.SearchRequest\x1a$.nexus.meta_api.proto.SearchResponse\"\x00\x62\x06proto3'
|
||||||
,
|
,
|
||||||
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
|
dependencies=[nexus_dot_models_dot_proto_dot_typed__document__pb2.DESCRIPTOR,])
|
||||||
|
|
||||||
@ -156,20 +156,6 @@ _SEARCHREQUEST = _descriptor.Descriptor(
|
|||||||
message_type=None, enum_type=None, containing_type=None,
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
is_extension=False, extension_scope=None,
|
is_extension=False, extension_scope=None,
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='user_id', full_name='nexus.meta_api.proto.SearchRequest.user_id', index=5,
|
|
||||||
number=6, type=3, cpp_type=2, label=1,
|
|
||||||
has_default_value=False, default_value=0,
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
_descriptor.FieldDescriptor(
|
|
||||||
name='session_id', full_name='nexus.meta_api.proto.SearchRequest.session_id', index=6,
|
|
||||||
number=7, type=9, cpp_type=9, label=1,
|
|
||||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
|
||||||
message_type=None, enum_type=None, containing_type=None,
|
|
||||||
is_extension=False, extension_scope=None,
|
|
||||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
|
||||||
],
|
],
|
||||||
extensions=[
|
extensions=[
|
||||||
],
|
],
|
||||||
@ -182,8 +168,8 @@ _SEARCHREQUEST = _descriptor.Descriptor(
|
|||||||
extension_ranges=[],
|
extension_ranges=[],
|
||||||
oneofs=[
|
oneofs=[
|
||||||
],
|
],
|
||||||
serialized_start=319,
|
serialized_start=318,
|
||||||
serialized_end=454,
|
serialized_end=416,
|
||||||
)
|
)
|
||||||
|
|
||||||
_SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
|
_SCOREDDOCUMENT.fields_by_name['typed_document'].message_type = nexus_dot_models_dot_proto_dot_typed__document__pb2._TYPEDDOCUMENT
|
||||||
@ -223,8 +209,8 @@ _SEARCH = _descriptor.ServiceDescriptor(
|
|||||||
index=0,
|
index=0,
|
||||||
serialized_options=None,
|
serialized_options=None,
|
||||||
create_key=_descriptor._internal_create_key,
|
create_key=_descriptor._internal_create_key,
|
||||||
serialized_start=456,
|
serialized_start=418,
|
||||||
serialized_end=551,
|
serialized_end=513,
|
||||||
methods=[
|
methods=[
|
||||||
_descriptor.MethodDescriptor(
|
_descriptor.MethodDescriptor(
|
||||||
name='search',
|
name='search',
|
||||||
|
@ -69,7 +69,7 @@ class DocumentsService(DocumentsServicer, BaseService):
|
|||||||
if self.learn_logger:
|
if self.learn_logger:
|
||||||
self.learn_logger.info({
|
self.learn_logger.info({
|
||||||
'action': 'get',
|
'action': 'get',
|
||||||
'session_id': request.session_id,
|
'session_id': metadata['session-id'],
|
||||||
'unixtime': time.time(),
|
'unixtime': time.time(),
|
||||||
'schema': request.schema,
|
'schema': request.schema,
|
||||||
'document_id': document['id'],
|
'document_id': document['id'],
|
||||||
@ -83,8 +83,8 @@ class DocumentsService(DocumentsServicer, BaseService):
|
|||||||
'position': request.position,
|
'position': request.position,
|
||||||
'request_id': metadata['request-id'],
|
'request_id': metadata['request-id'],
|
||||||
'schema': request.schema,
|
'schema': request.schema,
|
||||||
'session_id': request.session_id,
|
'session_id': metadata['session-id'],
|
||||||
'user_id': request.user_id,
|
'user_id': metadata['user-id'],
|
||||||
})
|
})
|
||||||
|
|
||||||
document_pb = pb_registry[request.schema](**document)
|
document_pb = pb_registry[request.schema](**document)
|
||||||
@ -109,8 +109,8 @@ class DocumentsService(DocumentsServicer, BaseService):
|
|||||||
'id': random_id,
|
'id': random_id,
|
||||||
'mode': 'roll',
|
'mode': 'roll',
|
||||||
'request_id': metadata['request-id'],
|
'request_id': metadata['request-id'],
|
||||||
'session_id': request.session_id,
|
'session_id': metadata['session-id'],
|
||||||
'user_id': request.user_id,
|
'user_id': metadata['user-id'],
|
||||||
})
|
})
|
||||||
|
|
||||||
return RollResponsePb(document_id=random_id)
|
return RollResponsePb(document_id=random_id)
|
||||||
|
@ -104,10 +104,11 @@ class Searcher(BaseService):
|
|||||||
cache_hit = True
|
cache_hit = True
|
||||||
page_size = request.page_size or 5
|
page_size = request.page_size or 5
|
||||||
schemas = tuple(sorted([schema for schema in request.schemas]))
|
schemas = tuple(sorted([schema for schema in request.schemas]))
|
||||||
|
user_id = metadata['user-id']
|
||||||
|
|
||||||
if (
|
if (
|
||||||
(request.user_id, request.language, schemas, request.query) not in self.query_cache
|
(user_id, request.language, schemas, request.query) not in self.query_cache
|
||||||
or len(self.query_cache[(request.user_id, request.language, schemas, request.query)].scored_documents) == 0
|
or len(self.query_cache[(user_id, request.language, schemas, request.query)].scored_documents) == 0
|
||||||
):
|
):
|
||||||
cache_hit = False
|
cache_hit = False
|
||||||
query = despace_full(request.query)
|
query = despace_full(request.query)
|
||||||
@ -145,12 +146,12 @@ class Searcher(BaseService):
|
|||||||
rescored_documents = await self.rescorer.rescore(
|
rescored_documents = await self.rescorer.rescore(
|
||||||
scored_documents=search_response['scored_documents'],
|
scored_documents=search_response['scored_documents'],
|
||||||
query=query,
|
query=query,
|
||||||
session_id=request.session_id,
|
session_id=metadata['session-id'],
|
||||||
language=request.language,
|
language=request.language,
|
||||||
)
|
)
|
||||||
search_response['scored_documents'] = rescored_documents
|
search_response['scored_documents'] = rescored_documents
|
||||||
search_response_pb = self.cast_search_response(search_response)
|
search_response_pb = self.cast_search_response(search_response)
|
||||||
self.query_cache[(request.user_id, request.language, schemas, request.query)] = search_response_pb
|
self.query_cache[(user_id, request.language, schemas, request.query)] = search_response_pb
|
||||||
|
|
||||||
logging.getLogger('query').info({
|
logging.getLogger('query').info({
|
||||||
'action': 'request',
|
'action': 'request',
|
||||||
@ -164,11 +165,11 @@ class Searcher(BaseService):
|
|||||||
'query_class': processor_response['class'].value if processor_response else None,
|
'query_class': processor_response['class'].value if processor_response else None,
|
||||||
'request_id': metadata['request-id'],
|
'request_id': metadata['request-id'],
|
||||||
'schemas': schemas,
|
'schemas': schemas,
|
||||||
'session_id': request.session_id,
|
'session_id': metadata['session-id'],
|
||||||
'user_id': request.user_id,
|
'user_id': user_id,
|
||||||
})
|
})
|
||||||
|
|
||||||
scored_documents = self.query_cache[(request.user_id, request.language, schemas, request.query)].scored_documents
|
scored_documents = self.query_cache[(user_id, request.language, schemas, request.query)].scored_documents
|
||||||
left_offset = request.page * page_size
|
left_offset = request.page * page_size
|
||||||
right_offset = left_offset + page_size
|
right_offset = left_offset + page_size
|
||||||
has_next = len(scored_documents) > right_offset
|
has_next = len(scored_documents) > right_offset
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
const ALNUMWHITESPACE_REGEX = /([^\s\w])/gu
|
const NON_ALNUMWHITESPACE_REGEX = /([^\s\p{L}\p{Nd}])/gu
|
||||||
const MULTIWHITESPACE_REGEX = /\s+/g
|
const MULTIWHITESPACE_REGEX = /\s+/g
|
||||||
|
|
||||||
export function castStringToSingleString (s) {
|
export function castStringToSingleString (s) {
|
||||||
let processed = s.replace(ALNUMWHITESPACE_REGEX, ' ')
|
let processed = s.replace(NON_ALNUMWHITESPACE_REGEX, ' ')
|
||||||
processed = processed.replace(MULTIWHITESPACE_REGEX, '-')
|
processed = processed.replace(MULTIWHITESPACE_REGEX, '-')
|
||||||
return processed
|
return processed
|
||||||
}
|
}
|
||||||
|
@ -7,4 +7,4 @@ There is a plenty of projects that are in need of your time or donation support
|
|||||||
- [IPFS](https://ipfs.io) - user-friendly replacement for torrent technology allowing you to exchange files without possibility for copyretards to ban exchange
|
- [IPFS](https://ipfs.io) - user-friendly replacement for torrent technology allowing you to exchange files without possibility for copyretards to ban exchange
|
||||||
- [TOR](https://www.torproject.org) / [I2P](https://geti2p.net) - tools for improving your anonymity in the Internet by hiding your IPs and other traits that could deanonymize you
|
- [TOR](https://www.torproject.org) / [I2P](https://geti2p.net) - tools for improving your anonymity in the Internet by hiding your IPs and other traits that could deanonymize you
|
||||||
- [Yggdrasil](https://yggdrasil-network.github.io) / [Cjdns](https://github.com/cjdelisle/cjdns) - tools for allowing you to route your Internet packages without relying on centralized state-controlled equipment. It could be useful to encounter Internet connectivity disruptions arranged by governments. It also allowes you to create mesh networks with your neighbors for keeping high connectivity. It would be useful in densely populated areas or even during peaceful demonstrations.
|
- [Yggdrasil](https://yggdrasil-network.github.io) / [Cjdns](https://github.com/cjdelisle/cjdns) - tools for allowing you to route your Internet packages without relying on centralized state-controlled equipment. It could be useful to encounter Internet connectivity disruptions arranged by governments. It also allowes you to create mesh networks with your neighbors for keeping high connectivity. It would be useful in densely populated areas or even during peaceful demonstrations.
|
||||||
- Our own [Nexus](https://github.com/nexus-stc/hyperboria) that is aimed to store important data and make them searchable.
|
- [Nexus STC](https://github.com/nexus-stc/hyperboria) that is aimed to store important data and make them searchable.
|
Loading…
Reference in New Issue
Block a user