mirror of
https://github.com/nexus-stc/hyperboria
synced 2025-02-11 01:46:51 +01:00
43be16e4bc
- [nexus] Remove outdated protos - [nexus] Development - [nexus] Development - [nexus] Development - [nexus] Development - [nexus] Development - [nexus] Refactor views - [nexus] Update aiosumma - [nexus] Add tags - [nexus] Development - [nexus] Update repository - [nexus] Update repository - [nexus] Update dependencies - [nexus] Update dependencies - [nexus] Fixes for MetaAPI - [nexus] Support for new queries - [nexus] Adopt new versions of search - [nexus] Improving Nexus - [nexus] Various fixes - [nexus] Add profile - [nexus] Fixes for ingestion - [nexus] Refactorings and bugfixes - [idm] Add profile methods - [nexus] Fix stalled nexus-meta bugs - [nexus] Various bugfixes - [nexus] Restore IDM API functionality GitOrigin-RevId: a0842345a6dde5b321279ab5510a50c0def0e71a
95 lines
3.7 KiB
YAML
95 lines
3.7 KiB
YAML
---
|
|
|
|
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
|
|
pipe:
|
|
bootstrap_servers: |
|
|
kafka-0.example.net
|
|
schema:
|
|
- consumers:
|
|
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
|
|
topics:
|
|
- name: cross_references
|
|
workers: 4
|
|
group_id: pipe
|
|
processors:
|
|
- class: nexus.pipe.processors.CrossReferencesProcessor
|
|
kwargs:
|
|
bootstrap_servers: |
|
|
kafka-0.example.net
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- consumers:
|
|
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
|
|
topics:
|
|
- name: operations
|
|
workers: 2
|
|
- class: nexus.pipe.consumers.DocumentOperationsConsumer
|
|
topics:
|
|
- name: operations_binary_hp
|
|
workers: 4
|
|
- name: operations_binary
|
|
workers: 14
|
|
group_id: pipe
|
|
processors:
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.FillFromExternalSourceAction
|
|
kwargs:
|
|
crossref:
|
|
ttl_dns_cache: 120
|
|
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.CleanAction
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.ToPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.ReferencesToKafkaAction
|
|
kwargs:
|
|
kafka:
|
|
bootstrap_servers: |
|
|
kafka-0.example.net
|
|
max_request_size: 1048576
|
|
topic: cross_references
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
document: scimag
|
|
operation: update_document
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.document_operations_pb.update_document_scitech_pb.CleanAction
|
|
- class: nexus.actions.document_operations_pb.update_document_scitech_pb.ToPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
document: scitech
|
|
operation: update_document
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.document_operations_pb.store_telegram_file_id_pb.ToPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
operation: store_telegram_file_id
|