95 lines
3.7 KiB
YAML
95 lines
3.7 KiB
YAML
---
|
|
|
|
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
|
|
pipe:
|
|
bootstrap_servers: |
|
|
kafka-0.example.net
|
|
schema:
|
|
- consumers:
|
|
- class: nexus.pipe.consumers.CrossReferencesConsumer
|
|
topics:
|
|
- name: cross_references
|
|
workers: 4
|
|
group_id: pipe
|
|
processors:
|
|
- class: nexus.pipe.processors.CrossReferencesProcessor
|
|
kwargs:
|
|
bootstrap_servers: |
|
|
kafka-0.example.net
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- consumers:
|
|
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
|
|
topics:
|
|
- name: operations
|
|
workers: 2
|
|
- class: nexus.pipe.consumers.DocumentOperationsConsumer
|
|
topics:
|
|
- name: operations_binary_hp
|
|
workers: 4
|
|
- name: operations_binary
|
|
workers: 14
|
|
group_id: pipe
|
|
processors:
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.FillFromExternalSourceAction
|
|
kwargs:
|
|
crossref:
|
|
ttl_dns_cache: 120
|
|
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.CleanAction
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.ToPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- class: nexus.actions.document_operations_pb.update_document_scimag_pb.ReferencesToKafkaAction
|
|
kwargs:
|
|
kafka:
|
|
bootstrap_servers: |
|
|
kafka-0.example.net
|
|
max_request_size: 1048576
|
|
topic: cross_references
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
document: scimag
|
|
operation: update_document
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.document_operations_pb.update_document_scitech_pb.CleanAction
|
|
- class: nexus.actions.document_operations_pb.update_document_scitech_pb.ToPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
document: scitech
|
|
operation: update_document
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.document_operations_pb.store_telegram_file_id_pb.ToPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
operation: store_telegram_file_id
|