mirror of
https://github.com/nexus-stc/hyperboria
synced 2025-01-12 19:47:36 +01:00
91 lines
3.4 KiB
YAML
91 lines
3.4 KiB
YAML
|
---
|
||
|
|
|
||
|
|
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
|
||
|
|
pipe:
|
||
|
|
brokers: |
|
||
|
|
kafka-0.example.net
|
||
|
|
schema:
|
||
|
|
- consumers:
|
||
|
|
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
|
||
|
|
topics:
|
||
|
|
- name: cross_references
|
||
|
|
workers: 4
|
||
|
|
group_id: pipe
|
||
|
|
processors:
|
||
|
|
- class: nexus.pipe.processors.CrossReferencesProcessor
|
||
|
|
kwargs:
|
||
|
|
brokers: |
|
||
|
|
kafka-0.example.net
|
||
|
|
database:
|
||
|
|
database: nexus
|
||
|
|
host: postgres.example.net
|
||
|
|
password: '{{ DATABASE_PASSWORD }}'
|
||
|
|
username: '{{ DATABASE_USERNAME }}'
|
||
|
|
- consumers:
|
||
|
|
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
|
||
|
|
topics:
|
||
|
|
- name: operations
|
||
|
|
workers: 2
|
||
|
|
- class: nexus.pipe.consumers.DocumentOperationsConsumer
|
||
|
|
topics:
|
||
|
|
- name: operations_binary_hp
|
||
|
|
workers: 4
|
||
|
|
- name: operations_binary
|
||
|
|
workers: 14
|
||
|
|
group_id: pipe
|
||
|
|
processors:
|
||
|
|
- class: nexus.pipe.processors.ActionProcessor
|
||
|
|
kwargs:
|
||
|
|
actions:
|
||
|
|
- class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
|
||
|
|
kwargs:
|
||
|
|
crossref:
|
||
|
|
rps: 50
|
||
|
|
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
|
||
|
|
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
|
||
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
|
||
|
|
kwargs:
|
||
|
|
database:
|
||
|
|
database: nexus
|
||
|
|
host: postgres.example.net
|
||
|
|
password: '{{ DATABASE_PASSWORD }}'
|
||
|
|
username: '{{ DATABASE_USERNAME }}'
|
||
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
|
||
|
|
kwargs:
|
||
|
|
brokers: |
|
||
|
|
kafka-0.example.net
|
||
|
|
topic: cross_references
|
||
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
|
||
|
|
kwargs:
|
||
|
|
summa:
|
||
|
|
base_url: http://summa.example.net
|
||
|
|
timeout: 15
|
||
|
|
ttl_dns_cache: 30
|
||
|
|
filter:
|
||
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
||
|
|
kwargs:
|
||
|
|
document: scimag
|
||
|
|
operation: update_document
|
||
|
|
- class: nexus.pipe.processors.ActionProcessor
|
||
|
|
kwargs:
|
||
|
|
actions:
|
||
|
|
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
|
||
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
|
||
|
|
kwargs:
|
||
|
|
database:
|
||
|
|
database: nexus
|
||
|
|
host: postgres.example.net
|
||
|
|
password: '{{ DATABASE_PASSWORD }}'
|
||
|
|
username: '{{ DATABASE_USERNAME }}'
|
||
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
|
||
|
|
kwargs:
|
||
|
|
summa:
|
||
|
|
base_url: http://summa.example.net
|
||
|
|
timeout: 15
|
||
|
|
ttl_dns_cache: 30
|
||
|
|
filter:
|
||
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
||
|
|
kwargs:
|
||
|
|
document: scitech
|
||
|
|
operation: update_document
|