mirror of
https://github.com/nexus-stc/hyperboria
synced 2024-06-06 13:18:08 +02:00
9ce67ec590
- feat(idm): Rename IDM-2 to IDM - feat(idm): Open IDM 3 internal commit(s) GitOrigin-RevId: e302e9b5cda18cca1adc4ae8a3d906714d222106
91 lines
3.4 KiB
YAML
91 lines
3.4 KiB
YAML
---
|
|
|
|
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
|
|
pipe:
|
|
brokers: |
|
|
kafka-0.example.net
|
|
schema:
|
|
- consumers:
|
|
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
|
|
topics:
|
|
- name: cross_references
|
|
workers: 4
|
|
group_id: pipe
|
|
processors:
|
|
- class: nexus.pipe.processors.CrossReferencesProcessor
|
|
kwargs:
|
|
brokers: |
|
|
kafka-0.example.net
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- consumers:
|
|
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
|
|
topics:
|
|
- name: operations
|
|
workers: 2
|
|
- class: nexus.pipe.consumers.DocumentOperationsConsumer
|
|
topics:
|
|
- name: operations_binary_hp
|
|
workers: 4
|
|
- name: operations_binary
|
|
workers: 14
|
|
group_id: pipe
|
|
processors:
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
|
|
kwargs:
|
|
crossref:
|
|
rps: 50
|
|
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
|
|
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
|
|
kwargs:
|
|
brokers: |
|
|
kafka-0.example.net
|
|
topic: cross_references
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
|
|
kwargs:
|
|
summa:
|
|
base_url: http://summa.example.net
|
|
timeout: 15
|
|
ttl_dns_cache: 30
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
document: scimag
|
|
operation: update_document
|
|
- class: nexus.pipe.processors.ActionProcessor
|
|
kwargs:
|
|
actions:
|
|
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
|
|
kwargs:
|
|
database:
|
|
database: nexus
|
|
host: postgres.example.net
|
|
password: '{{ DATABASE_PASSWORD }}'
|
|
username: '{{ DATABASE_USERNAME }}'
|
|
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
|
|
kwargs:
|
|
summa:
|
|
base_url: http://summa.example.net
|
|
timeout: 15
|
|
ttl_dns_cache: 30
|
|
filter:
|
|
class: nexus.pipe.filters.DocumentOperationFilter
|
|
kwargs:
|
|
document: scitech
|
|
operation: update_document
|