hyperboria/nexus/pipe/configs/base.yaml
the-superpirate 9ce67ec590 - feat(pylon): Refactor code
- feat(idm): Rename IDM-2 to IDM
  - feat(idm): Open IDM
3 internal commit(s)

GitOrigin-RevId: e302e9b5cda18cca1adc4ae8a3d906714d222106
2021-04-12 23:38:54 +03:00

91 lines
3.4 KiB
YAML

---
log_path: '/var/log/nexus-pipe/{{ ENV_TYPE }}'
pipe:
brokers: |
kafka-0.example.net
schema:
- consumers:
- class: nexus.pipe.consumers.CrossReferencesBulkConsumer
topics:
- name: cross_references
workers: 4
group_id: pipe
processors:
- class: nexus.pipe.processors.CrossReferencesProcessor
kwargs:
brokers: |
kafka-0.example.net
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- consumers:
- class: nexus.pipe.consumers.DocumentOperationsJsonConsumer
topics:
- name: operations
workers: 2
- class: nexus.pipe.consumers.DocumentOperationsConsumer
topics:
- name: operations_binary_hp
workers: 4
- name: operations_binary
workers: 14
group_id: pipe
processors:
- class: nexus.pipe.processors.ActionProcessor
kwargs:
actions:
- class: nexus.actions.FillDocumentOperationUpdateDocumentScimagPbFromExternalSourceAction
kwargs:
crossref:
rps: 50
user_agent: 'ScienceLegion/1.0 (Linux x86_64; ) ScienceLegion/1.0.0'
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScimagPbAction
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbToGoldenPostgresAction
kwargs:
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- class: nexus.actions.SendDocumentOperationUpdateDocumentScimagPbReferencesToKafkaAction
kwargs:
brokers: |
kafka-0.example.net
topic: cross_references
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:
summa:
base_url: http://summa.example.net
timeout: 15
ttl_dns_cache: 30
filter:
class: nexus.pipe.filters.DocumentOperationFilter
kwargs:
document: scimag
operation: update_document
- class: nexus.pipe.processors.ActionProcessor
kwargs:
actions:
- class: nexus.actions.CleanDocumentOperationUpdateDocumentScitechPbAction
- class: nexus.actions.SendDocumentOperationUpdateDocumentScitechPbToGoldenPostgresAction
kwargs:
database:
database: nexus
host: postgres.example.net
password: '{{ DATABASE_PASSWORD }}'
username: '{{ DATABASE_USERNAME }}'
- class: nexus.actions.SendDocumentOperationUpdateDocumentPbToSummaAction
kwargs:
summa:
base_url: http://summa.example.net
timeout: 15
ttl_dns_cache: 30
filter:
class: nexus.pipe.filters.DocumentOperationFilter
kwargs:
document: scitech
operation: update_document