change-propagation/config.jobqueue.wikimedia.yaml

Listens to events from Kafka and delivers them

spec: &spec
  x-sub-request-filters:
    - type: default
      name: http
      options:
        allow:
          - pattern: /^https?:\/\//
            forward_headers:
              user-agent: true
  title: The Change Propagation root
  paths:
    /sys/dedupe:
      x-modules:
        - path: sys/deduplicator.js
          options:
            redis_prefix: CPJQ
            redis:
              host: localhost
              port: 6379
    /sys/queue:
      x-modules:
        - path: sys/kafka.js
          options:
            metadata_broker_list: 127.0.0.1:9092
            dc_name: test_dc
            startup_delay: 0
            consumer:
              # These options should not be copied to puppet config.
              # We're using this config for testing, so need to configure
              # for minimal latency
              fetch.wait.max.ms: "1"
              fetch.min.bytes: "1"
              queue.buffering.max.ms: "1"
            producer:
              queue.buffering.max.messages: "10"
              compression.codec: snappy
            # Redis-mock does not support evalsha that rate limiting depend on.
            disable_ratelimit: '{env(MOCK_SERVICES)}'
            concurrency: 250
            reenqueue_delay: 5
            templates:
              normal_job_rule:
                topics:
                  - 'mediawiki.job.updateBetaFeaturesUserCounts'
                  - 'mediawiki.job.cdnPurge'
                exec:
                  method: post
                  uri: 'http://jobrunner.wikipedia.org/wiki/Special:RunSingleJob'
                  headers:
                    content-type: application/json
                    host: '{{message.meta.domain}}'
                  body: '{{globals.message}}'
              htmlCacheUpdate_partitioner:
                topic: 'mediawiki.job.htmlCacheUpdate'
                exec:
                  method: 'post'
                  uri: '/sys/partition/'
                  headers:
                    content-type: application/json
                  body: '{{globals.message}}'
              htmlCacheUpdate:
                topic: 'cpjobqueue.partitioned.mediawiki.job.htmlCacheUpdate'
                exec:
                  method: post
                  uri: 'http://jobrunner.wikipedia.org/wiki/Special:RunSingleJob'
                  headers:
                    content-type: application/json
                    host: '{{message.meta.domain}}'
                  body: '{{globals.message}}'
              refreshLinks_partitioner:
                topic: 'mediawiki.job.refreshLinks'
                exec:
                  method: 'post'
                  uri: '/sys/partition/'
                  headers:
                    content-type: application/json
                  body: '{{globals.message}}'
              refreshLinks:
                topic: 'cpjobqueue.partitioned.mediawiki.job.refreshLinks'
                exec:
                  method: post
                  uri: 'http://jobrunner.wikipedia.org/wiki/Special:RunSingleJob'
                  headers:
                    content-type: application/json
                    host: '{{message.meta.domain}}'
                  body: '{{globals.message}}'
    /sys/partition:
      x-modules:
        - path: sys/partitioner.js
          options:
            templates:
              partition_topic: 'cpjobqueue.partitioned.{{message.meta.topic}}'
            partition_key: database
            partition_default: 2
            partition_map:
              # s1: enwiki
              enwiki: 0
              # s2: large wikis
              bgwiki: 1
              bgwiktionary: 1
              cswiki: 1
              enwikiquote: 1
              enwiktionary: 1
              eowiki: 1
              fiwiki: 1
              idwiki: 1
              itwiki: 1
              nlwiki: 1
              nowiki: 1
              plwiki: 1
              ptwiki: 1
              svwiki: 1
              thwiki: 1
              trwiki: 1
              zhwiki: 1
              # s3 (default)
              # s4: commons
              commonswiki: 3
              # s5: dewiki
              dewiki: 4
              # s6: large wikis
              frwiki: 5
              jawiki: 5
              ruwiki: 5
              # s7: large wikis centralauth
              eswiki: 6
              huwiki: 6
              hewiki: 6
              ukwiki: 6
              frwiktionary: 6
              metawiki: 6
              arwiki: 6
              centralauth: 6
              cawiki: 6
              viwiki: 6
              fawiki: 6
              rowiki: 6
              kowiki: 6
              # s8: wikidata
              wikidatawiki: 7
num_workers: 0
logging:
  name: cpjobqueue
  level: fatal
  streams:
    - type: stdout
services:
  - name: changeprop
    module: hyperswitch
    conf:
      port: 7272
      user_agent: SampleChangePropInstance
      spec: *spec