cfk/ansible/services/master.yml

Finbox's PaaS

services:
  reporter:
    image: "finboxio/reporter:${CFK_VERSION}"
    container_name: reporter
    hostname: "${HOSTNAME}-reporter.weave.local"
    volumes:
      - "/var/run/weave/weave.sock:/var/run/docker.sock"
      - "/data/reporter:/srv/reporter"
    environment:
      CFK_START_AFTER: "discovery"
      HOSTNAME: "${HOSTNAME}"
      DOMAIN: "${DOMAIN}"
      SLACK_HOOK: "${SLACK_HOOK}"
      SLACK_CHANNEL: "${SLACK_CHANNEL}"

  router:
    image: "finboxio/router:${CFK_VERSION}"
    container_name: router
    hostname: "${HOSTNAME}-router.weave.local"
    dns:
      - "172.42.0.1"
      - "8.8.8.8"
      - "8.8.4.4"
    ports:
      - "80:80"
      - "81:81"
      - "9090:9090"
      - "${HOST_IP}:53:53"
      - "${HOST_IP}:53:53/udp"
    environment:
      CFK_START_AFTER: "reporter"
      CFK_ROLE: "${CFK_ROLE}"
      CFK_VERSION: "${CFK_VERSION}"
      CONSUL_MINWAIT: "${CONSUL_MINWAIT}"
      CONSUL_MAXWAIT: "${CONSUL_MAXWAIT}"
      CONSUL_MAXSTALE: "${CONSUL_MAXSTALE}"
      SERVICE_80_NAME: "router"
      SERVICE_80_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_81_NAME: "proxy-router"
      SERVICE_81_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_9090_NAME: "haproxy"
      SERVICE_9090_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_53_NAME: "router-dns"
      HOSTNAME: "${HOSTNAME}"
      HAPROXY_DOMAIN: "${DOMAIN}"
      CONSUL_URL: "${HOST_IP}:8500"
      ADMIN_USER: "${ADMIN_USER}"
      ADMIN_PASS: "${ADMIN_PASS}"
      FORCE_SSL: "${FORCE_SSL}"
      FALLBACK_URL: "${FALLBACK_URL}"
      REQUIRED_SERVICES: router-9090,consul-8500,zookeeper-8181,mesos-5050,marathon-8080,chronos-4400

  consul:
    image: gliderlabs/consul-server:0.6
    container_name: consul
    hostname: consul.weave.local
    ports:
      - "8300:8300"
      - "8301:8301"
      - "8301:8301/udp"
      - "8302:8302"
      - "8302:8302/udp"
      - "8400:8400"
      - "8500:8500"
      - "8600:8600"
      - "8600:8600/udp"
    command: "-node ${HOSTNAME} -bootstrap-expect ${NUM_MASTERS} -advertise ${HOST_IP}"
    volumes:
      - "/usr/bin/jq:/usr/bin/jq:ro"
    environment:
      CFK_START_PRECONDITION: >
        dig +short @${HOST_IP} ${HOSTNAME}-router.weave.local
        | grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b'
      CFK_START_AFTER: "router"
      GOMAXPROCS: 2
      SERVICE_8500_NAME: "consul"
      SERVICE_8500_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_8500_CHECK_HTTP: "/v1/status/leader"
      SERVICE_8500_CHECK_INTERVAL: "30s"
      SERVICE_8500_CHECK_TIMEOUT: "5s"
      SERVICE_8600_NAME: "consul-dns"
      SERVICE_8300_IGNORE: "true"
      SERVICE_8301_IGNORE: "true"
      SERVICE_8302_IGNORE: "true"
      SERVICE_8400_IGNORE: "true"
      SERVICE_53_IGNORE: "true"

  # elasticsearch:
  #   image: bdentino/elasticsearch
  #   container_name: elasticsearch
  #   hostname: "${HOSTNAME}-elasticsearch.weave.local"
  #   ports:
  #     - "9200:9200"
  #     - "9300:9300"
  #   volumes:
  #     - "/data/elasticsearch/data:/usr/share/elasticsearch/data"
  #     - "/data/elasticsearch/logs:/usr/share/elasticsearch/logs"
  #   command: "elasticsearch -Des.discovery.zen.ping.multicast.enabled=true -Des.discovery.zen.minimum_master_nodes=${QUORUM_NUM} -Des.discovery.zen.master_election.filter_client=true -Des.network.bind_host=0.0.0.0 -Des.network.publish_host=_ethwe:ipv4_ -Des.transport.tcp.port=9300 -Des.node.master=true -Des.gateway.recover_after_master_nodes=${NUM_MASTERS} -Des.gateway.expected_master_nodes=${NUM_MASTERS}"
  #   environment:
  #     CFK_START_AFTER: "consul"
  #     CFK_START_PRECONDITION: >
  #       curl -s -f localhost:8500/v1/health/service/consul
  #       | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
  #       | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
  #       | jq -s "."
  #       | jq -e "length == 0"
  #     ES_USE_IPV4: "true"
  #     SERVICE_9200_NAME: "elasticsearch"
  #     SERVICE_9200_TAGS: "${CFK_ROLE},http,admin"
  #     SERVICE_9200_CHECK_SCRIPT: >
  #       curl -s ${HOST_IP}:9200/_cluster/health
  #       | jq '.status' | grep -v 'red'
  #     SERVICE_9200_CHECK_INTERVAL: "30s"
  #     SERVICE_9200_CHECK_TIMEOUT: "5s"
  #     SERVICE_9300_IGNORE: "true"

  # logstash:
  #   image: logstash:2.1
  #   container_name: logstash
  #   hostname: "${HOSTNAME}-logstash.weave.local"
  #   volumes:
  #     - "/data/logstash:/srv/logstash"
  #     - "/data/reporter:/srv/reporter"
  #   command: "logstash -e \"input { file { path => '/srv/reporter/containers.log' codec => json sincedb_path => '/srv/logstash/sincedb' } } output { elasticsearch { hosts => 'elasticsearch' } }\""
  #   environment:
  #     CFK_START_AFTER: "elasticsearch"
  #     CFK_START_PRECONDITION: >
  #       curl -s -f localhost:8500/v1/health/service/elasticsearch
  #       | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
  #       | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
  #       | jq -s "."
  #       | jq -e "length == 0"

  # kibana:
  #   image: kibana:4.3
  #   container_name: kibana
  #   hostname: "${HOSTNAME}-kibana.weave.local"
  #   ports:
  #     - "5601:5601"
  #   environment:
  #     CFK_START_AFTER: "elasticsearch"
  #     CFK_START_PRECONDITION: >
  #       curl -s -f localhost:8500/v1/health/service/elasticsearch
  #       | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
  #       | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
  #       | jq -s "."
  #       | jq -e "length == 0"
  #     ELASTICSEARCH_URL: "http://elasticsearch:9200"
  #     SERVICE_TAGS: "${CFK_ROLE},http,admin"
  #     SERVICE_CHECK_HTTP: "/"
  #     SERVICE_CHECK_INTERVAL: "30s"
  #     SERVICE_CHECK_TIMEOUT: "5s"

  reaper:
    image: "finboxio/reaper:${CFK_VERSION}"
    container_name: reaper
    hostname: "${HOSTNAME}-reaper.weave.local"
    volumes:
      - "/var/run/weave/weave.sock:/var/run/docker.sock"
    environment:
      CFK_START_AFTER: "consul"
      CFK_START_PRECONDITION: >
        curl -s -f localhost:8500/v1/health/service/consul
        | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
        | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
        | jq -s "."
        | jq -e "length == 0"
      GRACE_PERIOD: "600"
      CONSUL_URL: "${HOST_IP}:8500"
      HOSTNAME: "${HOSTNAME}"
      DOMAIN: "${DOMAIN}"
      INTERVAL: "30"
      MAX_FAILURES: "9"
      SLACK_HOOK: "${SLACK_HOOK}"
      SLACK_CHANNEL: "${SLACK_CHANNEL}"

  registrator:
    # TODO: Switch back to original gliderlabs/registrator once https://github.com/gliderlabs/registrator/pull/245 is merged
    image: johnydays/registrator
    container_name: registrator
    hostname: "${HOSTNAME}-registrator.weave.local"
    volumes:
      - "/var/run/weave/weave.sock:/tmp/docker.sock"
    command: "-internal -retry-attempts -1 -deregister always -resync 30 consul://${HOST_IP}:8500"
    environment:
      CFK_START_AFTER: "consul"
      CFK_START_PRECONDITION: >
        curl -s -f localhost:8500/v1/health/service/consul
        | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
        | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
        | jq -s "."
        | jq -e "length == 0"
      HOSTNAME: "${HOSTNAME}"

  zookeeper:
    image: mbabineau/zookeeper-exhibitor
    container_name: zookeeper
    hostname: "${HOSTNAME}-zookeeper.weave.local"
    ports:
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
      - "8181:8181"
    environment:
      CFK_START_AFTER: "registrator"
      CFK_START_PRECONDITION: >
        curl -s -f localhost:8500/v1/health/service/consul
        | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
        | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
        | jq -s "."
        | jq -e "length == 0"
      S3_BUCKET: "${S3_BUCKET}"
      S3_PREFIX: "${S3_ZK_PREFIX}"
      AWS_REGION: "${S3_REGION}"
      ZK_ENSEMBLE_SIZE: "${NUM_MASTERS}"
      HOSTNAME: "${HOSTNAME}-zookeeper"
      SERVICE_8181_NAME: "zookeeper"
      SERVICE_8181_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_8181_CHECK_SCRIPT: >
        curl -s ${HOST_IP}:8181/exhibitor/v1/cluster/status
        | jq '.[] | select(.isLeader==true) | .hostname'
        | jq -se 'length == 1'
      SERVICE_8181_CHECK_INTERVAL: "30s"
      SERVICE_8181_CHECK_TIMEOUT: "5s"
      SERVICE_2181_IGNORE: "true"
      SERVICE_2888_IGNORE: "true"
      SERVICE_3888_IGNORE: "true"

  mesos:
    image: mesosphere/mesos-master:0.28.1
    container_name: mesos
    hostname: "${HOSTNAME}-mesos.weave.local"
    ports:
      - "5050:5050"
    environment:
      CFK_START_AFTER: "zookeeper"
      # Make sure all zookeeper nodes agree on leader before starting
      CFK_START_PRECONDITION: >
        curl -s -f localhost:8500/v1/catalog/service/zookeeper
        | jq -r ".[] | (.ServiceAddress + \":\" + (.ServicePort | tostring))"
        | xargs -i curl -s {}/exhibitor/v1/cluster/status
        | jq "map(select(.isLeader==true) | .hostname) | .[] // null"
        | sort -u
        | uniq
        | jq -se "length == 1 and .[0] != null"
        && curl -s -f localhost:8500/v1/catalog/service/zookeeper
        | jq -r ".[] | (.ServiceAddress + \":\" + (.ServicePort | tostring))"
        | xargs -i curl -s {}/exhibitor/v1/cluster/status
        | jq "length | . >= ${QUORUM_NUM}"
        | sort -u
        | uniq
        | jq -se "length == 1 and .[0] == true"
      MESOS_ZK: "zk://zookeeper:2181/mesos"
      MESOS_HOSTNAME: "${HOSTNAME}-mesos"
      MESOS_QUORUM: "${QUORUM_NUM}"
      MESOS_CLUSTER: "cfk"
      MESOS_IP: "0.0.0.0"
      SERVICE_NAME: "mesos"
      SERVICE_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_CHECK_SCRIPT: >
        curl -s -f ${HOST_IP}:8500/v1/catalog/service/mesos
        | jq -r ".[] | (.ServiceAddress + \":\" + (.ServicePort | tostring))"
        | xargs -i curl -s {}/master/state.json
        | jq -s ".[] | .elected_time // empty"
        | jq -se "length == 1"
      SERVICE_CHECK_INTERVAL: "30s"
      SERVICE_CHECK_TIMEOUT: "5s"

  marathon:
    image: mesosphere/marathon:v1.1.1
    container_name: marathon
    hostname: "${HOSTNAME}-marathon.weave.local"
    command: [ '--env_vars_prefix', 'CFK_' ]
    ports:
      - "8080:8080"
    environment:
      CFK_START_AFTER: "mesos"
      CFK_START_PRECONDITION: >
        curl -s -f localhost:8500/v1/health/service/mesos
        | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
        | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
        | jq -s "."
        | jq -e "length == 0"
        && curl -s -f localhost:8500/v1/health/service/mesos
        | jq -e ".[] | .Checks | .[] | select(.ServiceName==\"mesos\")"
        | jq -s ".[] | select(.Status==\"passing\")"
        | jq -s "."
        | jq -e "length >= ${QUORUM_NUM}"
      MARATHON_HOSTNAME: "${HOSTNAME}-marathon"
      MARATHON_MASTER: "zk://zookeeper:2181/mesos"
      MARATHON_ZK: "zk://zookeeper:2181/marathon"
      MARATHON_TASK_LAUNCH_TIMEOUT: "600000"
      MARATHON_WEBUI_URL: "http://marathon.${DOMAIN}"
      MARATHON_MESOS_LEADER_UI_URL: "http://mesos.${DOMAIN}"
      MARATHON_FRAMEWORK_NAME: "cfk-marathon"
      SERVICE_CHECK_SCRIPT: >
        curl -s -f http://${HOST_IP}:8080/v2/leader
        && curl -s -f mesos.router.local/state.json
        | jq -r  ".frameworks | .[].name"
        | grep "cfk-marathon"
      SERVICE_CHECK_INTERVAL: "30s"
      SERVICE_CHECK_TIMEOUT: "5s"
      SERVICE_TAGS: "${CFK_ROLE},http,admin"

  chronos:
    image: mesosphere/chronos:chronos-2.5.0-0.1.20160223054243.ubuntu1404-mesos-0.27.1-2.0.226.ubuntu1404
    container_name: chronos
    hostname: "${HOSTNAME}-chronos.weave.local"
    ports:
      - "4400:4400"
    command: >
      /usr/bin/chronos run_jar
      --http_port 4400
      --zk_hosts zookeeper:2181
      --hostname ${HOSTNAME}-chronos
      --cluster_name cfk
      --mesos_framework_name cfk-chronos
      --master zk://zookeeper:2181/mesos
      --webui_url http://chronos.${DOMAIN}
    environment:
      CFK_START_AFTER: "mesos"
      CFK_START_PRECONDITION: >
        curl -s -f localhost:8500/v1/health/service/mesos
        | jq -e ".[] | .Checks | .[] | select(.Node==\"${HOSTNAME}\")"
        | jq -s ".[] | select(.Status==\"warning\" or .Status==\"critical\")"
        | jq -s "."
        | jq -e "length == 0"
        && curl -s -f localhost:8500/v1/health/service/mesos
        | jq -e ".[] | .Checks | .[] | select(.ServiceName==\"mesos\")"
        | jq -s ".[] | select(.Status==\"passing\")"
        | jq -s "."
        | jq -e "length >= ${QUORUM_NUM}"
      SERVICE_TAGS: "${CFK_ROLE},http,admin"
      SERVICE_CHECK_SCRIPT: >
        curl -s -f http://${HOST_IP}:4400/scheduler/jobs
        && curl -s -f mesos.router.local/state.json
        | jq -r  ".frameworks | .[].name"
        | grep "cfk-chronos"
      SERVICE_CHECK_INTERVAL: "30s"
      SERVICE_CHECK_TIMEOUT: "5s"