cfk/ansible/upstart/scope-monitor.conf

Finbox's PaaS

description "monitor scope for failed startups"

start on starting scope
stop on stopping scope

script
  NO_PID=0
  NO_CONTAINER=0
  NO_HOST=0
  CONTAINER=weavescope
  eval $(cfk-env)
  while true; do
    sleep 10

    # Check for stalled startups (upstart thinks it's running but no pid)
    if (status scope | grep start/running | grep -v process > /dev/null); then
      NO_PID=$((NO_PID + 1))
      echo "$CONTAINER failed $NO_PID stall check(s)"
      if [ $NO_PID -gt 3 ]; then
        echo "$CONTAINER is stalled...Restarting"
        NO_PID=0
        restart scope
      fi

    # Check for missing container (upstart has pid but container isn't up)
    elif ((status scope | grep start/running) && [ $(docker ps --filter name=$CONTAINER --format "{{ .ID }}" | wc -w) -eq 0 ]); then
      NO_CONTAINER=$((NO_CONTAINER + 1))
      echo "$CONTAINER failed $NO_CONTAINER container check(s)"
      if [ $NO_CONTAINER -gt 5 ]; then
        echo "$CONTAINER is lost...Restarting"
        NO_CONTAINER=0
        restart scope
      fi

    # Check that host is actually registered in scope
    elif [ -z ${SCOPE_TOKEN} ]; then
      if (curl -s ${HOST_IP}:4040/api/topology/hosts | jq -r ".nodes | to_entries | map(.value) | .[].label_major" | grep ${HOSTNAME}); then
        NO_HOST=0
      else
        NO_HOST=$((NO_HOST + 1))
      fi
      if [ $NO_HOST -gt 5 ]; then
        echo "weavescope is not registering containers...Restarting"
        NO_HOST=0
        restart scope
      fi

    else
      NO_PID=0
      NO_CONTAINER=0
      NO_HOST=0
    fi
  done
end script