cfk/ansible/upstart/discovery-monitor.conf

Finbox's PaaS

description "monitor discovery for failed startups"

start on starting discovery
stop on stopping discovery

script
  NO_PID=0
  NO_CONTAINER=0
  CONTAINER=discovery
  while true; do
    sleep 10

    # Check for stalled startups (upstart thinks it's running but no pid)
    if (status $CONTAINER | grep start/running | grep -v process > /dev/null); then
      NO_PID=$((NO_PID + 1))
      echo "$CONTAINER failed $NO_PID stall check(s)"
      if [ $NO_PID -gt 3 ]; then
        echo "$CONTAINER is stalled...Restarting"
        NO_PID=0
        restart $CONTAINER
      fi

    # Check for missing container (upstart has pid but container isn't up)
    elif ((status $CONTAINER | grep start/running) && [ $(docker ps --filter name=$CONTAINER --format "{{ .ID }}" | wc -w) -eq 0 ]); then
      NO_CONTAINER=$((NO_CONTAINER + 1))
      echo "$CONTAINER failed $NO_CONTAINER container check(s)"
      if [ $NO_CONTAINER -gt 5 ]; then
        echo "$CONTAINER is lost...Restarting"
        NO_CONTAINER=0
        restart $CONTAINER
      fi

    else
      NO_PID=0
      NO_CONTAINER=0
    fi
  done
end script