Merge branch 'ramparts_and_ha' into devel

* ramparts_and_ha: (21 commits)
  Rename database migrations for devel integration
  Integrate callback receiver refactoring
  Fix an issue running jobs in the cluster
  Implement a more dynamic celery queue system
  Purge old munin monitors and tools
  Refactor Tower HA Instance logic and models
  Docker compose improvements
  Initial Docker Compose workflow for Tower cluster
  Add memcached role for setup playbook
  Removing qpid from deb packaging
  Refactor rabbitmq role
  Integrate memcached into setup playbook
  Remove mongodb shutdown task
  Remove dependency on erlang_sd_notify
  Add initial rabbitmq role
  Initial rabbitmq setup playbook integration
  Update development environment for rabbit
  Replace qpid with rabbitmq
  Remove redis role from setup playbook
  Update qpid packaging, remove migrations
  ...

tools/docker-compose-cluster.yml

@@ -0,0 +1,63 @@
+version: '2'
+services:
+  haproxy:
+    build:
+      context: ./docker-compose
+      dockerfile: Dockerfile-haproxy
+    depends_on:
+      - "tower_1"
+      - "tower_2"
+      - "tower_3"
+    ports:
+      - "8013:8013"
+      - "1936:1936"
+      - "5555:5555"
+  tower_1:
+    image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    hostname: tower_1
+    environment:
+      RABBITMQ_HOST: rabbitmq_1
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
+    volumes:
+      - "../:/tower_devel"
+  tower_2:
+    image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    hostname: tower_2
+    environment:
+      RABBITMQ_HOST: rabbitmq_2
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
+    volumes:
+      - "../:/tower_devel"
+  tower_3:
+    image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    hostname: tower_3
+    environment:
+      RABBITMQ_HOST: rabbitmq_3
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
+    volumes:
+      - "../:/tower_devel"
+  rabbitmq_1:
+    image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest
+    hostname: rabbitmq_1
+  rabbitmq_2:
+    image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest
+    hostname: rabbitmq_2
+    environment: 
+      - CLUSTERED=true
+      - CLUSTER_WITH=rabbitmq_1
+  rabbitmq_3:
+    image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest
+    hostname: rabbitmq_3
+    environment: 
+      - CLUSTERED=true
+      - CLUSTER_WITH=rabbitmq_1
+  postgres:
+    image: postgres:9.4.1
+  memcached:
+    image: memcached:alpine

tools/docker-compose.yml

@@ -3,9 +3,16 @@ services:
   # Primary Tower Development Container
   tower:
     image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    hostname: tower
+    environment:
+      RABBITMQ_HOST: rabbitmq
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
     ports:
       - "8080:8080"
       - "8013:8013"
+      - "5555:5555"
     links:
       - postgres
       - memcached
@@ -20,10 +27,8 @@ services:
   # Postgres Database Container
   postgres:
     image: postgres:9.4.1
-
   memcached:
     image: memcached:alpine
-
   rabbitmq:
     image: rabbitmq:3-management
 

tools/docker-compose/Dockerfile-haproxy

@@ -0,0 +1,2 @@
+FROM haproxy:1.6-alpine
+COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg

tools/docker-compose/haproxy.cfg

@@ -0,0 +1,53 @@
+global
+    debug
+    stats socket /tmp/admin.sock
+    stats timeout 30s
+
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+
+frontend localnodes
+    bind *:8013
+    mode http
+    default_backend nodes
+
+frontend flower
+    bind *:5555
+    mode http
+    default_backend flower_nodes
+
+backend nodes
+    mode http
+    balance roundrobin
+    option forwardfor
+    option http-pretend-keepalive
+    http-request set-header X-Forwarded-Port %[dst_port]
+    http-request add-header X-Forwarded-Proto https if { ssl_fc }
+    option httpchk HEAD / HTTP/1.1\r\nHost:localhost
+    server tower_1 tower_1:8013 check
+    server tower_2 tower_2:8013 check
+    server tower_3 tower_3:8013 check
+
+backend flower_nodes
+    mode http
+    balance roundrobin
+    option forwardfor
+    option http-pretend-keepalive
+    http-request set-header X-Forwarded-Port %[dst_port]
+    http-request add-header X-Forwarded-Proto https if { ssl_fc }
+    #option httpchk HEAD / HTTP/1.1\r\nHost:localhost
+    server tower_1 tower_1:5555
+    server tower_2 tower_2:5555
+    server tower_3 tower_3:5555
+
+listen stats
+    bind *:1936
+    stats enable
+    stats uri /
+

tools/docker-compose/start_development.sh

@@ -4,7 +4,7 @@ set +x
 # Wait for the databases to come up
 ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all
 ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all
-ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=rabbitmq port=5672" all
+ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=${RABBITMQ_HOST} port=5672" all
 
 # In case Tower in the container wants to connect to itself, use "docker exec" to attach to the container otherwise
 # TODO: FIX

tools/munin_monitors/callbackr_alive

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-graph_title Callback Receiver Processes
-graph_vlabel num processes
-graph_category tower
-callbackr.label Callback Receiver Processes
-EOM
-        exit 0;;
-esac
-
-printf "callbackr.value "
-ps ax | grep run_callback_receiver | grep -v grep | wc -l
-printf "\n"

tools/munin_monitors/celery_alive

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-graph_title Celery Processes
-graph_vlabel num processes
-graph_category tower
-celeryd.label Celery Processes
-EOM
-        exit 0;;
-esac
-
-printf "celeryd.value "
-ps ax | grep celeryd | grep -v grep | wc -l
-printf "\n"

tools/munin_monitors/postgres_alive

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-graph_title Postmaster Processes
-graph_vlabel num processes
-graph_category tower
-postmaster.label Postmaster Processes
-EOM
-        exit 0;;
-esac
-
-printf "postmaster.value "
-ps ax | grep postmaster | grep -v grep | wc -l
-printf "\n"

tools/munin_monitors/redis_alive

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-graph_title Redis Processes
-graph_vlabel num processes
-graph_category tower
-redis.label Redis Processes
-EOM
-        exit 0;;
-esac
-
-printf "redis.value "
-ps ax | grep redis | grep -v grep | wc -l
-printf "\n"

tools/munin_monitors/socketio_alive

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-graph_title SocketIO Service Processes
-graph_vlabel num processes
-graph_category tower
-socketio.label SocketIO Service Processes
-EOM
-        exit 0;;
-esac
-
-printf "socketio.value "
-ps ax | grep run_socketio_service | grep -v grep | wc -l
-printf "\n"

tools/munin_monitors/taskmanager_alive

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-graph_title Task Manager Processes
-graph_vlabel num processes
-graph_category tower
-taskm.label Task Manager Processes
-EOM
-        exit 0;;
-esac
-
-printf "taskm.value "
-ps ax | grep run_task_system | grep -v grep | wc -l
-printf "\n"

tools/munin_monitors/tower_jobs

@@ -1,27 +0,0 @@
-#!/bin/sh
-
-case $1 in
-   config)
-        cat <<'EOM'
-multigraph tower_jobs
-graph_title Running Jobs breakdown
-graph_vlabel job count
-graph_category tower
-running.label Running jobs
-waiting.label Waiting jobs
-pending.label Pending jobs
-EOM
-        exit 0;;
-esac
-
-printf "running.value "
-awx-manage stats --stat jobs_running
-printf "\n"
-
-printf "waiting.value "
-awx-manage stats --stat jobs_waiting
-printf "\n"
-
-printf "pending.value "
-awx-manage stats --stat jobs_pending
-printf "\n"