Initial Docker Compose workflow for Tower cluster

The goal is to share a common pattern with the existing development work

Makefile

@@ -731,6 +731,9 @@ docker-auth:
 docker-compose: docker-auth
 	TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml up --no-recreate
 
+docker-compose-cluster: docker-auth
+	TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose-cluster.yml up
+
 docker-compose-test: docker-auth
 	cd tools && TAG=$(COMPOSE_TAG) docker-compose run --rm --service-ports tower /bin/bash
 

awx/settings/local_settings.py.docker_compose

@@ -11,6 +11,36 @@
 ###############################################################################
 # MISC PROJECT SETTINGS
 ###############################################################################
+import os
+
+def patch_broken_pipe_error():
+    """Monkey Patch BaseServer.handle_error to not write
+    a stacktrace to stderr on broken pipe.
+    http://stackoverflow.com/a/22618740/362702"""
+    import sys
+    from SocketServer import BaseServer
+    from wsgiref import handlers
+
+    handle_error = BaseServer.handle_error
+    log_exception = handlers.BaseHandler.log_exception
+
+    def is_broken_pipe_error():
+        type, err, tb = sys.exc_info()
+        return "Connection reset by peer" in repr(err)
+
+    def my_handle_error(self, request, client_address):
+        if not is_broken_pipe_error():
+            handle_error(self, request, client_address)
+
+    def my_log_exception(self, exc_info):
+        if not is_broken_pipe_error():
+            log_exception(self, exc_info)
+
+    BaseServer.handle_error = my_handle_error
+    handlers.BaseHandler.log_exception = my_log_exception
+
+patch_broken_pipe_error()
+
 
 ADMINS = (
     # ('Your Name', 'your_email@domain.com'),
@@ -49,7 +79,10 @@ if is_testing(sys.argv):
     MONGO_DB = 'system_tracking_test'
 
 # Celery AMQP configuration.
-BROKER_URL = 'amqp://guest:guest@rabbitmq//'
+BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"),
+                                         os.environ.get("RABBITMQ_PASS"),
+                                         os.environ.get("RABBITMQ_HOST"),
+                                         os.environ.get("RABBITMQ_VHOST"))
 
 # Mongo host configuration
 MONGO_HOST = NotImplemented

tools/docker-compose-cluster.yml

@@ -0,0 +1,59 @@
+version: '2'
+services:
+  haproxy:
+    build:
+      context: ./docker-compose
+      dockerfile: Dockerfile-haproxy
+    depends_on:
+      - "tower_1"
+      - "tower_2"
+      - "tower_3"
+    ports:
+      - "8013:8013"
+      - "1936:1936"
+  tower_1:
+    image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    environment:
+      RABBITMQ_HOST: rabbitmq_1
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
+    volumes:
+      - "../:/tower_devel"
+  tower_2:
+    image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    environment:
+      RABBITMQ_HOST: rabbitmq_2
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
+    volumes:
+      - "../:/tower_devel"
+  tower_3:
+    image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    environment:
+      RABBITMQ_HOST: rabbitmq_3
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: /
+    volumes:
+      - "../:/tower_devel"
+  rabbitmq_1:
+    image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest
+    hostname: rabbitmq_1
+  rabbitmq_2:
+    image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest
+    hostname: rabbitmq_2
+    environment: 
+      - CLUSTERED=true
+      - CLUSTER_WITH=rabbitmq_1
+  rabbitmq_3:
+    image: gcr.io/ansible-tower-engineering/rabbit_cluster_node:latest
+    hostname: rabbitmq_3
+    environment: 
+      - CLUSTERED=true
+      - CLUSTER_WITH=rabbitmq_1
+  postgres:
+    image: postgres:9.4.1
+  memcached:
+    image: memcached:alpine

tools/docker-compose.yml

@@ -3,6 +3,11 @@ services:
   # Primary Tower Development Container
   tower:
     image: gcr.io/ansible-tower-engineering/tower_devel:${TAG}
+    environment:
+      RABBITMQ_HOST: rabbitmq
+      RABBITMQ_USER: guest
+      RABBITMQ_PASS: guest
+      RABBITMQ_VHOST: tower
     ports:
       - "8080:8080"
       - "8013:8013"

tools/docker-compose/Dockerfile-haproxy

@@ -0,0 +1,2 @@
+FROM haproxy:1.6-alpine
+COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg

tools/docker-compose/haproxy.cfg

@@ -0,0 +1,36 @@
+global
+    debug
+    stats socket /tmp/admin.sock
+    stats timeout 30s
+
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+
+frontend localnodes
+    bind *:8013
+    mode http
+    default_backend nodes
+
+backend nodes
+    mode http
+    balance roundrobin
+    option forwardfor
+    option http-pretend-keepalive
+    http-request set-header X-Forwarded-Port %[dst_port]
+    http-request add-header X-Forwarded-Proto https if { ssl_fc }
+    option httpchk HEAD / HTTP/1.1\r\nHost:localhost
+    server tower_1 tower_1:8013 check
+    server tower_2 tower_2:8013 check
+    server tower_3 tower_3:8013 check
+
+listen stats
+    bind *:1936
+    stats enable
+    stats uri /
+