Merge remote-tracking branch 'tower/release_3.3.0' into devel

tools/clusterdevel/bootstrap_minishift.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set +x
+
+# Wait for the databases to come up
+ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgresql port=5432" all
+ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=localhost port=11211" all
+ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=localhost port=5672" all
+ansible -i "127.0.0.1," -c local -v -m postgresql_db -U postgres -a "name=awx owner=awx login_user=awx login_password=awx login_host=postgresql" all
+
+# Move to the source directory so we can bootstrap
+if [ -f "/awx_devel/manage.py" ]; then
+    cd /awx_devel
+else
+    echo "Failed to find awx source tree, map your development tree volume"
+fi
+
+#make awx-link
+python setup.py develop
+ln -s /awx_devel/tools/rdb.py /venv/awx/lib/python2.7/site-packages/rdb.py || true
+yes | cp -rf /awx_devel/tools/docker-compose/supervisor.conf /supervisor.conf
+
+# AWX bootstrapping
+make version_file
+make migrate
+make init
+
+mkdir -p /awx_devel/awx/public/static
+mkdir -p /awx_devel/awx/ui/static
+
+cd /awx_devel
+# Start the services
+if [ -f "/awx_devel/tools/docker-compose/use_dev_supervisor.txt" ]; then
+    make supervisor
+else
+    honcho start -f "tools/docker-compose/Procfile"
+fi

tools/clusterdevel/roles/minishift/tasks/main.yml

@@ -0,0 +1,122 @@
+---
+- name: Get status of minishift
+  shell: minishift status
+  register: minishift_status
+
+- name: Echo minishift status so if verification fails we can see the results
+  debug:
+    var: minishift_status.stdout
+
+- name: Verify status of minishift
+  assert:
+    that:
+      - "'Minishift:  Running' == minishift_status.stdout_lines[0]"
+      - "'OpenShift:  Running' in minishift_status.stdout_lines[2]"
+
+- name: Get minishift ip
+  shell: minishift ip
+  register: minishift_ip
+
+- name: Get minishift oc location
+  shell: minishift oc-env
+  register: minishift_oc_env
+
+- name: Extract minishift binary path
+  set_fact:
+    minishift_oc_bin: "{{ minishift_oc_env.stdout_lines[0] | regex_replace('export PATH=\\\"(.*):\\$PATH\\\"', '\\1') }}/oc"
+
+- name: Dynamically found oc binary to be at
+  debug:
+    msg: "oc path: {{ minishift_oc_bin }}"
+
+- name: Login as admin
+  shell: "{{ minishift_oc_bin }} login {{ minishift_ip.stdout }}:8443 -u system:admin"
+
+- name: Create privileged user service account awx
+  shell: "{{ minishift_oc_bin }} adm policy add-scc-to-user privileged system:serviceaccount:{{ awx_dev_project }}:awx"
+
+- name: Authenticate with OpenShift via token
+  shell: "{{ minishift_oc_bin }} login {{ minishift_ip.stdout }}:8443 -u admin -p admin"
+
+- name: Unattach AWX dev tree volume locally
+  shell: "minishift hostfolder remove awx || true"
+
+- name: Attach AWX dev tree volume locally
+  shell: "minishift hostfolder add -t sshfs --source {{ devtree_directory }} --target /mnt/sda1/awx awx"
+
+- name: Unmount AWX dev volume
+  shell: "minishift hostfolder umount awx || true"
+
+- name: Mount AWX dev volume
+  shell: minishift hostfolder mount awx
+
+- name: Authenticate with OpenShift via token
+  shell: "{{ minishift_oc_bin }} login -u admin -p admin"
+
+- name: Get Project Detail
+  shell: "{{ minishift_oc_bin }} get project {{ awx_dev_project }}"
+  register: project_details
+  ignore_errors: yes
+
+- name: Get Postgres Service Detail
+  shell: "{{ minishift_oc_bin }} describe svc postgresql -n {{ awx_dev_project }}"
+  register: postgres_svc_details
+  ignore_errors: yes
+
+- name: Create AWX Openshift Project
+  shell: "{{ minishift_oc_bin }} new-project {{ awx_dev_project }}"
+  when: project_details.rc != 0
+
+- name: Stage serviceacct.yml
+  template:
+    src: serviceacct.yml.j2
+    dest: /tmp/serviceacct.yml
+
+- name: Apply svc account
+  shell: "{{ minishift_oc_bin }} apply -f /tmp/serviceacct.yml ; rm -rf /tmp/serviceaccount.yml"
+
+- name: Stage hostfolderpvc.yml
+  template:
+    src: hostfolderpvc.yml.j2
+    dest: /tmp/hostfolderpvc.yml
+
+- name: Create PV for host folder
+  shell: "{{ minishift_oc_bin }} apply -f /tmp/hostfolderpvc.yml ; rm -rf /tmp/hostfolderpvc.yml"
+
+- name: Stage volumeclaim.yml
+  template:
+    src: volumeclaim.yml.j2
+    dest: /tmp/volumeclaim.yml
+
+- name: Create PV for host folder
+  shell: "oc apply -f /tmp/volumeclaim.yml ; rm -rf /tmp/volumeclaim.yml"
+
+- name: Deploy and Activate Postgres
+  shell: "{{ minishift_oc_bin }} new-app --template=postgresql-persistent -e MEMORY_LIMIT={{ pg_memory_limit|default('512') }}Mi -e NAMESPACE=openshift -e DATABASE_SERVICE_NAME=postgresql  -e POSTGRESQL_USER={{ pg_username|default('awx') }} -e POSTGRESQL_PASSWORD={{ pg_password|default('awx') }} -e POSTGRESQL_DATABASE={{ pg_database|default('awx') }} -e VOLUME_CAPACITY={{ pg_volume_capacity|default('5')}}Gi -e POSTGRESQL_VERSION=9.5 -n {{ awx_dev_project }}"
+  when: postgres_svc_details is defined and postgres_svc_details.rc != 0
+  register: openshift_pg_activate
+
+- name: Wait for Postgres to activate
+  pause:
+    seconds: 15
+  when: openshift_pg_activate|changed
+
+- name: Template configmap
+  template:
+    src: configmap.yml.j2
+    dest: "/tmp/configmap.yml"
+
+- name: Create configmap
+  shell: "{{ minishift_oc_bin }} apply -f /tmp/configmap.yml ; rm -rf /tmp/configmap.yml"
+
+- name: Template deployment
+  template:
+    src: hostdev.yml.j2
+    dest: "/tmp/hostdev.yml"
+
+- name: Create deployment
+  shell: "{{ minishift_oc_bin }} apply -f /tmp/hostdev.yml ; rm -rf /tmp/hostdev.yml"
+
+- name: Please login
+  debug:
+    msg: "Login at https://{{ minishift_ip.stdout }}:8443 with admin / admin"

tools/clusterdevel/roles/minishift/templates/configmap.yml.j2

@@ -0,0 +1,128 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: awx-dev-config
+  namespace: {{ awx_dev_project }}
+data:
+  secret_key: isasekrit
+  awx_settings: |
+    import os
+    import socket
+    ADMINS = ()
+    
+    # Container environments don't like chroots
+    AWX_PROOT_ENABLED = False
+
+    # Automatically deprovision pods that go offline
+    AWX_AUTO_DEPROVISION_INSTANCES = True
+
+    SYSTEM_TASK_ABS_CPU = {{ ((awx_task_cpu_request|int / 1000) * 4)|int }}
+    SYSTEM_TASK_ABS_MEM = {{ ((awx_task_mem_request|int * 1024) / 100)|int }}
+
+    #Autoprovisioning should replace this
+    CLUSTER_HOST_ID = socket.gethostname()
+    SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
+    
+    SESSION_COOKIE_SECURE = False
+    CSRF_COOKIE_SECURE = False
+    
+    REMOTE_HOST_HEADERS = ['HTTP_X_FORWARDED_FOR']
+
+    STATIC_ROOT = '/var/lib/awx/public/static'
+    PROJECTS_ROOT = '/var/lib/awx/projects'
+    JOBOUTPUT_ROOT = '/var/lib/awx/job_status'
+    SECRET_KEY = file('/etc/tower/SECRET_KEY', 'rb').read().strip()
+    ALLOWED_HOSTS = ['*']
+    INTERNAL_API_URL = 'http://127.0.0.1:8052'
+    SERVER_EMAIL = 'root@localhost'
+    DEFAULT_FROM_EMAIL = 'webmaster@localhost'
+    EMAIL_SUBJECT_PREFIX = '[AWX] '
+    EMAIL_HOST = 'localhost'
+    EMAIL_PORT = 25
+    EMAIL_HOST_USER = ''
+    EMAIL_HOST_PASSWORD = ''
+    EMAIL_USE_TLS = False
+    
+    LOGGING['handlers']['console'] = {
+        '()': 'logging.StreamHandler',
+        'level': 'DEBUG',
+        'formatter': 'simple',
+    }
+    
+    LOGGING['loggers']['django.request']['handlers'] = ['console']
+    LOGGING['loggers']['rest_framework.request']['handlers'] = ['console']
+    LOGGING['loggers']['awx']['handlers'] = ['console']
+    LOGGING['loggers']['awx.main.commands.run_callback_receiver']['handlers'] = ['console']
+    LOGGING['loggers']['awx.main.commands.inventory_import']['handlers'] = ['console']
+    LOGGING['loggers']['awx.main.tasks']['handlers'] = ['console']
+    LOGGING['loggers']['awx.main.scheduler']['handlers'] = ['console']
+    LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']
+    LOGGING['loggers']['social']['handlers'] = ['console']
+    LOGGING['loggers']['system_tracking_migrations']['handlers'] = ['console']
+    LOGGING['loggers']['rbac_migrations']['handlers'] = ['console']
+    LOGGING['loggers']['awx.isolated.manager.playbooks']['handlers'] = ['console']
+    LOGGING['handlers']['callback_receiver'] = {'class': 'logging.NullHandler'}
+    LOGGING['handlers']['fact_receiver'] = {'class': 'logging.NullHandler'}
+    LOGGING['handlers']['task_system'] = {'class': 'logging.NullHandler'}
+    LOGGING['handlers']['tower_warnings'] = {'class': 'logging.NullHandler'}
+    LOGGING['handlers']['rbac_migrations'] = {'class': 'logging.NullHandler'}
+    LOGGING['handlers']['system_tracking_migrations'] = {'class': 'logging.NullHandler'}
+    LOGGING['handlers']['management_playbooks'] = {'class': 'logging.NullHandler'}
+    
+    DATABASES = {
+        'default': {
+            'ATOMIC_REQUESTS': True,
+            'ENGINE': 'django.db.backends.postgresql',
+            'NAME': "awx",
+            'USER': "awx",
+            'PASSWORD': "awx",
+            'HOST': "postgresql",
+            'PORT': "5432",
+        }
+    }
+    BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(
+        "awx",
+        "abcdefg",
+        "localhost",
+        "5672",
+        "awx")
+    CHANNEL_LAYERS = {
+        'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
+                    'ROUTING': 'awx.main.routing.channel_routing',
+                    'CONFIG': {'url': BROKER_URL}}
+    }
+    CACHES = {
+        'default': {
+            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+            'LOCATION': '{}:{}'.format("localhost", "11211")
+        },
+        'ephemeral': {
+            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        },
+    }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rabbitmq-config
+  namespace: {{ awx_dev_project }}
+data:
+  enabled_plugins: |
+      [rabbitmq_management,rabbitmq_peer_discovery_k8s].
+  rabbitmq.conf: |
+      default_user = awx
+      default_pass = abcdefg
+      default_vhost = awx
+
+      ## Clustering
+      cluster_formation.peer_discovery_backend  = rabbit_peer_discovery_k8s
+      cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
+      cluster_formation.k8s.address_type = ip
+      cluster_formation.node_cleanup.interval = 10
+      cluster_formation.node_cleanup.only_log_warning = false
+      cluster_partition_handling = autoheal
+      ## queue master locator
+      queue_master_locator=min-masters
+      ## enable guest user
+      loopback_users.guest = false

tools/clusterdevel/roles/minishift/templates/hostdev.yml.j2

@@ -0,0 +1,178 @@
+---
+apiVersion: apps/v1beta1 # for versions before 1.9.0 use apps/v1beta2
+kind: Deployment
+metadata:
+  name: awx
+  namespace: {{ awx_dev_project }}
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: awx
+        app: rabbitmq
+    spec:
+      serviceAccountName: awx
+      containers:
+        - image: ansible/awx-dev:latest
+          name: awx
+          command: ["/awx_devel/tools/clusterdevel/bootstrap_minishift.sh"]
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: localdev
+              mountPath: /awx_devel
+            - name: awx-application-config
+              mountPath: /etc/tower
+        - name: awx-rabbit
+          image: ansible/awx_rabbitmq:latest
+          imagePullPolicy: Always
+          ports:
+            - name: http
+              protocol: TCP
+              containerPort: 15672
+            - name: amqp
+              protocol: TCP
+              containerPort: 5672
+          livenessProbe:
+            exec:
+              command: ["rabbitmqctl", "status"]
+            initialDelaySeconds: 30
+            timeoutSeconds: 10
+          readinessProbe:
+            exec:
+              command: ["rabbitmqctl", "status"]
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+          env:
+            # For consupmption by rabbitmq-env.conf
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: RABBITMQ_USE_LONGNAME
+              value: "true"
+            - name: RABBITMQ_NODENAME
+              value: "rabbit@$(MY_POD_IP)"
+            - name: K8S_SERVICE_NAME
+              value: "rabbitmq"
+            - name: RABBITMQ_ERLANG_COOKIE
+              value: "cookiemonster"
+          volumeMounts:
+            - name: rabbitmq-config
+              mountPath: /etc/rabbitmq
+        - name: awx-memcached
+          image: memcached
+        - name: postgres
+          image: postgres:9.6
+      volumes:
+        - name: localdev
+          persistentVolumeClaim:
+            claimName: devtest
+        - name: awx-application-config
+          configMap:
+            name: awx-dev-config
+            items:
+              - key: awx_settings
+                path: settings.py
+              - key: secret_key
+                path: SECRET_KEY
+        - name: rabbitmq-config
+          configMap:
+            name: rabbitmq-config
+            items:
+            - key: rabbitmq.conf
+              path: rabbitmq.conf
+            - key: enabled_plugins
+              path: enabled_plugins
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: awx-web-svc
+  namespace: {{ awx_dev_project }}
+  labels:
+    name: awx-web-svc
+spec:
+  type: "NodePort"
+  ports:
+    - name: https
+      port: 8043
+    - name: http
+      port: 8013
+    - name: notsure
+      port: 8080
+  selector:
+    name: awx
+---
+kind: Service
+apiVersion: v1
+metadata:
+  namespace: {{ awx_dev_project }}
+  name: rabbitmq
+  labels:
+    app: rabbitmq
+    type: LoadBalancer
+spec:
+  type: NodePort
+  ports:
+   - name: http
+     protocol: TCP
+     port: 15672
+     targetPort: 15672
+   - name: amqp
+     protocol: TCP
+     port: 5672
+     targetPort: 5672
+  selector:
+    app: rabbitmq
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: awx-rmq-mgmt
+  namespace: {{ awx_dev_project }}
+  labels:
+    name: awx-rmq-mgmt
+spec:
+  type: ClusterIP
+  ports:
+    - name: rmqmgmt
+      port: 15672
+      targetPort: 15672
+  selector:
+    name: awx
+---
+apiVersion: v1
+kind: Route
+metadata:
+  name: awx-web-svc
+  namespace: {{ awx_dev_project }}
+spec:
+  port:
+    targetPort: http
+  tls:
+    insecureEdgeTerminationPolicy: Allow
+    termination: edge
+  to:
+    kind: Service
+    name: awx-web-svc
+    weight: 100
+  wildcardPolicy: None
+---
+apiVersion: v1
+kind: Route
+metadata:
+  name: awx-rmq-mgmt
+  namespace: {{ awx_dev_project }}
+spec:
+  port:
+    targetPort: rmqmgmt
+  tls:
+    insecureEdgeTerminationPolicy: Allow
+    termination: edge
+  to:
+    kind: Service
+    name: rabbitmq
+    weight: 100
+  wildcardPolicy: None

tools/clusterdevel/roles/minishift/templates/hostfolderpvc.yml.j2

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: projdata
+  namespace: {{ awx_dev_project }}
+  labels:
+    project: awx
+spec:
+  capacity:
+    storage: 10Gi 
+  accessModes:
+    - ReadWriteMany 
+  persistentVolumeReclaimPolicy: Retain 
+  hostPath:
+    path: /mnt/sda1/awx

tools/clusterdevel/roles/minishift/templates/serviceacct.yml.j2

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: awx
+  namespace: {{ awx_dev_project }}
+---
+kind: Role
+apiVersion: v1
+metadata:
+  name: endpoint-reader
+  namespace: {{ awx_dev_project }}
+rules:
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get"]
+---
+kind: RoleBinding
+apiVersion: v1
+metadata:
+  name: endpoint-reader
+  namespace: {{ awx_dev_project }}
+roleRef:
+  name: endpoint-reader
+  namespace: {{ awx_dev_project }}
+subjects:
+  - kind: ServiceAccount
+    name: awx
+    namespace: {{ awx_dev_project }}
+userNames:
+  - system:serviceaccount:{{ awx_dev_project }}:awx

tools/clusterdevel/roles/minishift/templates/volumeclaim.yml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: devtest
+  namespace: {{ awx_dev_project }}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+  selector:
+    matchLabels:
+      project: awx

tools/clusterdevel/start_minishift_dev.yml

@@ -0,0 +1,11 @@
+---
+- name: Setup minishift dev environment
+  hosts: localhost
+  gather_facts: false
+  vars:
+    - ansible_connection: local
+    - awx_dev_project: awx
+    - awx_task_cpu_request: 500
+    - awx_task_mem_request: 512
+  roles:
+    - { role: minishift }

tools/docker-compose-cluster.yml

@@ -24,10 +24,12 @@ services:
       RABBITMQ_PASS: guest
       RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
+      SDB_PORT: 5899
       AWX_GROUP_QUEUES: alpha,tower
     volumes:
       - "../:/awx_devel"
-
+    ports:
+      - "5899-5999:5899-5999"
   awx_2:
     privileged: true
     image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG}
@@ -38,9 +40,12 @@ services:
       RABBITMQ_PASS: guest
       RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
+      SDB_PORT: 6899
       AWX_GROUP_QUEUES: bravo,tower
     volumes:
       - "../:/awx_devel"
+    ports:
+      - "6899-6999:6899-6999"
   awx_3:
     privileged: true
     image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG}
@@ -51,9 +56,12 @@ services:
       RABBITMQ_PASS: guest
       RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
+      SDB_PORT: 7899
       AWX_GROUP_QUEUES: charlie,tower
     volumes:
       - "../:/awx_devel"
+    ports:
+      - "7899-7999:7899-7999"
   rabbitmq_1:
     image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest
     hostname: rabbitmq_1
@@ -73,7 +81,3 @@ services:
     image: postgres:9.6
   memcached:
     image: memcached:alpine
-  logstash:
-    build:
-      context: ./docker-compose
-      dockerfile: Dockerfile-logstash

tools/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2'
 services:
   # Primary AWX Development Container
   awx:
@@ -22,17 +22,18 @@ services:
       - postgres
       - memcached
       - rabbitmq
-      - logstash
     #   - sync
     # volumes_from:
     #   - sync
     volumes:
       - "../:/awx_devel"
     privileged: true
-  logstash:
-    build:
-      context: ./docker-compose
-      dockerfile: Dockerfile-logstash
+  # A useful container that simply passes through log messages to the console
+  # helpful for testing awx/tower logging
+  # logstash:
+  #   build:
+  #     context: ./docker-compose
+  #     dockerfile: Dockerfile-logstash
   # Postgres Database Container
   postgres:
     image: postgres:9.6

tools/docker-compose/README

@@ -8,7 +8,7 @@ authentication set up inside of the logstash configuration file).
 
 ```
 {
-    "LOG_AGGREGATOR_HOST": "logstash",
+    "LOG_AGGREGATOR_HOST": "http://logstash",
     "LOG_AGGREGATOR_PORT": 8085,
     "LOG_AGGREGATOR_TYPE": "logstash",
     "LOG_AGGREGATOR_USERNAME": "awx_logger",
@@ -25,6 +25,8 @@ authentication set up inside of the logstash configuration file).
 }
 ```
 
+> Note: HTTP must be specified in the `LOG_AGGREGATOR_HOST` if you are using the docker development environment.  
+
 An example of how to view the most recent logs from the container:
 
 ```

tools/docker-compose/nginx.vh.default.conf

@@ -30,7 +30,7 @@ server {
         sendfile off;
     }
 
-    location ~ ^/(websocket|network_ui/topology/) {
+    location ~ ^/websocket {
         # Pass request to the upstream alias
         proxy_pass http://daphne;
         # Require http version 1.1 to allow for upgrade requests
@@ -90,7 +90,7 @@ server {
         sendfile off;
     }
 
-    location ~ ^/(websocket|network_ui/topology/) {
+    location ~ ^/websocket {
         # Pass request to the upstream alias
         proxy_pass http://daphne;
         # Require http version 1.1 to allow for upgrade requests

tools/docker-compose/start_development.sh

@@ -8,5 +8,6 @@ cd /awx_devel
 if [ -f "/awx_devel/tools/docker-compose/use_dev_supervisor.txt" ]; then
     make supervisor
 else
+    export PYTHONIOENCODING=utf_8
     honcho start -f "tools/docker-compose/Procfile"
 fi