Consolidate the Local Docker installer and the dev env

- removes local_docker installer and points community users to our development environment (make docker-compose)
  - provides a migration path from Local Docker Compose installations --> the dev environment
  - the dev env can now be configured to use an external database
  - consolidated the Local Docker and dev env docker-compose.yml files into one template file, used by the dockerfile role
  - added a 'sources' role to template out config files
  - the postgres data dir is no longer a bind-mount, it is a docker volume
  - the redis socket is not longer a bind-mount, it is a docker volume
  - the local_settings.py.docker-compose file no longer needs to be copied over in the dev env
  - Create tmp rsyslog.conf in rsyslog volume to avoid cross-linking. Previously, the tmp code-generated rsyslog.conf was being written to /tmp (by default).  As a result, we were attempting to shutil.move() across volumes.
  - move k8s image build and push roles under tools/ansible
  - See tools/docker-compose/README.md for usage of these changes
This commit is contained in:
Christian M. Adams
2021-01-27 11:01:17 -05:00
parent 0f6d2c36a0
commit 9672e72834
52 changed files with 1325 additions and 808 deletions

8
tools/ansible/build.yml Normal file
View File

@@ -0,0 +1,8 @@
---
- name: Build AWX Docker Images
hosts: localhost
gather_facts: true
roles:
- {role: dockerfile}
- {role: image_build}
- {role: image_push, when: "docker_registry is defined"}

View File

@@ -0,0 +1,6 @@
---
- name: Render AWX Dockerfile and sources
hosts: localhost
gather_facts: true
roles:
- {role: dockerfile}

View File

@@ -0,0 +1,6 @@
---
build_dev: false
kube_dev: false
dockerfile_dest: '../..'
dockerfile_name: 'Dockerfile'
template_dest: '_build'

View File

@@ -0,0 +1,52 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG/MacGPG2 v2.0.19 (Darwin)
mQINBFVfhqABEAC6EEEPv57spTUSQvtgqbVZI7d5ooCTMXEo5KJGVPVSfKtO8+BV
ZTKPukUazbtplDlIe9csfbP7CBaaBn5CtDgIrbROzazxoWv7mIP6hjUaTQSd5tvv
ONDQvnCDD5SKcy+XhqkmALSvREsN9tNtKETGXgNOLwJAlzxcpt8JLXnuiCCbefum
gaDoPQsIkegFa/r6XhY6kLi2lpQOJ3v72IXNDpdau1vtp/xPHclfCI1iQ7gnfEdw
rRJRGeOx1qikyqAVFgXXiI/NAQrsyIsO0ECGSBLQeDna/bGrqpCGKnrbJhfGAIWA
aXUTRCQRemiansk0Whu4ATZz8iM9zJPi1R7CeMXgwe7VtD4KOd1y7UBHKwAhIWdu
4Q4lsOpm2tzYFQUrY6mQ/3BkywDHkdVqmQKTGCuwcNO9PMOBLSE99yCIjxXL04VM
dPWIqMvh15TLjd6UahNFucowX3312z4JpWFHWA075MdkvVVcqfMxohViOLUCYt/C
74xFmT+uZUKnSQFYT/JaGqxFLjkYHmnFrb710fBjniDlaB4Ii3Tft/yXsgx8P9xb
y2cWA/W6yFeRqXM49C3/KA6RhDWU90P55O/MWbYUSGiGu+eYT3rMAV2cI6r4+U7e
YgQvntpc9GbAzab6co8ceJ3lpTHtSl+QZJUhSoPYg5VbSilf0AqZgUSUUQARAQAB
tC5BbnNpYmxlLCBJbmMuIChyZWxlYXNlKSA8c2VjdXJpdHlAYW5zaWJsZS5jb20+
iQI4BBMBAgAiBQJVX4agAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRC4
TjOcRCZnqfN+D/9LvJVLW/zMPcZ4qK+/KpNiD+MXducBpQiUfj2AoEqkR2LwL4/G
v5N3GCpBHSrzK1PBp5uW1+6TcdotBO5ePtmvJlSjVMVxHkcTTBfuzqtErcw/zequ
sNsagllPlCePT1Osi34onGm7zMdillh/uw30bojYRwtDpfXiXJAqPc3vqXrER8EE
r+ZFj4MJcqDGWdIguWX8hnIFkzYZ7Gyvwo4ETWrdRhRfHoIdwyiAawnxkgpMVHxv
2+JlqDb+qqY2Wjffd5WC2uaxK88sCsScJF+aE+WlBIVRA4POu4gZneLfuGzzpg2e
9PSWmtDo5X7ECRnfTIMvAbbpt08x/zSZxwRUwLdQ+F9vN7RZ/ibaXE2rG1xrWOO/
wel8cfhDl7YgZhKw8R/RURliOB5FFJ336zGWm5HTHyhblbiNn+LcUAy8ipYp6y+C
ATLnHzF6J5CtIRpG4Bs5Ar3xePNGUnEHXiLv3wYeq3uUkrFcRpmcMUSBrtk6QHbD
fmJvWpdA4twmoBFBMyvvTJmBb52teNzoqBgeNflXl+SVAMT2eZSezqbvevuHQTOX
uRw0GXsKCQ/hyR9f1fd0yGRMtPqNTRLwBb4lzpU70/rRmU9gHzY4Yhwg3E9Tv+rM
a5Lj3YmlJRax5gUVQN02E1zlBDsiNrGqpmDG1Mxo7YPpbxgu0PFPqeqcFbkCDQRV
X4agARAAwO4MA+7uIRV+oHmyMPLFWqiKp2nFy5McQByJxSQchn93/9qud4JYd1i6
8pIiKN6XJqtpt16UCTewcZHM5oOJQVNwAS8TP9imfg73TfUaoOoUbp0qfGKub/Q4
6Ktnwe940qEqYG1/QsPWNE/4G1O3b/O7m6qlozEEmxep8bRviRChz4/Mw75S1W6i
jlKYI8yZOUco9oiFJcKqyYtaKkgEg18cNuY8uvAlvULezaZyCqVjoVbKGUUAPSVg
CBixqLQ7UmBMA6xxptVuBvaRJAaF0VvvcyBZo4SzybtrHbUD1VWIzmWKKD/sDS2J
MQbnQ0FnhRzTjhvQhAp2LVPeAQVbQNFdG7y+ROCHeE9mqutTZLOilut+CQ8HDWuQ
/eCQU5yV7vh3FL/SVYS0ahZj+FdfTq8rbeIsDT42Z/MjDB54jxB5ajCHLomi4LhC
09zeb7HgwUc5wzoN7nU1OLmmn0AFwKJVD1R5UgySggv2xJym1H/mjJiR0MDweJDc
xj3bf4qGRDLVFRkZcO3cmMDLhL9gb1MIU3zBVotOBt2dig/Je+K6CUFHAA237Vcg
VKUrLIi6OdG3ecAdflGsaNKQ5XPv2mfhbieXu9N/S7HBvjeHIBD2xjWNz9UE1ymu
QPwR6+zTxD4Nx1xIiink0MN1PaCkGJ03YBSsXnHoyiOhqAfceRcAEQEAAYkCHwQY
AQIACQUCVV+GoAIbDAAKCRC4TjOcRCZnqSvGD/wP8y6fz2PsrgspHCraNuWTJaVA
DesQgOxJS6uHskW/jnHkvAMTNzlVhov1hN7g+QjPMISQDCn+913kyqZ0lU3lYmvz
nByPAbgzZvmAaTqb8v79zY6UH4NzbBuz4dhYN65dxhiMpNrXVvMRQjPFRXG0GG5d
7ypM1b9eoRTRlJNAwQ/ONoQxZdzVpmpXjcMOaifs75lkGAfNT0bcG/o/Qh/p4MRF
t/VSmH8tM8jJuHbIPcs8FWP4J8xzum8uhF2ZlKEQsR2C9cBJSBrs5jdOjgMqwFv5
2qCg0PpEKKNQdu9MabapBprFMwJWIl+dOjUE3fdMrOSJBZZusQq9nwtDNAaaLcD7
RwStw7AXi6CxYuB/uikKRviLqRCwASdj5Cdjtu6mohS8DdVkpEYbpuPjEdqc7UyW
fAZQqYMkwIfaxE25/S+FxqISSCFIOCL3QNTk0Q9u2W6Fh+KUACZobtwUL/XytPBz
7Fn5wXeOCPoAbOXoiT7kPsFGvIsFHpF3K7Fy+cMrqr5dqhywGK5ckIKXRKmCAu8H
iDeBqVjBn143WJPZ8uiu+7TiaGLuOqDdiDSchM24W4hs5DbD9zdVYy6IFi1OWSot
HUQyZisiIgD1hSHhkn2LTYrJqIdvJ/q8buMKywB9Avs5fwP/CnsrSP9z+RWJ8HKP
OwWvTVGXCPUZTxHiYg==
=msBf
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -0,0 +1,25 @@
#!/usr/bin/env bash
if [ `id -u` -ge 500 ]; then
echo "awx:x:`id -u`:`id -g`:,,,:/var/lib/awx:/bin/bash" >> /tmp/passwd
cat /tmp/passwd > /etc/passwd
rm /tmp/passwd
fi
if [ -n "${AWX_KUBE_DEVEL}" ]; then
pushd /awx_devel
make awx-link
popd
export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}')
fi
source /etc/tower/conf.d/environment.sh
ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all
ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all
awx-manage collectstatic --noinput --clear
unset $(cut -d = -f -1 /etc/tower/conf.d/environment.sh)
supervisord -c /etc/supervisord.conf

View File

@@ -0,0 +1,41 @@
#!/usr/bin/env bash
if [ `id -u` -ge 500 ]; then
echo "awx:x:`id -u`:`id -g`:,,,:/var/lib/awx:/bin/bash" >> /tmp/passwd
cat /tmp/passwd > /etc/passwd
rm /tmp/passwd
fi
if [ -n "${AWX_KUBE_DEVEL}" ]; then
pushd /awx_devel
make awx-link
popd
export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}')
fi
source /etc/tower/conf.d/environment.sh
ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all
ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all
if [ -z "$AWX_SKIP_MIGRATIONS" ]; then
echo "Running migrations..."
awx-manage migrate --noinput
fi
if [ -z "$AWX_SKIP_PROVISION_INSTANCE" ]; then
awx-manage provision_instance --hostname=$(hostname)
fi
if [ -z "$AWX_SKIP_REGISTERQUEUE" ]; then
awx-manage register_queue --queuename=tower --instance_percent=100
fi
if [ ! -z "$AWX_ADMIN_USER" ]&&[ ! -z "$AWX_ADMIN_PASSWORD" ]; then
echo "from django.contrib.auth.models import User; User.objects.create_superuser('$AWX_ADMIN_USER', 'root@localhost', '$AWX_ADMIN_PASSWORD')" | awx-manage shell
fi
echo 'from django.conf import settings; x = settings.AWX_TASK_ENV; x["HOME"] = "/var/lib/awx"; settings.AWX_TASK_ENV = x' | awx-manage shell
unset $(cut -d = -f -1 /etc/tower/conf.d/environment.sh)
supervisord -c /etc/supervisord_task.conf

View File

@@ -0,0 +1,7 @@
$WorkDirectory /var/lib/awx/rsyslog
$MaxMessageSize 700000
$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf
module(load="imuxsock" SysSock.Use="off")
input(type="imuxsock" Socket="/var/run/awx-rsyslog/rsyslog.sock" unlink="on")
template(name="awx" type="string" string="%msg%")
action(type="omfile" file="/dev/null")

View File

@@ -0,0 +1,89 @@
# AWX settings file
import os
def get_secret():
if os.path.exists("/etc/tower/SECRET_KEY"):
return open('/etc/tower/SECRET_KEY', 'rb').read().strip()
ADMINS = ()
STATIC_ROOT = '/var/lib/awx/public/static'
PROJECTS_ROOT = '/var/lib/awx/projects'
AWX_ANSIBLE_COLLECTIONS_PATHS = '/var/lib/awx/vendor/awx_ansible_collections'
JOBOUTPUT_ROOT = '/var/lib/awx/job_status'
SECRET_KEY = get_secret()
ALLOWED_HOSTS = ['*']
# Container environments don't like chroots
AWX_PROOT_ENABLED = False
CLUSTER_HOST_ID = "awx"
SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
CSRF_COOKIE_SECURE = False
SESSION_COOKIE_SECURE = False
###############################################################################
# EMAIL SETTINGS
###############################################################################
SERVER_EMAIL = 'root@localhost'
DEFAULT_FROM_EMAIL = 'webmaster@localhost'
EMAIL_SUBJECT_PREFIX = '[AWX] '
EMAIL_HOST = 'localhost'
EMAIL_PORT = 25
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
EMAIL_USE_TLS = False
LOGGING['handlers']['console'] = {
'()': 'logging.StreamHandler',
'level': 'DEBUG',
'formatter': 'simple',
}
LOGGING['loggers']['django.request']['handlers'] = ['console']
LOGGING['loggers']['rest_framework.request']['handlers'] = ['console']
LOGGING['loggers']['awx']['handlers'] = ['console', 'external_logger']
LOGGING['loggers']['awx.main.commands.run_callback_receiver']['handlers'] = ['console']
LOGGING['loggers']['awx.main.tasks']['handlers'] = ['console', 'external_logger']
LOGGING['loggers']['awx.main.scheduler']['handlers'] = ['console', 'external_logger']
LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']
LOGGING['loggers']['social']['handlers'] = ['console']
LOGGING['loggers']['system_tracking_migrations']['handlers'] = ['console']
LOGGING['loggers']['rbac_migrations']['handlers'] = ['console']
LOGGING['loggers']['awx.isolated.manager.playbooks']['handlers'] = ['console']
LOGGING['handlers']['callback_receiver'] = {'class': 'logging.NullHandler'}
LOGGING['handlers']['task_system'] = {'class': 'logging.NullHandler'}
LOGGING['handlers']['tower_warnings'] = {'class': 'logging.NullHandler'}
LOGGING['handlers']['rbac_migrations'] = {'class': 'logging.NullHandler'}
LOGGING['handlers']['system_tracking_migrations'] = {'class': 'logging.NullHandler'}
LOGGING['handlers']['management_playbooks'] = {'class': 'logging.NullHandler'}
DATABASES = {
'default': {
'ATOMIC_REQUESTS': True,
'ENGINE': 'awx.main.db.profiled_pg',
'NAME': os.getenv("DATABASE_NAME", None),
'USER': os.getenv("DATABASE_USER", None),
'PASSWORD': os.getenv("DATABASE_PASSWORD", None),
'HOST': os.getenv("DATABASE_HOST", None),
'PORT': os.getenv("DATABASE_PORT", None),
}
}
if os.getenv("DATABASE_SSLMODE", False):
DATABASES['default']['OPTIONS'] = {'sslmode': os.getenv("DATABASE_SSLMODE")}
USE_X_FORWARDED_HOST = True
USE_X_FORWARDED_PORT = True

View File

@@ -0,0 +1,19 @@
---
- name: Create _build directory
file:
path: "{{ dockerfile_dest }}/{{ template_dest }}"
state: directory
- name: Render supervisor configs
template:
src: "{{ item }}.j2"
dest: "{{ dockerfile_dest }}/{{ template_dest }}/{{ item }}"
with_items:
- "supervisor.conf"
- "supervisor_task.conf"
- name: Render Dockerfile
template:
src: Dockerfile.j2
dest: "{{ dockerfile_dest }}/{{ dockerfile_name }}"

View File

@@ -0,0 +1,268 @@
### This file is generated from
### tools/ansible/roles/dockerfile/templates/Dockerfile.j2
###
### DO NOT EDIT
###
# Locations - set globally to be used across stages
ARG COLLECTION_BASE="/var/lib/awx/vendor/awx_ansible_collections"
# Build container
FROM centos:8 as builder
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
USER root
# Install build dependencies
RUN dnf -y module enable 'postgresql:12'
RUN dnf -y update && \
dnf -y install epel-release 'dnf-command(config-manager)' && \
dnf module -y enable 'postgresql:12' && \
dnf config-manager --set-enabled powertools && \
dnf -y install ansible \
gcc \
gcc-c++ \
git-core \
glibc-langpack-en \
libcurl-devel \
libffi-devel \
libtool-ltdl-devel \
make \
nodejs \
nss \
openldap-devel \
patch \
@postgresql:12 \
postgresql-devel \
python3-devel \
python3-pip \
python3-psycopg2 \
python3-setuptools \
swig \
unzip \
xmlsec1-devel \
xmlsec1-openssl-devel
RUN python3 -m ensurepip && pip3 install "virtualenv < 20"
# Install & build requirements
ADD Makefile /tmp/Makefile
RUN mkdir /tmp/requirements
ADD requirements/requirements_ansible.txt \
requirements/requirements_ansible_uninstall.txt \
requirements/requirements_ansible_git.txt \
requirements/requirements.txt \
requirements/requirements_tower_uninstall.txt \
requirements/requirements_git.txt \
requirements/collections_requirements.yml \
/tmp/requirements/
RUN cd /tmp && make requirements_awx requirements_ansible_py3
RUN cd /tmp && make requirements_collections
{% if (build_dev|bool) or (kube_dev|bool) %}
ADD requirements/requirements_dev.txt /tmp/requirements
RUN cd /tmp && make requirements_awx_dev requirements_ansible_dev
{% else %}
# Use the distro provided npm to bootstrap our required version of node
RUN npm install -g n && n 14.15.1 && dnf remove -y nodejs
# Copy source into builder, build sdist, install it into awx venv
COPY . /tmp/src/
WORKDIR /tmp/src/
RUN make sdist && \
/var/lib/awx/venv/awx/bin/pip install dist/awx-$(cat VERSION).tar.gz
{% endif %}
# Final container(s)
FROM centos:8
ARG COLLECTION_BASE
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
USER root
# Install runtime requirements
RUN dnf -y module enable 'postgresql:12'
RUN dnf -y update && \
dnf -y install epel-release 'dnf-command(config-manager)' && \
dnf module -y enable 'postgresql:12' && \
dnf config-manager --set-enabled powertools && \
dnf -y install acl \
ansible \
bubblewrap \
git-core \
git-lfs \
glibc-langpack-en \
krb5-workstation \
libcgroup-tools \
nginx \
@postgresql:12 \
python3-devel \
python3-libselinux \
python3-pip \
python3-psycopg2 \
python3-setuptools \
rsync \
subversion \
sudo \
vim-minimal \
which \
unzip \
xmlsec1-openssl && \
dnf -y install centos-release-stream && dnf -y install "rsyslog >= 8.1911.0" && dnf -y remove centos-release-stream && \
dnf -y clean all
# Install kubectl
RUN curl -L -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.17.8/bin/linux/{{ kubectl_architecture | default('amd64') }}/kubectl && \
chmod a+x /usr/bin/kubectl
RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && \
chmod 700 get_helm.sh && \
./get_helm.sh
# Install tini
RUN curl -L -o /usr/bin/tini https://github.com/krallin/tini/releases/download/v0.19.0/tini-{{ tini_architecture | default('amd64') }} && \
chmod +x /usr/bin/tini
RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if build_dev|bool %}flake8{% endif %}
RUN rm -rf /root/.cache && rm -rf /tmp/*
# Install OpenShift CLI
RUN cd /usr/local/bin && \
curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \
tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
{% if (build_dev|bool) or (kube_dev|bool) %}
# Install development/test requirements
RUN dnf -y install \
gdb \
gtk3 \
gettext \
alsa-lib \
libX11-xcb \
libXScrnSaver \
strace \
vim \
nmap-ncat \
nodejs \
nss \
make \
patch \
socat \
tmux \
wget \
diffutils \
unzip && \
npm install -g n && n 14.15.1 && dnf remove -y nodejs
# This package randomly fails to download.
# It is nice to have in the dev env, but not necessary.
# Add it back to the list above if the repo ever straighten up.
RUN dnf --enablerepo=debuginfo -y install python3-debuginfo || :
{% endif %}
# Copy app from builder
COPY --from=builder /var/lib/awx /var/lib/awx
RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage
{%if build_dev|bool %}
RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr \
-subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \
openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \
chmod 640 /etc/nginx/nginx.{csr,key,crt}
{% endif %}
# Create default awx rsyslog config
ADD tools/ansible/roles/dockerfile/files/rsyslog.conf /var/lib/awx/rsyslog/rsyslog.conf
## File mappings
{% if build_dev|bool %}
ADD tools/docker-compose/launch_awx.sh /usr/bin/launch_awx.sh
ADD tools/docker-compose/nginx.conf /etc/nginx/nginx.conf
ADD tools/docker-compose/nginx.vh.default.conf /etc/nginx/conf.d/nginx.vh.default.conf
ADD tools/docker-compose/start_tests.sh /start_tests.sh
ADD tools/docker-compose/bootstrap_development.sh /usr/bin/bootstrap_development.sh
ADD tools/docker-compose/entrypoint.sh /entrypoint.sh
{% else %}
ADD tools/ansible/roles/dockerfile/files/launch_awx.sh /usr/bin/launch_awx.sh
ADD tools/ansible/roles/dockerfile/files/launch_awx_task.sh /usr/bin/launch_awx_task.sh
ADD tools/ansible/roles/dockerfile/files/settings.py /etc/tower/settings.py
ADD {{ template_dest }}/supervisor.conf /etc/supervisord.conf
ADD {{ template_dest }}/supervisor_task.conf /etc/supervisord_task.conf
ADD tools/scripts/config-watcher /usr/bin/config-watcher
{% endif %}
{% if (build_dev|bool) or (kube_dev|bool) %}
ADD tools/docker-compose/awx.egg-link /tmp/awx.egg-link
ADD tools/docker-compose/awx-manage /usr/local/bin/awx-manage
ADD tools/scripts/awx-python /usr/bin/awx-python
{% endif %}
# Pre-create things we need to access
RUN for dir in \
/var/lib/awx \
/var/lib/awx/rsyslog \
/var/lib/awx/rsyslog/conf.d \
/var/run/awx-rsyslog \
/var/log/tower \
/var/log/nginx \
/var/lib/postgresql \
/var/run/supervisor \
/var/lib/nginx ; \
do mkdir -m 0775 -p $dir ; chmod g+rw $dir ; chgrp root $dir ; done && \
for file in \
/etc/passwd \
/var/lib/awx/rsyslog/rsyslog.conf ; \
do touch $file ; chmod g+rw $file ; chgrp root $file ; done
# Adjust any remaining permissions
RUN chmod u+s /usr/bin/bwrap ; \
chgrp -R root ${COLLECTION_BASE} ; \
chmod -R g+rw ${COLLECTION_BASE}
{% if (build_dev|bool) or (kube_dev|bool) %}
RUN for dir in \
/var/lib/awx/venv \
/var/lib/awx/venv/awx/lib/python3.6 \
/var/lib/awx/projects \
/var/lib/awx/rsyslog \
/var/run/awx-rsyslog \
/.ansible \
/var/lib/awx/vendor ; \
do mkdir -m 0775 -p $dir ; chmod g+rw $dir ; chgrp root $dir ; done && \
for file in \
/var/run/nginx.pid \
/var/lib/awx/venv/awx/lib/python3.6/site-packages/awx.egg-link ; \
do touch $file ; chmod g+rw $file ; done
{% endif %}
{% if not build_dev|bool %}
RUN ln -sf /dev/stdout /var/log/nginx/access.log && \
ln -sf /dev/stderr /var/log/nginx/error.log
{% endif %}
ENV HOME="/var/lib/awx"
ENV PATH="/usr/pgsql-10/bin:${PATH}"
{% if build_dev|bool %}
EXPOSE 8043 8013 8080 22
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/bin/bash"]
{% else %}
USER 1000
EXPOSE 8052
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD /usr/bin/launch_awx.sh
VOLUME /var/lib/nginx
{% endif %}

View File

@@ -0,0 +1,117 @@
[supervisord]
nodaemon = True
umask = 022
logfile = /dev/stdout
logfile_maxbytes = 0
pidfile = /var/run/supervisor/supervisor.web.pid
[program:nginx]
{% if kube_dev | bool %}
command = make nginx
directory = /awx_devel
{% else %}
command = nginx -g "daemon off;"
{% endif %}
autostart = true
autorestart = true
stopwaitsecs = 5
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:uwsgi]
{% if kube_dev | bool %}
command = make uwsgi
directory = /awx_devel
environment =
UWSGI_DEV_RELOAD_COMMAND='supervisorctl -c /etc/supervisord_task.conf restart all; supervisorctl restart tower-processes:daphne tower-processes:wsbroadcast'
{% else %}
command = /var/lib/awx/venv/awx/bin/uwsgi --socket 127.0.0.1:8050 --module=awx.wsgi:application --vacuum --processes=5 --harakiri=120 --no-orphans --master --max-requests=1000 --master-fifo=/var/lib/awx/awxfifo --lazy-apps -b 32768
directory = /var/lib/awx
{% endif %}
autostart = true
autorestart = true
stopwaitsecs = 15
stopasgroup=true
killasgroup=true
stopsignal=KILL
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:daphne]
{% if kube_dev | bool %}
command = make daphne
directory = /awx_devel
{% else %}
command = /var/lib/awx/venv/awx/bin/daphne -b 127.0.0.1 -p 8051 --websocket_timeout -1 awx.asgi:channel_layer
directory = /var/lib/awx
{% endif %}
autostart = true
stopsignal=KILL
autorestart = true
stopwaitsecs = 5
stopasgroup=true
killasgroup=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:wsbroadcast]
{% if kube_dev | bool %}
command = make wsbroadcast
directory = /awx_devel
{% else %}
command = awx-manage run_wsbroadcast
directory = /var/lib/awx
{% endif %}
autostart = true
autorestart = true
stopwaitsecs = 5
stopasgroup=true
killasgroup=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:awx-rsyslogd]
command = rsyslogd -n -i /var/run/awx-rsyslog/rsyslog.pid -f /var/lib/awx/rsyslog/rsyslog.conf
autostart = true
autorestart = true
startretries = 10
stopwaitsecs = 5
stopsignal=TERM
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/stderr
stdout_logfile_maxbytes=0
[group:tower-processes]
programs=nginx,uwsgi,daphne,wsbroadcast,awx-rsyslogd
priority=5
# TODO: Exit Handler
[eventlistener:awx-config-watcher]
command=/usr/bin/config-watcher
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
events=TICK_60
priority=0
[unix_http_server]
file=/var/run/supervisor/supervisor.web.sock
[supervisorctl]
serverurl=unix:///var/run/supervisor/supervisor.web.sock ; use a unix:// URL for a unix socket
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

View File

@@ -0,0 +1,62 @@
[supervisord]
nodaemon = True
umask = 022
logfile = /dev/stdout
logfile_maxbytes = 0
pidfile = /var/run/supervisor/supervisor.pid
[program:dispatcher]
{% if kube_dev | bool %}
command = make dispatcher
directory = /awx_devel
{% else %}
command = awx-manage run_dispatcher
directory = /var/lib/awx
{% endif %}
autostart = true
autorestart = true
stopwaitsecs = 5
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:callback-receiver]
{% if kube_dev | bool %}
command = make receiver
directory = /awx_devel
{% else %}
command = awx-manage run_callback_receiver
directory = /var/lib/awx
{% endif %}
autostart = true
autorestart = true
stopwaitsecs = 5
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[group:tower-processes]
programs=dispatcher,callback-receiver
priority=5
# TODO: Exit Handler
[eventlistener:awx-config-watcher]
command=/usr/bin/config-watcher
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
events=TICK_60
priority=0
[unix_http_server]
file=/var/run/supervisor/supervisor.sock
[supervisorctl]
serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

View File

@@ -0,0 +1,6 @@
---
create_preload_data: true
# Helper vars to construct the proper download URL for the current architecture
tini_architecture: '{{ { "x86_64": "amd64", "aarch64": "arm64", "armv7": "arm" }[ansible_facts.architecture] }}'
kubectl_architecture: '{{ { "x86_64": "amd64", "aarch64": "arm64", "armv7": "arm" }[ansible_facts.architecture] }}'

View File

@@ -0,0 +1,31 @@
---
- name: Set global version if not provided
set_fact:
awx_version: "{{ lookup('file', playbook_dir + '/../../VERSION') }}"
when: awx_version is not defined
- name: Verify awx-logos directory exists for official install
stat:
path: "../../awx-logos"
register: logosdir
failed_when: logosdir.stat.isdir is not defined or not logosdir.stat.isdir
when: awx_official|default(false)|bool
- name: Copy logos for inclusion in sdist
copy:
src: "../../awx-logos/awx/ui/client/assets/"
dest: "../awx/ui_next/public/static/media/"
when: awx_official|default(false)|bool
- name: Set awx image name
set_fact:
awx_image: "{{ awx_image|default('awx') }}"
# Calling Docker directly because docker-py doesnt support BuildKit
- name: Build AWX image
command: docker build -t {{ awx_image }}:{{ awx_version }} -f ../../{{ dockerfile_name }} ../..
- name: Tag awx images as latest
command: "docker tag {{ item }}:{{ awx_version }} {{ item }}:latest"
with_items:
- "{{ awx_image }}"

View File

@@ -0,0 +1,33 @@
---
- name: Authenticate with Docker registry if registry password given
docker_login:
registry: "{{ docker_registry }}"
username: "{{ docker_registry_username }}"
password: "{{ docker_registry_password }}"
reauthorize: true
when: docker_registry is defined and docker_registry_password is defined
- name: Remove local images to ensure proper push behavior
block:
- name: Remove awx image
docker_image:
name: "{{ docker_registry }}/{{ docker_registry_repository }}/{{ awx_image }}"
tag: "{{ awx_version }}"
state: absent
- name: Tag and Push Container Images
block:
- name: Tag and push awx image to registry
docker_image:
name: "{{ awx_image }}"
repository: "{{ docker_registry }}/{{ docker_registry_repository }}/{{ awx_image }}"
tag: "{{ item }}"
push: true
with_items:
- "latest"
- "{{ awx_version }}"
- name: Set full image path for Registry
set_fact:
awx_docker_actual_image: >-
{{ docker_registry }}/{{ docker_registry_repository }}/{{ awx_image }}:{{ awx_version }}

View File

@@ -1,86 +1,171 @@
# Docker Compose for Dev container
# Docker Compose for Development
## How to start the Dev container
## Overview
In the root directory of your awx clone, run the following to build your docker image.
This step takes a while, but once your image is built, you don't need to do this again
unless you make changes to the Dockerfile or any of the files used by the Dockerfile.
Here are the main make targets:
```
make docker-compose-build
* `docker-compose-build` - used for building the development image, which is used by both `docker-compose`
* `docker-compose` - Make target for development, passes awx_devel image and tag.
Notable files:
* `tools/docker-compose/inventory` file - used to configure the local AWX development deployment.
* `migrate.yml` - playbook for migrating data from Local Docker to the Development Environment.
### Prerequisites
- [Docker](https://docs.docker.com/engine/installation/) on the host where AWX will be deployed. After installing Docker, the Docker service must be started (depending on your OS, you may have to add the local user that uses Docker to the ``docker`` group, refer to the documentation for details)
- [docker-compose](https://pypi.org/project/docker-compose/) Python module.
+ This also installs the `docker` Python module, which is incompatible with `docker-py`. If you have previously installed `docker-py`, please uninstall it.
- [Docker Compose](https://docs.docker.com/compose/install/).
## Configuration
In the `inventory` file, set your `pg_password`, `broadcast_websocket_secret`, `secret_key`, and any other settings you need for your deployment.
AWX requires access to a PostgreSQL database, and by default, one will be created and deployed in a container, and data will be persisted to a docker volume. When the container is stopped, the database files will still exist in the docker volume. An external database can be used by setting the `pg_host`, `pg_hostname`, and `pg_username`.
> If you are coming from a Local Docker installation of AWX, consider migrating your data first, see the [data migration section](#migrating-data-from-local-docker) below.
## Starting the Development Environment
### Build the image
The AWX base container image (defined in the Dockerfile templated from [Dockerfile.j2](./../ansible/roles/dockerfile/templates/Dockerfile.j2)) contains basic OS dependencies and symbolic links into the development environment that make running the services easy.
Run the following to build the image:
```bash
$ make docker-compose-build
```
> Note: By default, this image will be tagged with your branch name.
> The image will need to be rebuilt if there are any changes to Dockerfile.j2 or any of the files used by the templated Dockerfile.
Copy over your local settings
Once the build completes, you will have a `ansible/awx_devel` image in your local image cache. Use the `docker images` command to view it, as follows:
```
cp awx/settings/local_settings.py.docker_compose awx/settings/local_settings.py
```bash
(host)$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
ansible/awx_devel latest ba9ec3e8df74 26 minutes ago 1.42GB
```
Build the UI
> By default, this image will be tagged with your branch name. You can specify a custom tag by setting an environment variable, for example: `DEVEL_IMAGE_NAME=quay.io/your_user/awx_devel:17.0.1`
```
make ui-devel
### Run AWX
##### Start the containers
Run the awx, postgres and redis containers. This utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, your session will be attached to the awx container, and you'll be able to watch log messages and events in real time. You will see messages from Django and the front end build process.
```bash
$ make docker-compose
```
Run the container
> The make target assumes that the image you built is tagged with your current branch. This allows you to build images for different contexts or branches. When starting the containers, you can choose a specific branch by setting `COMPOSE_TAG=<branch name> `in your environment. For example, you might be working in a feature branch, but you want to run the containers using the devel image you built previously. To do that, start the containers using the following command: `$ COMPOSE_TAG=devel make docker-compose`
```
make docker-compose
```
##### Wait for migrations to complete
> Note: You may need to add `COMPOSE_TAG=<yourbranch>` at the end of this to use the correct image for your branch.
The first time you start the environment, database migrations need to run in order to build the PostgreSQL database. It will take few moments, but eventually you will see output in your terminal session that looks like the following:
The app should now be accessible in your browser at `https://localhost:8043/#/home`
## How to use the logstash container
#### Modify the docker-compose.yml
Uncomment the following lines in the `docker-compose.yml`
```
#- logstash
```bash
awx_1 | Operations to perform:
awx_1 | Synchronize unmigrated apps: solo, api, staticfiles, debug_toolbar, messages, channels, django_extensions, ui, rest_framework, polymorphic
awx_1 | Apply all migrations: sso, taggit, sessions, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
awx_1 | Synchronizing apps without migrations:
awx_1 | Creating tables...
awx_1 | Running deferred SQL...
awx_1 | Installing custom SQL...
awx_1 | Running migrations:
awx_1 | Rendering model states... DONE
awx_1 | Applying contenttypes.0001_initial... OK
awx_1 | Applying contenttypes.0002_remove_content_type_name... OK
awx_1 | Applying auth.0001_initial... OK
...
#logstash:
# build:
# context: ./docker-compose
# dockerfile: Dockerfile-logstash
```
POST the following content to `/api/v2/settings/logging/` (this uses
authentication set up inside of the logstash configuration file).
```
{
"LOG_AGGREGATOR_HOST": "http://logstash",
"LOG_AGGREGATOR_PORT": 8085,
"LOG_AGGREGATOR_TYPE": "logstash",
"LOG_AGGREGATOR_USERNAME": "awx_logger",
"LOG_AGGREGATOR_PASSWORD": "workflows",
"LOG_AGGREGATOR_LOGGERS": [
"awx",
"activity_stream",
"job_events",
"system_tracking"
],
"LOG_AGGREGATOR_INDIVIDUAL_FACTS": false,
"LOG_AGGREGATOR_TOWER_UUID": "991ac7e9-6d68-48c8-bbde-7ca1096653c6",
"LOG_AGGREGATOR_ENABLED": true
}
##### Clean and Build UI
```bash
$ docker exec tools_awx_1 make clean-ui ui-devel
```
> Note: HTTP must be specified in the `LOG_AGGREGATOR_HOST` if you are using the docker development environment.
See [the ui development documentation](../../awx/ui_next/README.md) for more information on using the frontend development, build, and test tooling.
An example of how to view the most recent logs from the container:
Once migrations are completed and the UI is built, you can begin using AWX. The UI can be reached in your browser at `https://localhost:8043/#/home`, and the API can be found at `https://localhost:8043/api/v2`.
```
docker exec -i -t $(docker ps -aqf "name=tools_logstash_1") tail -n 50 /logstash.log
##### Create an admin user
Before you can log into AWX, you need to create an admin user. With this user you will be able to create more users, and begin configuring the server. From within the container shell, run the following command:
```bash
$ docker exec -ti tools_awx_1 awx-manage createsuperuser
```
#### How to add logstash plugins
> Remember the username and password, as you will use them to log into the web interface for the first time.
Add any plugins you need in `tools/elastic/logstash/Dockerfile` before running the container.
##### Load Demo Data
Optionally, you may also want to load some demo data. This will create a demo project, inventory, and job template.
```bash
$ docker exec tools_awx_1 awx-manage create_preload_data
```
> This information will persist in the database running in the `tools_postgres_1` container, until the container is removed. You may periodically need to recreate
this container, and thus the database, if the database schema changes in an upstream commit.
## Migrating Data from Local Docker
If you are migrating data from a Local Docker installation (17.0.1 and prior), you can
migrate your data to the development environment via the migrate.yml playbook using the steps described [here](./docs/data_migration.md).
## Upgrading the Development Environment
Upgrading AWX involves checking out the new source code and re-running the make target. Download a newer release from [https://github.com/ansible/awx/releases](https://github.com/ansible/awx/releases) and re-populate the inventory file with your customized variables.
After updating the inventory file with any custom values, run the make target from the root of your AWX clone.
```bash
$ make docker-compose
```
## Extras
* [Start a shell](#start-a-shell)
* [Start AWX from the container shell](#start-awx-from-the-container-shell)
* [Using Logstash](./docs/logstash.md)
### Start a Shell
To run `awx-manage` commands and modify things inside the container, you will want to start a shell session on the *awx* container. In a new terminal session, use the `docker exec` command to start the shell session:
```bash
(host)$ docker exec -it tools_awx_1 bash
```
This creates a session in the *awx* containers, just as if you were using `ssh`, and allows you execute commands within the running container.
### Start AWX from the container shell
Often times you'll want to start the development environment without immediately starting all of the services in the *awx* container, and instead be taken directly to a shell. You can do this with the following:
```bash
(host)$ make docker-compose-test
```
Using `docker exec`, this will create a session in the running *awx* container, and place you at a command prompt, where you can run shell commands inside the container.
If you want to start and use the development environment, you'll first need to bootstrap it by running the following command:
```bash
(container)# /usr/bin/bootstrap_development.sh
```
The above will do all the setup tasks, including running database migrations, so it may take a couple minutes. Once it's done it
will drop you back to the shell.
In order to launch all developer services:
```bash
(container)# /usr/bin/launch_awx.sh
```
`launch_awx.sh` also calls `bootstrap_development.sh` so if all you are doing is launching the supervisor to start all services, you don't
need to call `bootstrap_development.sh` first.

View File

@@ -0,0 +1,7 @@
---
- name: Migrate data for upgrades and from Local Docker installs
hosts: localhost
gather_facts: true
roles:
- {role: sources}
- {role: migrate}

View File

@@ -0,0 +1,5 @@
---
postgres_data_dir: "~/.awx/pgdocker"
migrate_local_docker: false
old_docker_compose_dir: "~/.awx/awxcompose"
pg_volume_name: "postgres_data"

View File

@@ -0,0 +1,5 @@
---
# Migrate data from a bind-mount to a volume
- import_tasks: migrate-from-local-docker.yml
when: migrate_local_docker

View File

@@ -0,0 +1,46 @@
---
# Migrate data from old Local Docker to a fresh development environment
- name: Remove awx_postgres to ensure consistent start state
shell: |
docker rm -f awx_postgres
- name: Start Local Docker database container
docker_compose:
project_src: "{{ old_docker_compose_dir }}"
services:
- postgres
state: present
recreate: always
- name: Database dump to local filesystem
shell: |
docker-compose -f {{ old_docker_compose_dir }}/docker-compose.yml exec -T postgres pg_dumpall -U {{ pg_username }} > awx_dump.sql
- name: Stop AWX containers so the old postgres container does not get used
docker_compose:
project_src: "{{ old_docker_compose_dir }}"
state: absent
ignore_errors: true
- name: Start dev env database container
docker_compose:
project_src: "{{ playbook_dir }}/../_sources"
files: "docker-compose.yml"
services:
- postgres
state: present
recreate: always
- name: Wait for postgres to initialize
wait_for:
timeout: 3
- name: Restore to new postgres container
shell: |
docker-compose -f {{ playbook_dir }}/../_sources/docker-compose.yml exec -T postgres psql -U {{ pg_username }} -d {{ pg_database }} -p {{ pg_port }} < awx_dump.sql
- name: Clean up temporary awx db dump
file:
path: awx_dump.sql
state: absent

View File

@@ -0,0 +1,7 @@
---
sources_dest: '../_sources'
compose_name: 'docker-compose.yml'
awx_image: 'quay.io/ansible/awx_devel'
pg_port: 5432
pg_username: 'awx'
pg_database: 'awx'

View File

@@ -0,0 +1,68 @@
# Copyright (c) 2015 Ansible, Inc. (formerly AnsibleWorks, Inc.)
# All Rights Reserved.
# Local Django settings for AWX project. Rename to "local_settings.py" and
# edit as needed for your development environment.
# All variables defined in awx/settings/development.py will already be loaded
# into the global namespace before this file is loaded, to allow for reading
# and updating the default settings as needed.
###############################################################################
# MISC PROJECT SETTINGS
###############################################################################
import os
import sys
# Enable the following lines and install the browser extension to use Django debug toolbar
# if your deployment method is not VMWare of Docker-for-Mac you may
# need a different IP address from request.META['REMOTE_ADDR']
# INTERNAL_IPS = ('172.19.0.1', '172.18.0.1', '192.168.100.1')
# ALLOWED_HOSTS = ['*']
# Use SQLite for unit tests instead of PostgreSQL. If the lines below are
# commented out, Django will create the test_awx-dev database in PostgreSQL to
# run unit tests.
if "pytest" in sys.modules:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'awx.sqlite3'),
'TEST': {
# Test database cannot be :memory: for inventory tests.
'NAME': os.path.join(BASE_DIR, 'awx_test.sqlite3'),
},
}
}
# Location for cross-development of inventory plugins
AWX_ANSIBLE_COLLECTIONS_PATHS = '/var/lib/awx/vendor/awx_ansible_collections'
# The UUID of the system, for HA.
SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
# If set, use -vvv for project updates instead of -v for more output.
# PROJECT_UPDATE_VVV=True
###############################################################################
# LOGGING SETTINGS
###############################################################################
# Enable logging to syslog. Setting level to ERROR captures 500 errors,
# WARNING also logs 4xx responses.
# Enable the following lines to turn on lots of permissions-related logging.
#LOGGING['loggers']['awx.main.access']['level'] = 'DEBUG'
#LOGGING['loggers']['awx.main.signals']['level'] = 'DEBUG'
#LOGGING['loggers']['awx.main.permissions']['level'] = 'DEBUG'
# Enable the following line to turn on database settings logging.
#LOGGING['loggers']['awx.conf']['level'] = 'DEBUG'
# Enable the following lines to turn on LDAP auth logging.
#LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']
#LOGGING['loggers']['django_auth_ldap']['level'] = 'DEBUG'
BROADCAST_WEBSOCKET_PORT = 8013
BROADCAST_WEBSOCKET_VERIFY_CERT = False
BROADCAST_WEBSOCKET_PROTOCOL = 'http'

View File

@@ -0,0 +1,76 @@
---
- name: Create _sources directory
file:
path: "{{ sources_dest }}/secrets"
state: 'directory'
mode: '0700'
- name: Detect secrets
stat:
path: "{{ sources_dest }}/secrets/{{ item }}.yml"
register: secrets
when: not lookup('vars', item, default='')
loop:
- pg_password
- secret_key
- broadcast_websocket_secret
- name: Generate secrets if needed
template:
src: 'secrets.yml.j2'
dest: '{{ sources_dest }}/secrets/{{ item.item }}.yml'
mode: '0600'
when: not lookup('vars', item.item, default='') and not item.stat.exists
loop: "{{ secrets.results }}"
- name: Include generated secrets unless they are explicitly passed in
include_vars: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
no_log: true
when: not lookup('vars', item.item, default='')
loop: "{{ secrets.results }}"
- name: Render configuration templates
template:
src: "{{ item }}.j2"
dest: "{{ sources_dest }}/{{ item }}"
mode: '0600'
with_items:
- "database.py"
- "websocket_secret.py"
- name: Delete old local_settings.py
file:
path: "{{ playbook_dir }}/../../../awx/settings/local_settings.py"
state: absent
- name: Copy local_settings.py
copy:
src: "local_settings.py"
dest: "{{ sources_dest }}/local_settings.py"
- name: Get OS info for sdb
shell: |
docker info | grep 'Operating System'
register: os_info
changed_when: false
- name: Get user UID
shell: id -u
register: current_user
changed_when: false
- name: Set fact with user UID
set_fact:
user_id: "'{{ current_user.stdout }}'"
- name: Set global version if not provided
set_fact:
awx_image_tag: "{{ lookup('file', playbook_dir + '/../../../VERSION') }}"
when: awx_image_tag is not defined
- name: Render Docker-Compose
template:
src: docker-compose.yml.j2
dest: "{{ sources_dest }}/{{ compose_name }}"
mode: '0600'

View File

@@ -0,0 +1,11 @@
DATABASES = {
'default': {
'ATOMIC_REQUESTS': True,
'ENGINE': 'awx.main.db.profiled_pg',
'NAME': "{{ pg_database }}",
'USER': "{{ pg_username }}",
'PASSWORD': "{{ pg_password }}",
'HOST': "{{ pg_hostname | default('postgres') }}",
'PORT': "{{ pg_port }}",
}
}

View File

@@ -3,14 +3,13 @@ version: '2'
services:
# Primary AWX Development Container
awx:
user: ${CURRENT_UID}
image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG}
user: "{{ ansible_user_uid }}"
image: "{{ awx_image }}:{{ awx_image_tag }}"
container_name: tools_awx_1
hostname: awx
command: launch_awx.sh
environment:
CURRENT_UID:
OS:
OS: "{{ os_info.stdout }}"
SDB_HOST: 0.0.0.0
SDB_PORT: 7899
AWX_GROUP_QUEUES: tower
@@ -24,15 +23,14 @@ services:
links:
- postgres
- redis
# - sync
# volumes_from:
# - sync
working_dir: "/awx_devel"
volumes:
- "../:/awx_devel"
- "../awx/projects/:/var/lib/awx/projects/"
- "./redis/redis_socket_standalone:/var/run/redis/"
- "./docker-compose/supervisor.conf:/etc/supervisord.conf"
- "../../../:/awx_devel"
- "../../docker-compose/supervisor.conf:/etc/supervisord.conf"
- "../../docker-compose/_sources/database.py:/etc/tower/conf.d/database.py"
- "../../docker-compose/_sources/websocket_secret.py:/etc/tower/conf.d/websocket_secret.py"
- "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py"
- "redis_socket:/var/run/redis/:rw"
privileged: true
tty: true
# A useful container that simply passes through log messages to the console
@@ -48,16 +46,20 @@ services:
container_name: tools_postgres_1
environment:
POSTGRES_HOST_AUTH_METHOD: trust
POSTGRES_USER: {{ pg_username }}
POSTGRES_DB: {{ pg_database }}
POSTGRES_PASSWORD: {{ pg_password }}
volumes:
- "awx_db:/var/lib/postgresql/data"
redis:
image: redis:latest
container_name: tools_redis_1
user: ${CURRENT_UID}
volumes:
- "./redis/redis.conf:/usr/local/etc/redis/redis.conf"
- "./redis/redis_socket_standalone:/var/run/redis/"
- "../../redis/redis.conf:/usr/local/etc/redis/redis.conf"
- "redis_socket:/var/run/redis/:rw"
entrypoint: ["redis-server"]
command: ["/usr/local/etc/redis/redis.conf"]
volumes:
awx_db:
redis_socket:

View File

@@ -0,0 +1,31 @@
---
version: '2'
services:
# Primary Tower Development Container link
awx:
links:
- hashivault
- conjur
hashivault:
image: vault
container_name: tools_hashivault_1
ports:
- '8200:8200'
cap_add:
- IPC_LOCK
environment:
VAULT_DEV_ROOT_TOKEN_ID: 'vaultdev'
conjur:
image: cyberark/conjur
container_name: tools_conjur_1
command: server -p 8300
environment:
DATABASE_URL: postgres://awx@postgres/postgres
CONJUR_DATA_KEY: 'dveUwOI/71x9BPJkIgvQRRBF3SdASc+HP4CUGL7TKvM='
depends_on:
- postgres
links:
- postgres
ports:
- "8300:8300"

View File

@@ -0,0 +1,10 @@
DATABASE_USER={{ pg_username|quote }}
DATABASE_NAME={{ pg_database|quote }}
DATABASE_HOST={{ pg_hostname|default('postgres')|quote }}
DATABASE_PORT={{ pg_port|default('5432')|quote }}
DATABASE_PASSWORD={{ pg_password|default('awxpass')|quote }}
{% if pg_admin_password is defined %}
DATABASE_ADMIN_PASSWORD={{ pg_admin_password|quote }}
{% endif %}
AWX_ADMIN_USER={{ admin_user|quote }}
AWX_ADMIN_PASSWORD={{ admin_password|quote }}

View File

@@ -0,0 +1,122 @@
#user awx;
worker_processes 1;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_tokens off;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout main;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
sendfile on;
#tcp_nopush on;
#gzip on;
upstream uwsgi {
server 127.0.0.1:8050;
}
upstream daphne {
server 127.0.0.1:8051;
}
{% if ssl_certificate is defined %}
server {
listen 8052 default_server;
server_name _;
# Redirect all HTTP links to the matching HTTPS page
return 301 https://$host$request_uri;
}
{%endif %}
server {
{% if (ssl_certificate is defined) and (ssl_certificate_key is defined) %}
listen 8053 ssl;
ssl_certificate /etc/nginx/awxweb.pem;
ssl_certificate_key /etc/nginx/awxweb_key.pem;
{% elif (ssl_certificate is defined) and (ssl_certificate_key is not defined) %}
listen 8053 ssl;
ssl_certificate /etc/nginx/awxweb.pem;
ssl_certificate_key /etc/nginx/awxweb.pem;
{% else %}
listen 8052 default_server;
{% endif %}
# If you have a domain name, this is where to add it
server_name _;
keepalive_timeout 65;
# HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
add_header Strict-Transport-Security max-age=15768000;
# Protect against click-jacking https://www.owasp.org/index.php/Testing_for_Clickjacking_(OTG-CLIENT-009)
add_header X-Frame-Options "DENY";
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1;
deny all;
}
location /static/ {
alias /var/lib/awx/public/static/;
}
location /favicon.ico { alias /var/lib/awx/public/static/favicon.ico; }
location /websocket {
# Pass request to the upstream alias
proxy_pass http://daphne;
# Require http version 1.1 to allow for upgrade requests
proxy_http_version 1.1;
# We want proxy_buffering off for proxying to websockets.
proxy_buffering off;
# http://en.wikipedia.org/wiki/X-Forwarded-For
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# enable this if you use HTTPS:
proxy_set_header X-Forwarded-Proto https;
# pass the Host: header from the client for the sake of redirects
proxy_set_header Host $http_host;
# We've set the Host header, so we don't need Nginx to muddle
# about with redirects
proxy_redirect off;
# Depending on the request value, set the Upgrade and
# connection headers
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
}
location / {
# Add trailing / if missing
rewrite ^(.*)$http_host(.*[^/])$ $1$http_host$2/ permanent;
uwsgi_read_timeout 120s;
uwsgi_pass uwsgi;
include /etc/nginx/uwsgi_params;
{%- if extra_nginx_include is defined %}
include {{ extra_nginx_include }};
{%- endif %}
proxy_set_header X-Forwarded-Port 443;
uwsgi_param HTTP_X_FORWARDED_PORT 443;
}
}
}

View File

@@ -0,0 +1 @@
{{ item.item }}: '{{ lookup('vars', item.item, default='') or lookup('password', '/dev/null chars=ascii_letters') }}'

View File

@@ -0,0 +1 @@
BROADCAST_WEBSOCKET_SECRET = "{{ broadcast_websocket_secret | b64encode }}"

View File

@@ -0,0 +1,6 @@
---
- name: Render AWX Dockerfile and sources
hosts: localhost
gather_facts: true
roles:
- {role: sources}

View File

@@ -9,9 +9,6 @@ ansible -i "127.0.0.1," -c local -v -m wait_for -a "path=/var/run/redis/redis.so
# TODO: FIX
#/etc/init.d/ssh start
ansible -i "127.0.0.1," -c local -v -m postgresql_user --become-user postgres -a "name=awx-dev password=AWXsome1 encrypted=yes login_user=postgres login_password=postgrespass login_host=postgres" all
ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user postgres -a "name=awx-dev owner=awx-dev login_user=postgres login_password=postgrespass login_host=postgres" all
# Move to the source directory so we can bootstrap
if [ -f "/awx_devel/manage.py" ]; then
cd /awx_devel

View File

@@ -0,0 +1,60 @@
# Migrating Data from Local Docker
If you are migrating data from a Local Docker installation (17.0.1 and prior), you can
migrate your data to the development environment via the migrate.yml playbook, or by using the manual steps described below.
> Note: This will also convert your postgresql bind-mount into a docker volume.
### Migrate data with migrate.yml
If you had a custom pgdocker or awxcompose location, you will need to set the `postgres_data_dir` and `old_docker_compose_dir` variables.
1. Run the [migrate playbook](./ansible/migrate.yml) to migrate your data to the new postgresql container and convert the data directory to a volume mount.
```bash
$ ansible-playbook migrate.yml -e "migrate_local_docker=true" -e "postgres_data_dir=~/.awx/pgdocker" -e "old_docker_compose_dir=~/.awx/awxcompose"
```
2. Change directory to the top of your awx checkout and start your containers
```bash
$ make docker-compose
```
3. After ensuring your data has been successfully migrated, you may delete your old data directory (typically stored at `~/.awx/pgdocker`).
### Migrating data manually
1. With Local Docker still running, perform a pg_dumpall:
> Note: If Local Docker is no longer running
`docker-compose -f ~/.awx/awxcompose/docker-compose.yml up postgres`
```bash
$ docker-compose -f ~/.awx/awxcompose/docker-compose.yml exec postgres pg_dumpall -U awx > awx_dump.sql
```
2. Remove all local docker containers (specifically awx_postgres)
```bash
$ docker -f rm awx_postgres
```
3. Template the new docker-compose.yml
```bash
$ ansible-playbook -i tools/ansible/inventory tools/ansible/sources.yml
```
4. Start a container with a volume (using the new tools/docker-compose/_sources/docker-compose.yml)
```bash
$ docker-compose -f ../docker-compose/_sources/docker-compose.yml up postgres
```
5. Restore to new `awx_postgres`
```bash
$ docker-compose -f ../docker-compose/_sources/docker-compose.yml exec -T postgres psql -U awx -d awx -p 5432 < awx_dump.sql
```
6. Run the docker-compose.yml to start the containers
```bash
$ docker-compose -f ../docker-compose/_sources/docker-compose.yml up task
```
7. Check to ensure your data migration was successful, then you can delete your the `awx_dump.sql` backup and your old data directory.

View File

@@ -0,0 +1,49 @@
# How to use the logstash container
#### Modify the docker-compose.yml
Uncomment the following lines in the `docker-compose.yml`
```
#- logstash
...
#logstash:
# build:
# context: ./docker-compose
# dockerfile: Dockerfile-logstash
```
POST the following content to `/api/v2/settings/logging/` (this uses
authentication set up inside of the logstash configuration file).
```
{
"LOG_AGGREGATOR_HOST": "http://logstash",
"LOG_AGGREGATOR_PORT": 8085,
"LOG_AGGREGATOR_TYPE": "logstash",
"LOG_AGGREGATOR_USERNAME": "awx_logger",
"LOG_AGGREGATOR_PASSWORD": "workflows",
"LOG_AGGREGATOR_LOGGERS": [
"awx",
"activity_stream",
"job_events",
"system_tracking"
],
"LOG_AGGREGATOR_INDIVIDUAL_FACTS": false,
"LOG_AGGREGATOR_TOWER_UUID": "991ac7e9-6d68-48c8-bbde-7ca1096653c6",
"LOG_AGGREGATOR_ENABLED": true
}
```
> Note: HTTP must be specified in the `LOG_AGGREGATOR_HOST` if you are using the docker development environment.
An example of how to view the most recent logs from the container:
```
docker exec -i -t $(docker ps -aqf "name=tools_logstash_1") tail -n 50 /logstash.log
```
#### How to add logstash plugins
Add any plugins you need in `tools/elastic/logstash/Dockerfile` before running the container.

View File

@@ -0,0 +1,19 @@
localhost ansible_connection=local ansible_python_interpreter="/usr/bin/env python3"
[all:vars]
# AWX-Managed Database Settings
# If left blank, these will be generated upon install.
# Values are written out to tools/docker-compose/_sources/secrets/
# pg_password=""
# broadcast_websocket_secret=""
# secret_key=""
# External Database Settings
# pg_host=""
# pg_password=""
# pg_username=""
# pg_hostname=""
# awx_image="ansible/awx"
# migrate_local_docker=false

View File

@@ -5,5 +5,5 @@ cd /awx_devel
make clean
make awx-link
cp awx/settings/local_settings.py.docker_compose awx/settings/local_settings.py
cp tools/docker-compose/ansible/roles/sources/files/local_settings.py awx/settings/local_settings.py
make "${1:-test}"

View File

@@ -18,9 +18,10 @@ services:
conjur:
image: cyberark/conjur
container_name: tools_conjur_1
command: server -p 8300
environment:
DATABASE_URL: postgres://postgres@postgres/postgres
DATABASE_URL: postgres://awx@postgres/postgres
CONJUR_DATA_KEY: 'dveUwOI/71x9BPJkIgvQRRBF3SdASc+HP4CUGL7TKvM='
depends_on:
- postgres

View File

@@ -10,7 +10,7 @@ services:
# Components of ELK stack for logging
elasticsearch:
build: elastic/elasticsearch/
build: ../../elastic/elasticsearch/
ports:
- "9200:9200"
- "9300:9300"
@@ -18,12 +18,11 @@ services:
ES_JAVA_OPTS: "-Xms1g -Xmx1g"
# networks: # add back in when a connection to tower_tools is possible
# - docker_elk
logstash:
build: elastic/logstash/
build: ../../elastic/logstash/
command: -f /etc/logstash/conf.d/
volumes:
- ./elastic/logstash/config:/etc/logstash/conf.d
- ../../elastic/logstash/config:/etc/logstash/conf.d
ports:
- "8085:8085"
links:
@@ -33,9 +32,9 @@ services:
depends_on:
- elasticsearch
kibana:
build: elastic/kibana/
build: ../../elastic/kibana/
volumes:
- ./elastic/kibana/config/:/opt/kibana/config/
- ../../elastic/kibana/config/:/opt/kibana/config/
ports:
- "5601:5601"
links:

View File

@@ -1 +0,0 @@
This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root.