External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-03 09:48:51 -03:30
Code Issues Packages Projects Releases Wiki Activity

Consolidate the Local Docker installer and the dev env

Browse Source 
- removes local_docker installer and points community users to our development environment (make docker-compose)
  - provides a migration path from Local Docker Compose installations --> the dev environment
  - the dev env can now be configured to use an external database
  - consolidated the Local Docker and dev env docker-compose.yml files into one template file, used by the dockerfile role
  - added a 'sources' role to template out config files
  - the postgres data dir is no longer a bind-mount, it is a docker volume
  - the redis socket is not longer a bind-mount, it is a docker volume
  - the local_settings.py.docker-compose file no longer needs to be copied over in the dev env
  - Create tmp rsyslog.conf in rsyslog volume to avoid cross-linking. Previously, the tmp code-generated rsyslog.conf was being written to /tmp (by default).  As a result, we were attempting to shutil.move() across volumes.
  - move k8s image build and push roles under tools/ansible
  - See tools/docker-compose/README.md for usage of these changes
This commit is contained in:
Christian M. Adams
2021-01-27 11:01:17 -05:00
parent 0f6d2c36a0
commit 9672e72834
52 changed files with 1325 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tools/docker-compose/ansible/migrate.yml Normal file
View File

@@ -0,0 +1,7 @@
---
- name: Migrate data for upgrades and from Local Docker installs
hosts: localhost
gather_facts: true
roles:
- {role: sources}
- {role: migrate}

5
tools/docker-compose/ansible/roles/migrate/defaults/main.yml Normal file
View File

@@ -0,0 +1,5 @@
---
postgres_data_dir: "~/.awx/pgdocker"
migrate_local_docker: false
old_docker_compose_dir: "~/.awx/awxcompose"
pg_volume_name: "postgres_data"

5
tools/docker-compose/ansible/roles/migrate/tasks/main.yml Normal file
View File

@@ -0,0 +1,5 @@
---
# Migrate data from a bind-mount to a volume
- import_tasks: migrate-from-local-docker.yml
when: migrate_local_docker

46
tools/docker-compose/ansible/roles/migrate/tasks/migrate-from-local-docker.yml Normal file
View File

@@ -0,0 +1,46 @@
---
# Migrate data from old Local Docker to a fresh development environment
- name: Remove awx_postgres to ensure consistent start state
shell: |
docker rm -f awx_postgres
- name: Start Local Docker database container
docker_compose:
project_src: "{{ old_docker_compose_dir }}"
services:
- postgres
state: present
recreate: always
- name: Database dump to local filesystem
shell: |
docker-compose -f {{ old_docker_compose_dir }}/docker-compose.yml exec -T postgres pg_dumpall -U {{ pg_username }} > awx_dump.sql
- name: Stop AWX containers so the old postgres container does not get used
docker_compose:
project_src: "{{ old_docker_compose_dir }}"
state: absent
ignore_errors: true
- name: Start dev env database container
docker_compose:
project_src: "{{ playbook_dir }}/../_sources"
files: "docker-compose.yml"
services:
- postgres
state: present
recreate: always
- name: Wait for postgres to initialize
wait_for:
timeout: 3
- name: Restore to new postgres container
shell: |
docker-compose -f {{ playbook_dir }}/../_sources/docker-compose.yml exec -T postgres psql -U {{ pg_username }} -d {{ pg_database }} -p {{ pg_port }} < awx_dump.sql
- name: Clean up temporary awx db dump
file:
path: awx_dump.sql
state: absent

7
tools/docker-compose/ansible/roles/sources/defaults/main.yml Normal file
View File

@@ -0,0 +1,7 @@
---
sources_dest: '../_sources'
compose_name: 'docker-compose.yml'
awx_image: 'quay.io/ansible/awx_devel'
pg_port: 5432
pg_username: 'awx'
pg_database: 'awx'

68
tools/docker-compose/ansible/roles/sources/files/local_settings.py Normal file
View File

@@ -0,0 +1,68 @@
# Copyright (c) 2015 Ansible, Inc. (formerly AnsibleWorks, Inc.)
# All Rights Reserved.
# Local Django settings for AWX project. Rename to "local_settings.py" and
# edit as needed for your development environment.
# All variables defined in awx/settings/development.py will already be loaded
# into the global namespace before this file is loaded, to allow for reading
# and updating the default settings as needed.
###############################################################################
# MISC PROJECT SETTINGS
###############################################################################
import os
import sys
# Enable the following lines and install the browser extension to use Django debug toolbar
# if your deployment method is not VMWare of Docker-for-Mac you may
# need a different IP address from request.META['REMOTE_ADDR']
# INTERNAL_IPS = ('172.19.0.1', '172.18.0.1', '192.168.100.1')
# ALLOWED_HOSTS = ['*']
# Use SQLite for unit tests instead of PostgreSQL. If the lines below are
# commented out, Django will create the test_awx-dev database in PostgreSQL to
# run unit tests.
if "pytest" in sys.modules:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'awx.sqlite3'),
'TEST': {
# Test database cannot be :memory: for inventory tests.
'NAME': os.path.join(BASE_DIR, 'awx_test.sqlite3'),
},
}
}
# Location for cross-development of inventory plugins
AWX_ANSIBLE_COLLECTIONS_PATHS = '/var/lib/awx/vendor/awx_ansible_collections'
# The UUID of the system, for HA.
SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
# If set, use -vvv for project updates instead of -v for more output.
# PROJECT_UPDATE_VVV=True
###############################################################################
# LOGGING SETTINGS
###############################################################################
# Enable logging to syslog. Setting level to ERROR captures 500 errors,
# WARNING also logs 4xx responses.
# Enable the following lines to turn on lots of permissions-related logging.
#LOGGING['loggers']['awx.main.access']['level'] = 'DEBUG'
#LOGGING['loggers']['awx.main.signals']['level'] = 'DEBUG'
#LOGGING['loggers']['awx.main.permissions']['level'] = 'DEBUG'
# Enable the following line to turn on database settings logging.
#LOGGING['loggers']['awx.conf']['level'] = 'DEBUG'
# Enable the following lines to turn on LDAP auth logging.
#LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']
#LOGGING['loggers']['django_auth_ldap']['level'] = 'DEBUG'
BROADCAST_WEBSOCKET_PORT = 8013
BROADCAST_WEBSOCKET_VERIFY_CERT = False
BROADCAST_WEBSOCKET_PROTOCOL = 'http'

76
tools/docker-compose/ansible/roles/sources/tasks/main.yml Normal file
View File

@@ -0,0 +1,76 @@
---
- name: Create _sources directory
file:
path: "{{ sources_dest }}/secrets"
state: 'directory'
mode: '0700'
- name: Detect secrets
stat:
path: "{{ sources_dest }}/secrets/{{ item }}.yml"
register: secrets
when: not lookup('vars', item, default='')
loop:
- pg_password
- secret_key
- broadcast_websocket_secret
- name: Generate secrets if needed
template:
src: 'secrets.yml.j2'
dest: '{{ sources_dest }}/secrets/{{ item.item }}.yml'
mode: '0600'
when: not lookup('vars', item.item, default='') and not item.stat.exists
loop: "{{ secrets.results }}"
- name: Include generated secrets unless they are explicitly passed in
include_vars: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
no_log: true
when: not lookup('vars', item.item, default='')
loop: "{{ secrets.results }}"
- name: Render configuration templates
template:
src: "{{ item }}.j2"
dest: "{{ sources_dest }}/{{ item }}"
mode: '0600'
with_items:
- "database.py"
- "websocket_secret.py"
- name: Delete old local_settings.py
file:
path: "{{ playbook_dir }}/../../../awx/settings/local_settings.py"
state: absent
- name: Copy local_settings.py
copy:
src: "local_settings.py"
dest: "{{ sources_dest }}/local_settings.py"
- name: Get OS info for sdb
shell: |
docker info | grep 'Operating System'
register: os_info
changed_when: false
- name: Get user UID
shell: id -u
register: current_user
changed_when: false
- name: Set fact with user UID
set_fact:
user_id: "'{{ current_user.stdout }}'"
- name: Set global version if not provided
set_fact:
awx_image_tag: "{{ lookup('file', playbook_dir + '/../../../VERSION') }}"
when: awx_image_tag is not defined
- name: Render Docker-Compose
template:
src: docker-compose.yml.j2
dest: "{{ sources_dest }}/{{ compose_name }}"
mode: '0600'

11
tools/docker-compose/ansible/roles/sources/templates/database.py.j2 Normal file
View File

@@ -0,0 +1,11 @@
DATABASES = {
'default': {
'ATOMIC_REQUESTS': True,
'ENGINE': 'awx.main.db.profiled_pg',
'NAME': "{{ pg_database }}",
'USER': "{{ pg_username }}",
'PASSWORD': "{{ pg_password }}",
'HOST': "{{ pg_hostname | default('postgres') }}",
'PORT': "{{ pg_port }}",
}
}

65
tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 Normal file
View File

@@ -0,0 +1,65 @@
---
version: '2'
services:
# Primary AWX Development Container
awx:
user: "{{ ansible_user_uid }}"
image: "{{ awx_image }}:{{ awx_image_tag }}"
container_name: tools_awx_1
hostname: awx
command: launch_awx.sh
environment:
OS: "{{ os_info.stdout }}"
SDB_HOST: 0.0.0.0
SDB_PORT: 7899
AWX_GROUP_QUEUES: tower
ports:
- "8888:8888"
- "8080:8080"
- "8013:8013"
- "8043:8043"
- "6899:6899" # default port range for sdb-listen
- "7899-7999:7899-7999" # default port range for sdb-listen
links:
- postgres
- redis
working_dir: "/awx_devel"
volumes:
- "../../../:/awx_devel"
- "../../docker-compose/supervisor.conf:/etc/supervisord.conf"
- "../../docker-compose/_sources/database.py:/etc/tower/conf.d/database.py"
- "../../docker-compose/_sources/websocket_secret.py:/etc/tower/conf.d/websocket_secret.py"
- "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py"
- "redis_socket:/var/run/redis/:rw"
privileged: true
tty: true
# A useful container that simply passes through log messages to the console
# helpful for testing awx/tower logging
# logstash:
# build:
# context: ./docker-compose
# dockerfile: Dockerfile-logstash
# Postgres Database Container
postgres:
image: postgres:12
container_name: tools_postgres_1
environment:
POSTGRES_HOST_AUTH_METHOD: trust
POSTGRES_USER: {{ pg_username }}
POSTGRES_DB: {{ pg_database }}
POSTGRES_PASSWORD: {{ pg_password }}
volumes:
- "awx_db:/var/lib/postgresql/data"
redis:
image: redis:latest
container_name: tools_redis_1
volumes:
- "../../redis/redis.conf:/usr/local/etc/redis/redis.conf"
- "redis_socket:/var/run/redis/:rw"
entrypoint: ["redis-server"]
command: ["/usr/local/etc/redis/redis.conf"]
volumes:
awx_db:
redis_socket:

31
tools/docker-compose/ansible/roles/sources/templates/docker-credential-plugins-override.yml Normal file
View File

@@ -0,0 +1,31 @@
---
version: '2'
services:
# Primary Tower Development Container link
awx:
links:
- hashivault
- conjur
hashivault:
image: vault
container_name: tools_hashivault_1
ports:
- '8200:8200'
cap_add:
- IPC_LOCK
environment:
VAULT_DEV_ROOT_TOKEN_ID: 'vaultdev'
conjur:
image: cyberark/conjur
container_name: tools_conjur_1
command: server -p 8300
environment:
DATABASE_URL: postgres://awx@postgres/postgres
CONJUR_DATA_KEY: 'dveUwOI/71x9BPJkIgvQRRBF3SdASc+HP4CUGL7TKvM='
depends_on:
- postgres
links:
- postgres
ports:
- "8300:8300"

10
tools/docker-compose/ansible/roles/sources/templates/environment.sh.j2 Normal file
View File

@@ -0,0 +1,10 @@
DATABASE_USER={{ pg_username|quote }}
DATABASE_NAME={{ pg_database|quote }}
DATABASE_HOST={{ pg_hostname|default('postgres')|quote }}
DATABASE_PORT={{ pg_port|default('5432')|quote }}
DATABASE_PASSWORD={{ pg_password|default('awxpass')|quote }}
{% if pg_admin_password is defined %}
DATABASE_ADMIN_PASSWORD={{ pg_admin_password|quote }}
{% endif %}
AWX_ADMIN_USER={{ admin_user|quote }}
AWX_ADMIN_PASSWORD={{ admin_password|quote }}

122
tools/docker-compose/ansible/roles/sources/templates/nginx.conf.j2 Normal file
View File

@@ -0,0 +1,122 @@
#user awx;
worker_processes 1;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_tokens off;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout main;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
sendfile on;
#tcp_nopush on;
#gzip on;
upstream uwsgi {
server 127.0.0.1:8050;
}
upstream daphne {
server 127.0.0.1:8051;
}
{% if ssl_certificate is defined %}
server {
listen 8052 default_server;
server_name _;
# Redirect all HTTP links to the matching HTTPS page
return 301 https://$host$request_uri;
}
{%endif %}
server {
{% if (ssl_certificate is defined) and (ssl_certificate_key is defined) %}
listen 8053 ssl;
ssl_certificate /etc/nginx/awxweb.pem;
ssl_certificate_key /etc/nginx/awxweb_key.pem;
{% elif (ssl_certificate is defined) and (ssl_certificate_key is not defined) %}
listen 8053 ssl;
ssl_certificate /etc/nginx/awxweb.pem;
ssl_certificate_key /etc/nginx/awxweb.pem;
{% else %}
listen 8052 default_server;
{% endif %}
# If you have a domain name, this is where to add it
server_name _;
keepalive_timeout 65;
# HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
add_header Strict-Transport-Security max-age=15768000;
# Protect against click-jacking https://www.owasp.org/index.php/Testing_for_Clickjacking_(OTG-CLIENT-009)
add_header X-Frame-Options "DENY";
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1;
deny all;
}
location /static/ {
alias /var/lib/awx/public/static/;
}
location /favicon.ico { alias /var/lib/awx/public/static/favicon.ico; }
location /websocket {
# Pass request to the upstream alias
proxy_pass http://daphne;
# Require http version 1.1 to allow for upgrade requests
proxy_http_version 1.1;
# We want proxy_buffering off for proxying to websockets.
proxy_buffering off;
# http://en.wikipedia.org/wiki/X-Forwarded-For
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# enable this if you use HTTPS:
proxy_set_header X-Forwarded-Proto https;
# pass the Host: header from the client for the sake of redirects
proxy_set_header Host $http_host;
# We've set the Host header, so we don't need Nginx to muddle
# about with redirects
proxy_redirect off;
# Depending on the request value, set the Upgrade and
# connection headers
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
}
location / {
# Add trailing / if missing
rewrite ^(.*)$http_host(.*[^/])$ $1$http_host$2/ permanent;
uwsgi_read_timeout 120s;
uwsgi_pass uwsgi;
include /etc/nginx/uwsgi_params;
{%- if extra_nginx_include is defined %}
include {{ extra_nginx_include }};
{%- endif %}
proxy_set_header X-Forwarded-Port 443;
uwsgi_param HTTP_X_FORWARDED_PORT 443;
}
}
}

1
tools/docker-compose/ansible/roles/sources/templates/secrets.yml.j2 Normal file
View File

@@ -0,0 +1 @@
{{ item.item }}: '{{ lookup('vars', item.item, default='') or lookup('password', '/dev/null chars=ascii_letters') }}'

1
tools/docker-compose/ansible/roles/sources/templates/websocket_secret.py.j2 Normal file
View File

@@ -0,0 +1 @@
BROADCAST_WEBSOCKET_SECRET = "{{ broadcast_websocket_secret | b64encode }}"

6
tools/docker-compose/ansible/sources.yml Normal file
View File

@@ -0,0 +1,6 @@
---
- name: Render AWX Dockerfile and sources
hosts: localhost
gather_facts: true
roles:
- {role: sources}