Bump VERSION to 9.1.0

Don't reload template panel on tab change

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.github/BOTMETA.yml

@@ -1,3 +1,4 @@
+---
 files:
     awx/ui/:
         labels: component:ui

.gitignore

@@ -133,4 +133,13 @@ awx/lib/site-packages
 venv/*
 use_dev_supervisor.txt
 
+
+# Ansible module tests
+/awx_collection_test_venv/
+/awx_collection/*.tar.gz
+/awx_collection/galaxy.yml
+/sanity/
+
 .idea/*
+*.unison.tmp
+*.#

.yamllint

@@ -0,0 +1,12 @@
+---
+ignore: |
+  .tox
+  awx/main/tests/data/inventory/plugins/**
+  # vault files
+  awx/main/tests/data/ansible_utils/playbooks/valid/vault.yml
+  awx/ui/test/e2e/tests/smoke-vars.yml
+
+extends: default
+
+rules:
+  line-length: disable

CONTRIBUTING.md

@@ -156,8 +156,8 @@ If you start a second terminal session, you can take a look at the running conta
 
 $ docker ps
 CONTAINER ID        IMAGE                                              COMMAND                  CREATED             STATUS              PORTS                                                                                                                                      NAMES
-aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:6899-6999->6899-6999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
-e4c0afeb548c        postgres:9.6                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
+aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:7899-7999->7899-7999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
+e4c0afeb548c        postgres:10                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
 0089699d5afd        tools_logstash                                     "/docker-entrypoin..."   26 seconds ago      Up 25 seconds                                                                                                                                                  tools_logstash_1
 4d4ff0ced266        memcached:alpine                                   "docker-entrypoint..."   26 seconds ago      Up 25 seconds       0.0.0.0:11211->11211/tcp                                                                                                                   tools_memcached_1
 92842acd64cd        rabbitmq:3-management                              "docker-entrypoint..."   26 seconds ago      Up 24 seconds       4369/tcp, 5671-5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp                                                                    tools_rabbitmq_1

INSTALL.md

@@ -4,41 +4,45 @@ This document provides a guide for installing AWX.
 
 ## Table of contents
 
-- [Getting started](#getting-started)
-  - [Clone the repo](#clone-the-repo)
-  - [AWX branding](#awx-branding)
-  - [Prerequisites](#prerequisites)
-  - [System Requirements](#system-requirements)
-  - [AWX Tunables](#awx-tunables)
-  - [Choose a deployment platform](#choose-a-deployment-platform)
-  - [Official vs Building Images](#official-vs-building-images)
-- [OpenShift](#openshift)
-  - [Prerequisites](#prerequisites-1)
-    - [Deploying to Minishift](#deploying-to-minishift)
-  - [Pre-build steps](#pre-build-steps)
-  - [PostgreSQL](#postgresql)
-  - [Start the build](#start-the-build)
-  - [Post build](#post-build)
-  - [Accessing AWX](#accessing-awx)
-- [Kubernetes](#kubernetes)
-  - [Prerequisites](#prerequisites-2)
-  - [Pre-build steps](#pre-build-steps-1)
-  - [Configuring Helm](#configuring-helm)
-  - [Start the build](#start-the-build-1)
-  - [Accessing AWX](#accessing-awx-1)
-  - [SSL Termination](#ssl-termination)
-- [Docker Compose](#docker-compose)
-  - [Prerequisites](#prerequisites-3)
-  - [Pre-build steps](#pre-build-steps-2)
-    - [Deploying to a remote host](#deploying-to-a-remote-host)
-    - [Inventory variables](#inventory-variables)
+- [Installing AWX](#installing-awx)
+  * [Getting started](#getting-started)
+    + [Clone the repo](#clone-the-repo)
+    + [AWX branding](#awx-branding)
+    + [Prerequisites](#prerequisites)
+    + [System Requirements](#system-requirements)
+    + [AWX Tunables](#awx-tunables)
+    + [Choose a deployment platform](#choose-a-deployment-platform)
+    + [Official vs Building Images](#official-vs-building-images)
+  * [Upgrading from previous versions](#upgrading-from-previous-versions)
+  * [OpenShift](#openshift)
+    + [Prerequisites](#prerequisites-1)
+    + [Pre-install steps](#pre-install-steps)
+      - [Deploying to Minishift](#deploying-to-minishift)
+      - [PostgreSQL](#postgresql)
+    + [Run the installer](#run-the-installer)
+    + [Post-install](#post-install)
+    + [Accessing AWX](#accessing-awx)
+  * [Kubernetes](#kubernetes)
+    + [Prerequisites](#prerequisites-2)
+    + [Pre-install steps](#pre-install-steps-1)
+    + [Configuring Helm](#configuring-helm)
+    + [Run the installer](#run-the-installer-1)
+    + [Post-install](#post-install-1)
+    + [Accessing AWX](#accessing-awx-1)
+    + [SSL Termination](#ssl-termination)
+  * [Docker-Compose](#docker-compose)
+    + [Prerequisites](#prerequisites-3)
+    + [Pre-install steps](#pre-install-steps-2)
+      - [Deploying to a remote host](#deploying-to-a-remote-host)
+      - [Inventory variables](#inventory-variables)
       - [Docker registry](#docker-registry)
-      - [PostgreSQL](#postgresql-1)
       - [Proxy settings](#proxy-settings)
-  - [Start the build](#start-the-build-2)
-  - [Post build](#post-build-2)
-  - [Accessing AWX](#accessing-awx-2)
+      - [PostgreSQL](#postgresql-1)
+    + [Run the installer](#run-the-installer-2)
+    + [Post-install](#post-install-2)
+    + [Accessing AWX](#accessing-awx-2)
 
+    
 ## Getting started
 
 ### Clone the repo
@@ -57,7 +61,7 @@ To install the assets, clone the `awx-logos` repo so that it is next to your `aw
 
 Before you can run a deployment, you'll need the following installed in your local environment:
 
-- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) Requires Version 2.4+
+- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) Requires Version 2.8+
 - [Docker](https://docs.docker.com/engine/installation/)
     + A recent version
 - [docker](https://pypi.org/project/docker/) Python module
@@ -114,12 +118,34 @@ If these variables are present then all deployments will use these hosted images
 
 > Multiple versions are provided. `latest` always pulls the most recent. You may also select version numbers at different granularities: 1, 1.0, 1.0.1, 1.0.0.123
 
+
+## Upgrading from previous versions
+
+Upgrading AWX involves rerunning the install playbook. Download a newer release from [https://github.com/ansible/awx/releases](https://github.com/ansible/awx/releases) and re-populate the inventory file with your customized variables.
+
+For convenience, you can create a file called `vars.yml`:
+
+```
+admin_password: 'adminpass'
+pg_password: 'pgpass'
+rabbitmq_password: 'rabbitpass'
+secret_key: 'mysupersecret'
+```
+
+And pass it to the installer:
+
+```
+$ ansible-playbook -i inventory install.yml -e @vars.yml
+```
+
 ## OpenShift
 
 ### Prerequisites
 
 To complete a deployment to OpenShift, you will obviously need access to an OpenShift cluster. For demo and testing purposes, you can use [Minishift](https://github.com/minishift/minishift) to create a single node cluster running inside a virtual machine.
 
+When using OpenShift for deploying AWX make sure you have correct privileges to add the security context 'privileged', otherwise the installation will fail. The privileged context is needed because of the use of [the bubblewrap tool](https://github.com/containers/bubblewrap) to add an additional layer of security when using containers.
+
 You will also need to have the `oc` command in your PATH. The `install.yml` playbook will call out to `oc` when logging into, and creating objects on the cluster.
 
 The default resource requests per-deployment requires:
@@ -131,9 +157,9 @@ This can be tuned by overriding the variables found in [/installer/roles/kuberne
 
 For more detail on how resource requests are formed see: [https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources](https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources)
 
-### Pre-build steps
+### Pre-install steps
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
+Before starting the install, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
 
 *openshift_host*
 
@@ -193,22 +219,22 @@ $ eval $(minishift docker-env)
 
 By default, AWX will deploy a PostgreSQL pod inside of your cluster. You will need to create a [Persistent Volume Claim](https://docs.openshift.org/latest/dev_guide/persistent_volumes.html) which is named `postgresql` by default, and can be overridden by setting the `openshift_pg_pvc_name` variable. For testing and demo purposes, you may set `openshift_pg_emptydir=yes`.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
 
-### Start the build
+### Run the installer
 
-To start the build, you will pass two *extra* variables on the command line. The first is *openshift_password*, which is the password for the *openshift_user*, and the second is *docker_registry_password*, which is the password associated with *docker_registry_username*.
+To start the install, you will pass two *extra* variables on the command line. The first is *openshift_password*, which is the password for the *openshift_user*, and the second is *docker_registry_password*, which is the password associated with *docker_registry_username*.
 
 If you're using the OpenShift internal registry, then you'll pass an access token for the *docker_registry_password* value, rather than a password. The `oc whoami -t` command will generate the required token, as long as you're logged into the cluster via `oc cluster login`.
 
-To start the build and deployment, run the following (docker_registry_password is optional if using official images):
+Run the following command (docker_registry_password is optional if using official images):
 
 ```bash
-# Start the build and deployment
+# Start the install
 $ ansible-playbook -i inventory install.yml -e openshift_password=developer  -e docker_registry_password=$(oc whoami -t)
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, check the status of the deployment by running `oc get pods`:
 
@@ -325,9 +351,9 @@ This can be tuned by overriding the variables found in [/installer/roles/kuberne
 
 For more detail on how resource requests are formed see: [https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)
 
-### Pre-build steps
+### Pre-install steps
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section uncommenting when necessary. Make sure the openshift and standalone docker sections are commented out:
+Before starting the install process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section uncommenting when necessary. Make sure the openshift and standalone docker sections are commented out:
 
 *kubernetes_context*
 
@@ -347,7 +373,7 @@ If you want the AWX installer to manage creating the database pod (rather than i
 
 Newer Kubernetes clusters with RBAC enabled will need to make sure a service account is created, make sure to follow the instructions here [https://docs.helm.sh/using_helm/#role-based-access-control](https://docs.helm.sh/using_helm/#role-based-access-control)
 
-### Start the build
+### Run the installer
 
 After making changes to the `inventory` file use `ansible-playbook` to begin the install
 
@@ -355,7 +381,7 @@ After making changes to the `inventory` file use `ansible-playbook` to begin the
 $ ansible-playbook -i inventory install.yml
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, check the status of the deployment by running `kubectl get pods --namespace awx` (replace awx with the namespace you used):
 
@@ -403,7 +429,7 @@ Unlike Openshift's `Route` the Kubernetes `Ingress` doesn't yet handle SSL termi
     + This also installs the `docker` Python module, which is incompatible with `docker-py`. If you have previously installed `docker-py`, please uninstall it.
 - [Docker Compose](https://docs.docker.com/compose/install/).
 
-### Pre-build steps
+### Pre-install steps
 
 #### Deploying to a remote host
 
@@ -434,7 +460,7 @@ If you choose to use the official images then the remote host will be the one to
 
 #### Inventory variables
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
+Before starting the install process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
 
 *postgres_data_dir*
 
@@ -456,6 +482,10 @@ Before starting the build process, review the [inventory](./installer/inventory)
 
 > When using docker-compose, the `docker-compose.yml` file will be created there (default `/tmp/awxcompose`).
 
+*custom_venv_dir*
+
+> Adds the custom venv environments from the local host to be passed into the containers at install.
+
 *ca_trust_dir*
 
 > If you're using a non trusted CA, provide a path where the untrusted Certs are stored on your Host.
@@ -503,11 +533,11 @@ If you wish to tag and push built images to a Docker registry, set the following
 
 AWX requires access to a PostgreSQL database, and by default, one will be created and deployed in a container, and data will be persisted to a host volume. In this scenario, you must set the value of `postgres_data_dir` to a path that can be mounted to the container. When the container is stopped, the database files will still exist in the specified path.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information.
 
-### Start the build
+### Run the installer
 
-If you are not pushing images to a Docker registry, start the build by running the following:
+If you are not pushing images to a Docker registry, start the install by running the following:
 
 ```bash
 # Set the working directory to installer
@@ -527,7 +557,7 @@ $ cd installer
 $ ansible-playbook -i inventory -e docker_registry_password=password install.yml
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, Docker will report up to 5 running containers. If you chose to use an existing PostgresSQL database, then it will report 4. You can view the running containers using the `docker ps` command, as follows:
 
@@ -604,14 +634,3 @@ Added instance awx to tower
 The AWX web server is accessible on the deployment host, using the *host_port* value set in the *inventory* file. The default URL is [http://localhost](http://localhost).
 
 You will prompted with a login dialog. The default administrator username is `admin`, and the password is `password`.
-
-### Maintenance using docker-compose
-
-After the installation, maintenance operations with docker-compose can be done by using the  `docker-compose.yml` file created at the location pointed by `docker_compose_dir`.
-
-Among the possible operations, you may:
-
-- Stop AWX : `docker-compose stop`
-- Upgrade AWX : `docker-compose pull && docker-compose up --force-recreate`
-
-See the [docker-compose documentation](https://docs.docker.com/compose/) for details.

Makefile

@@ -18,6 +18,7 @@ COMPOSE_TAG ?= $(GIT_BRANCH)
 COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
+COLLECTION_VENV ?= /awx_devel/awx_collection_test_venv
 SCL_PREFIX ?=
 CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
@@ -99,20 +100,26 @@ clean-languages:
 	find . -type f -regex ".*\.mo$$" -delete
 
 # Remove temporary build files, compiled Python files.
-clean: clean-ui clean-dist
+clean: clean-ui clean-api clean-awxkit clean-dist
 	rm -rf awx/public
 	rm -rf awx/lib/site-packages
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3*
-	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
 	mkdir tmp
+
+clean-api:
 	rm -rf build $(NAME)-$(VERSION) *.egg-info
 	find . -type f -regex ".*\.py[co]$$" -delete
 	find . -type d -name "__pycache__" -delete
+	rm -f awx/awx_test.sqlite3*
+	rm -rf requirements/vendor
+	rm -rf awx/projects
+
+clean-awxkit:
+	rm -rf awxkit/*.egg-info awxkit/.tox
 
 # convenience target to assert environment variables are defined
 guard-%:
@@ -186,14 +193,14 @@ requirements_awx: virtualenv_awx
 	    cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
 	fi
 	echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg
-	#$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
+	$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
 
 requirements_awx_dev:
 	$(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt
 
 requirements: requirements_ansible requirements_awx
 
-requirements_dev: requirements requirements_awx_dev requirements_ansible_dev
+requirements_dev: requirements_awx requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev
 
 requirements_test: requirements
 
@@ -361,7 +368,7 @@ check: flake8 pep8 # pyflakes pylint
 
 awx-link:
 	cp -R /tmp/awx.egg-info /awx_devel/ || true
-	sed -i "s/placeholder/$(shell git describe --long | sed 's/\./\\./g')/" /awx_devel/awx.egg-info/PKG-INFO
+	sed -i "s/placeholder/$(shell cat VERSION)/" /awx_devel/awx.egg-info/PKG-INFO
 	cp -f /tmp/awx.egg-link /venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.egg-link
 
 TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests
@@ -375,6 +382,37 @@ test:
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
 	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
+prepare_collection_venv:
+	rm -rf $(COLLECTION_VENV)
+	mkdir $(COLLECTION_VENV)
+	$(VENV_BASE)/awx/bin/pip install --target=$(COLLECTION_VENV) git+https://github.com/ansible/tower-cli.git
+
+COLLECTION_TEST_DIRS ?= awx_collection/test/awx
+COLLECTION_PACKAGE ?= awx
+COLLECTION_NAMESPACE ?= awx
+
+test_collection:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	PYTHONPATH=$(COLLECTION_VENV):/awx_devel/awx_collection:$PYTHONPATH py.test $(COLLECTION_TEST_DIRS)
+
+flake8_collection:
+	flake8 awx_collection/  # Different settings, in main exclude list
+
+test_collection_all: prepare_collection_venv test_collection flake8_collection
+
+test_collection_sanity:
+	rm -rf sanity
+	mkdir -p sanity/ansible_collections/awx
+	cp -Ra awx_collection sanity/ansible_collections/awx/awx  # symlinks do not work
+	cd sanity/ansible_collections/awx/awx && git init && git add . # requires both this file structure and a git repo, so there you go
+	cd sanity/ansible_collections/awx/awx && ansible-test sanity
+
+build_collection:
+	ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION)
+	ansible-galaxy collection build awx_collection --output-path=awx_collection
+
 test_unit:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
@@ -516,6 +554,12 @@ jshint: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) run --prefix awx/ui jshint
 	$(NPM_BIN) run --prefix awx/ui lint
 
+ui-zuul-lint-and-test: $(UI_DEPS_FLAG_FILE)
+	$(NPM_BIN) run --prefix awx/ui jshint
+	$(NPM_BIN) run --prefix awx/ui lint
+	$(NPM_BIN) --prefix awx/ui run test:ci
+	$(NPM_BIN) --prefix awx/ui run unit
+
 # END UI TASKS
 # --------------------------------------
 
@@ -531,6 +575,12 @@ ui-next-test:
 	$(NPM_BIN) --prefix awx/ui_next install
 	$(NPM_BIN) run --prefix awx/ui_next test
 
+ui-next-zuul-lint-and-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+	$(NPM_BIN) run --prefix awx/ui_next test
+
 # END UI NEXT TASKS
 # --------------------------------------
 
@@ -569,28 +619,34 @@ docker-auth:
 		echo "$(IMAGE_REPOSITORY_AUTH)" | docker login -u oauth2accesstoken --password-stdin $(IMAGE_REPOSITORY_BASE); \
 	fi;
 
+# This directory is bind-mounted inside of the development container and
+# needs to be pre-created for permissions to be set correctly. Otherwise,
+# Docker will create this directory as root.
+awx/projects:
+	@mkdir -p $@
+
 # Docker isolated rampart
-docker-compose-isolated:
+docker-compose-isolated: awx/projects
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml up
 
 # Docker Compose Development environment
-docker-compose: docker-auth
+docker-compose: docker-auth awx/projects
 	CURRENT_UID=$(shell id -u) OS="$(shell docker info | grep 'Operating System')" TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml up --no-recreate awx
 
-docker-compose-cluster: docker-auth
+docker-compose-cluster: docker-auth awx/projects
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml up
 
-docker-compose-credential-plugins: docker-auth
+docker-compose-credential-plugins: docker-auth awx/projects
 	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx
 
-docker-compose-test: docker-auth
+docker-compose-test: docker-auth awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) OS="$(shell docker info | grep 'Operating System')" TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /bin/bash
 
-docker-compose-runtest:
+docker-compose-runtest: awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh
 
-docker-compose-build-swagger:
+docker-compose-build-swagger: awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh swagger
 
 detect-schema-change: genschema
@@ -598,7 +654,7 @@ detect-schema-change: genschema
 	# Ignore differences in whitespace with -b
 	diff -u -b reference-schema.json schema.json
 
-docker-compose-clean:
+docker-compose-clean: awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm -w /awx_devel --service-ports awx make clean
 	cd tools && TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose rm -sf
 
@@ -607,7 +663,6 @@ docker-compose-build: awx-devel-build
 # Base development image build
 awx-devel-build:
 	docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile \
-		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:devel \
 		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 	docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
@@ -627,10 +682,10 @@ docker-clean:
 docker-refresh: docker-clean docker-compose
 
 # Docker Development Environment with Elastic Stack Connected
-docker-compose-elk: docker-auth
+docker-compose-elk: docker-auth awx/projects
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
-docker-compose-cluster-elk: docker-auth
+docker-compose-cluster-elk: docker-auth awx/projects
 	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
 prometheus:
@@ -648,7 +703,7 @@ clean-elk:
 	docker rm tools_kibana_1
 
 psql-container:
-	docker run -it --net tools_default --rm postgres:9.6 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
+	docker run -it --net tools_default --rm postgres:10 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
 
 VERSION:
 	@echo "awx: $(VERSION)"

VERSION

@@ -1 +1 @@
-7.0.0
+9.1.0

awx/__init__.py

@@ -82,6 +82,23 @@ def find_commands(management_dir):
     return commands
 
 
+def oauth2_getattribute(self, attr):
+    # Custom method to override
+    # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__
+    from django.conf import settings
+    val = None
+    if 'migrate' not in sys.argv:
+        # certain Django OAuth Toolkit migrations actually reference
+        # setting lookups for references to model classes (e.g.,
+        # oauth2_settings.REFRESH_TOKEN_MODEL)
+        # If we're doing an OAuth2 setting lookup *while running* a migration,
+        # don't do our usual "Configure Tower in Tower" database setting lookup
+        val = settings.OAUTH2_PROVIDER.get(attr)
+    if val is None:
+        val = object.__getattribute__(self, attr)
+    return val
+
+
 def prepare_env():
     # Update the default settings environment variable based on current mode.
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
@@ -93,6 +110,12 @@ def prepare_env():
     # Monkeypatch Django find_commands to also work with .pyc files.
     import django.core.management
     django.core.management.find_commands = find_commands
+
+    # Monkeypatch Oauth2 toolkit settings class to check for settings
+    # in django.conf settings each time, not just once during import
+    import oauth2_provider.settings
+    oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ = oauth2_getattribute
+
     # Use the AWX_TEST_DATABASE_* environment variables to specify the test
     # database settings to use when management command is run as an external
     # program via unit tests.

awx/api/conf.py

@@ -38,12 +38,15 @@ register(
     'OAUTH2_PROVIDER',
     field_class=OAuth2ProviderField,
     default={'ACCESS_TOKEN_EXPIRE_SECONDS': oauth2_settings.ACCESS_TOKEN_EXPIRE_SECONDS,
-             'AUTHORIZATION_CODE_EXPIRE_SECONDS': 600},
+             'AUTHORIZATION_CODE_EXPIRE_SECONDS': oauth2_settings.AUTHORIZATION_CODE_EXPIRE_SECONDS,
+             'REFRESH_TOKEN_EXPIRE_SECONDS': oauth2_settings.REFRESH_TOKEN_EXPIRE_SECONDS},
     label=_('OAuth 2 Timeout Settings'),
     help_text=_('Dictionary for customizing OAuth 2 timeouts, available items are '
                 '`ACCESS_TOKEN_EXPIRE_SECONDS`, the duration of access tokens in the number '
-                'of seconds, and `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
-                'authorization codes in the number of seconds.'),
+                'of seconds, `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
+                'authorization codes in the number of seconds, and `REFRESH_TOKEN_EXPIRE_SECONDS`, '
+                'the duration of refresh tokens, after expired access tokens, '
+                'in the number of seconds.'),
     category=_('Authentication'),
     category_slug='authentication',
 )
@@ -59,3 +62,14 @@ register(
     category=_('Authentication'),
     category_slug='authentication',
 )
+register(
+    'LOGIN_REDIRECT_OVERRIDE',
+    field_class=fields.CharField,
+    allow_blank=True,
+    required=False,
+    label=_('Login redirect override URL'),
+    help_text=_('URL to which unauthorized users will be redirected to log in. '
+                'If blank, users will be sent to the Tower login page.'),
+    category=_('Authentication'),
+    category_slug='authentication',
+)

awx/api/fields.py

@@ -80,7 +80,7 @@ class OAuth2ProviderField(fields.DictField):
     default_error_messages = {
         'invalid_key_names': _('Invalid key names: {invalid_key_names}'),
     }
-    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS'}
+    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS', 'REFRESH_TOKEN_EXPIRE_SECONDS'}
     child = fields.IntegerField(min_value=1)
 
     def to_internal_value(self, data):

awx/api/filters.py

@@ -126,7 +126,7 @@ class FieldLookupBackend(BaseFilterBackend):
     '''
 
     RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
-                      'search', 'type', 'host_filter')
+                      'search', 'type', 'host_filter', 'count_disabled', 'no_truncate')
 
     SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                          'startswith', 'istartswith', 'endswith', 'iendswith',

awx/api/generics.py

@@ -92,7 +92,7 @@ class LoggedLoginView(auth_views.LoginView):
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
         if request.user.is_authenticated:
-            logger.info(smart_text(u"User {} logged in.".format(self.request.user.username)))
+            logger.info(smart_text(u"User {} logged in from {}".format(self.request.user.username,request.META.get('REMOTE_ADDR', None))))
             ret.set_cookie('userLoggedIn', 'true')
             current_user = UserSerializer(self.request.user)
             current_user = smart_text(JSONRenderer().render(current_user.data))
@@ -205,6 +205,9 @@ class APIView(views.APIView):
             response['X-API-Query-Count'] = len(q_times)
             response['X-API-Query-Time'] = '%0.3fs' % sum(q_times)
 
+        if getattr(self, 'deprecated', False):
+            response['Warning'] = '299 awx "This resource has been deprecated and will be removed in a future release."'  # noqa
+
         return response
 
     def get_authenticate_header(self, request):
@@ -489,9 +492,12 @@ class SubListAPIView(ParentMixin, ListAPIView):
         parent = self.get_parent_object()
         self.check_parent_access(parent)
         qs = self.request.user.get_queryset(self.model).distinct()
-        sublist_qs = getattrd(parent, self.relationship).distinct()
+        sublist_qs = self.get_sublist_queryset(parent)
         return qs & sublist_qs
 
+    def get_sublist_queryset(self, parent):
+        return getattrd(parent, self.relationship).distinct()
+
 
 class DestroyAPIView(generics.DestroyAPIView):
 
@@ -568,7 +574,7 @@ class SubListCreateAPIView(SubListAPIView, ListCreateAPIView):
                             status=status.HTTP_400_BAD_REQUEST)
 
         # Verify we have permission to add the object as given.
-        if not request.user.can_access(self.model, 'add', serializer.initial_data):
+        if not request.user.can_access(self.model, 'add', serializer.validated_data):
             raise PermissionDenied()
 
         # save the object through the serializer, reload and returned the saved

awx/api/metadata.py

@@ -20,8 +20,9 @@ from rest_framework.fields import JSONField as DRFJSONField
 from rest_framework.request import clone_request
 
 # AWX
-from awx.main.fields import JSONField
+from awx.main.fields import JSONField, ImplicitRoleField
 from awx.main.models import InventorySource, NotificationTemplate
+from awx.main.scheduler.kubernetes import PodManager
 
 
 class Metadata(metadata.SimpleMetadata):
@@ -157,9 +158,16 @@ class Metadata(metadata.SimpleMetadata):
             isinstance(field, JSONField) or
             isinstance(model_field, JSONField) or
             isinstance(field, DRFJSONField) or
-            isinstance(getattr(field, 'model_field', None), JSONField)
+            isinstance(getattr(field, 'model_field', None), JSONField) or
+            field.field_name == 'credential_passwords'
         ):
             field_info['type'] = 'json'
+        elif (
+            isinstance(field, ManyRelatedField) and
+            field.field_name == 'credentials'
+            # launch-time credentials
+        ):
+            field_info['type'] = 'list_of_ids'
         elif isinstance(model_field, BooleanField):
             field_info['type'] = 'boolean'
 
@@ -200,6 +208,9 @@ class Metadata(metadata.SimpleMetadata):
                 if not isinstance(meta, dict):
                     continue
 
+                if field == "pod_spec_override":
+                    meta['default'] = PodManager().pod_definition
+
                 # Add type choices if available from the serializer.
                 if field == 'type' and hasattr(serializer, 'get_type_choices'):
                     meta['choices'] = serializer.get_type_choices()
@@ -252,6 +263,16 @@ class Metadata(metadata.SimpleMetadata):
         if getattr(view, 'related_search_fields', None):
             metadata['related_search_fields'] = view.related_search_fields
 
+        # include role names in metadata
+        roles = []
+        model = getattr(view, 'model', None)
+        if model:
+            for field in model._meta.get_fields():
+                if type(field) is ImplicitRoleField:
+                    roles.append(field.name)
+        if len(roles) > 0:
+            metadata['object_roles'] = roles
+
         from rest_framework import generics
         if isinstance(view, generics.ListAPIView) and hasattr(view, 'paginator'):
             metadata['max_page_size'] = view.paginator.max_page_size

awx/api/pagination.py

@@ -3,14 +3,28 @@
 
 # Django REST Framework
 from django.conf import settings
+from django.core.paginator import Paginator as DjangoPaginator
 from rest_framework import pagination
+from rest_framework.response import Response
 from rest_framework.utils.urls import replace_query_param
 
 
+class DisabledPaginator(DjangoPaginator):
+
+    @property
+    def num_pages(self):
+        return 1
+
+    @property
+    def count(self):
+        return 200
+
+
 class Pagination(pagination.PageNumberPagination):
 
     page_size_query_param = 'page_size'
     max_page_size = settings.MAX_PAGE_SIZE
+    count_disabled = False
 
     def get_next_link(self):
         if not self.page.has_next():
@@ -39,3 +53,17 @@ class Pagination(pagination.PageNumberPagination):
                                  for pl in context['page_links']]
 
         return context
+
+    def paginate_queryset(self, queryset, request, **kwargs):
+        self.count_disabled = 'count_disabled' in request.query_params
+        try:
+            if self.count_disabled:
+                self.django_paginator_class = DisabledPaginator
+            return super(Pagination, self).paginate_queryset(queryset, request, **kwargs)
+        finally:
+            self.django_paginator_class = DjangoPaginator
+
+    def get_paginated_response(self, data):
+        if self.count_disabled:
+            return Response({'results': data})
+        return super(Pagination, self).get_paginated_response(data)

awx/api/permissions.py

@@ -249,3 +249,8 @@ class InstanceGroupTowerPermission(ModelAccessPermission):
         if request.method == 'DELETE' and obj.name == "tower":
             return False
         return super(InstanceGroupTowerPermission, self).has_object_permission(request, view, obj)
+
+
+class WebhookKeyPermission(permissions.BasePermission):
+    def has_object_permission(self, request, view, obj):
+        return request.user.can_access(view.model, 'admin', obj, request.data)

awx/api/serializers.py

@@ -45,7 +45,6 @@ from polymorphic.models import PolymorphicModel
 from awx.main.access import get_user_capabilities
 from awx.main.constants import (
     SCHEDULEABLE_PROVIDERS,
-    ANSI_SGR_PATTERN,
     ACTIVE_STATES,
     CENSOR_VALUE,
 )
@@ -70,7 +69,8 @@ from awx.main.utils import (
     get_type_for_model, get_model_for_type,
     camelcase_to_underscore, getattrd, parse_yaml_or_json,
     has_model_field_prefetched, extract_ansible_vars, encrypt_dict,
-    prefetch_page_capabilities, get_external_account)
+    prefetch_page_capabilities, get_external_account, truncate_stdout,
+)
 from awx.main.utils.filters import SmartFilter
 from awx.main.redact import UriCleaner, REPLACE_STR
 
@@ -116,7 +116,7 @@ SUMMARIZABLE_FK_FIELDS = {
     'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
     'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
     'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',),
-    'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
+    'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'kubernetes', 'credential_type_id'),
     'job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'elapsed', 'type'),
     'job_template': DEFAULT_SUMMARY_FIELDS,
     'workflow_job_template': DEFAULT_SUMMARY_FIELDS,
@@ -135,10 +135,12 @@ SUMMARIZABLE_FK_FIELDS = {
     'source_script': ('name', 'description'),
     'role': ('id', 'role_field'),
     'notification_template': DEFAULT_SUMMARY_FIELDS,
-    'instance_group': {'id', 'name', 'controller_id'},
+    'instance_group': ('id', 'name', 'controller_id', 'is_containerized'),
     'insights_credential': DEFAULT_SUMMARY_FIELDS,
     'source_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
     'target_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
+    'webhook_credential': DEFAULT_SUMMARY_FIELDS,
+    'approved_or_denied_by': ('id', 'username', 'first_name', 'last_name'),
 }
 
 
@@ -1261,6 +1263,7 @@ class OrganizationSerializer(BaseSerializer):
             notification_templates_started = self.reverse('api:organization_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:organization_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:organization_notification_templates_error_list', kwargs={'pk': obj.pk}),
+            notification_templates_approvals = self.reverse('api:organization_notification_templates_approvals_list', kwargs={'pk': obj.pk}),
             object_roles = self.reverse('api:organization_object_roles_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:organization_access_list', kwargs={'pk': obj.pk}),
             instance_groups = self.reverse('api:organization_instance_groups_list', kwargs={'pk': obj.pk}),
@@ -1469,7 +1472,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer):
 
     class Meta:
         model = ProjectUpdate
-        fields = ('*', 'project', 'job_type', '-controller_node')
+        fields = ('*', 'project', 'job_type', 'job_tags', '-controller_node')
 
     def get_related(self, obj):
         res = super(ProjectUpdateSerializer, self).get_related(obj)
@@ -2453,12 +2456,18 @@ class CredentialTypeSerializer(BaseSerializer):
             raise PermissionDenied(
                 detail=_("Modifications not allowed for managed credential types")
             )
+
+        old_inputs = {}
+        if self.instance:
+            old_inputs = copy.deepcopy(self.instance.inputs)
+
+        ret = super(CredentialTypeSerializer, self).validate(attrs)
+
         if self.instance and self.instance.credentials.exists():
-            if 'inputs' in attrs and attrs['inputs'] != self.instance.inputs:
+            if 'inputs' in attrs and old_inputs != self.instance.inputs:
                 raise PermissionDenied(
                     detail= _("Modifications to inputs are not allowed for credential types that are in use")
                 )
-        ret = super(CredentialTypeSerializer, self).validate(attrs)
 
         if 'kind' in attrs and attrs['kind'] not in ('cloud', 'net'):
             raise serializers.ValidationError({
@@ -2513,7 +2522,7 @@ class CredentialSerializer(BaseSerializer):
 
     class Meta:
         model = Credential
-        fields = ('*', 'organization', 'credential_type', 'inputs', 'kind', 'cloud')
+        fields = ('*', 'organization', 'credential_type', 'inputs', 'kind', 'cloud', 'kubernetes')
         extra_kwargs = {
             'credential_type': {
                 'label': _('Credential Type'),
@@ -2825,6 +2834,25 @@ class JobTemplateMixin(object):
         d['recent_jobs'] = self._recent_jobs(obj)
         return d
 
+    def validate(self, attrs):
+        webhook_service = attrs.get('webhook_service', getattr(self.instance, 'webhook_service', None))
+        webhook_credential = attrs.get('webhook_credential', getattr(self.instance, 'webhook_credential', None))
+
+        if webhook_credential:
+            if webhook_credential.credential_type.kind != 'token':
+                raise serializers.ValidationError({
+                    'webhook_credential': _("Must be a Personal Access Token."),
+                })
+
+            msg = {'webhook_credential': _("Must match the selected webhook service.")}
+            if webhook_service:
+                if webhook_credential.credential_type.namespace != '{}_token'.format(webhook_service):
+                    raise serializers.ValidationError(msg)
+            else:
+                raise serializers.ValidationError(msg)
+
+        return super().validate(attrs)
+
 
 class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobOptionsSerializer):
     show_capabilities = ['start', 'schedule', 'copy', 'edit', 'delete']
@@ -2837,30 +2865,39 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
 
     class Meta:
         model = JobTemplate
-        fields = ('*', 'host_config_key', 'ask_scm_branch_on_launch', 'ask_diff_mode_on_launch', 'ask_variables_on_launch',
-                  'ask_limit_on_launch', 'ask_tags_on_launch',
-                  'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch',
-                  'ask_credential_on_launch', 'survey_enabled', 'become_enabled', 'diff_mode',
-                  'allow_simultaneous', 'custom_virtualenv', 'job_slice_count')
+        fields = (
+            '*', 'host_config_key', 'ask_scm_branch_on_launch', 'ask_diff_mode_on_launch',
+            'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch',
+            'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch',
+            'ask_inventory_on_launch', 'ask_credential_on_launch', 'survey_enabled',
+            'become_enabled', 'diff_mode', 'allow_simultaneous', 'custom_virtualenv',
+            'job_slice_count', 'webhook_service', 'webhook_credential',
+        )
 
     def get_related(self, obj):
         res = super(JobTemplateSerializer, self).get_related(obj)
-        res.update(dict(
-            jobs = self.reverse('api:job_template_jobs_list', kwargs={'pk': obj.pk}),
-            schedules = self.reverse('api:job_template_schedules_list', kwargs={'pk': obj.pk}),
-            activity_stream = self.reverse('api:job_template_activity_stream_list', kwargs={'pk': obj.pk}),
-            launch = self.reverse('api:job_template_launch', kwargs={'pk': obj.pk}),
-            notification_templates_started = self.reverse('api:job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
-            notification_templates_success = self.reverse('api:job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
-            notification_templates_error = self.reverse('api:job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
-            access_list = self.reverse('api:job_template_access_list',      kwargs={'pk': obj.pk}),
-            survey_spec = self.reverse('api:job_template_survey_spec', kwargs={'pk': obj.pk}),
-            labels = self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}),
-            object_roles = self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}),
-            instance_groups = self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}),
-            slice_workflow_jobs = self.reverse('api:job_template_slice_workflow_jobs_list', kwargs={'pk': obj.pk}),
-            copy = self.reverse('api:job_template_copy', kwargs={'pk': obj.pk}),
-        ))
+        res.update(
+            jobs=self.reverse('api:job_template_jobs_list', kwargs={'pk': obj.pk}),
+            schedules=self.reverse('api:job_template_schedules_list', kwargs={'pk': obj.pk}),
+            activity_stream=self.reverse('api:job_template_activity_stream_list', kwargs={'pk': obj.pk}),
+            launch=self.reverse('api:job_template_launch', kwargs={'pk': obj.pk}),
+            webhook_key=self.reverse('api:webhook_key', kwargs={'model_kwarg': 'job_templates', 'pk': obj.pk}),
+            webhook_receiver=(
+                self.reverse('api:webhook_receiver_{}'.format(obj.webhook_service),
+                             kwargs={'model_kwarg': 'job_templates', 'pk': obj.pk})
+                if obj.webhook_service else ''
+            ),
+            notification_templates_started=self.reverse('api:job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
+            notification_templates_success=self.reverse('api:job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
+            notification_templates_error=self.reverse('api:job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
+            access_list=self.reverse('api:job_template_access_list',      kwargs={'pk': obj.pk}),
+            survey_spec=self.reverse('api:job_template_survey_spec', kwargs={'pk': obj.pk}),
+            labels=self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}),
+            object_roles=self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}),
+            instance_groups=self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}),
+            slice_workflow_jobs=self.reverse('api:job_template_slice_workflow_jobs_list', kwargs={'pk': obj.pk}),
+            copy=self.reverse('api:job_template_copy', kwargs={'pk': obj.pk}),
+        )
         if obj.host_config_key:
             res['callback'] = self.reverse('api:job_template_callback', kwargs={'pk': obj.pk})
         return res
@@ -2888,7 +2925,6 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
     def validate_extra_vars(self, value):
         return vars_validate_or_raise(value)
 
-
     def get_summary_fields(self, obj):
         summary_fields = super(JobTemplateSerializer, self).get_summary_fields(obj)
         all_creds = []
@@ -2929,9 +2965,11 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer):
 
     class Meta:
         model = Job
-        fields = ('*', 'job_template', 'passwords_needed_to_start',
-                  'allow_simultaneous', 'artifacts', 'scm_revision',
-                  'instance_group', 'diff_mode', 'job_slice_number', 'job_slice_count')
+        fields = (
+            '*', 'job_template', 'passwords_needed_to_start', 'allow_simultaneous',
+            'artifacts', 'scm_revision', 'instance_group', 'diff_mode', 'job_slice_number',
+            'job_slice_count', 'webhook_service', 'webhook_credential', 'webhook_guid',
+        )
 
     def get_related(self, obj):
         res = super(JobSerializer, self).get_related(obj)
@@ -3314,29 +3352,42 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
         'admin', 'execute',
         {'copy': 'organization.workflow_admin'}
     ]
+    limit = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+    scm_branch = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
 
     class Meta:
         model = WorkflowJobTemplate
-        fields = ('*', 'extra_vars', 'organization', 'survey_enabled', 'allow_simultaneous',
-                  'ask_variables_on_launch', 'inventory', 'ask_inventory_on_launch',)
+        fields = (
+            '*', 'extra_vars', 'organization', 'survey_enabled', 'allow_simultaneous',
+            'ask_variables_on_launch', 'inventory', 'limit', 'scm_branch',
+            'ask_inventory_on_launch', 'ask_scm_branch_on_launch', 'ask_limit_on_launch',
+            'webhook_service', 'webhook_credential',
+        )
 
     def get_related(self, obj):
         res = super(WorkflowJobTemplateSerializer, self).get_related(obj)
-        res.update(dict(
+        res.update(
             workflow_jobs = self.reverse('api:workflow_job_template_jobs_list', kwargs={'pk': obj.pk}),
             schedules = self.reverse('api:workflow_job_template_schedules_list', kwargs={'pk': obj.pk}),
             launch = self.reverse('api:workflow_job_template_launch', kwargs={'pk': obj.pk}),
+            webhook_key=self.reverse('api:webhook_key', kwargs={'model_kwarg': 'workflow_job_templates', 'pk': obj.pk}),
+            webhook_receiver=(
+                self.reverse('api:webhook_receiver_{}'.format(obj.webhook_service),
+                             kwargs={'model_kwarg': 'workflow_job_templates', 'pk': obj.pk})
+                if obj.webhook_service else ''
+            ),
             workflow_nodes = self.reverse('api:workflow_job_template_workflow_nodes_list', kwargs={'pk': obj.pk}),
             labels = self.reverse('api:workflow_job_template_label_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:workflow_job_template_activity_stream_list', kwargs={'pk': obj.pk}),
             notification_templates_started = self.reverse('api:workflow_job_template_notification_templates_started_list', kwargs={'pk': obj.pk}),
             notification_templates_success = self.reverse('api:workflow_job_template_notification_templates_success_list', kwargs={'pk': obj.pk}),
             notification_templates_error = self.reverse('api:workflow_job_template_notification_templates_error_list', kwargs={'pk': obj.pk}),
+            notification_templates_approvals = self.reverse('api:workflow_job_template_notification_templates_approvals_list', kwargs={'pk': obj.pk}),
             access_list = self.reverse('api:workflow_job_template_access_list', kwargs={'pk': obj.pk}),
             object_roles = self.reverse('api:workflow_job_template_object_roles_list', kwargs={'pk': obj.pk}),
             survey_spec = self.reverse('api:workflow_job_template_survey_spec', kwargs={'pk': obj.pk}),
             copy = self.reverse('api:workflow_job_template_copy', kwargs={'pk': obj.pk}),
-        ))
+        )
         if obj.organization:
             res['organization'] = self.reverse('api:organization_detail',   kwargs={'pk': obj.organization.pk})
         return res
@@ -3344,6 +3395,22 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
     def validate_extra_vars(self, value):
         return vars_validate_or_raise(value)
 
+    def validate(self, attrs):
+        attrs = super(WorkflowJobTemplateSerializer, self).validate(attrs)
+
+        # process char_prompts, these are not direct fields on the model
+        mock_obj = self.Meta.model()
+        for field_name in ('scm_branch', 'limit'):
+            if field_name in attrs:
+                setattr(mock_obj, field_name, attrs[field_name])
+                attrs.pop(field_name)
+
+        # Model `.save` needs the container dict, not the pseudo fields
+        if mock_obj.char_prompts:
+            attrs['char_prompts'] = mock_obj.char_prompts
+
+        return attrs
+
 
 class WorkflowJobTemplateWithSpecSerializer(WorkflowJobTemplateSerializer):
     '''
@@ -3356,13 +3423,16 @@ class WorkflowJobTemplateWithSpecSerializer(WorkflowJobTemplateSerializer):
 
 
 class WorkflowJobSerializer(LabelsListMixin, UnifiedJobSerializer):
+    limit = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+    scm_branch = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
 
     class Meta:
         model = WorkflowJob
-        fields = ('*', 'workflow_job_template', 'extra_vars', 'allow_simultaneous',
-                  'job_template', 'is_sliced_job',
-                  '-execution_node', '-event_processing_finished', '-controller_node',
-                  'inventory',)
+        fields = (
+            '*', 'workflow_job_template', 'extra_vars', 'allow_simultaneous', 'job_template',
+            'is_sliced_job', '-execution_node', '-event_processing_finished', '-controller_node',
+            'inventory', 'limit', 'scm_branch', 'webhook_service', 'webhook_credential', 'webhook_guid',
+        )
 
     def get_related(self, obj):
         res = super(WorkflowJobSerializer, self).get_related(obj)
@@ -3438,6 +3508,8 @@ class WorkflowApprovalSerializer(UnifiedJobSerializer):
                                                              kwargs={'pk': obj.workflow_approval_template.pk})
         res['approve'] = self.reverse('api:workflow_approval_approve', kwargs={'pk': obj.pk})
         res['deny'] = self.reverse('api:workflow_approval_deny', kwargs={'pk': obj.pk})
+        if obj.approved_or_denied_by:
+            res['approved_or_denied_by'] = self.reverse('api:user_detail', kwargs={'pk': obj.approved_or_denied_by.pk})
         return res
 
 
@@ -3469,7 +3541,7 @@ class WorkflowApprovalTemplateSerializer(UnifiedJobTemplateSerializer):
         if 'last_job' in res:
             del res['last_job']
 
-        res.update(dict(jobs = self.reverse('api:workflow_approval_template_jobs_list', kwargs={'pk': obj.pk}),))
+        res.update(jobs = self.reverse('api:workflow_approval_template_jobs_list', kwargs={'pk': obj.pk}))
         return res
 
 
@@ -3596,7 +3668,7 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
         if errors:
             raise serializers.ValidationError(errors)
 
-        # Model `.save` needs the container dict, not the psuedo fields
+        # Model `.save` needs the container dict, not the pseudo fields
         if mock_obj.char_prompts:
             attrs['char_prompts'] = mock_obj.char_prompts
 
@@ -3788,25 +3860,17 @@ class JobEventSerializer(BaseSerializer):
         return d
 
     def to_representation(self, obj):
-        ret = super(JobEventSerializer, self).to_representation(obj)
-        # Show full stdout for event detail view, truncate only for list view.
-        if hasattr(self.context.get('view', None), 'retrieve'):
-            return ret
+        data = super(JobEventSerializer, self).to_representation(obj)
         # Show full stdout for playbook_on_* events.
         if obj and obj.event.startswith('playbook_on'):
-            return ret
+            return data
+        # If the view logic says to not trunctate (request was to the detail view or a param was used)
+        if self.context.get('no_truncate', False):
+            return data
         max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY
-        if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes:
-            ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026'
-            set_count = 0
-            reset_count = 0
-            for m in ANSI_SGR_PATTERN.finditer(ret['stdout']):
-                if m.string[m.start():m.end()] == u'\u001b[0m':
-                    reset_count += 1
-                else:
-                    set_count += 1
-            ret['stdout'] += u'\u001b[0m' * (set_count - reset_count)
-        return ret
+        if 'stdout' in data:
+            data['stdout'] = truncate_stdout(data['stdout'], max_bytes)
+        return data
 
 
 class JobEventWebSocketSerializer(JobEventSerializer):
@@ -3901,22 +3965,14 @@ class AdHocCommandEventSerializer(BaseSerializer):
         return res
 
     def to_representation(self, obj):
-        ret = super(AdHocCommandEventSerializer, self).to_representation(obj)
-        # Show full stdout for event detail view, truncate only for list view.
-        if hasattr(self.context.get('view', None), 'retrieve'):
-            return ret
+        data = super(AdHocCommandEventSerializer, self).to_representation(obj)
+        # If the view logic says to not trunctate (request was to the detail view or a param was used)
+        if self.context.get('no_truncate', False):
+            return data
         max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY
-        if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes:
-            ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026'
-            set_count = 0
-            reset_count = 0
-            for m in ANSI_SGR_PATTERN.finditer(ret['stdout']):
-                if m.string[m.start():m.end()] == u'\u001b[0m':
-                    reset_count += 1
-                else:
-                    set_count += 1
-            ret['stdout'] += u'\u001b[0m' * (set_count - reset_count)
-        return ret
+        if 'stdout' in data:
+            data['stdout'] = truncate_stdout(data['stdout'], max_bytes)
+        return data
 
 
 class AdHocCommandEventWebSocketSerializer(AdHocCommandEventSerializer):
@@ -4180,12 +4236,16 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
         queryset=Inventory.objects.all(),
         required=False, write_only=True
     )
+    limit = serializers.CharField(required=False, write_only=True, allow_blank=True)
+    scm_branch = serializers.CharField(required=False, write_only=True, allow_blank=True)
     workflow_job_template_data = serializers.SerializerMethodField()
 
     class Meta:
         model = WorkflowJobTemplate
-        fields = ('ask_inventory_on_launch', 'can_start_without_user_input', 'defaults', 'extra_vars',
-                  'inventory', 'survey_enabled', 'variables_needed_to_start',
+        fields = ('ask_inventory_on_launch', 'ask_limit_on_launch', 'ask_scm_branch_on_launch',
+                  'can_start_without_user_input', 'defaults', 'extra_vars',
+                  'inventory', 'limit', 'scm_branch',
+                  'survey_enabled', 'variables_needed_to_start',
                   'node_templates_missing', 'node_prompts_rejected',
                   'workflow_job_template_data', 'survey_enabled', 'ask_variables_on_launch')
         read_only_fields = ('ask_inventory_on_launch', 'ask_variables_on_launch')
@@ -4225,9 +4285,14 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
 
         WFJT_extra_vars = template.extra_vars
         WFJT_inventory = template.inventory
+        WFJT_limit = template.limit
+        WFJT_scm_branch = template.scm_branch
         super(WorkflowJobLaunchSerializer, self).validate(attrs)
         template.extra_vars = WFJT_extra_vars
         template.inventory = WFJT_inventory
+        template.limit = WFJT_limit
+        template.scm_branch = WFJT_scm_branch
+
         return accepted
 
 
@@ -4279,13 +4344,30 @@ class NotificationTemplateSerializer(BaseSerializer):
         error_list = []
         collected_messages = []
 
+        def check_messages(messages):
+            for message_type in messages:
+                if message_type not in ('message', 'body'):
+                    error_list.append(_("Message type '{}' invalid, must be either 'message' or 'body'").format(message_type))
+                    continue
+                message = messages[message_type]
+                if message is None:
+                    continue
+                if not isinstance(message, str):
+                    error_list.append(_("Expected string for '{}', found {}, ").format(message_type, type(message)))
+                    continue
+                if message_type == 'message':
+                    if '\n' in message:
+                        error_list.append(_("Messages cannot contain newlines (found newline in {} event)".format(event)))
+                        continue
+                collected_messages.append(message)
+
         # Validate structure / content types
         if not isinstance(messages, dict):
             error_list.append(_("Expected dict for 'messages' field, found {}".format(type(messages))))
         else:
             for event in messages:
-                if event not in ['started', 'success', 'error']:
-                    error_list.append(_("Event '{}' invalid, must be one of 'started', 'success', or 'error'").format(event))
+                if event not in ('started', 'success', 'error', 'workflow_approval'):
+                    error_list.append(_("Event '{}' invalid, must be one of 'started', 'success', 'error', or 'workflow_approval'").format(event))
                     continue
                 event_messages = messages[event]
                 if event_messages is None:
@@ -4293,21 +4375,21 @@ class NotificationTemplateSerializer(BaseSerializer):
                 if not isinstance(event_messages, dict):
                     error_list.append(_("Expected dict for event '{}', found {}").format(event, type(event_messages)))
                     continue
-                for message_type in event_messages:
-                    if message_type not in ['message', 'body']:
-                        error_list.append(_("Message type '{}' invalid, must be either 'message' or 'body'").format(message_type))
-                        continue
-                    message = event_messages[message_type]
-                    if message is None:
-                        continue
-                    if not isinstance(message, str):
-                        error_list.append(_("Expected string for '{}', found {}, ").format(message_type, type(message)))
-                        continue
-                    if message_type == 'message':
-                        if '\n' in message:
-                            error_list.append(_("Messages cannot contain newlines (found newline in {} event)".format(event)))
+                if event == 'workflow_approval':
+                    for subevent in event_messages:
+                        if subevent not in ('running', 'approved', 'timed_out', 'denied'):
+                            error_list.append(_("Workflow Approval event '{}' invalid, must be one of "
+                                                "'running', 'approved', 'timed_out', or 'denied'").format(subevent))
                             continue
-                    collected_messages.append(message)
+                        subevent_messages = event_messages[subevent]
+                        if subevent_messages is None:
+                            continue
+                        if not isinstance(subevent_messages, dict):
+                            error_list.append(_("Expected dict for workflow approval event '{}', found {}").format(subevent, type(subevent_messages)))
+                            continue
+                        check_messages(subevent_messages)
+                else:
+                    check_messages(event_messages)
 
         # Subclass to return name of undefined field
         class DescriptiveUndefined(StrictUndefined):
@@ -4347,6 +4429,8 @@ class NotificationTemplateSerializer(BaseSerializer):
             for event in messages:
                 if not messages[event]:
                     continue
+                if not isinstance(messages[event], dict):
+                    continue
                 body = messages[event].get('body', {})
                 if body:
                     try:
@@ -4436,8 +4520,18 @@ class NotificationSerializer(BaseSerializer):
                   'notification_type', 'recipients', 'subject', 'body')
 
     def get_body(self, obj):
-        if obj.notification_type == 'webhook' and 'body' in obj.body:
-            return obj.body['body']
+        if obj.notification_type in ('webhook', 'pagerduty'):
+            if isinstance(obj.body, dict):
+                if 'body' in obj.body:
+                    return obj.body['body']
+            elif isinstance(obj.body, str):
+                # attempt to load json string
+                try:
+                    potential_body = json.loads(obj.body)
+                    if isinstance(potential_body, dict):
+                        return potential_body
+                except json.JSONDecodeError:
+                    pass
         return obj.body
 
     def get_related(self, obj):
@@ -4570,6 +4664,10 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
 
     def get_summary_fields(self, obj):
         summary_fields = super(ScheduleSerializer, self).get_summary_fields(obj)
+
+        if isinstance(obj.unified_job_template, SystemJobTemplate):
+            summary_fields['unified_job_template']['job_type'] = obj.unified_job_template.job_type
+
         if 'inventory' in summary_fields:
             return summary_fields
 
@@ -4658,6 +4756,11 @@ class InstanceGroupSerializer(BaseSerializer):
                     'Isolated groups have a designated controller group.'),
         read_only=True
     )
+    is_containerized = serializers.BooleanField(
+        help_text=_('Indicates whether instances in this group are containerized.'
+                    'Containerized groups have a designated Openshift or Kubernetes cluster.'),
+        read_only=True
+    )
     # NOTE: help_text is duplicated from field definitions, no obvious way of
     # both defining field details here and also getting the field's help_text
     policy_instance_percentage = serializers.IntegerField(
@@ -4683,8 +4786,9 @@ class InstanceGroupSerializer(BaseSerializer):
         fields = ("id", "type", "url", "related", "name", "created", "modified",
                   "capacity", "committed_capacity", "consumed_capacity",
                   "percent_capacity_remaining", "jobs_running", "jobs_total",
-                  "instances", "controller", "is_controller", "is_isolated",
-                  "policy_instance_percentage", "policy_instance_minimum", "policy_instance_list")
+                  "instances", "controller", "is_controller", "is_isolated", "is_containerized", "credential",
+                  "policy_instance_percentage", "policy_instance_minimum", "policy_instance_list",
+                  "pod_spec_override", "summary_fields")
 
     def get_related(self, obj):
         res = super(InstanceGroupSerializer, self).get_related(obj)
@@ -4692,6 +4796,9 @@ class InstanceGroupSerializer(BaseSerializer):
         res['instances'] = self.reverse('api:instance_group_instance_list', kwargs={'pk': obj.pk})
         if obj.controller_id:
             res['controller'] = self.reverse('api:instance_group_detail', kwargs={'pk': obj.controller_id})
+        if obj.credential:
+            res['credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.credential_id})
+
         return res
 
     def validate_policy_instance_list(self, value):
@@ -4704,6 +4811,18 @@ class InstanceGroupSerializer(BaseSerializer):
                 raise serializers.ValidationError(_('Isolated instances may not be added or removed from instances groups via the API.'))
             if self.instance and self.instance.controller_id is not None:
                 raise serializers.ValidationError(_('Isolated instance group membership may not be managed via the API.'))
+        if value and self.instance and self.instance.is_containerized:
+            raise serializers.ValidationError(_('Containerized instances may not be managed via the API'))
+        return value
+
+    def validate_policy_instance_percentage(self, value):
+        if value and self.instance and self.instance.is_containerized:
+            raise serializers.ValidationError(_('Containerized instances may not be managed via the API'))
+        return value
+
+    def validate_policy_instance_minimum(self, value):
+        if value and self.instance and self.instance.is_containerized:
+            raise serializers.ValidationError(_('Containerized instances may not be managed via the API'))
         return value
 
     def validate_name(self, value):
@@ -4711,6 +4830,11 @@ class InstanceGroupSerializer(BaseSerializer):
             raise serializers.ValidationError(_('tower instance group name may not be changed.'))
         return value
 
+    def validate_credential(self, value):
+        if value and not value.kubernetes:
+            raise serializers.ValidationError(_('Only Kubernetes credentials can be associated with an Instance Group'))
+        return value
+
     def get_capacity_dict(self):
         # Store capacity values (globally computed) in the context
         if 'capacity_map' not in self.context:

awx/api/templates/api/inventory_source_cancel.md

@@ -1,7 +1,7 @@
 # Cancel Inventory Update
 
 Make a GET request to this resource to determine if the inventory update can be
-cancelled.  The response will include the following field:
+canceled.  The response will include the following field:
 
 * `can_cancel`: Indicates whether this update can be canceled (boolean,
   read-only)

awx/api/templates/api/job_cancel.md

@@ -1,7 +1,7 @@
 {% ifmeth GET %}
-# Determine if a Job can be cancelled
+# Determine if a Job can be canceled
 
-Make a GET request to this resource to determine if the job can be cancelled.
+Make a GET request to this resource to determine if the job can be canceled.
 The response will include the following field:
 
 * `can_cancel`: Indicates whether this job can be canceled (boolean, read-only)

awx/api/templates/api/project_update_cancel.md

@@ -1,7 +1,7 @@
 # Cancel Project Update
 
 Make a GET request to this resource to determine if the project update can be
-cancelled.  The response will include the following field:
+canceled.  The response will include the following field:
 
 * `can_cancel`: Indicates whether this update can be canceled (boolean,
   read-only)

awx/api/templates/api/team_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a Team:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected team.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/user_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a User:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected user.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/webhook_key_view.md

@@ -0,0 +1,12 @@
+Webhook Secret Key:
+
+Make a GET request to this resource to obtain the secret key for a job
+template or workflow job template configured to be triggered by
+webhook events.  The response will include the following fields:
+
+* `webhook_key`: Secret key that needs to be copied and added to the
+  webhook configuration of the service this template will be receiving
+  webhook events from (string, read-only)
+
+Make an empty POST request to this resource to generate a new
+replacement `webhook_key`.

awx/api/urls/job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     JobTemplateList,
@@ -45,6 +45,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/organization.py

@@ -18,6 +18,7 @@ from awx.api.views import (
     OrganizationNotificationTemplatesErrorList,
     OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
+    OrganizationNotificationTemplatesApprovalList,
     OrganizationInstanceGroupsList,
     OrganizationObjectRolesList,
     OrganizationAccessList,
@@ -43,6 +44,8 @@ urls = [
         name='organization_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', OrganizationNotificationTemplatesSuccessList.as_view(),
         name='organization_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', OrganizationNotificationTemplatesApprovalList.as_view(),
+        name='organization_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/instance_groups/$', OrganizationInstanceGroupsList.as_view(), name='organization_instance_groups_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', OrganizationObjectRolesList.as_view(), name='organization_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', OrganizationAccessList.as_view(), name='organization_access_list'),

awx/api/urls/urls.py

@@ -14,6 +14,7 @@ from awx.api.views import (
     ApiV2RootView,
     ApiV2PingView,
     ApiV2ConfigView,
+    ApiV2SubscriptionView,
     AuthView,
     UserMeList,
     DashboardView,
@@ -94,6 +95,7 @@ v2_urls = [
     url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
     url(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'),
     url(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'),
+    url(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'),
     url(r'^auth/$', AuthView.as_view()),
     url(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     url(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),

awx/api/urls/webhooks.py

@@ -0,0 +1,14 @@
+from django.conf.urls import url
+
+from awx.api.views import (
+    WebhookKeyView,
+    GithubWebhookReceiver,
+    GitlabWebhookReceiver,
+)
+
+
+urlpatterns = [
+    url(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
+    url(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
+    url(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
+]

awx/api/urls/workflow_job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     WorkflowJobTemplateList,
@@ -16,6 +16,7 @@ from awx.api.views import (
     WorkflowJobTemplateNotificationTemplatesErrorList,
     WorkflowJobTemplateNotificationTemplatesStartedList,
     WorkflowJobTemplateNotificationTemplatesSuccessList,
+    WorkflowJobTemplateNotificationTemplatesApprovalList,
     WorkflowJobTemplateAccessList,
     WorkflowJobTemplateObjectRolesList,
     WorkflowJobTemplateLabelList,
@@ -38,9 +39,12 @@ urls = [
         name='workflow_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', WorkflowJobTemplateNotificationTemplatesSuccessList.as_view(),
         name='workflow_job_template_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', WorkflowJobTemplateNotificationTemplatesApprovalList.as_view(),
+        name='workflow_job_template_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', WorkflowJobTemplateAccessList.as_view(), name='workflow_job_template_access_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', WorkflowJobTemplateObjectRolesList.as_view(), name='workflow_job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobTemplateLabelList.as_view(), name='workflow_job_template_label_list'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'workflow_job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/views/__init__.py

@@ -72,12 +72,11 @@ from awx.api.generics import (
     SubListDestroyAPIView
 )
 from awx.api.versioning import reverse
-from awx.conf.license import get_license
 from awx.main import models
 from awx.main.utils import (
     camelcase_to_underscore,
     extract_ansible_vars,
-    get_awx_version,
+    get_awx_http_client_headers,
     get_object_or_400,
     getattrd,
     get_pk_from_dict,
@@ -102,7 +101,7 @@ from awx.main.scheduler.dag_workflow import WorkflowDAG
 from awx.api.views.mixin import (
     ControlledByScmMixin, InstanceGroupMembershipMixin,
     OrganizationCountsMixin, RelatedJobsPreventDeleteMixin,
-    UnifiedJobDeletionMixin,
+    UnifiedJobDeletionMixin, NoTruncateMixin,
 )
 from awx.api.views.organization import ( # noqa
     OrganizationList,
@@ -119,6 +118,7 @@ from awx.api.views.organization import ( # noqa
     OrganizationNotificationTemplatesErrorList,
     OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
+    OrganizationNotificationTemplatesApprovalList,
     OrganizationInstanceGroupsList,
     OrganizationAccessList,
     OrganizationObjectRolesList,
@@ -147,6 +147,12 @@ from awx.api.views.root import ( # noqa
     ApiV2RootView,
     ApiV2PingView,
     ApiV2ConfigView,
+    ApiV2SubscriptionView,
+)
+from awx.api.views.webhooks import ( # noqa
+    WebhookKeyView,
+    GithubWebhookReceiver,
+    GitlabWebhookReceiver,
 )
 
 
@@ -245,13 +251,6 @@ class DashboardView(APIView):
                                    'total': hg_projects.count(),
                                    'failed': hg_failed_projects.count()}
 
-        user_jobs = get_user_queryset(request.user, models.Job)
-        user_failed_jobs = user_jobs.filter(failed=True)
-        data['jobs'] = {'url': reverse('api:job_list', request=request),
-                        'failure_url': reverse('api:job_list', request=request) + "?failed=True",
-                        'total': user_jobs.count(),
-                        'failed': user_failed_jobs.count()}
-
         user_list = get_user_queryset(request.user, models.User)
         team_list = get_user_queryset(request.user, models.Team)
         credential_list = get_user_queryset(request.user, models.Credential)
@@ -383,6 +382,13 @@ class InstanceGroupDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAP
     serializer_class = serializers.InstanceGroupSerializer
     permission_classes = (InstanceGroupTowerPermission,)
 
+    def update_raw_data(self, data):
+        if self.get_object().is_containerized:
+            data.pop('policy_instance_percentage', None)
+            data.pop('policy_instance_minimum', None)
+            data.pop('policy_instance_list', None)
+        return super(InstanceGroupDetail, self).update_raw_data(data)
+
     def destroy(self, request, *args, **kwargs):
         instance = self.get_object()
         if instance.controller is not None:
@@ -568,6 +574,7 @@ class TeamUsersList(BaseUsersList):
     serializer_class = serializers.UserSerializer
     parent_model = models.Team
     relationship = 'member_role.members'
+    ordering = ('username',)
 
 
 class TeamRolesList(SubListAttachDetachAPIView):
@@ -904,6 +911,7 @@ class UserList(ListCreateAPIView):
     model = models.User
     serializer_class = serializers.UserSerializer
     permission_classes = (UserPermission,)
+    ordering = ('username',)
 
 
 class UserMeList(ListAPIView):
@@ -911,6 +919,7 @@ class UserMeList(ListAPIView):
     model = models.User
     serializer_class = serializers.UserSerializer
     name = _('Me')
+    ordering = ('username',)
 
     def get_queryset(self):
         return self.model.objects.filter(pk=self.request.user.pk)
@@ -1254,6 +1263,7 @@ class CredentialOwnerUsersList(SubListAPIView):
     serializer_class = serializers.UserSerializer
     parent_model = models.Credential
     relationship = 'admin_role.members'
+    ordering = ('username',)
 
 
 class CredentialOwnerTeamsList(SubListAPIView):
@@ -1375,6 +1385,7 @@ class CredentialExternalTest(SubDetailAPIView):
 
     model = models.Credential
     serializer_class = serializers.EmptySerializer
+    obj_permission_type = 'use'
 
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
@@ -1632,18 +1643,6 @@ class HostInsights(GenericAPIView):
 
         return session
 
-    def _get_headers(self):
-        license = get_license(show_key=False).get('license_type', 'UNLICENSED')
-        headers = {
-            'Content-Type': 'application/json',
-            'User-Agent': '{} {} ({})'.format(
-                'AWX' if license == 'open' else 'Red Hat Ansible Tower',
-                get_awx_version(),
-                license
-            )
-        }
-
-        return headers
 
     def _get_platform_info(self, host, session, headers):
         url = '{}/api/inventory/v1/hosts?insights_id={}'.format(
@@ -1710,7 +1709,7 @@ class HostInsights(GenericAPIView):
         username = cred.get_input('username', default='')
         password = cred.get_input('password', default='')
         session = self._get_session(username, password)
-        headers = self._get_headers()
+        headers = get_awx_http_client_headers()
 
         data = self._get_insights(host, session, headers)
         return Response(data, status=status.HTTP_200_OK)
@@ -2136,12 +2135,21 @@ class InventorySourceHostsList(HostRelatedSearchMixin, SubListDestroyAPIView):
     def perform_list_destroy(self, instance_list):
         inv_source = self.get_parent_object()
         with ignore_inventory_computed_fields():
-            # Activity stream doesn't record disassociation here anyway
-            # no signals-related reason to not bulk-delete
-            models.Host.groups.through.objects.filter(
-                host__inventory_sources=inv_source
-            ).delete()
-            r = super(InventorySourceHostsList, self).perform_list_destroy(instance_list)
+            if not settings.ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC:
+                from awx.main.signals import disable_activity_stream
+                with disable_activity_stream():
+                    # job host summary deletion necessary to avoid deadlock
+                    models.JobHostSummary.objects.filter(host__inventory_sources=inv_source).update(host=None)
+                    models.Host.objects.filter(inventory_sources=inv_source).delete()
+                    r = super(InventorySourceHostsList, self).perform_list_destroy([])
+            else:
+                # Advance delete of group-host memberships to prevent deadlock
+                # Activity stream doesn't record disassociation here anyway
+                # no signals-related reason to not bulk-delete
+                models.Host.groups.through.objects.filter(
+                    host__inventory_sources=inv_source
+                ).delete()
+                r = super(InventorySourceHostsList, self).perform_list_destroy(instance_list)
         update_inventory_computed_fields.delay(inv_source.inventory_id, True)
         return r
 
@@ -2157,11 +2165,18 @@ class InventorySourceGroupsList(SubListDestroyAPIView):
     def perform_list_destroy(self, instance_list):
         inv_source = self.get_parent_object()
         with ignore_inventory_computed_fields():
-            # Same arguments for bulk delete as with host list
-            models.Group.hosts.through.objects.filter(
-                group__inventory_sources=inv_source
-            ).delete()
-            r = super(InventorySourceGroupsList, self).perform_list_destroy(instance_list)
+            if not settings.ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC:
+                from awx.main.signals import disable_activity_stream
+                with disable_activity_stream():
+                    models.Group.objects.filter(inventory_sources=inv_source).delete()
+                    r = super(InventorySourceGroupsList, self).perform_list_destroy([])
+            else:
+                # Advance delete of group-host memberships to prevent deadlock
+                # Same arguments for bulk delete as with host list
+                models.Group.hosts.through.objects.filter(
+                    group__inventory_sources=inv_source
+                ).delete()
+                r = super(InventorySourceGroupsList, self).perform_list_destroy(instance_list)
         update_inventory_computed_fields.delay(inv_source.inventory_id, True)
         return r
 
@@ -2568,10 +2583,34 @@ class JobTemplateSurveySpec(GenericAPIView):
                         return Response(dict(error=_(
                             "The {min_or_max} limit in survey question {idx} expected to be integer."
                         ).format(min_or_max=key, **context)))
-            if qtype in ['multiplechoice', 'multiselect'] and 'choices' not in survey_item:
-                return Response(dict(error=_(
-                    "Survey question {idx} of type {survey_item[type]} must specify choices.".format(**context)
-                )))
+            # if it's a multiselect or multiple choice, it must have coices listed
+            # choices and defualts must come in as strings seperated by /n characters.
+            if qtype == 'multiselect' or qtype == 'multiplechoice':
+                if 'choices' in survey_item:
+                    if isinstance(survey_item['choices'], str):
+                        survey_item['choices'] = '\n'.join(choice for choice in survey_item['choices'].splitlines() if choice.strip() != '')
+                else:
+                    return Response(dict(error=_(
+                        "Survey question {idx} of type {survey_item[type]} must specify choices.".format(**context)
+                    )))
+                # If there is a default string split it out removing extra /n characters.
+                # Note: There can still be extra newline characters added in the API, these are sanitized out using .strip()
+                if 'default' in survey_item:
+                    if isinstance(survey_item['default'], str):
+                        survey_item['default'] = '\n'.join(choice for choice in survey_item['default'].splitlines() if choice.strip() != '')
+                        list_of_defaults = survey_item['default'].splitlines()
+                    else:
+                        list_of_defaults = survey_item['default']
+                    if qtype == 'multiplechoice':
+                        # Multiplechoice types should only have 1 default.
+                        if len(list_of_defaults) > 1:
+                            return Response(dict(error=_(
+                                "Multiple Choice (Single Select) can only have one default value.".format(**context)
+                            )))
+                    if any(item not in survey_item['choices'] for item in list_of_defaults):
+                        return Response(dict(error=_(
+                            "Default choice must be answered from the choices listed.".format(**context)
+                        )))
 
             # Process encryption substitution
             if ("default" in survey_item and isinstance(survey_item['default'], str) and
@@ -3117,6 +3156,17 @@ class WorkflowJobTemplateCopy(CopyAPIView):
             data.update(messages)
         return Response(data)
 
+    def _build_create_dict(self, obj):
+        """Special processing of fields managed by char_prompts
+        """
+        r = super(WorkflowJobTemplateCopy, self)._build_create_dict(obj)
+        field_names = set(f.name for f in obj._meta.get_fields())
+        for field_name, ask_field_name in obj.get_ask_mapping().items():
+            if field_name in r and field_name not in field_names:
+                r.setdefault('char_prompts', {})
+                r['char_prompts'][field_name] = r.pop(field_name)
+        return r
+
     @staticmethod
     def deep_copy_permission_check_func(user, new_objs):
         for obj in new_objs:
@@ -3145,7 +3195,6 @@ class WorkflowJobTemplateLabelList(JobTemplateLabelList):
 
 class WorkflowJobTemplateLaunch(RetrieveAPIView):
 
-
     model = models.WorkflowJobTemplate
     obj_permission_type = 'start'
     serializer_class = serializers.WorkflowJobLaunchSerializer
@@ -3162,10 +3211,15 @@ class WorkflowJobTemplateLaunch(RetrieveAPIView):
                 extra_vars.setdefault(v, u'')
             if extra_vars:
                 data['extra_vars'] = extra_vars
-            if obj.ask_inventory_on_launch:
-                data['inventory'] = obj.inventory_id
-            else:
-                data.pop('inventory', None)
+            modified_ask_mapping = models.WorkflowJobTemplate.get_ask_mapping()
+            modified_ask_mapping.pop('extra_vars')
+            for field_name, ask_field_name in obj.get_ask_mapping().items():
+                if not getattr(obj, ask_field_name):
+                    data.pop(field_name, None)
+                elif field_name == 'inventory':
+                    data[field_name] = getattrd(obj, "%s.%s" % (field_name, 'id'), None)
+                else:
+                    data[field_name] = getattr(obj, field_name)
         return data
 
     def post(self, request, *args, **kwargs):
@@ -3279,6 +3333,11 @@ class WorkflowJobTemplateNotificationTemplatesSuccessList(WorkflowJobTemplateNot
     relationship = 'notification_templates_success'
 
 
+class WorkflowJobTemplateNotificationTemplatesApprovalList(WorkflowJobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_approvals'
+
+
 class WorkflowJobTemplateAccessList(ResourceAccessList):
 
     model = models.User # needs to be User for AccessLists's
@@ -3364,6 +3423,11 @@ class WorkflowJobNotificationsList(SubListAPIView):
     relationship = 'notifications'
     search_fields = ('subject', 'notification_type', 'body',)
 
+    def get_sublist_queryset(self, parent):
+        return self.model.objects.filter(Q(unifiedjob_notifications=parent) |
+                                         Q(unifiedjob_notifications__unified_job_node__workflow_job=parent,
+                                         unifiedjob_notifications__workflowapproval__isnull=False)).distinct()
+
 
 class WorkflowJobActivityStreamList(SubListAPIView):
 
@@ -3713,7 +3777,7 @@ class JobHostSummaryDetail(RetrieveAPIView):
     serializer_class = serializers.JobHostSummarySerializer
 
 
-class JobEventList(ListAPIView):
+class JobEventList(NoTruncateMixin, ListAPIView):
 
     model = models.JobEvent
     serializer_class = serializers.JobEventSerializer
@@ -3725,8 +3789,13 @@ class JobEventDetail(RetrieveAPIView):
     model = models.JobEvent
     serializer_class = serializers.JobEventSerializer
 
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context.update(no_truncate=True)
+        return context
 
-class JobEventChildrenList(SubListAPIView):
+
+class JobEventChildrenList(NoTruncateMixin, SubListAPIView):
 
     model = models.JobEvent
     serializer_class = serializers.JobEventSerializer
@@ -3751,7 +3820,7 @@ class JobEventHostsList(HostRelatedSearchMixin, SubListAPIView):
     name = _('Job Event Hosts List')
 
 
-class BaseJobEventsList(SubListAPIView):
+class BaseJobEventsList(NoTruncateMixin, SubListAPIView):
 
     model = models.JobEvent
     serializer_class = serializers.JobEventSerializer
@@ -3947,7 +4016,7 @@ class AdHocCommandRelaunch(GenericAPIView):
             return Response(data, status=status.HTTP_201_CREATED, headers=headers)
 
 
-class AdHocCommandEventList(ListAPIView):
+class AdHocCommandEventList(NoTruncateMixin, ListAPIView):
 
     model = models.AdHocCommandEvent
     serializer_class = serializers.AdHocCommandEventSerializer
@@ -3959,8 +4028,13 @@ class AdHocCommandEventDetail(RetrieveAPIView):
     model = models.AdHocCommandEvent
     serializer_class = serializers.AdHocCommandEventSerializer
 
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context.update(no_truncate=True)
+        return context
 
-class BaseAdHocCommandEventsList(SubListAPIView):
+
+class BaseAdHocCommandEventsList(NoTruncateMixin, SubListAPIView):
 
     model = models.AdHocCommandEvent
     serializer_class = serializers.AdHocCommandEventSerializer
@@ -4226,8 +4300,15 @@ class NotificationTemplateTest(GenericAPIView):
 
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
-        notification = obj.generate_notification("Tower Notification Test {} {}".format(obj.id, settings.TOWER_URL_BASE),
-                                                 {"body": "Ansible Tower Test Notification {} {}".format(obj.id, settings.TOWER_URL_BASE)})
+        msg = "Tower Notification Test {} {}".format(obj.id, settings.TOWER_URL_BASE)
+        if obj.notification_type in ('email', 'pagerduty'):
+            body = "Ansible Tower Test Notification {} {}".format(obj.id, settings.TOWER_URL_BASE)
+        elif obj.notification_type == 'webhook':
+            body = '{{"body": "Ansible Tower Test Notification {} {}"}}'.format(obj.id, settings.TOWER_URL_BASE)
+        else:
+            body = {"body": "Ansible Tower Test Notification {} {}".format(obj.id, settings.TOWER_URL_BASE)}
+        notification = obj.generate_notification(msg, body)
+
         if not notification:
             return Response({}, status=status.HTTP_400_BAD_REQUEST)
         else:

awx/api/views/inventory.py

@@ -70,12 +70,16 @@ class InventoryUpdateEventsList(SubListAPIView):
 
 class InventoryScriptList(ListCreateAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     serializer_class = CustomInventoryScriptSerializer
 
 
 class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     serializer_class = CustomInventoryScriptSerializer
 
@@ -92,6 +96,8 @@ class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
 
 class InventoryScriptObjectRolesList(SubListAPIView):
 
+    deprecated = True
+
     model = Role
     serializer_class = RoleSerializer
     parent_model = CustomInventoryScript
@@ -105,6 +111,8 @@ class InventoryScriptObjectRolesList(SubListAPIView):
 
 class InventoryScriptCopy(CopyAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     copy_return_serializer_class = CustomInventoryScriptSerializer
 

awx/api/views/mixin.py

@@ -270,3 +270,11 @@ class ControlledByScmMixin(object):
         obj = super(ControlledByScmMixin, self).get_parent_object()
         self._reset_inv_src_rev(obj)
         return obj
+
+
+class NoTruncateMixin(object):
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.query_params.get('no_truncate'):
+            context.update(no_truncate=True)
+        return context

awx/api/views/organization.py

@@ -195,6 +195,11 @@ class OrganizationNotificationTemplatesSuccessList(OrganizationNotificationTempl
     relationship = 'notification_templates_success'
 
 
+class OrganizationNotificationTemplatesApprovalList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_approvals'
+
+
 class OrganizationInstanceGroupsList(SubListAttachDetachAPIView):
 
     model = InstanceGroup

awx/api/views/root.py

@@ -17,6 +17,8 @@ from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 from rest_framework import status
 
+import requests
+
 from awx.api.generics import APIView
 from awx.main.ha import is_ha_environment
 from awx.main.utils import (
@@ -58,6 +60,7 @@ class ApiRootView(APIView):
         data['oauth2'] = drf_reverse('api:oauth_authorization_root_view')
         data['custom_logo'] = settings.CUSTOM_LOGO
         data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
+        data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
         return Response(data)
 
 
@@ -169,6 +172,45 @@ class ApiV2PingView(APIView):
         return Response(response)
 
 
+class ApiV2SubscriptionView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    name = _('Configuration')
+    swagger_topic = 'System Configuration'
+
+    def check_permissions(self, request):
+        super(ApiV2SubscriptionView, self).check_permissions(request)
+        if not request.user.is_superuser and request.method.lower() not in {'options', 'head'}:
+            self.permission_denied(request)  # Raises PermissionDenied exception.
+
+    def post(self, request):
+        from awx.main.utils.common import get_licenser
+        data = request.data.copy()
+        if data.get('rh_password') == '$encrypted$':
+            data['rh_password'] = settings.REDHAT_PASSWORD
+        try:
+            user, pw = data.get('rh_username'), data.get('rh_password')
+            validated = get_licenser().validate_rh(user, pw)
+            if user:
+                settings.REDHAT_USERNAME = data['rh_username']
+            if pw:
+                settings.REDHAT_PASSWORD = data['rh_password']
+        except Exception as exc:
+            msg = _("Invalid License")
+            if (
+                isinstance(exc, requests.exceptions.HTTPError) and
+                getattr(getattr(exc, 'response', None), 'status_code', None) == 401
+            ):
+                msg = _("The provided credentials are invalid (HTTP 401).")
+            if isinstance(exc, (ValueError, OSError)) and exc.args:
+                msg = exc.args[0]
+            logger.exception(smart_text(u"Invalid license submitted."),
+                             extra=dict(actor=request.user.username))
+            return Response({"error": msg}, status=status.HTTP_400_BAD_REQUEST)
+
+        return Response(validated)
+
+
 class ApiV2ConfigView(APIView):
 
     permission_classes = (IsAuthenticated,)

awx/api/views/webhooks.py

@@ -0,0 +1,246 @@
+from hashlib import sha1
+import hmac
+import logging
+import urllib.parse
+
+from django.utils.encoding import force_bytes
+from django.utils.translation import ugettext_lazy as _
+from django.views.decorators.csrf import csrf_exempt
+
+from rest_framework import status
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+
+from awx.api import serializers
+from awx.api.generics import APIView, GenericAPIView
+from awx.api.permissions import WebhookKeyPermission
+from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
+
+
+logger = logging.getLogger('awx.api.views.webhooks')
+
+
+class WebhookKeyView(GenericAPIView):
+    serializer_class = serializers.EmptySerializer
+    permission_classes = (WebhookKeyPermission,)
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        self.model = qs_models.get(self.kwargs['model_kwarg'])
+
+        return super().get_queryset()
+
+    def get(self, request, *args, **kwargs):
+        obj = self.get_object()
+
+        return Response({'webhook_key': obj.webhook_key})
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        obj.rotate_webhook_key()
+        obj.save(update_fields=['webhook_key'])
+
+        return Response({'webhook_key': obj.webhook_key}, status=status.HTTP_201_CREATED)
+
+
+class WebhookReceiverBase(APIView):
+    lookup_url_kwarg = None
+    lookup_field = 'pk'
+
+    permission_classes = (AllowAny,)
+    authentication_classes = ()
+
+    ref_keys = {}
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        model = qs_models.get(self.kwargs['model_kwarg'])
+        if model is None:
+            raise PermissionDenied
+
+        return model.objects.filter(webhook_service=self.service).exclude(webhook_key='')
+
+    def get_object(self):
+        queryset = self.get_queryset()
+        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
+        filter_kwargs = {self.lookup_field: self.kwargs[lookup_url_kwarg]}
+
+        obj = queryset.filter(**filter_kwargs).first()
+        if obj is None:
+            raise PermissionDenied
+
+        return obj
+
+    def get_event_type(self):
+        raise NotImplementedError
+
+    def get_event_guid(self):
+        raise NotImplementedError
+
+    def get_event_status_api(self):
+        raise NotImplementedError
+
+    def get_event_ref(self):
+        key = self.ref_keys.get(self.get_event_type(), '')
+        value = self.request.data
+        for element in key.split('.'):
+            try:
+                if element.isdigit():
+                    value = value[int(element)]
+                else:
+                    value = (value or {}).get(element)
+            except Exception:
+                value = None
+        if value == '0000000000000000000000000000000000000000':  # a deleted ref
+            value = None
+        return value
+
+    def get_signature(self):
+        raise NotImplementedError
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        logger.debug("header signature: %s", self.get_signature())
+        logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
+            raise PermissionDenied
+
+    @csrf_exempt
+    def post(self, request, *args, **kwargs):
+        # Ensure that the full contents of the request are captured for multiple uses.
+        request.body
+
+        logger.debug(
+            "headers: {}\n"
+            "data: {}\n".format(request.headers, request.data)
+        )
+        obj = self.get_object()
+        self.check_signature(obj)
+
+        event_type = self.get_event_type()
+        event_guid = self.get_event_guid()
+        event_ref = self.get_event_ref()
+        status_api = self.get_event_status_api()
+
+        kwargs = {
+            'unified_job_template_id': obj.id,
+            'webhook_service': obj.webhook_service,
+            'webhook_guid': event_guid,
+        }
+        if WorkflowJob.objects.filter(**kwargs).exists() or Job.objects.filter(**kwargs).exists():
+            # Short circuit if this webhook has already been received and acted upon.
+            logger.debug("Webhook previously received, returning without action.")
+            return Response({'message': _("Webhook previously received, aborting.")},
+                            status=status.HTTP_202_ACCEPTED)
+
+        kwargs = {
+            '_eager_fields': {
+                'launch_type': 'webhook',
+                'webhook_service': obj.webhook_service,
+                'webhook_credential': obj.webhook_credential,
+                'webhook_guid': event_guid,
+            },
+            'extra_vars': {
+                'tower_webhook_event_type': event_type,
+                'tower_webhook_event_guid': event_guid,
+                'tower_webhook_event_ref': event_ref,
+                'tower_webhook_status_api': status_api,
+                'tower_webhook_payload': request.data,
+            }
+        }
+
+        new_job = obj.create_unified_job(**kwargs)
+        new_job.signal_start()
+
+        return Response({'message': "Job queued."}, status=status.HTTP_202_ACCEPTED)
+
+
+class GithubWebhookReceiver(WebhookReceiverBase):
+    service = 'github'
+
+    ref_keys = {
+        'pull_request': 'pull_request.head.sha',
+        'pull_request_review': 'pull_request.head.sha',
+        'pull_request_review_comment': 'pull_request.head.sha',
+        'push': 'after',
+        'release': 'release.tag_name',
+        'commit_comment': 'comment.commit_id',
+        'create': 'ref',
+        'page_build': 'build.commit',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITHUB_EVENT')
+
+    def get_event_guid(self):
+        return self.request.META.get('HTTP_X_GITHUB_DELIVERY')
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'pull_request':
+            return
+        return self.request.data.get('pull_request', {}).get('statuses_url')
+
+    def get_signature(self):
+        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
+        if not header_sig:
+            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
+            raise PermissionDenied
+        hash_alg, signature = header_sig.split('=')
+        if hash_alg != 'sha1':
+            logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
+            raise PermissionDenied
+        return force_bytes(signature)
+
+
+class GitlabWebhookReceiver(WebhookReceiverBase):
+    service = 'gitlab'
+
+    ref_keys = {
+        'Push Hook': 'checkout_sha',
+        'Tag Push Hook': 'checkout_sha',
+        'Merge Request Hook': 'object_attributes.last_commit.id',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITLAB_EVENT')
+
+    def get_event_guid(self):
+        # GitLab does not provide a unique identifier on events, so construct one.
+        h = sha1()
+        h.update(force_bytes(self.request.body))
+        return h.hexdigest()
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'Merge Request Hook':
+            return
+        project = self.request.data.get('project', {})
+        repo_url = project.get('web_url')
+        if not repo_url:
+            return
+        parsed = urllib.parse.urlparse(repo_url)
+
+        return "{}://{}/api/v4/projects/{}/statuses/{}".format(
+            parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
+
+    def get_signature(self):
+        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        # GitLab only returns the secret token, not an hmac hash.  Use
+        # the hmac `compare_digest` helper function to prevent timing
+        # analysis by attackers.
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
+            raise PermissionDenied

awx/conf/fields.py

@@ -1,17 +1,18 @@
 # Python
 import os
+import re
 import logging
 import urllib.parse as urlparse
 from collections import OrderedDict
 
 # Django
-from django.core.validators import URLValidator
+from django.core.validators import URLValidator, _lazy_re_compile
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework.fields import (  # noqa
-    BooleanField, CharField, ChoiceField, DictField, EmailField, IntegerField,
-    ListField, NullBooleanField
+    BooleanField, CharField, ChoiceField, DictField, EmailField,
+    IntegerField, ListField, NullBooleanField
 )
 
 logger = logging.getLogger('awx.conf.fields')
@@ -118,14 +119,42 @@ class StringListPathField(StringListField):
 
 
 class URLField(CharField):
+    # these lines set up a custom regex that allow numbers in the
+    # top-level domain
+    tld_re = (
+        r'\.'                                              # dot
+        r'(?!-)'                                           # can't start with a dash
+        r'(?:[a-z' + URLValidator.ul + r'0-9' + '-]{2,63}' # domain label, this line was changed from the original URLValidator
+        r'|xn--[a-z0-9]{1,59})'                            # or punycode label
+        r'(?<!-)'                                          # can't end with a dash
+        r'\.?'                                             # may have a trailing dot
+    )
+
+    host_re = '(' + URLValidator.hostname_re + URLValidator.domain_re + tld_re + '|localhost)'
+
+    regex = _lazy_re_compile(
+        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
+        r'(?:[^\s:@/]+(?::[^\s:@/]*)?@)?'  # user:pass authentication
+        r'(?:' + URLValidator.ipv4_re + '|' + URLValidator.ipv6_re + '|' + host_re + ')'
+        r'(?::\d{2,5})?'  # port
+        r'(?:[/?#][^\s]*)?'  # resource path
+        r'\Z', re.IGNORECASE)
 
     def __init__(self, **kwargs):
         schemes = kwargs.pop('schemes', None)
+        regex = kwargs.pop('regex', None)
         self.allow_plain_hostname = kwargs.pop('allow_plain_hostname', False)
+        self.allow_numbers_in_top_level_domain = kwargs.pop('allow_numbers_in_top_level_domain', True)
         super(URLField, self).__init__(**kwargs)
         validator_kwargs = dict(message=_('Enter a valid URL'))
         if schemes is not None:
             validator_kwargs['schemes'] = schemes
+        if regex is not None:
+            validator_kwargs['regex'] = regex
+        if self.allow_numbers_in_top_level_domain and regex is None:
+            # default behavior is to allow numbers in the top level domain
+            # if a custom regex isn't provided
+            validator_kwargs['regex'] = URLField.regex
         self.validators.append(URLValidator(**validator_kwargs))
 
     def to_representation(self, value):

awx/conf/license.py

@@ -1,13 +1,12 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
-# Tower
-from awx.main.utils.common import get_licenser
 
 __all__ = ['get_license']
 
 
 def _get_validated_license_data():
+    from awx.main.utils.common import get_licenser
     return get_licenser().validate()
 
 

awx/conf/migrations/0004_v320_reencrypt.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from django.db import migrations
-from awx.conf.migrations import _reencrypt
 
 
 class Migration(migrations.Migration):
@@ -12,5 +11,8 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(_reencrypt.replace_aesecb_fernet),
+        # This list is intentionally empty.
+        # Tower 3.2 included several data migrations that are no longer
+        # necessary (this list is now empty because Tower 3.2 is past EOL and
+        # cannot be directly upgraded to modern versions of Tower)
     ]

awx/conf/migrations/_reencrypt.py

@@ -1,30 +1,13 @@
 import base64
 import hashlib
 
-from django.utils.encoding import smart_str
-
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.ciphers import Cipher
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
 from cryptography.hazmat.primitives.ciphers.modes import ECB
 
-from awx.conf import settings_registry
 
-
-__all__ = ['replace_aesecb_fernet', 'get_encryption_key', 'encrypt_field',
-           'decrypt_value', 'decrypt_value', 'should_decrypt_field']
-
-
-def replace_aesecb_fernet(apps, schema_editor):
-    from awx.main.utils.encryption import encrypt_field
-    Setting = apps.get_model('conf', 'Setting')
-
-    for setting in Setting.objects.filter().order_by('pk'):
-        if settings_registry.is_setting_encrypted(setting.key):
-            if should_decrypt_field(setting.value):
-                setting.value = decrypt_field(setting, 'value')
-                setting.value = encrypt_field(setting, 'value')
-            setting.save()
+__all__ = ['get_encryption_key', 'decrypt_field']
 
 
 def get_encryption_key(field_name, pk=None):
@@ -76,38 +59,3 @@ def decrypt_field(instance, field_name, subfield=None):
     key = get_encryption_key(field_name, getattr(instance, 'pk', None))
 
     return decrypt_value(key, value)
-
-
-def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False):
-    '''
-    Return content of the given instance and field name encrypted.
-    '''
-    value = getattr(instance, field_name)
-    if isinstance(value, dict) and subfield is not None:
-        value = value[subfield]
-    if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'):
-        return value
-    if skip_utf8:
-        utf8 = False
-    else:
-        utf8 = type(value) == str
-    value = smart_str(value)
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
-    encryptor = Cipher(AES(key), ECB(), default_backend()).encryptor()
-    block_size = 16
-    while len(value) % block_size != 0:
-        value += '\x00'
-    encrypted = encryptor.update(value) + encryptor.finalize()
-    b64data = base64.b64encode(encrypted)
-    tokens = ['$encrypted', 'AES', b64data]
-    if utf8:
-        # If the value to encrypt is utf-8, we need to add a marker so we
-        # know to decode the data when it's decrypted later
-        tokens.insert(1, 'UTF8')
-    return '$'.join(tokens)
-
-
-def should_decrypt_field(value):
-    if hasattr(value, 'startswith'):
-        return value.startswith('$encrypted$') and '$AESCBC$' not in value
-    return False

awx/conf/tests/unit/test_fields.py

@@ -1,7 +1,7 @@
 import pytest
 
 from rest_framework.fields import ValidationError
-from awx.conf.fields import StringListBooleanField, StringListPathField, ListTuplesField
+from awx.conf.fields import StringListBooleanField, StringListPathField, ListTuplesField, URLField
 
 
 class TestStringListBooleanField():
@@ -62,7 +62,7 @@ class TestListTuplesField():
     FIELD_VALUES = [
         ([('a', 'b'), ('abc', '123')], [("a", "b"), ("abc", "123")]),
     ]
-    
+
     FIELD_VALUES_INVALID = [
         ("abc", type("abc")),
         ([('a', 'b', 'c'), ('abc', '123', '456')], type(('a',))),
@@ -130,3 +130,25 @@ class TestStringListPathField():
             field.to_internal_value([value])
         assert e.value.detail[0] == "{} is not a valid path choice.".format(value)
 
+
+class TestURLField():
+    regex = "^https://www.example.org$"
+
+    @pytest.mark.parametrize("url,schemes,regex, allow_numbers_in_top_level_domain, expect_no_error",[
+        ("ldap://www.example.org42", "ldap", None, True, True),
+        ("https://www.example.org42", "https", None, False, False),
+        ("https://www.example.org", None,  regex, None, True),
+        ("https://www.example3.org", None, regex, None, False),
+        ("ftp://www.example.org", "https", None, None, False)
+    ])
+    def test_urls(self, url, schemes, regex, allow_numbers_in_top_level_domain, expect_no_error):
+        kwargs = {}
+        kwargs.setdefault("allow_numbers_in_top_level_domain", allow_numbers_in_top_level_domain)
+        kwargs.setdefault("schemes", schemes)
+        kwargs.setdefault("regex", regex)
+        field = URLField(**kwargs)
+        if expect_no_error:
+            field.run_validators(url)
+        else:
+            with pytest.raises(ValidationError):
+                field.run_validators(url)

awx/main/access.py

@@ -317,10 +317,19 @@ class BaseAccess(object):
             validation_info['time_remaining'] = 99999999
             validation_info['grace_period_remaining'] = 99999999
 
+        report_violation = lambda message: logger.error(message)
+
+        if (
+            validation_info.get('trial', False) is True or
+            validation_info['instance_count'] == 10  # basic 10 license
+        ):
+            def report_violation(message):
+                raise PermissionDenied(message)
+
         if check_expiration and validation_info.get('time_remaining', None) is None:
             raise PermissionDenied(_("License is missing."))
-        if check_expiration and validation_info.get("grace_period_remaining") <= 0:
-            raise PermissionDenied(_("License has expired."))
+        elif check_expiration and validation_info.get("grace_period_remaining") <= 0:
+            report_violation(_("License has expired."))
 
         free_instances = validation_info.get('free_instances', 0)
         available_instances = validation_info.get('available_instances', 0)
@@ -328,11 +337,11 @@ class BaseAccess(object):
         if add_host_name:
             host_exists = Host.objects.filter(name=add_host_name).exists()
             if not host_exists and free_instances == 0:
-                raise PermissionDenied(_("License count of %s instances has been reached.") % available_instances)
+                report_violation(_("License count of %s instances has been reached.") % available_instances)
             elif not host_exists and free_instances < 0:
-                raise PermissionDenied(_("License count of %s instances has been exceeded.") % available_instances)
+                report_violation(_("License count of %s instances has been exceeded.") % available_instances)
         elif not add_host_name and free_instances < 0:
-            raise PermissionDenied(_("Host count exceeds available instances."))
+            report_violation(_("Host count exceeds available instances."))
 
     def check_org_host_limit(self, data, add_host_name=None):
         validation_info = get_licenser().validate()
@@ -456,7 +465,7 @@ class BaseAccess(object):
                 else:
                     relationship = 'members'
                 return access_method(obj, parent_obj, relationship, skip_sub_obj_read_check=True, data={})
-        except (ParseError, ObjectDoesNotExist):
+        except (ParseError, ObjectDoesNotExist, PermissionDenied):
             return False
         return False
 
@@ -652,7 +661,7 @@ class UserAccess(BaseAccess):
         if obj.is_superuser and super_users.count() == 1:
             # cannot delete the last active superuser
             return False
-        if self.user.is_superuser:
+        if self.can_admin(obj, None, allow_orphans=True):
             return True
         return False
 
@@ -1651,26 +1660,19 @@ class JobAccess(BaseAccess):
         except JobLaunchConfig.DoesNotExist:
             config = None
 
+        if obj.job_template and (self.user not in obj.job_template.execute_role):
+            return False
+
         # Check if JT execute access (and related prompts) is sufficient
-        if obj.job_template is not None:
-            if config is None:
-                prompts_access = False
-            elif not config.has_user_prompts(obj.job_template):
-                prompts_access = True
-            elif obj.created_by_id != self.user.pk and vars_are_encrypted(config.extra_data):
-                prompts_access = False
-                if self.save_messages:
-                    self.messages['detail'] = _('Job was launched with secret prompts provided by another user.')
-            else:
-                prompts_access = (
-                    JobLaunchConfigAccess(self.user).can_add({'reference_obj': config}) and
-                    not config.has_unprompted(obj.job_template)
-                )
-            jt_access = self.user in obj.job_template.execute_role
-            if prompts_access and jt_access:
+        if config and obj.job_template:
+            if not config.has_user_prompts(obj.job_template):
                 return True
-            elif not jt_access:
-                return False
+            elif obj.created_by_id != self.user.pk and vars_are_encrypted(config.extra_data):
+                # never allowed, not even for org admins
+                raise PermissionDenied(_('Job was launched with secret prompts provided by another user.'))
+            elif not config.has_unprompted(obj.job_template):
+                if JobLaunchConfigAccess(self.user).can_add({'reference_obj': config}):
+                    return True
 
         org_access = bool(obj.inventory) and self.user in obj.inventory.organization.inventory_admin_role
         project_access = obj.project is None or self.user in obj.project.admin_role
@@ -2089,23 +2091,20 @@ class WorkflowJobAccess(BaseAccess):
                 self.messages['detail'] = _('Workflow Job was launched with unknown prompts.')
             return False
 
+        # execute permission to WFJT is mandatory for any relaunch
+        if self.user not in template.execute_role:
+            return False
+
         # Check if access to prompts to prevent relaunch
         if config.prompts_dict():
             if obj.created_by_id != self.user.pk and vars_are_encrypted(config.extra_data):
-                if self.save_messages:
-                    self.messages['detail'] = _('Job was launched with secret prompts provided by another user.')
-                return False
+                raise PermissionDenied(_("Job was launched with secret prompts provided by another user."))
             if not JobLaunchConfigAccess(self.user).can_add({'reference_obj': config}):
-                if self.save_messages:
-                    self.messages['detail'] = _('Job was launched with prompts you lack access to.')
-                return False
+                raise PermissionDenied(_('Job was launched with prompts you lack access to.'))
             if config.has_unprompted(template):
-                if self.save_messages:
-                    self.messages['detail'] = _('Job was launched with prompts no longer accepted.')
-                return False
+                raise PermissionDenied(_('Job was launched with prompts no longer accepted.'))
 
-        # execute permission to WFJT is mandatory for any relaunch
-        return (self.user in template.execute_role)
+        return True  # passed config checks
 
     def can_recreate(self, obj):
         node_qs = obj.workflow_job_nodes.all().prefetch_related('inventory', 'credentials', 'unified_job_template')

awx/main/analytics/collectors.py

@@ -166,6 +166,8 @@ def instance_info(since, include_hostnames=False):
     instances = models.Instance.objects.values_list('hostname').values(
         'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled')
     for instance in instances:
+        consumed_capacity = sum(x.task_impact for x in models.UnifiedJob.objects.filter(execution_node=instance['hostname'],
+                                status__in=('running', 'waiting')))
         instance_info = {
             'uuid': instance['uuid'],
             'version': instance['version'],
@@ -174,7 +176,9 @@ def instance_info(since, include_hostnames=False):
             'memory': instance['memory'],
             'managed_by_policy': instance['managed_by_policy'],
             'last_isolated_check': _get_isolated_datetime(instance['last_isolated_check']),
-            'enabled': instance['enabled']
+            'enabled': instance['enabled'],
+            'consumed_capacity': consumed_capacity,
+            'remaining_capacity': instance['capacity'] - consumed_capacity
         }
         if include_hostnames is True:
             instance_info['hostname'] = instance['hostname']

awx/main/analytics/core.py

@@ -5,10 +5,9 @@ import os
 import os.path
 import tempfile
 import shutil
-import subprocess
+import requests
 
 from django.conf import settings
-from django.utils.encoding import smart_str
 from django.utils.timezone import now, timedelta
 from rest_framework.exceptions import PermissionDenied
 
@@ -16,6 +15,7 @@ from awx.conf.license import get_license
 from awx.main.models import Job
 from awx.main.access import access_registry
 from awx.main.models.ha import TowerAnalyticsState
+from awx.main.utils import get_awx_http_client_headers
 
 
 __all__ = ['register', 'gather', 'ship', 'table_version']
@@ -81,17 +81,16 @@ def gather(dest=None, module=None, collection_type='scheduled'):
     last_run = state.last_run
     logger.debug("Last analytics run was: {}".format(last_run))
     
-    max_interval = now() - timedelta(days=7)
+    max_interval = now() - timedelta(weeks=4)
     if last_run < max_interval or not last_run:
         last_run = max_interval
 
-
     if _valid_license() is False:
         logger.exception("Invalid License provided, or No License Provided")
         return "Error: Invalid License provided, or No License Provided"
-    
-    if not settings.INSIGHTS_TRACKING_STATE:
-        logger.error("Insights analytics not enabled")
+
+    if collection_type != 'dry-run' and not settings.INSIGHTS_TRACKING_STATE:
+        logger.error("Automation Analytics not enabled. Use --dry-run to gather locally without sending.")
         return
 
     if module is None:
@@ -146,30 +145,43 @@ def gather(dest=None, module=None, collection_type='scheduled'):
 
 def ship(path):
     """
-    Ship gathered metrics via the Insights agent
+    Ship gathered metrics to the Insights API
     """
+    if not path:
+        logger.error('Automation Analytics TAR not found')
+        return
+    if "Error:" in str(path):
+        return
     try:
-        agent = 'insights-client'
-        if shutil.which(agent) is None:
-            logger.error('could not find {} on PATH'.format(agent))
-            return
         logger.debug('shipping analytics file: {}'.format(path))
-        try:
-            cmd = [
-                agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
-            ]
-            output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
-            logger.debug(output)
-            # reset the `last_run` when data is shipped
-            run_now = now()
-            state = TowerAnalyticsState.get_solo()
-            state.last_run = run_now
-            state.save()
-
-        except subprocess.CalledProcessError:
-            logger.exception('{} failure:'.format(cmd))
-        except subprocess.TimeoutExpired:
-            logger.exception('{} timeout:'.format(cmd))
+        url = getattr(settings, 'AUTOMATION_ANALYTICS_URL', None)
+        if not url:
+            logger.error('AUTOMATION_ANALYTICS_URL is not set')
+            return
+        rh_user = getattr(settings, 'REDHAT_USERNAME', None)
+        rh_password = getattr(settings, 'REDHAT_PASSWORD', None)
+        if not rh_user:
+            return logger.error('REDHAT_USERNAME is not set')
+        if not rh_password:
+            return logger.error('REDHAT_PASSWORD is not set')
+        with open(path, 'rb') as f:
+            files = {'file': (os.path.basename(path), f, settings.INSIGHTS_AGENT_MIME)}
+            s = requests.Session()
+            s.headers = get_awx_http_client_headers()
+            s.headers.pop('Content-Type')
+            response = s.post(url, 
+                              files=files,
+                              verify="/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
+                              auth=(rh_user, rh_password),
+                              headers=s.headers,
+                              timeout=(31, 31))
+            if response.status_code != 202:
+                return logger.exception('Upload failed with status {}, {}'.format(response.status_code,
+                                                                                  response.text))
+        run_now = now()
+        state = TowerAnalyticsState.get_solo()
+        state.last_run = run_now
+        state.save()
     finally:
         # cleanup tar.gz
         os.remove(path)

awx/main/analytics/metrics.py

@@ -46,6 +46,8 @@ INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower
 INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['hostname', 'instance_uuid',])
 INSTANCE_LAUNCH_TYPE = Gauge('awx_instance_launch_type_total', 'Type of Job launched', ['node', 'launch_type',])
 INSTANCE_STATUS = Gauge('awx_instance_status_total', 'Status of Job launched', ['node', 'status',])
+INSTANCE_CONSUMED_CAPACITY = Gauge('awx_instance_consumed_capacity', 'Consumed capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_REMAINING_CAPACITY = Gauge('awx_instance_remaining_capacity', 'Remaining capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
 
 LICENSE_INSTANCE_TOTAL = Gauge('awx_license_instance_total', 'Total number of managed hosts provided by your license')
 LICENSE_INSTANCE_FREE = Gauge('awx_license_instance_free', 'Number of remaining managed hosts provided by your license')
@@ -104,6 +106,8 @@ def metrics():
         INSTANCE_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['capacity'])
         INSTANCE_CPU.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['cpu'])
         INSTANCE_MEMORY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['memory'])
+        INSTANCE_CONSUMED_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['consumed_capacity'])
+        INSTANCE_REMAINING_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['remaining_capacity'])
         INSTANCE_INFO.labels(hostname=hostname, instance_uuid=uuid).info({
             'enabled': str(instance_data[uuid]['enabled']),
             'last_isolated_check': getattr(instance_data[uuid], 'last_isolated_check', 'None'),

awx/main/apps.py

@@ -1,8 +1,17 @@
 from django.apps import AppConfig
+from django.db.models.signals import pre_migrate
 from django.utils.translation import ugettext_lazy as _
 
 
+def raise_migration_flag(**kwargs):
+    from awx.main.tasks import set_migration_flag
+    set_migration_flag.delay()
+
+
 class MainConfig(AppConfig):
 
     name = 'awx.main'
     verbose_name = _('Main')
+
+    def ready(self):
+        pre_migrate.connect(raise_migration_flag, sender=self)

awx/main/conf.py

@@ -2,12 +2,14 @@
 import json
 import logging
 import os
+from distutils.version import LooseVersion as Version
 
 # Django
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework import serializers
+from rest_framework.fields import FloatField
 
 # Tower
 from awx.conf import fields, register, register_validate
@@ -52,15 +54,6 @@ register(
     category_slug='system',
 )
 
-register(
-    'TOWER_ADMIN_ALERTS',
-    field_class=fields.BooleanField,
-    label=_('Enable Administrator Alerts'),
-    help_text=_('Email Admin users for system events that may require attention.'),
-    category=_('System'),
-    category_slug='system',
-)
-
 register(
     'TOWER_URL_BASE',
     field_class=fields.URLField,
@@ -153,7 +146,7 @@ register(
 register(
     'AUTOMATION_ANALYTICS_URL',
     field_class=fields.URLField,
-    default='https://cloud.redhat.com',
+    default='https://example.com',
     schemes=('http', 'https'),
     allow_plain_hostname=True,  # Allow hostname only without TLD.
     label=_('Automation Analytics upload URL.'),
@@ -298,6 +291,16 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_ISOLATED_HOST_KEY_CHECKING',
+    field_class=fields.BooleanField,
+    label=_('Isolated host key checking'),
+    help_text=_('When set to True, AWX will enforce strict host key checking for communication with isolated nodes.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    default=False
+)
+
 register(
     'AWX_ISOLATED_KEY_GENERATION',
     field_class=fields.BooleanField,
@@ -335,6 +338,53 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_RESOURCE_PROFILING_ENABLED',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Enable detailed resource profiling on all playbook runs'),
+    help_text=_('If set, detailed resource profiling data will be collected on all jobs. '
+                'This data can be gathered with `sosreport`.'),  # noqa
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
+register(
+    'AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL',
+    field_class=FloatField,
+    default='0.25',
+    label=_('Interval (in seconds) between polls for cpu usage.'),
+    help_text=_('Interval (in seconds) between polls for cpu usage. '
+                'Setting this lower than the default will affect playbook performance.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    required=False,
+)
+
+register(
+    'AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL',
+    field_class=FloatField,
+    default='0.25',
+    label=_('Interval (in seconds) between polls for memory usage.'),
+    help_text=_('Interval (in seconds) between polls for memory usage. '
+                'Setting this lower than the default will affect playbook performance.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    required=False,
+)
+
+register(
+    'AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL',
+    field_class=FloatField,
+    default='0.25',
+    label=_('Interval (in seconds) between polls for PID count.'),
+    help_text=_('Interval (in seconds) between polls for PID count. '
+                'Setting this lower than the default will affect playbook performance.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    required=False,
+)
+
 register(
     'AWX_TASK_ENV',
     field_class=fields.KeyValueField,
@@ -350,12 +400,21 @@ register(
     'INSIGHTS_TRACKING_STATE',
     field_class=fields.BooleanField,
     default=False,
-    label=_('Gather data for Automation Insights'),
-    help_text=_('Enables Tower to gather data on automation and send it to Red Hat Insights.'),
+    label=_('Gather data for Automation Analytics'),
+    help_text=_('Enables Tower to gather data on automation and send it to Red Hat.'),
     category=_('System'),
     category_slug='system',
 )
 
+register(
+    'PROJECT_UPDATE_VVV',
+    field_class=fields.BooleanField,
+    label=_('Run Project Updates With Higher Verbosity'),
+    help_text=_('Adds the CLI -vvv flag to ansible-playbook runs of project_update.yml used for project updates.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'AWX_ROLES_ENABLED',
     field_class=fields.BooleanField,
@@ -376,6 +435,96 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'PRIMARY_GALAXY_URL',
+    field_class=fields.URLField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server URL'),
+    help_text=_(
+        'For organizations that run their own Galaxy service, this gives the option to specify a '
+        'host as the primary galaxy server. Requirements will be downloaded from the primary if the '
+        'specific role or collection is available there. If the content is not avilable in the primary, '
+        'or if this field is left blank, it will default to galaxy.ansible.com.'
+    ),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_USERNAME',
+    field_class=fields.CharField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Username'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The username to use for basic authentication against the Galaxy instance, '
+                'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_PASSWORD',
+    field_class=fields.CharField,
+    encrypted=True,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Password'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The password to use for basic authentication against the Galaxy instance, '
+                'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_TOKEN',
+    field_class=fields.CharField,
+    encrypted=True,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Token'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The token to use for connecting with the Galaxy instance, '
+                'this is mutually exclusive with corresponding username and password settings.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_AUTH_URL',
+    field_class=fields.CharField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Authentication URL'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The token_endpoint of a Keycloak server.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PUBLIC_GALAXY_ENABLED',
+    field_class=fields.BooleanField,
+    default=True,
+    label=_('Allow Access to Public Galaxy'),
+    help_text=_('Allow or deny access to the public Ansible Galaxy during project updates.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'GALAXY_IGNORE_CERTS',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Ignore Ansible Galaxy SSL Certificate Verification'),
+    help_text=_('If set to true, certificate validation will not be done when'
+                'installing content from any Galaxy server.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
 register(
     'STDOUT_MAX_BYTES_DISPLAY',
     field_class=fields.IntegerField,
@@ -616,6 +765,16 @@ register(
     category=_('Logging'),
     category_slug='logging',
 )
+register(
+    'LOG_AGGREGATOR_AUDIT',
+    field_class=fields.BooleanField,
+    allow_null=True,
+    default=False,
+    label=_('Enabled external log aggregation auditing'),
+    help_text=_('When enabled, all external logs emitted by Tower will also be written to /var/log/tower/external.log.  This is an experimental setting intended to be used for debugging external log aggregation issues (and may be subject to change in the future).'),  # noqa
+    category=_('Logging'),
+    category_slug='logging',
+)
 
 
 register(
@@ -646,4 +805,75 @@ def logging_validate(serializer, attrs):
     return attrs
 
 
+def galaxy_validate(serializer, attrs):
+    """Ansible Galaxy config options have mutual exclusivity rules, these rules
+    are enforced here on serializer validation so that users will not be able
+    to save settings which obviously break all project updates.
+    """
+    prefix = 'PRIMARY_GALAXY_'
+
+    from awx.main.constants import GALAXY_SERVER_FIELDS
+    if not any('{}{}'.format(prefix, subfield.upper()) in attrs for subfield in GALAXY_SERVER_FIELDS):
+        return attrs
+
+    def _new_value(setting_name):
+        if setting_name in attrs:
+            return attrs[setting_name]
+        elif not serializer.instance:
+            return ''
+        return getattr(serializer.instance, setting_name, '')
+
+    galaxy_data = {}
+    for subfield in GALAXY_SERVER_FIELDS:
+        galaxy_data[subfield] = _new_value('{}{}'.format(prefix, subfield.upper()))
+    errors = {}
+    if not galaxy_data['url']:
+        for k, v in galaxy_data.items():
+            if v:
+                setting_name = '{}{}'.format(prefix, k.upper())
+                errors.setdefault(setting_name, [])
+                errors[setting_name].append(_(
+                    'Cannot provide field if PRIMARY_GALAXY_URL is not set.'
+                ))
+    for k in GALAXY_SERVER_FIELDS:
+        if galaxy_data[k]:
+            setting_name = '{}{}'.format(prefix, k.upper())
+            if (not serializer.instance) or (not getattr(serializer.instance, setting_name, '')):
+                # new auth is applied, so check if compatible with version
+                from awx.main.utils import get_ansible_version
+                current_version = get_ansible_version()
+                min_version = '2.9'
+                if Version(current_version) < Version(min_version):
+                    errors.setdefault(setting_name, [])
+                    errors[setting_name].append(_(
+                        'Galaxy server settings are not available until Ansible {min_version}, '
+                        'you are running {current_version}.'
+                    ).format(min_version=min_version, current_version=current_version))
+    if (galaxy_data['password'] or galaxy_data['username']) and (galaxy_data['token'] or galaxy_data['auth_url']):
+        for k in ('password', 'username', 'token', 'auth_url'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            if setting_name in attrs:
+                errors.setdefault(setting_name, [])
+                errors[setting_name].append(_(
+                    'Setting Galaxy token and authentication URL is mutually exclusive with username and password.'
+                ))
+    if bool(galaxy_data['username']) != bool(galaxy_data['password']):
+        msg = _('If authenticating via username and password, both must be provided.')
+        for k in ('username', 'password'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            errors.setdefault(setting_name, [])
+            errors[setting_name].append(msg)
+    if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']):
+        msg = _('If authenticating via token, both token and authentication URL must be provided.')
+        for k in ('token', 'auth_url'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            errors.setdefault(setting_name, [])
+            errors[setting_name].append(msg)
+
+    if errors:
+        raise serializers.ValidationError(errors)
+    return attrs
+
+
 register_validate('logging', logging_validate)
+register_validate('jobs', galaxy_validate)

awx/main/constants.py

@@ -51,3 +51,7 @@ LOGGER_BLACKLIST = (
     # loggers that may be called getting logging settings
     'awx.conf'
 )
+
+# these correspond to both AWX and Ansible settings to keep naming consistent
+# for instance, settings.PRIMARY_GALAXY_AUTH_URL vs env var ANSIBLE_GALAXY_SERVER_FOO_AUTH_URL
+GALAXY_SERVER_FIELDS = ('url', 'username', 'password', 'token', 'auth_url')

awx/main/credential_plugins/aim.py

@@ -101,7 +101,7 @@ def aim_backend(**kwargs):
 
 
 aim_plugin = CredentialPlugin(
-    'CyberArk AIM Secret Lookup',
+    'CyberArk AIM Central Credential Provider Lookup',
     inputs=aim_inputs,
     backend=aim_backend
 )

awx/main/credential_plugins/hashivault.py

@@ -103,6 +103,8 @@ def kv_backend(**kwargs):
 
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
+    # Compatability header for older installs of Hashicorp Vault
+    sess.headers['X-Vault-Token'] = token
 
     if api_version == 'v2':
         if kwargs.get('secret_version'):
@@ -158,6 +160,8 @@ def ssh_backend(**kwargs):
 
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
+    # Compatability header for older installs of Hashicorp Vault
+    sess.headers['X-Vault-Token'] = token
     # https://www.vaultproject.io/api/secret/ssh/index.html#sign-ssh-key
     request_url = '/'.join([url, secret_path, 'sign', role]).rstrip('/')
     resp = sess.post(request_url, **request_kwargs)

awx/main/dispatch/pool.py

@@ -123,8 +123,16 @@ class PoolWorker(object):
         # if any tasks were finished, removed them from the managed tasks for
         # this worker
         for uuid in finished:
-            self.messages_finished += 1
-            del self.managed_tasks[uuid]
+            try:
+                del self.managed_tasks[uuid]
+                self.messages_finished += 1
+            except KeyError:
+                # ansible _sometimes_ appears to send events w/ duplicate UUIDs;
+                # UUIDs for ansible events are *not* actually globally unique
+                # when this occurs, it's _fine_ to ignore this KeyError because
+                # the purpose of self.managed_tasks is to just track internal
+                # state of which events are *currently* being processed.
+                pass
 
     @property
     def current_task(self):

awx/main/dispatch/reaper.py

@@ -33,7 +33,11 @@ def reap(instance=None, status='failed', excluded_uuids=[]):
     '''
     Reap all jobs in waiting|running for this instance.
     '''
-    me = instance or Instance.objects.me()
+    me = instance
+    if me is None:
+        (changed, me) = Instance.objects.get_or_register()
+        if changed:
+            logger.info("Registered tower node '{}'".format(me.hostname))
     now = tz_now()
     workflow_ctype_id = ContentType.objects.get_for_model(WorkflowJob).id
     jobs = UnifiedJob.objects.filter(

awx/main/dispatch/worker/task.py

@@ -4,6 +4,7 @@ import importlib
 import sys
 import traceback
 
+from kubernetes.config import kube_config
 
 from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown
 
@@ -107,6 +108,14 @@ class TaskWorker(BaseWorker):
             for callback in body.get('errbacks', []) or []:
                 callback['uuid'] = body['uuid']
                 self.perform_work(callback)
+        finally:
+            # It's frustrating that we have to do this, but the python k8s
+            # client leaves behind cacert files in /tmp, so we must clean up
+            # the tmpdir per-dispatcher process every time a new task comes in
+            try:
+                kube_config._cleanup_temp_files()
+            except Exception:
+                logger.exception('failed to cleanup k8s client tmp files')
 
         for callback in body.get('callbacks', []) or []:
             callback['uuid'] = body['uuid']

awx/main/isolated/manager.py

@@ -6,12 +6,15 @@ import stat
 import tempfile
 import time
 import logging
+import yaml
 
 from django.conf import settings
 import ansible_runner
 
 import awx
-from awx.main.utils import get_system_task_capacity
+from awx.main.utils import (
+    get_system_task_capacity
+)
 from awx.main.queue import CallbackQueueDispatcher
 
 logger = logging.getLogger('awx.isolated.manager')
@@ -29,22 +32,46 @@ def set_pythonpath(venv_libdir, env):
 
 class IsolatedManager(object):
 
-    def __init__(self, cancelled_callback=None, check_callback=None):
+    def __init__(self, canceled_callback=None, check_callback=None, pod_manager=None):
         """
-        :param cancelled_callback:  a callable - which returns `True` or `False`
+        :param canceled_callback:  a callable - which returns `True` or `False`
                                     - signifying if the job has been prematurely
-                                      cancelled
+                                      canceled
         """
-        self.cancelled_callback = cancelled_callback
+        self.canceled_callback = canceled_callback
         self.check_callback = check_callback
-        self.idle_timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT)
         self.started_at = None
         self.captured_command_artifact = False
+        self.instance = None
+        self.pod_manager = pod_manager
+
+    def build_inventory(self, hosts):
+        if self.instance and self.instance.is_containerized:
+            inventory = {'all': {'hosts': {}}}
+            fd, path = tempfile.mkstemp(
+                prefix='.kubeconfig', dir=self.private_data_dir
+            )
+            with open(path, 'wb') as temp:
+                temp.write(yaml.dump(self.pod_manager.kube_config).encode())
+                temp.flush()
+                os.chmod(temp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+            for host in hosts:
+                inventory['all']['hosts'][host] = {
+                    "ansible_connection": "kubectl",
+                    "ansible_kubectl_config": path,
+                }
+        else:
+            inventory = '\n'.join([
+                '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME)
+                for host in hosts
+            ])
+
+        return inventory
 
     def build_runner_params(self, hosts, verbosity=1):
         env = dict(os.environ.items())
         env['ANSIBLE_RETRY_FILES_ENABLED'] = 'False'
-        env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        env['ANSIBLE_HOST_KEY_CHECKING'] = str(settings.AWX_ISOLATED_HOST_KEY_CHECKING)
         env['ANSIBLE_LIBRARY'] = os.path.join(os.path.dirname(awx.__file__), 'plugins', 'isolated')
         set_pythonpath(os.path.join(settings.ANSIBLE_VENV_PATH, 'lib'), env)
 
@@ -69,23 +96,17 @@ class IsolatedManager(object):
             else:
                 playbook_logger.info(runner_obj.stdout.read())
 
-        inventory = '\n'.join([
-            '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME)
-            for host in hosts
-        ])
-
         return {
             'project_dir': os.path.abspath(os.path.join(
                 os.path.dirname(awx.__file__),
                 'playbooks'
             )),
-            'inventory': inventory,
+            'inventory': self.build_inventory(hosts),
             'envvars': env,
             'finished_callback': finished_callback,
             'verbosity': verbosity,
-            'cancel_callback': self.cancelled_callback,
+            'cancel_callback': self.canceled_callback,
             'settings': {
-                'idle_timeout': self.idle_timeout,
                 'job_timeout': settings.AWX_ISOLATED_LAUNCH_TIMEOUT,
                 'pexpect_timeout': getattr(settings, 'PEXPECT_TIMEOUT', 5),
                 'suppress_ansible_output': True,
@@ -95,7 +116,7 @@ class IsolatedManager(object):
     def path_to(self, *args):
         return os.path.join(self.private_data_dir, *args)
 
-    def run_management_playbook(self, playbook, private_data_dir, **kw):
+    def run_management_playbook(self, playbook, private_data_dir, idle_timeout=None, **kw):
         iso_dir = tempfile.mkdtemp(
             prefix=playbook,
             dir=private_data_dir
@@ -103,6 +124,10 @@ class IsolatedManager(object):
         params = self.runner_params.copy()
         params['playbook'] = playbook
         params['private_data_dir'] = iso_dir
+        if idle_timeout:
+            params['settings']['idle_timeout'] = idle_timeout
+        else:
+            params['settings'].pop('idle_timeout', None)
         params.update(**kw)
         if all([
             getattr(settings, 'AWX_ISOLATED_KEY_GENERATION', False) is True,
@@ -128,6 +153,8 @@ class IsolatedManager(object):
             '- /artifacts/job_events/*-partial.json.tmp',
             # don't rsync the ssh_key FIFO
             '- /env/ssh_key',
+            # don't rsync kube config files
+            '- .kubeconfig*'
         ]
 
         for filename, data in (
@@ -152,7 +179,14 @@ class IsolatedManager(object):
         logger.debug('Starting job {} on isolated host with `run_isolated.yml` playbook.'.format(self.instance.id))
         runner_obj = self.run_management_playbook('run_isolated.yml',
                                                   self.private_data_dir,
+                                                  idle_timeout=max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT),
                                                   extravars=extravars)
+
+        if runner_obj.status == 'failed':
+            self.instance.result_traceback = runner_obj.stdout.read()
+            self.instance.save(update_fields=['result_traceback'])
+            return 'error', runner_obj.rc
+
         return runner_obj.status, runner_obj.rc
 
     def check(self, interval=None):
@@ -175,15 +209,16 @@ class IsolatedManager(object):
         rc = None
         last_check = time.time()
         dispatcher = CallbackQueueDispatcher()
+
         while status == 'failed':
-            canceled = self.cancelled_callback() if self.cancelled_callback else False
+            canceled = self.canceled_callback() if self.canceled_callback else False
             if not canceled and time.time() - last_check < interval:
-                # If the job isn't cancelled, but we haven't waited `interval` seconds, wait longer
+                # If the job isn't canceled, but we haven't waited `interval` seconds, wait longer
                 time.sleep(1)
                 continue
 
             if canceled:
-                logger.warning('Isolated job {} was manually cancelled.'.format(self.instance.id))
+                logger.warning('Isolated job {} was manually canceled.'.format(self.instance.id))
 
             logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id))
             runner_obj = self.run_management_playbook('check_isolated.yml',
@@ -279,7 +314,6 @@ class IsolatedManager(object):
 
 
     def cleanup(self):
-        # If the job failed for any reason, make a last-ditch effort at cleanup
         extravars = {
             'private_data_dir': self.private_data_dir,
             'cleanup_dirs': [
@@ -393,6 +427,7 @@ class IsolatedManager(object):
             [instance.execution_node],
             verbosity=min(5, self.instance.verbosity)
         )
+
         status, rc = self.dispatch(playbook, module, module_args)
         if status == 'successful':
             status, rc = self.check()

awx/main/management/commands/check_db.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+class Command(BaseCommand):
+    """Checks connection to the database, and prints out connection info if not connected"""
+
+    def handle(self, *args, **options):
+
+        with connection.cursor() as cursor:
+            cursor.execute("SELECT version()")
+            version = str(cursor.fetchone()[0])
+
+        return "Database Version: {}".format(version)

awx/main/management/commands/gather_analytics.py

@@ -11,8 +11,10 @@ class Command(BaseCommand):
     help = 'Gather AWX analytics data'
 
     def add_arguments(self, parser):
+        parser.add_argument('--dry-run', dest='dry-run', action='store_true',
+                            help='Gather analytics without shipping. Works even if analytics are disabled in settings.')
         parser.add_argument('--ship', dest='ship', action='store_true',
-                            help='Enable to ship metrics via insights-client')
+                            help='Enable to ship metrics to the Red Hat Cloud')
 
     def init_logging(self):
         self.logger = logging.getLogger('awx.main.analytics')
@@ -23,9 +25,14 @@ class Command(BaseCommand):
         self.logger.propagate = False
 
     def handle(self, *args, **options):
-        tgz = gather(collection_type='manual')
         self.init_logging()
+        opt_ship = options.get('ship')
+        opt_dry_run = options.get('dry-run')
+        if opt_ship and opt_dry_run:
+            self.logger.error('Both --ship and --dry-run cannot be processed at the same time.')
+            return
+        tgz = gather(collection_type='manual' if not opt_dry_run else 'dry-run')
         if tgz:
             self.logger.debug(tgz)
-        if options.get('ship'):
+        if opt_ship:
             ship(tgz)

awx/main/management/commands/inventory_import.py

@@ -28,6 +28,7 @@ from awx.main.models.inventory import (
     Host
 )
 from awx.main.utils.mem_inventory import MemInventory, dict_to_mem_data
+from awx.main.utils.safe_yaml import sanitize_jinja
 
 # other AWX imports
 from awx.main.models.rbac import batch_role_ancestor_rebuilding
@@ -795,6 +796,10 @@ class Command(BaseCommand):
             if self.instance_id_var:
                 instance_id = self._get_instance_id(mem_host.variables)
                 host_attrs['instance_id'] = instance_id
+            try:
+                sanitize_jinja(mem_host_name)
+            except ValueError as e:
+                raise ValueError(str(e) + ': {}'.format(mem_host_name))
             db_host = self.inventory.hosts.update_or_create(name=mem_host_name, defaults=host_attrs)[0]
             if enabled is False:
                 logger.debug('Host "%s" added (disabled)', mem_host_name)
@@ -919,7 +924,8 @@ class Command(BaseCommand):
         new_count = Host.objects.active_count()
         if time_remaining <= 0 and not license_info.get('demo', False):
             logger.error(LICENSE_EXPIRED_MESSAGE)
-            raise CommandError("License has expired!")
+            if license_info.get('trial', False) is True:
+                raise CommandError("License has expired!")
         # special check for tower-type inventory sources
         # but only if running the plugin
         TOWER_SOURCE_FILES = ['tower.yml', 'tower.yaml']
@@ -936,7 +942,11 @@ class Command(BaseCommand):
                 logger.error(DEMO_LICENSE_MESSAGE % d)
             else:
                 logger.error(LICENSE_MESSAGE % d)
-            raise CommandError('License count exceeded!')
+            if (
+                license_info.get('trial', False) is True or
+                license_info['instance_count'] == 10  # basic 10 license
+            ):
+                raise CommandError('License count exceeded!')
 
     def check_org_host_limit(self):
         license_info = get_licenser().validate()

awx/main/management/commands/regenerate_secret_key.py

@@ -0,0 +1,129 @@
+import base64
+import json
+import os
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.db import transaction
+from django.db.models.signals import post_save
+
+from awx.conf import settings_registry
+from awx.conf.models import Setting
+from awx.conf.signals import on_post_save_setting
+from awx.main.models import (
+    UnifiedJob, Credential, NotificationTemplate, Job, JobTemplate, WorkflowJob,
+    WorkflowJobTemplate, OAuth2Application
+)
+from awx.main.utils.encryption import (
+    encrypt_field, decrypt_field, encrypt_value, decrypt_value, get_encryption_key
+)
+
+
+class Command(BaseCommand):
+    """
+    Regenerate a new SECRET_KEY value and re-encrypt every secret in the
+    Tower database.
+    """
+
+    @transaction.atomic
+    def handle(self, **options):
+        self.old_key = settings.SECRET_KEY
+        self.new_key = base64.encodebytes(os.urandom(33)).decode().rstrip()
+        self._notification_templates()
+        self._credentials()
+        self._unified_jobs()
+        self._oauth2_app_secrets()
+        self._settings()
+        self._survey_passwords()
+        return self.new_key
+
+    def _notification_templates(self):
+        for nt in NotificationTemplate.objects.iterator():
+            CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NotificationTemplate.NOTIFICATION_TYPES])
+            notification_class = CLASS_FOR_NOTIFICATION_TYPE[nt.notification_type]
+            for field in filter(lambda x: notification_class.init_parameters[x]['type'] == "password",
+                                notification_class.init_parameters):
+                nt.notification_configuration[field] = decrypt_field(nt, 'notification_configuration', subfield=field, secret_key=self.old_key)
+                nt.notification_configuration[field] = encrypt_field(nt, 'notification_configuration', subfield=field, secret_key=self.new_key)
+            nt.save()
+
+    def _credentials(self):
+        for credential in Credential.objects.iterator():
+            for field_name in credential.credential_type.secret_fields:
+                if field_name in credential.inputs:
+                    credential.inputs[field_name] = decrypt_field(
+                        credential,
+                        field_name,
+                        secret_key=self.old_key
+                    )
+                    credential.inputs[field_name] = encrypt_field(
+                        credential,
+                        field_name,
+                        secret_key=self.new_key
+                    )
+                credential.save()
+
+    def _unified_jobs(self):
+        for uj in UnifiedJob.objects.iterator():
+            if uj.start_args:
+                uj.start_args = decrypt_field(
+                    uj,
+                    'start_args',
+                    secret_key=self.old_key
+                )
+                uj.start_args = encrypt_field(uj, 'start_args', secret_key=self.new_key)
+                uj.save()
+
+    def _oauth2_app_secrets(self):
+        for app in OAuth2Application.objects.iterator():
+            raw = app.client_secret
+            app.client_secret = raw
+            encrypted = encrypt_value(raw, secret_key=self.new_key)
+            OAuth2Application.objects.filter(pk=app.pk).update(client_secret=encrypted)
+
+    def _settings(self):
+        # don't update memcached, the *actual* value isn't changing
+        post_save.disconnect(on_post_save_setting, sender=Setting)
+        for setting in Setting.objects.filter().order_by('pk'):
+            if settings_registry.is_setting_encrypted(setting.key):
+                setting.value = decrypt_field(setting, 'value', secret_key=self.old_key)
+                setting.value = encrypt_field(setting, 'value', secret_key=self.new_key)
+                setting.save()
+
+    def _survey_passwords(self):
+        for _type in (JobTemplate, WorkflowJobTemplate):
+            for jt in _type.objects.exclude(survey_spec={}):
+                changed = False
+                if jt.survey_spec.get('spec', []):
+                    for field in jt.survey_spec['spec']:
+                        if field.get('type') == 'password' and field.get('default', ''):
+                            raw = decrypt_value(
+                                get_encryption_key('value', None, secret_key=self.old_key),
+                                field['default']
+                            )
+                            field['default'] = encrypt_value(
+                                raw,
+                                pk=None,
+                                secret_key=self.new_key
+                            )
+                            changed = True
+                if changed:
+                    jt.save(update_fields=["survey_spec"])
+
+        for _type in (Job, WorkflowJob):
+            for job in _type.objects.exclude(survey_passwords={}).iterator():
+                changed = False
+                for key in job.survey_passwords:
+                    if key in job.extra_vars:
+                        extra_vars = json.loads(job.extra_vars)
+                        if not extra_vars.get(key):
+                            continue
+                        raw = decrypt_value(
+                            get_encryption_key('value', None, secret_key=self.old_key),
+                            extra_vars[key]
+                        )
+                        extra_vars[key] = encrypt_value(raw, pk=None, secret_key=self.new_key)
+                        job.extra_vars = json.dumps(extra_vars)
+                        changed = True
+                if changed:
+                    job.save(update_fields=["extra_vars"])

awx/main/management/commands/test_isolated_connection.py

@@ -33,6 +33,7 @@ class Command(BaseCommand):
             ]):
                 ssh_key = settings.AWX_ISOLATED_PRIVATE_KEY
             env = dict(os.environ.items())
+            env['ANSIBLE_HOST_KEY_CHECKING'] = str(settings.AWX_ISOLATED_HOST_KEY_CHECKING)
             set_pythonpath(os.path.join(settings.ANSIBLE_VENV_PATH, 'lib'), env)
             res = ansible_runner.interface.run(
                 private_data_dir=path,

awx/main/managers.py

@@ -221,8 +221,9 @@ class InstanceGroupManager(models.Manager):
             elif t.status == 'running':
                 # Subtract capacity from all groups that contain the instance
                 if t.execution_node not in instance_ig_mapping:
-                    logger.warning('Detected %s running inside lost instance, '
-                                   'may still be waiting for reaper.', t.log_format)
+                    if not t.is_containerized:
+                        logger.warning('Detected %s running inside lost instance, '
+                                       'may still be waiting for reaper.', t.log_format)
                     if t.instance_group:
                         impacted_groups = [t.instance_group.name]
                     else:

awx/main/middleware.py

@@ -13,8 +13,7 @@ import urllib.parse
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db.models.signals import post_save
-from django.db.migrations.executor import MigrationExecutor
-from django.db import IntegrityError, connection
+from django.db import IntegrityError
 from django.utils.functional import curry
 from django.shortcuts import get_object_or_404, redirect
 from django.apps import apps
@@ -24,6 +23,7 @@ from django.urls import reverse, resolve
 
 from awx.main.models import ActivityStream
 from awx.main.utils.named_url_graph import generate_graph, GraphNode
+from awx.main.utils.db import migration_in_progress_check_or_relase
 from awx.conf import fields, register
 
 
@@ -213,8 +213,7 @@ class URLModificationMiddleware(MiddlewareMixin):
 class MigrationRanCheckMiddleware(MiddlewareMixin):
 
     def process_request(self, request):
-        executor = MigrationExecutor(connection)
-        plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
-        if bool(plan) and \
-                getattr(resolve(request.path), 'url_name', '') != 'migrations_notran':
+        if migration_in_progress_check_or_relase():
+            if getattr(resolve(request.path), 'url_name', '') == 'migrations_notran':
+                return
             return redirect(reverse("ui:migrations_notran"))

awx/main/migrations/0007_v320_data_migrations.py

@@ -7,12 +7,6 @@ from django.db import migrations, models
 
 # AWX
 from awx.main.migrations import ActivityStreamDisabledMigration
-from awx.main.migrations import _inventory_source as invsrc
-from awx.main.migrations import _migration_utils as migration_utils
-from awx.main.migrations import _reencrypt as reencrypt
-from awx.main.migrations import _scan_jobs as scan_jobs
-from awx.main.migrations import _credentialtypes as credentialtypes
-from awx.main.migrations import _azure_credentials as azurecreds
 import awx.main.fields
 
 
@@ -23,16 +17,8 @@ class Migration(ActivityStreamDisabledMigration):
     ]
 
     operations = [
-        # Inventory Refresh
-        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
-        migrations.RunPython(invsrc.remove_rax_inventory_sources),
-        migrations.RunPython(azurecreds.remove_azure_credentials),
-        migrations.RunPython(invsrc.remove_azure_inventory_sources),
-        migrations.RunPython(invsrc.remove_inventory_source_with_no_inventory_link),
-        migrations.RunPython(invsrc.rename_inventory_sources),
-        migrations.RunPython(reencrypt.replace_aesecb_fernet),
-        migrations.RunPython(scan_jobs.migrate_scan_job_templates),
-
-        migrations.RunPython(credentialtypes.migrate_to_v2_credentials),
-        migrations.RunPython(credentialtypes.migrate_job_credentials),
+        # This list is intentionally empty.
+        # Tower 3.2 included several data migrations that are no longer
+        # necessary (this list is now empty because Tower 3.2 is past EOL and
+        # cannot be directly upgraded to modern versions of Tower)
     ]

awx/main/migrations/0010_v322_add_ovirt4_tower_inventory.py

@@ -15,8 +15,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
-        migrations.RunPython(credentialtypes.create_rhv_tower_credtype),
         migrations.AlterField(
             model_name='inventorysource',
             name='source',

awx/main/migrations/0011_v322_encrypt_survey_passwords.py

@@ -3,8 +3,6 @@ from __future__ import unicode_literals
 
 from django.db import migrations
 from awx.main.migrations import ActivityStreamDisabledMigration
-from awx.main.migrations import _reencrypt as reencrypt
-from awx.main.migrations import _migration_utils as migration_utils
 
 
 class Migration(ActivityStreamDisabledMigration):
@@ -14,6 +12,8 @@ class Migration(ActivityStreamDisabledMigration):
     ]
 
     operations = [
-        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
-        migrations.RunPython(reencrypt.encrypt_survey_passwords),
+        # This list is intentionally empty.
+        # Tower 3.2 included several data migrations that are no longer
+        # necessary (this list is now empty because Tower 3.2 is past EOL and
+        # cannot be directly upgraded to modern versions of Tower)
     ]

awx/main/migrations/0012_v322_update_cred_types.py

@@ -1,10 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-# AWX
-from awx.main.migrations import _migration_utils as migration_utils
-from awx.main.migrations import _credentialtypes as credentialtypes
-
 from django.db import migrations
 
 
@@ -15,6 +11,8 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
-        migrations.RunPython(credentialtypes.add_azure_cloud_environment_field),
+        # This list is intentionally empty.
+        # Tower 3.2 included several data migrations that are no longer
+        # necessary (this list is now empty because Tower 3.2 is past EOL and
+        # cannot be directly upgraded to modern versions of Tower)
     ]

awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py

@@ -19,11 +19,11 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='systemjob',
             name='job_type',
-            field=models.CharField(blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('clearsessions', 'Removes expired browser sessions from the database'), ('cleartokens', 'Removes expired OAuth 2 access tokens and refresh tokens')], default='', max_length=32),
+            field=models.CharField(blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_sessions', 'Removes expired browser sessions from the database'), ('cleanup_tokens', 'Removes expired OAuth 2 access tokens and refresh tokens')], default='', max_length=32),
         ),
         migrations.AlterField(
             model_name='systemjobtemplate',
             name='job_type',
-            field=models.CharField(blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('clearsessions', 'Removes expired browser sessions from the database'), ('cleartokens', 'Removes expired OAuth 2 access tokens and refresh tokens')], default='', max_length=32),
+            field=models.CharField(blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_sessions', 'Removes expired browser sessions from the database'), ('cleanup_tokens', 'Removes expired OAuth 2 access tokens and refresh tokens')], default='', max_length=32),
         ),
     ]

awx/main/migrations/0088_v360_dashboard_optimizations.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.4 on 2019-09-10 21:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0087_v360_update_credential_injector_help_text'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='finished',
+            field=models.DateTimeField(db_index=True, default=None, editable=False, help_text='The date and time the job finished execution.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='launch_type',
+            field=models.CharField(choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency'), ('workflow', 'Workflow'), ('sync', 'Sync'), ('scm', 'SCM Update')], db_index=True, default='manual', editable=False, max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='created',
+            field=models.DateTimeField(db_index=True, default=None, editable=False),
+        ),
+    ]

awx/main/migrations/0089_v360_new_job_event_types.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.4 on 2019-09-12 13:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0088_v360_dashboard_optimizations'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='jobevent',
+            name='event',
+            field=models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_start', 'Host Started'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='projectupdateevent',
+            name='event',
+            field=models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_start', 'Host Started'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100),
+        ),
+    ]

awx/main/migrations/0090_v360_WFJT_prompts.py

@@ -0,0 +1,59 @@
+# Generated by Django 2.2.2 on 2019-07-23 17:56
+
+import awx.main.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0089_v360_new_job_event_types'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_limit_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_scm_branch_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='char_prompts',
+            field=awx.main.fields.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='joblaunchconfig',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='joblaunchconfigs', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='schedule',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='schedules', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjob',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobs', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobnodes', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplate',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobtemplates', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobtemplatenodes', to='main.Inventory'),
+        ),
+    ]

awx/main/migrations/0091_v360_approval_node_notifications.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.4 on 2019-09-11 13:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0090_v360_WFJT_prompts'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organization',
+            name='notification_templates_approvals',
+            field=models.ManyToManyField(blank=True, related_name='organization_notification_templates_for_approvals', to='main.NotificationTemplate'),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='notification_templates_approvals',
+            field=models.ManyToManyField(blank=True, related_name='workflowjobtemplate_notification_templates_for_approvals', to='main.NotificationTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='do_not_run',
+            field=models.BooleanField(default=False, help_text='Indicates that a job will not be created when True. Workflow runtime semantics will mark this True if the node is in a path that will decidedly not be ran. A value of False means the node may not run.'),
+        ),
+    ]

awx/main/migrations/0092_v360_webhook_mixin.py

@@ -0,0 +1,49 @@
+# Generated by Django 2.2.4 on 2019-09-12 14:49
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0091_v360_approval_node_notifications'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='jobtemplates', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='webhook_key',
+            field=models.CharField(blank=True, help_text='Shared secret that the webhook service will use to sign requests', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobtemplates', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='webhook_key',
+            field=models.CharField(blank=True, help_text='Shared secret that the webhook service will use to sign requests', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='launch_type',
+            field=models.CharField(choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency'), ('workflow', 'Workflow'), ('webhook', 'Webhook'), ('sync', 'Sync'), ('scm', 'SCM Update')], db_index=True, default='manual', editable=False, max_length=20),
+        ),
+    ]

awx/main/migrations/0093_v360_personal_access_tokens.py

@@ -0,0 +1,27 @@
+# Generated by Django 2.2.4 on 2019-09-12 14:50
+
+from django.db import migrations, models
+
+from awx.main.models import CredentialType
+from awx.main.utils.common import set_current_apps
+
+
+def setup_tower_managed_defaults(apps, schema_editor):
+    set_current_apps(apps)
+    CredentialType.setup_tower_managed_defaults()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0092_v360_webhook_mixin'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='kind',
+            field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External')], max_length=32),
+        ),
+        migrations.RunPython(setup_tower_managed_defaults),
+    ]

awx/main/migrations/0094_v360_webhook_mixin2.py

@@ -0,0 +1,44 @@
+# Generated by Django 2.2.4 on 2019-09-12 14:52
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0093_v360_personal_access_tokens'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='job',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='jobs', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='webhook_guid',
+            field=models.CharField(blank=True, help_text='Unique identifier of the event that triggered this webhook', max_length=128),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobs', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='webhook_guid',
+            field=models.CharField(blank=True, help_text='Unique identifier of the event that triggered this webhook', max_length=128),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+    ]

awx/main/migrations/0095_v360_increase_instance_version_length.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-04 00:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0094_v360_webhook_mixin2'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='version',
+            field=models.CharField(blank=True, max_length=120),
+        ),
+    ]

awx/main/migrations/0096_v360_container_groups.py

@@ -0,0 +1,38 @@
+# Generated by Django 2.2.4 on 2019-09-16 23:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+from awx.main.models import CredentialType
+from awx.main.utils.common import set_current_apps
+
+
+def create_new_credential_types(apps, schema_editor):
+    set_current_apps(apps)
+    CredentialType.setup_tower_managed_defaults()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0095_v360_increase_instance_version_length'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instancegroup',
+            name='credential',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instancegroups', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='pod_spec_override',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='kind',
+            field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External'), ('kubernetes', 'Kubernetes')], max_length=32),
+        ),
+        migrations.RunPython(create_new_credential_types)
+    ]

awx/main/migrations/0097_v360_workflowapproval_approved_or_denied_by.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.4 on 2019-10-11 15:40
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('main', '0096_v360_container_groups'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='workflowapproval',
+            name='approved_or_denied_by',
+            field=models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'workflowapproval', 'model_name': 'workflowapproval', 'app_label': 'main'}(class)s_approved+", to=settings.AUTH_USER_MODEL),
+        ),
+    ]

awx/main/migrations/0098_v360_rename_cyberark_aim_credential_type.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.4 on 2019-10-16 19:51
+
+from django.db import migrations
+from awx.main.models import CredentialType
+
+
+def update_cyberark_aim_name(apps, schema_editor):
+    CredentialType.setup_tower_managed_defaults()
+    aim_types = apps.get_model('main', 'CredentialType').objects.filter(
+        namespace='aim'
+    ).order_by('id')
+
+    if aim_types.count() == 2:
+        original, renamed = aim_types.all()
+        apps.get_model('main', 'Credential').objects.filter(
+            credential_type_id=original.id
+        ).update(
+            credential_type_id=renamed.id
+        )
+        original.delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0097_v360_workflowapproval_approved_or_denied_by'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_cyberark_aim_name)
+    ]

awx/main/migrations/0099_v361_license_cleanup.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def _cleanup_license_setting(apps, schema_editor):
+    Setting = apps.get_model('conf', 'Setting')
+    for license in Setting.objects.filter(key='LICENSE').all():
+        for k in ('rh_username', 'rh_password'):
+            license.value.pop(k, None)
+        license.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0098_v360_rename_cyberark_aim_credential_type'),
+    ]
+
+    operations = [migrations.RunPython(_cleanup_license_setting)]

awx/main/migrations/0100_v370_projectupdate_job_tags.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-11-01 18:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0099_v361_license_cleanup'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='projectupdate',
+            name='job_tags',
+            field=models.CharField(blank=True, default='', help_text='Parts of the project update playbook that will be run.', max_length=1024),
+        ),
+    ]

awx/main/migrations/_azure_credentials.py

@@ -1,15 +0,0 @@
-import logging
-
-from django.db.models import Q
-
-logger = logging.getLogger('awx.main.migrations')
-
-
-def remove_azure_credentials(apps, schema_editor):
-    '''Azure is not supported as of 3.2 and greater. Instead, azure_rm is
-    supported.
-    '''
-    Credential = apps.get_model('main', 'Credential')
-    logger.debug("Removing all Azure Credentials from database.")
-    Credential.objects.filter(kind='azure').delete()
-

awx/main/migrations/_credentialtypes.py

@@ -1,6 +1,4 @@
-from awx.main import utils
 from awx.main.models import CredentialType
-from awx.main.utils.encryption import encrypt_field, decrypt_field
 from django.db.models import Q
 
 
@@ -61,16 +59,6 @@ def _disassociate_non_insights_projects(apps, cred):
     apps.get_model('main', 'Project').objects.filter(~Q(scm_type='insights') & Q(credential=cred)).update(credential=None)
 
 
-def migrate_to_v2_credentials(apps, schema_editor):
-    # TODO: remove once legacy/EOL'd Towers no longer support this upgrade path
-    pass
-
-
-def migrate_job_credentials(apps, schema_editor):
-    # TODO: remove once legacy/EOL'd Towers no longer support this upgrade path
-    pass
-
-
 def add_vault_id_field(apps, schema_editor):
     # this is no longer necessary; schemas are defined in code
     pass
@@ -81,21 +69,11 @@ def remove_vault_id_field(apps, schema_editor):
     pass
 
 
-def create_rhv_tower_credtype(apps, schema_editor):
-    # this is no longer necessary; schemas are defined in code
-    pass
-
-
 def add_tower_verify_field(apps, schema_editor):
     # this is no longer necessary; schemas are defined in code
     pass
 
 
-def add_azure_cloud_environment_field(apps, schema_editor):
-    # this is no longer necessary; schemas are defined in code
-    pass
-
-
 def remove_become_methods(apps, schema_editor):
     # this is no longer necessary; schemas are defined in code
     pass

awx/main/migrations/_inventory_source.py

@@ -1,6 +1,5 @@
 import logging
 
-from django.db.models import Q
 from django.utils.encoding import smart_text
 
 from awx.main.utils.common import parse_yaml_or_json
@@ -8,64 +7,6 @@ from awx.main.utils.common import parse_yaml_or_json
 logger = logging.getLogger('awx.main.migrations')
 
 
-def remove_manual_inventory_sources(apps, schema_editor):
-    '''Previously we would automatically create inventory sources after
-    Group creation and we would use the parent Group as our interface for the user.
-    During that process we would create InventorySource that had a source of "manual".
-    '''
-    # TODO: use this in the 3.3 data migrations
-    InventorySource = apps.get_model('main', 'InventorySource')
-    # see models/inventory.py SOURCE_CHOICES - ('', _('Manual'))
-    logger.debug("Removing all Manual InventorySource from database.")
-    InventorySource.objects.filter(source='').delete()
-
-
-def remove_rax_inventory_sources(apps, schema_editor):
-    '''Rackspace inventory sources are not supported since 3.2, remove them.
-    '''
-    InventorySource = apps.get_model('main', 'InventorySource')
-    logger.debug("Removing all Rackspace InventorySource from database.")
-    InventorySource.objects.filter(source='rax').delete()
-
-
-def rename_inventory_sources(apps, schema_editor):
-    '''Rename existing InventorySource entries using the following format.
-    {{ inventory_source.name }} - {{ inventory.module }} - {{ number }}
-    The number will be incremented for each InventorySource for the organization.
-    '''
-    Organization = apps.get_model('main', 'Organization')
-    InventorySource = apps.get_model('main', 'InventorySource')
-
-    for org in Organization.objects.iterator():
-        for i, invsrc in enumerate(InventorySource.objects.filter(Q(inventory__organization=org) |
-                                                                  Q(deprecated_group__inventory__organization=org)).distinct().all()):
-
-            inventory = invsrc.deprecated_group.inventory if invsrc.deprecated_group else invsrc.inventory
-            name = '{0} - {1} - {2}'.format(invsrc.name, inventory.name, i)
-            logger.debug("Renaming InventorySource({0}) {1} -> {2}".format(
-                invsrc.pk, invsrc.name, name
-            ))
-            invsrc.name = name
-            invsrc.save()
-
-
-def remove_inventory_source_with_no_inventory_link(apps, schema_editor):
-    '''If we cannot determine the Inventory for which an InventorySource exists
-    we can safely remove it.
-    '''
-    InventorySource = apps.get_model('main', 'InventorySource')
-    logger.debug("Removing all InventorySource that have no link to an Inventory from database.")
-    InventorySource.objects.filter(Q(inventory__organization=None) & Q(deprecated_group__inventory=None)).delete()
-
-
-def remove_azure_inventory_sources(apps, schema_editor):
-    '''Azure inventory sources are not supported since 3.2, remove them.
-    '''
-    InventorySource = apps.get_model('main', 'InventorySource')
-    logger.debug("Removing all Azure InventorySource from database.")
-    InventorySource.objects.filter(source='azure').delete()
-
-
 def _get_instance_id(from_dict, new_id, default=''):
     '''logic mostly duplicated with inventory_import command Command._get_instance_id
     frozen in time here, for purposes of migrations

awx/main/migrations/_reencrypt.py

@@ -1,79 +1,12 @@
 import logging
-import json
-from django.utils.translation import ugettext_lazy as _
 
 from awx.conf.migrations._reencrypt import (
     decrypt_field,
-    should_decrypt_field,
 )
-from awx.main.utils.encryption import encrypt_field
-
-from awx.main.notifications.email_backend import CustomEmailBackend
-from awx.main.notifications.slack_backend import SlackBackend
-from awx.main.notifications.twilio_backend import TwilioBackend
-from awx.main.notifications.pagerduty_backend import PagerDutyBackend
-from awx.main.notifications.hipchat_backend import HipChatBackend
-from awx.main.notifications.mattermost_backend import MattermostBackend
-from awx.main.notifications.webhook_backend import WebhookBackend
-from awx.main.notifications.irc_backend import IrcBackend
 
 logger = logging.getLogger('awx.main.migrations')
 
-__all__ = ['replace_aesecb_fernet']
-
-
-NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend),
-                      ('slack', _('Slack'), SlackBackend),
-                      ('twilio', _('Twilio'), TwilioBackend),
-                      ('pagerduty', _('Pagerduty'), PagerDutyBackend),
-                      ('hipchat', _('HipChat'), HipChatBackend),
-                      ('mattermost', _('Mattermost'), MattermostBackend),
-                      ('webhook', _('Webhook'), WebhookBackend),
-                      ('irc', _('IRC'), IrcBackend)]
-
-
-PASSWORD_FIELDS = ('password', 'security_token', 'ssh_key_data', 'ssh_key_unlock',
-                   'become_password', 'vault_password', 'secret', 'authorize_password')
-
-
-def replace_aesecb_fernet(apps, schema_editor):
-    _notification_templates(apps)
-    _credentials(apps)
-    _unified_jobs(apps)
-
-
-def _notification_templates(apps):
-    NotificationTemplate = apps.get_model('main', 'NotificationTemplate')
-    for nt in NotificationTemplate.objects.all():
-        CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES])
-        notification_class = CLASS_FOR_NOTIFICATION_TYPE[nt.notification_type]
-        for field in filter(lambda x: notification_class.init_parameters[x]['type'] == "password",
-                            notification_class.init_parameters):
-            if should_decrypt_field(nt.notification_configuration[field]):
-                nt.notification_configuration[field] = decrypt_field(nt, 'notification_configuration', subfield=field)
-                nt.notification_configuration[field] = encrypt_field(nt, 'notification_configuration', subfield=field)
-        nt.save()
-
-
-def _credentials(apps):
-    for credential in apps.get_model('main', 'Credential').objects.all():
-        for field_name in PASSWORD_FIELDS:
-            value = getattr(credential, field_name)
-            if should_decrypt_field(value):
-                value = decrypt_field(credential, field_name)
-                setattr(credential, field_name, value)
-                setattr(credential, field_name, encrypt_field(credential, field_name))
-        credential.save()
-
-
-def _unified_jobs(apps):
-    UnifiedJob = apps.get_model('main', 'UnifiedJob')
-    for uj in UnifiedJob.objects.all():
-        if uj.start_args is not None:
-            if should_decrypt_field(uj.start_args):
-                uj.start_args = decrypt_field(uj, 'start_args')
-                uj.start_args = encrypt_field(uj, 'start_args')
-                uj.save()
+__all__ = []
 
 
 def blank_old_start_args(apps, schema_editor):
@@ -91,53 +24,3 @@ def blank_old_start_args(apps, schema_editor):
             logger.debug('Blanking job args for %s', uj.pk)
             uj.start_args = ''
             uj.save()
-
-
-def encrypt_survey_passwords(apps, schema_editor):
-    _encrypt_survey_passwords(
-        apps.get_model('main', 'Job'),
-        apps.get_model('main', 'JobTemplate'),
-        apps.get_model('main', 'WorkflowJob'),
-        apps.get_model('main', 'WorkflowJobTemplate'),
-    )
-
-
-def _encrypt_survey_passwords(Job, JobTemplate, WorkflowJob, WorkflowJobTemplate):
-    from awx.main.utils.encryption import encrypt_value
-    for _type in (JobTemplate, WorkflowJobTemplate):
-        for jt in _type.objects.exclude(survey_spec={}):
-            changed = False
-            if jt.survey_spec.get('spec', []):
-                for field in jt.survey_spec['spec']:
-                    if field.get('type') == 'password' and field.get('default', ''):
-                        default = field['default']
-                        if default.startswith('$encrypted$'):
-                            if default == '$encrypted$':
-                                # If you have a survey_spec with a literal
-                                # '$encrypted$' as the default, you have
-                                # encountered a known bug in awx/Tower
-                                # https://github.com/ansible/ansible-tower/issues/7800
-                                logger.error(
-                                    '{}.pk={} survey_spec has ambiguous $encrypted$ default for {}, needs attention...'.format(jt, jt.pk, field['variable'])
-                                )
-                                field['default'] = ''
-                                changed = True
-                            continue
-                        field['default'] = encrypt_value(field['default'], pk=None)
-                        changed = True
-            if changed:
-                jt.save()
-
-    for _type in (Job, WorkflowJob):
-        for job in _type.objects.defer('result_stdout_text').exclude(survey_passwords={}).iterator():
-            changed = False
-            for key in job.survey_passwords:
-                if key in job.extra_vars:
-                    extra_vars = json.loads(job.extra_vars)
-                    if not extra_vars.get(key, '') or extra_vars[key].startswith('$encrypted$'):
-                        continue
-                    extra_vars[key] = encrypt_value(extra_vars[key], pk=None)
-                    job.extra_vars = json.dumps(extra_vars)
-                    changed = True
-            if changed:
-                job.save()

awx/main/migrations/_scan_jobs.py

@@ -1,89 +1,9 @@
 import logging
 
-from django.utils.timezone import now
-from django.utils.text import slugify
-
-from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY
-from awx.main import utils
-
 
 logger = logging.getLogger('awx.main.migrations')
 
 
-def _create_fact_scan_project(ContentType, Project, org):
-    ct = ContentType.objects.get_for_model(Project)
-    name = u"Tower Fact Scan - {}".format(org.name if org else "No Organization")
-    proj = Project(name=name,
-                   scm_url='https://github.com/ansible/awx-facts-playbooks',
-                   scm_type='git',
-                   scm_update_on_launch=True,
-                   scm_update_cache_timeout=86400,
-                   organization=org,
-                   created=now(),
-                   modified=now(),
-                   polymorphic_ctype=ct)
-    proj.save()
-
-    slug_name = slugify(str(name)).replace(u'-', u'_')
-    proj.local_path = u'_%d__%s' % (int(proj.pk), slug_name)
-
-    proj.save()
-    return proj
-
-
-def _create_fact_scan_projects(ContentType, Project, orgs):
-    return {org.id : _create_fact_scan_project(ContentType, Project, org) for org in orgs}
-
-
-def _get_tower_scan_job_templates(JobTemplate):
-    return JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=True) \
-                              .prefetch_related('inventory__organization')
-
-
-def _get_orgs(Organization, job_template_ids):
-    return Organization.objects.filter(inventories__jobtemplates__in=job_template_ids).distinct()
-
-
-def _migrate_scan_job_templates(apps):
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Organization = apps.get_model('main', 'Organization')
-    ContentType = apps.get_model('contenttypes', 'ContentType')
-    Project = apps.get_model('main', 'Project')
-
-    project_no_org = None
-
-    # A scan job template with a custom project will retain the custom project.
-    JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=False).update(use_fact_cache=True, job_type=PERM_INVENTORY_DEPLOY)
-
-    # Scan jobs templates using Tower's default scan playbook will now point at
-    # the same playbook but in a github repo.
-    jts = _get_tower_scan_job_templates(JobTemplate)
-    if jts.count() == 0:
-        return
-
-    orgs = _get_orgs(Organization, jts.values_list('id'))
-    if orgs.count() == 0:
-        return
-
-    org_proj_map = _create_fact_scan_projects(ContentType, Project, orgs)
-    for jt in jts:
-        if jt.inventory and jt.inventory.organization:
-            jt.project_id = org_proj_map[jt.inventory.organization.id].id
-        # Job Templates without an Organization; through related Inventory
-        else:
-            if not project_no_org:
-                project_no_org = _create_fact_scan_project(ContentType, Project, None)
-            jt.project_id = project_no_org.id
-        jt.job_type = PERM_INVENTORY_DEPLOY
-        jt.playbook = "scan_facts.yml"
-        jt.use_fact_cache = True
-        jt.save()
-
-
-def migrate_scan_job_templates(apps, schema_editor):
-    _migrate_scan_job_templates(apps)
-
-
 def remove_scan_type_nodes(apps, schema_editor):
     WorkflowJobTemplateNode = apps.get_model('main', 'WorkflowJobTemplateNode')
     WorkflowJobNode = apps.get_model('main', 'WorkflowJobNode')

awx/main/models/ad_hoc_commands.py

@@ -150,6 +150,14 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
     def supports_isolation(cls):
         return True
 
+    @property
+    def is_containerized(self):
+        return bool(self.instance_group and self.instance_group.is_containerized)
+
+    @property
+    def can_run_containerized(self):
+        return True
+
     def get_absolute_url(self, request=None):
         return reverse('api:ad_hoc_command_detail', kwargs={'pk': self.pk}, request=request)
 

awx/main/models/base.py

@@ -295,7 +295,10 @@ class PrimordialModel(HasEditsMixin, CreatedModifiedModel):
 
     def __init__(self, *args, **kwargs):
         r = super(PrimordialModel, self).__init__(*args, **kwargs)
-        self._prior_values_store = self._get_fields_snapshot()
+        if self.pk:
+            self._prior_values_store = self._get_fields_snapshot()
+        else:
+            self._prior_values_store = {}
         return r
 
     def save(self, *args, **kwargs):

awx/main/models/credential/__init__.py

@@ -64,7 +64,7 @@ def build_safe_env(env):
     for k, v in safe_env.items():
         if k == 'AWS_ACCESS_KEY_ID':
             continue
-        elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET'):
+        elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET') and not k.startswith('ANSIBLE_GALAXY_SERVER'):
             continue
         elif hidden_re.search(k):
             safe_env[k] = HIDDEN_PASSWORD
@@ -86,6 +86,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
         unique_together = (('organization', 'name', 'credential_type'))
 
     PASSWORD_FIELDS = ['inputs']
+    FIELDS_TO_PRESERVE_AT_COPY = ['input_sources']
 
     credential_type = models.ForeignKey(
         'CredentialType',
@@ -135,6 +136,10 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     def cloud(self):
         return self.credential_type.kind == 'cloud'
 
+    @property
+    def kubernetes(self):
+        return self.credential_type.kind == 'kubernetes'
+
     def get_absolute_url(self, request=None):
         return reverse('api:credential_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -151,7 +156,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     @property
     def has_encrypted_ssh_key_data(self):
         try:
-            ssh_key_data = decrypt_field(self, 'ssh_key_data')
+            ssh_key_data = self.get_input('ssh_key_data')
         except AttributeError:
             return False
 
@@ -322,8 +327,10 @@ class CredentialType(CommonModelNameNotUnique):
         ('net', _('Network')),
         ('scm', _('Source Control')),
         ('cloud', _('Cloud')),
+        ('token', _('Personal Access Token')),
         ('insights', _('Insights')),
         ('external', _('External')),
+        ('kubernetes', _('Kubernetes')),
     )
 
     kind = models.CharField(
@@ -633,9 +640,6 @@ ManagedCredentialType(
             'secret': True,
             'ask_at_runtime': True
         }],
-        'dependencies': {
-            'ssh_key_unlock': ['ssh_key_data'],
-        }
     }
 )
 
@@ -667,9 +671,6 @@ ManagedCredentialType(
             'type': 'string',
             'secret': True
         }],
-        'dependencies': {
-            'ssh_key_unlock': ['ssh_key_data'],
-        }
     }
 )
 
@@ -738,7 +739,6 @@ ManagedCredentialType(
             'secret': True,
         }],
         'dependencies': {
-            'ssh_key_unlock': ['ssh_key_data'],
             'authorize_password': ['authorize'],
         },
         'required': ['username'],
@@ -975,6 +975,40 @@ ManagedCredentialType(
     }
 )
 
+ManagedCredentialType(
+    namespace='github_token',
+    kind='token',
+    name=ugettext_noop('GitHub Personal Access Token'),
+    managed_by_tower=True,
+    inputs={
+        'fields': [{
+            'id': 'token',
+            'label': ugettext_noop('Token'),
+            'type': 'string',
+            'secret': True,
+            'help_text': ugettext_noop('This token needs to come from your profile settings in GitHub')
+        }],
+        'required': ['token'],
+    },
+)
+
+ManagedCredentialType(
+    namespace='gitlab_token',
+    kind='token',
+    name=ugettext_noop('GitLab Personal Access Token'),
+    managed_by_tower=True,
+    inputs={
+        'fields': [{
+            'id': 'token',
+            'label': ugettext_noop('Token'),
+            'type': 'string',
+            'secret': True,
+            'help_text': ugettext_noop('This token needs to come from your profile settings in GitLab')
+        }],
+        'required': ['token'],
+    },
+)
+
 ManagedCredentialType(
     namespace='insights',
     kind='insights',
@@ -1090,6 +1124,38 @@ ManagedCredentialType(
 )
 
 
+ManagedCredentialType(
+    namespace='kubernetes_bearer_token',
+    kind='kubernetes',
+    name=ugettext_noop('OpenShift or Kubernetes API Bearer Token'),
+    inputs={
+        'fields': [{
+            'id': 'host',
+            'label': ugettext_noop('OpenShift or Kubernetes API Endpoint'),
+            'type': 'string',
+            'help_text': ugettext_noop('The OpenShift or Kubernetes API Endpoint to authenticate with.')
+        },{
+            'id': 'bearer_token',
+            'label': ugettext_noop('API authentication bearer token.'),
+            'type': 'string',
+            'secret': True,
+        },{
+            'id': 'verify_ssl',
+            'label': ugettext_noop('Verify SSL'),
+            'type': 'boolean',
+            'default': True,
+        },{
+            'id': 'ssl_ca_cert',
+            'label': ugettext_noop('Certificate Authority data'),
+            'type': 'string',
+            'secret': True,
+            'multiline': True,
+        }],
+        'required': ['host', 'bearer_token'],
+    }
+)
+
+
 class CredentialInputSource(PrimordialModel):
 
     class Meta:
@@ -1097,6 +1163,8 @@ class CredentialInputSource(PrimordialModel):
         unique_together = (('target_credential', 'input_field_name'),)
         ordering = ('target_credential', 'source_credential', 'input_field_name',)
 
+    FIELDS_TO_PRESERVE_AT_COPY = ['source_credential', 'metadata', 'input_field_name']
+
     target_credential = models.ForeignKey(
         'Credential',
         related_name='input_sources',

awx/main/models/events.py

@@ -83,6 +83,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     #       - runner_on*
     #     - playbook_on_task_start (once for each task within a play)
     #       - runner_on_failed
+    #       - runner_on_start
     #       - runner_on_ok
     #       - runner_on_error (not used for v2)
     #       - runner_on_skipped
@@ -102,6 +103,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     EVENT_TYPES = [
         # (level, event, verbose name, failed)
         (3, 'runner_on_failed', _('Host Failed'), True),
+        (3, 'runner_on_start', _('Host Started'), False),
         (3, 'runner_on_ok', _('Host OK'), False),
         (3, 'runner_on_error', _('Host Failure'), True),
         (3, 'runner_on_skipped', _('Host Skipped'), False),
@@ -322,7 +324,10 @@ class BasePlaybookEvent(CreatedModifiedModel):
             kwargs.pop('created', None)
 
         sanitize_event_keys(kwargs, cls.VALID_KEYS)
+        workflow_job_id = kwargs.pop('workflow_job_id', None)
         job_event = cls.objects.create(**kwargs)
+        if workflow_job_id:
+            setattr(job_event, 'workflow_job_id', workflow_job_id)
         analytics_logger.info('Event data saved.', extra=dict(python_objects=dict(job_event=job_event)))
         return job_event
 
@@ -394,7 +399,7 @@ class JobEvent(BasePlaybookEvent):
     An event/message logged from the callback when running a job.
     '''
 
-    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id']
+    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'
@@ -528,7 +533,7 @@ class JobEvent(BasePlaybookEvent):
 
 class ProjectUpdateEvent(BasePlaybookEvent):
 
-    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['project_update_id']
+    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['project_update_id', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'
@@ -614,6 +619,7 @@ class BaseCommandEvent(CreatedModifiedModel):
             kwargs.pop('created', None)
 
         sanitize_event_keys(kwargs, cls.VALID_KEYS)
+        kwargs.pop('workflow_job_id', None)
         event = cls.objects.create(**kwargs)
         if isinstance(event, AdHocCommandEvent):
             analytics_logger.info(
@@ -637,7 +643,7 @@ class BaseCommandEvent(CreatedModifiedModel):
 
 class AdHocCommandEvent(BaseCommandEvent):
 
-    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['ad_hoc_command_id', 'event']
+    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['ad_hoc_command_id', 'event', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'
@@ -745,7 +751,7 @@ class AdHocCommandEvent(BaseCommandEvent):
 
 class InventoryUpdateEvent(BaseCommandEvent):
 
-    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['inventory_update_id']
+    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['inventory_update_id', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'

awx/main/models/ha.py

@@ -18,7 +18,7 @@ from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
 from awx.main.managers import InstanceManager, InstanceGroupManager
 from awx.main.fields import JSONField
-from awx.main.models.base import BaseModel, HasEditsMixin
+from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.utils import get_cpu_capacity, get_mem_capacity, get_system_task_capacity
 from awx.main.models.mixins import RelatedJobsMixin
@@ -59,7 +59,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         null=True,
         editable=False,
     )
-    version = models.CharField(max_length=24, blank=True)
+    version = models.CharField(max_length=120, blank=True)
     capacity = models.PositiveIntegerField(
         default=100,
         editable=False,
@@ -176,6 +176,18 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
         null=True,
         on_delete=models.CASCADE
     )
+    credential = models.ForeignKey(
+        'Credential',
+        related_name='%(class)ss',
+        blank=True,
+        null=True,
+        default=None,
+        on_delete=models.SET_NULL,
+    )
+    pod_spec_override = prevent_search(models.TextField(
+        blank=True,
+        default='',
+    ))
     policy_instance_percentage = models.IntegerField(
         default=0,
         help_text=_("Percentage of Instances to automatically assign to this group")
@@ -218,6 +230,10 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
     def is_isolated(self):
         return bool(self.controller)
 
+    @property
+    def is_containerized(self):
+        return bool(self.credential and self.credential.kubernetes)
+
     '''
     RelatedJobsMixin
     '''
@@ -254,6 +270,11 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
                                       .filter(capacity__gt=0, enabled=True)
                                       .values_list('hostname', flat=True)))
 
+    def set_default_policy_fields(self):
+        self.policy_instance_list = []
+        self.policy_instance_minimum = 0
+        self.policy_instance_percentage = 0
+
 
 class TowerScheduleState(SingletonModel):
     schedule_last_run = models.DateTimeField(auto_now_add=True)
@@ -271,7 +292,10 @@ def schedule_policy_task():
 @receiver(post_save, sender=InstanceGroup)
 def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs):
     if created or instance.has_policy_changes():
-        schedule_policy_task()
+        if not instance.is_containerized:
+            schedule_policy_task()
+    elif created or instance.is_containerized:
+        instance.set_default_policy_fields()
 
 
 @receiver(post_save, sender=Instance)
@@ -282,7 +306,8 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
 
 @receiver(post_delete, sender=InstanceGroup)
 def on_instance_group_deleted(sender, instance, using, **kwargs):
-    schedule_policy_task()
+    if not instance.is_containerized:
+        schedule_policy_task()
 
 
 @receiver(post_delete, sender=Instance)

awx/main/models/inventory.py

@@ -61,6 +61,7 @@ from awx.main.models.notifications import (
 )
 from awx.main.models.credential.injectors import _openstack_data
 from awx.main.utils import _inventory_updates, region_sorting, get_licenser
+from awx.main.utils.safe_yaml import sanitize_jinja
 
 
 __all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate',
@@ -754,6 +755,13 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
                 update_host_smart_inventory_memberships.delay()
             connection.on_commit(on_commit)
 
+    def clean_name(self):
+        try:
+            sanitize_jinja(self.name)
+        except ValueError as e:
+            raise ValidationError(str(e) + ": {}".format(self.name))
+        return self.name
+
     def save(self, *args, **kwargs):
         self._update_host_smart_inventory_memeberships()
         super(Host, self).save(*args, **kwargs)
@@ -1501,7 +1509,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in InventorySourceOptions._meta.fields) | set(
-            ['name', 'description', 'schedule', 'credentials', 'inventory']
+            ['name', 'description', 'credentials', 'inventory']
         )
 
     def save(self, *args, **kwargs):
@@ -2036,9 +2044,25 @@ class azure_rm(PluginFileInjector):
         for key, loc in old_filterables:
             value = source_vars.get(key, None)
             if value and isinstance(value, str):
-                user_filters.append('{} not in {}'.format(
-                    loc, value.split(',')
-                ))
+                # tags can be list of key:value pairs
+                #  e.g. 'Creator:jmarshall, peanutbutter:jelly'
+                # or tags can be a list of keys
+                #  e.g. 'Creator, peanutbutter'
+                if key == "tags":
+                    # grab each key value pair
+                    for kvpair in value.split(','):
+                        # split into key and value
+                        kv = kvpair.split(':')
+                        # filter out any host that does not have key
+                        # in their tags.keys() variable
+                        user_filters.append('"{}" not in tags.keys()'.format(kv[0].strip()))
+                        # if a value is provided, check that the key:value pair matches
+                        if len(kv) > 1:
+                            user_filters.append('tags["{}"] != "{}"'.format(kv[0].strip(), kv[1].strip()))
+                else:
+                    user_filters.append('{} not in {}'.format(
+                        loc, value.split(',')
+                    ))
         if user_filters:
             ret.setdefault('exclude_host_filters', [])
             ret['exclude_host_filters'].extend(user_filters)

awx/main/models/jobs.py

@@ -39,7 +39,7 @@ from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin,
 )
-from awx.main.utils import parse_yaml_or_json, getattr_dne
+from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
 from awx.main.fields import ImplicitRoleField, JSONField, AskForField
 from awx.main.models.mixins import (
     ResourceMixin,
@@ -48,6 +48,8 @@ from awx.main.models.mixins import (
     TaskManagerJobMixin,
     CustomVirtualEnvMixin,
     RelatedJobsMixin,
+    WebhookMixin,
+    WebhookTemplateMixin,
 )
 
 
@@ -187,7 +189,7 @@ class JobOptions(BaseModel):
         return needed
 
 
-class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin):
+class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin, WebhookTemplateMixin):
     '''
     A job template is a reusable job definition for applying a project (with
     playbook) to an inventory source with a given credential.
@@ -271,7 +273,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in JobOptions._meta.fields) | set(
-            ['name', 'description', 'schedule', 'survey_passwords', 'labels', 'credentials',
+            ['name', 'description', 'survey_passwords', 'labels', 'credentials',
              'job_slice_number', 'job_slice_count']
         )
 
@@ -484,7 +486,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         return UnifiedJob.objects.filter(unified_job_template=self)
 
 
-class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin):
+class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin, WebhookMixin):
     '''
     A job applies a project (with playbook) to an inventory source with a given
     credential.  It represents a single invocation of ansible-playbook with the
@@ -627,15 +629,17 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
 
     @property
     def task_impact(self):
-        # NOTE: We sorta have to assume the host count matches and that forks default to 5
-        from awx.main.models.inventory import Host
         if self.launch_type == 'callback':
             count_hosts = 2
         else:
-            count_hosts = Host.objects.filter(inventory__jobs__pk=self.pk).count()
-            if self.job_slice_count > 1:
-                # Integer division intentional
-                count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count
+            # If for some reason we can't count the hosts then lets assume the impact as forks
+            if self.inventory is not None:
+                count_hosts = self.inventory.total_hosts
+                if self.job_slice_count > 1:
+                    # Integer division intentional
+                    count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count
+            else:
+                count_hosts = 5 if self.forks == 0 else self.forks
         return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1
 
     @property
@@ -666,6 +670,14 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     def processed_hosts(self):
         return self._get_hosts(job_host_summaries__processed__gt=0)
 
+    @property
+    def ignored_hosts(self):
+        return self._get_hosts(job_host_summaries__ignored__gt=0)
+
+    @property
+    def rescued_hosts(self):
+        return self._get_hosts(job_host_summaries__rescued__gt=0)
+
     def notification_data(self, block=5):
         data = super(Job, self).notification_data()
         all_hosts = {}
@@ -684,7 +696,9 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
                                           failures=h.failures,
                                           ok=h.ok,
                                           processed=h.processed,
-                                          skipped=h.skipped)  # TODO: update with rescued, ignored (see https://github.com/ansible/awx/issues/4394)
+                                          skipped=h.skipped,
+                                          rescued=h.rescued,
+                                          ignored=h.ignored)
         data.update(dict(inventory=self.inventory.name if self.inventory else None,
                          project=self.project.name if self.project else None,
                          playbook=self.playbook,
@@ -706,6 +720,14 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
             return "$hidden due to Ansible no_log flag$"
         return artifacts
 
+    @property
+    def can_run_containerized(self):
+        return any([ig for ig in self.preferred_instance_groups if ig.is_containerized])
+
+    @property
+    def is_containerized(self):
+        return bool(self.instance_group and self.instance_group.is_containerized)
+
     @property
     def preferred_instance_groups(self):
         if self.project is not None and self.project.organization is not None:
@@ -829,25 +851,6 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
                 host.save()
 
 
-# Add on aliases for the non-related-model fields
-class NullablePromptPsuedoField(object):
-    """
-    Interface for psuedo-property stored in `char_prompts` dict
-    Used in LaunchTimeConfig and submodels
-    """
-    def __init__(self, field_name):
-        self.field_name = field_name
-
-    def __get__(self, instance, type=None):
-        return instance.char_prompts.get(self.field_name, None)
-
-    def __set__(self, instance, value):
-        if value in (None, {}):
-            instance.char_prompts.pop(self.field_name, None)
-        else:
-            instance.char_prompts[self.field_name] = value
-
-
 class LaunchTimeConfigBase(BaseModel):
     '''
     Needed as separate class from LaunchTimeConfig because some models
@@ -868,6 +871,7 @@ class LaunchTimeConfigBase(BaseModel):
         null=True,
         default=None,
         on_delete=models.SET_NULL,
+        help_text=_('Inventory applied as a prompt, assuming job template prompts for inventory')
     )
     # All standard fields are stored in this dictionary field
     # This is a solution to the nullable CharField problem, specific to prompting
@@ -896,6 +900,9 @@ class LaunchTimeConfigBase(BaseModel):
                         data[prompt_name] = self.display_extra_vars()
                     else:
                         data[prompt_name] = self.extra_vars
+                    # Depending on model, field type may save and return as string
+                    if isinstance(data[prompt_name], str):
+                        data[prompt_name] = parse_yaml_or_json(data[prompt_name])
                 if self.survey_passwords and not display:
                     data['survey_passwords'] = self.survey_passwords
             else:
@@ -904,21 +911,14 @@ class LaunchTimeConfigBase(BaseModel):
                     data[prompt_name] = prompt_val
         return data
 
-    def display_extra_vars(self):
-        '''
-        Hides fields marked as passwords in survey.
-        '''
-        if self.survey_passwords:
-            extra_vars = parse_yaml_or_json(self.extra_vars).copy()
-            for key, value in self.survey_passwords.items():
-                if key in extra_vars:
-                    extra_vars[key] = value
-            return extra_vars
-        else:
-            return self.extra_vars
 
-    def display_extra_data(self):
-        return self.display_extra_vars()
+for field_name in JobTemplate.get_ask_mapping().keys():
+    if field_name == 'extra_vars':
+        continue
+    try:
+        LaunchTimeConfigBase._meta.get_field(field_name)
+    except FieldDoesNotExist:
+        setattr(LaunchTimeConfigBase, field_name, NullablePromptPseudoField(field_name))
 
 
 class LaunchTimeConfig(LaunchTimeConfigBase):
@@ -953,14 +953,21 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
     def extra_vars(self, extra_vars):
         self.extra_data = extra_vars
 
+    def display_extra_vars(self):
+        '''
+        Hides fields marked as passwords in survey.
+        '''
+        if hasattr(self, 'survey_passwords') and self.survey_passwords:
+            extra_vars = parse_yaml_or_json(self.extra_vars).copy()
+            for key, value in self.survey_passwords.items():
+                if key in extra_vars:
+                    extra_vars[key] = value
+            return extra_vars
+        else:
+            return self.extra_vars
 
-for field_name in JobTemplate.get_ask_mapping().keys():
-    if field_name == 'extra_vars':
-        continue
-    try:
-        LaunchTimeConfig._meta.get_field(field_name)
-    except FieldDoesNotExist:
-        setattr(LaunchTimeConfig, field_name, NullablePromptPsuedoField(field_name))
+    def display_extra_data(self):
+        return self.display_extra_vars()
 
 
 class JobLaunchConfig(LaunchTimeConfig):
@@ -1099,8 +1106,8 @@ class SystemJobOptions(BaseModel):
     SYSTEM_JOB_TYPE = [
         ('cleanup_jobs', _('Remove jobs older than a certain number of days')),
         ('cleanup_activitystream', _('Remove activity stream entries older than a certain number of days')),
-        ('clearsessions', _('Removes expired browser sessions from the database')),
-        ('cleartokens', _('Removes expired OAuth 2 access tokens and refresh tokens'))
+        ('cleanup_sessions', _('Removes expired browser sessions from the database')),
+        ('cleanup_tokens', _('Removes expired OAuth 2 access tokens and refresh tokens'))
     ]
 
     class Meta:
@@ -1175,18 +1182,19 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
             for key in unallowed_vars:
                 rejected[key] = data.pop(key)
 
-        if 'days' in data:
-            try:
-                if type(data['days']) is bool:
-                    raise ValueError
-                if float(data['days']) != int(data['days']):
-                    raise ValueError
-                days = int(data['days'])
-                if days < 0:
-                    raise ValueError
-            except ValueError:
-                errors_list.append(_("days must be a positive integer."))
-                rejected['days'] = data.pop('days')
+        if self.job_type in ('cleanup_jobs', 'cleanup_activitystream'):
+            if 'days' in data:
+                try:
+                    if isinstance(data['days'], (bool, type(None))):
+                        raise ValueError
+                    if float(data['days']) != int(data['days']):
+                        raise ValueError
+                    days = int(data['days'])
+                    if days < 0:
+                        raise ValueError
+                except ValueError:
+                    errors_list.append(_("days must be a positive integer."))
+                    rejected['days'] = data.pop('days')
 
         if errors_list:
             errors['extra_vars'] = errors_list

awx/main/models/mixins.py

@@ -1,31 +1,37 @@
 # Python
-import os
-import json
 from copy import copy, deepcopy
+import json
+import logging
+import os
 
+import requests
 
 # Django
 from django.apps import apps
 from django.conf import settings
-from django.db import models
-from django.contrib.contenttypes.models import ContentType
 from django.contrib.auth.models import User # noqa
-from django.utils.translation import ugettext_lazy as _
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
+from django.db import models
 from django.db.models.query import QuerySet
+from django.utils.crypto import get_random_string
+from django.utils.translation import ugettext_lazy as _
 
 # AWX
 from awx.main.models.base import prevent_search
 from awx.main.models.rbac import (
     Role, RoleAncestorEntry, get_roles_on_resource
 )
-from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices
+from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
 from awx.main.fields import JSONField, AskForField
 from awx.main.constants import ACTIVE_STATES
 
 
+logger = logging.getLogger('awx.main.models.mixins')
+
+
 __all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin',
            'TaskManagerUnifiedJobMixin', 'TaskManagerJobMixin', 'TaskManagerProjectUpdateMixin',
            'TaskManagerInventoryUpdateMixin', 'CustomVirtualEnvMixin']
@@ -247,7 +253,7 @@ class SurveyJobTemplateMixin(models.Model):
                 else:
                     choice_list = copy(survey_element['choices'])
                     if isinstance(choice_list, str):
-                        choice_list = choice_list.split('\n')
+                        choice_list = [choice for choice in choice_list.splitlines() if choice.strip() != '']
                     for val in data[survey_element['variable']]:
                         if val not in choice_list:
                             errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'],
@@ -255,7 +261,7 @@ class SurveyJobTemplateMixin(models.Model):
         elif survey_element['type'] == 'multiplechoice':
             choice_list = copy(survey_element['choices'])
             if isinstance(choice_list, str):
-                choice_list = choice_list.split('\n')
+                choice_list = [choice for choice in choice_list.splitlines() if choice.strip() != '']
             if survey_element['variable'] in data:
                 if data[survey_element['variable']] not in choice_list:
                     errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']],
@@ -483,3 +489,139 @@ class RelatedJobsMixin(object):
             raise RuntimeError("Programmer error. Expected _get_active_jobs() to return a QuerySet.")
 
         return [dict(id=t[0], type=mapping[t[1]]) for t in jobs.values_list('id', 'polymorphic_ctype_id')]
+
+
+class WebhookTemplateMixin(models.Model):
+    class Meta:
+        abstract = True
+
+    SERVICES = [
+        ('github', "GitHub"),
+        ('gitlab', "GitLab"),
+    ]
+
+    webhook_service = models.CharField(
+        max_length=16,
+        choices=SERVICES,
+        blank=True,
+        help_text=_('Service that webhook requests will be accepted from')
+    )
+    webhook_key = prevent_search(models.CharField(
+        max_length=64,
+        blank=True,
+        help_text=_('Shared secret that the webhook service will use to sign requests')
+    ))
+    webhook_credential = models.ForeignKey(
+        'Credential',
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name='%(class)ss',
+        help_text=_('Personal Access Token for posting back the status to the service API')
+    )
+
+    def rotate_webhook_key(self):
+        self.webhook_key = get_random_string(length=50)
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get('update_fields')
+
+        if not self.pk or self._values_have_edits({'webhook_service': self.webhook_service}):
+            if self.webhook_service:
+                self.rotate_webhook_key()
+            else:
+                self.webhook_key = ''
+
+            if update_fields and 'webhook_service' in update_fields:
+                update_fields.add('webhook_key')
+
+        super().save(*args, **kwargs)
+
+
+class WebhookMixin(models.Model):
+    class Meta:
+        abstract = True
+
+    SERVICES = WebhookTemplateMixin.SERVICES
+
+    webhook_service = models.CharField(
+        max_length=16,
+        choices=SERVICES,
+        blank=True,
+        help_text=_('Service that webhook requests will be accepted from')
+    )
+    webhook_credential = models.ForeignKey(
+        'Credential',
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name='%(class)ss',
+        help_text=_('Personal Access Token for posting back the status to the service API')
+    )
+    webhook_guid = models.CharField(
+        blank=True,
+        max_length=128,
+        help_text=_('Unique identifier of the event that triggered this webhook')
+    )
+
+    def update_webhook_status(self, status):
+        if not self.webhook_credential:
+            logger.debug("No credential configured to post back webhook status, skipping.")
+            return
+
+        status_api = self.extra_vars_dict.get('tower_webhook_status_api')
+        if not status_api:
+            logger.debug("Webhook event did not have a status API endpoint associated, skipping.")
+            return
+
+        service_header = {
+            'github': ('Authorization', 'token {}'),
+            'gitlab': ('PRIVATE-TOKEN', '{}'),
+        }
+        service_statuses = {
+            'github': {
+                'pending': 'pending',
+                'successful': 'success',
+                'failed': 'failure',
+                'canceled': 'failure',  # GitHub doesn't have a 'canceled' status :(
+                'error': 'error',
+            },
+            'gitlab': {
+                'pending': 'pending',
+                'running': 'running',
+                'successful': 'success',
+                'failed': 'failed',
+                'error': 'failed',  # GitLab doesn't have an 'error' status distinct from 'failed' :(
+                'canceled': 'canceled',
+            },
+        }
+
+        statuses = service_statuses[self.webhook_service]
+        if status not in statuses:
+            logger.debug("Skipping webhook job status change: '{}'".format(status))
+            return
+        try:
+            license_type = get_licenser().validate().get('license_type')
+            data = {
+                'state': statuses[status],
+                'context': 'ansible/awx' if license_type == 'open' else 'ansible/tower',
+                'target_url': self.get_ui_url(),
+            }
+            k, v = service_header[self.webhook_service]
+            headers = {
+                k: v.format(self.webhook_credential.get_input('token')),
+                'Content-Type': 'application/json'
+            }
+            response = requests.post(status_api, data=json.dumps(data), headers=headers, timeout=30)
+        except Exception:
+            logger.exception("Posting webhook status caused an error.")
+            return
+
+        if response.status_code < 400:
+            logger.debug("Webhook status update sent.")
+        else:
+            logger.error(
+                "Posting webhook status failed, code: {}\n"
+                "{}\n"
+                "Payload sent: {}".format(response.status_code, response.text, json.dumps(data))
+            )

awx/main/models/notifications.py

@@ -17,7 +17,7 @@ from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel
+from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
 from awx.main.utils import encrypt_field, decrypt_field, set_environ
 from awx.main.notifications.email_backend import CustomEmailBackend
 from awx.main.notifications.slack_backend import SlackBackend
@@ -70,10 +70,10 @@ class NotificationTemplate(CommonModelNameNotUnique):
         choices=NOTIFICATION_TYPE_CHOICES,
     )
 
-    notification_configuration = JSONField(blank=False)
+    notification_configuration = prevent_search(JSONField(blank=False))
 
     def default_messages():
-        return {'started': None, 'success': None, 'error': None}
+        return {'started': None, 'success': None, 'error': None, 'workflow_approval': None}
 
     messages = JSONField(
         null=True,
@@ -92,25 +92,6 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def get_message(self, condition):
         return self.messages.get(condition, {})
 
-    def build_notification_message(self, event_type, context):
-        env = sandbox.ImmutableSandboxedEnvironment()
-        templates = self.get_message(event_type)
-        msg_template = templates.get('message', {})
-
-        try:
-            notification_subject = env.from_string(msg_template).render(**context)
-        except (TemplateSyntaxError, UndefinedError, SecurityError):
-            notification_subject = ''
-
-
-        msg_body = templates.get('body', {})
-        try:
-            notification_body = env.from_string(msg_body).render(**context)
-        except (TemplateSyntaxError, UndefinedError, SecurityError):
-            notification_body = ''
-
-        return (notification_subject, notification_body)
-
     def get_absolute_url(self, request=None):
         return reverse('api:notification_template_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -128,19 +109,34 @@ class NotificationTemplate(CommonModelNameNotUnique):
             old_messages = old_nt.messages
             new_messages = self.messages
 
+            def merge_messages(local_old_messages, local_new_messages, local_event):
+                if local_new_messages.get(local_event, {}) and local_old_messages.get(local_event, {}):
+                    local_old_event_msgs = local_old_messages[local_event]
+                    local_new_event_msgs = local_new_messages[local_event]
+                    for msg_type in ['message', 'body']:
+                        if msg_type not in local_new_event_msgs and local_old_event_msgs.get(msg_type, None):
+                            local_new_event_msgs[msg_type] = local_old_event_msgs[msg_type]
             if old_messages is not None and new_messages is not None:
-                for event in ['started', 'success', 'error']:
+                for event in ('started', 'success', 'error', 'workflow_approval'):
                     if not new_messages.get(event, {}) and old_messages.get(event, {}):
                         new_messages[event] = old_messages[event]
                         continue
-                    if new_messages.get(event, {}) and old_messages.get(event, {}):
-                        old_event_msgs = old_messages[event]
-                        new_event_msgs = new_messages[event]
-                        for msg_type in ['message', 'body']:
-                            if msg_type not in new_event_msgs and old_event_msgs.get(msg_type, None):
-                                new_event_msgs[msg_type] = old_event_msgs[msg_type]
+
+                    if event == 'workflow_approval' and old_messages.get('workflow_approval', None):
+                        new_messages.setdefault('workflow_approval', {})
+                        for subevent in ('running', 'approved', 'timed_out', 'denied'):
+                            old_wfa_messages = old_messages['workflow_approval']
+                            new_wfa_messages = new_messages['workflow_approval']
+                            if not new_wfa_messages.get(subevent, {}) and old_wfa_messages.get(subevent, {}):
+                                new_wfa_messages[subevent] = old_wfa_messages[subevent]
+                                continue
+                            if old_wfa_messages:
+                                merge_messages(old_wfa_messages, new_wfa_messages, subevent)
+                    else:
+                        merge_messages(old_messages, new_messages, event)
                     new_messages.setdefault(event, None)
 
+
         for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
                             self.notification_class.init_parameters):
             if self.notification_configuration[field].startswith("$encrypted$"):
@@ -169,12 +165,12 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def recipients(self):
         return self.notification_configuration[self.notification_class.recipient_parameter]
 
-    def generate_notification(self, subject, message):
+    def generate_notification(self, msg, body):
         notification = Notification(notification_template=self,
                                     notification_type=self.notification_type,
                                     recipients=smart_str(self.recipients),
-                                    subject=subject,
-                                    body=message)
+                                    subject=msg,
+                                    body=body)
         notification.save()
         return notification
 
@@ -273,6 +269,7 @@ class JobNotificationMixin(object):
                             'timeout', 'use_fact_cache', 'launch_type', 'status', 'failed', 'started', 'finished',
                             'elapsed', 'job_explanation', 'execution_node', 'controller_node', 'allow_simultaneous',
                             'scm_revision', 'diff_mode', 'job_slice_number', 'job_slice_count', 'custom_virtualenv',
+                            'approval_status', 'approval_node_name', 'workflow_url',
                             {'host_status_counts': ['skipped', 'ok', 'changed', 'failures', 'dark']},
                             {'playbook_counts': ['play_count', 'task_count']},
                             {'summary_fields': [{'inventory': ['id', 'name', 'description', 'has_active_failures',
@@ -370,7 +367,10 @@ class JobNotificationMixin(object):
                            'verbosity': 0},
                    'job_friendly_name': 'Job',
                    'url': 'https://towerhost/#/jobs/playbook/1010',
-                   'job_summary_dict': """{'url': 'https://towerhost/$/jobs/playbook/13',
+                   'approval_status': 'approved',
+                   'approval_node_name': 'Approve Me',
+                   'workflow_url': 'https://towerhost/#/workflows/1010',
+                   'job_metadata': """{'url': 'https://towerhost/$/jobs/playbook/13',
  'traceback': '',
  'status': 'running',
  'started': '2019-08-07T21:46:38.362630+00:00',
@@ -389,14 +389,14 @@ class JobNotificationMixin(object):
         return context
 
     def context(self, serialized_job):
-        """Returns a context that can be used for rendering notification messages.
-        Context contains whitelisted content retrieved from a serialized job object
+        """Returns a dictionary that can be used for rendering notification messages.
+        The context will contain whitelisted content retrieved from a serialized job object
         (see JobNotificationMixin.JOB_FIELDS_WHITELIST), the job's friendly name,
         and a url to the job run."""
         context = {'job': {},
                    'job_friendly_name': self.get_notification_friendly_name(),
                    'url': self.get_ui_url(),
-                   'job_summary_dict': json.dumps(self.notification_data(), indent=4)}
+                   'job_metadata': json.dumps(self.notification_data(), indent=4)}
 
         def build_context(node, fields, whitelisted_fields):
             for safe_field in whitelisted_fields:
@@ -434,32 +434,33 @@ class JobNotificationMixin(object):
         context = self.context(job_serialization)
 
         msg_template = body_template = None
+        msg = body = ''
 
+        # Use custom template if available
         if nt.messages:
-            templates = nt.messages.get(self.STATUS_TO_TEMPLATE_TYPE[status], {}) or {}
-            msg_template = templates.get('message', {})
-            body_template = templates.get('body', {})
+            template = nt.messages.get(self.STATUS_TO_TEMPLATE_TYPE[status], {}) or {}
+            msg_template = template.get('message', None)
+            body_template = template.get('body', None)
+        # If custom template not provided, look up default template
+        default_template = nt.notification_class.default_messages[self.STATUS_TO_TEMPLATE_TYPE[status]]
+        if not msg_template:
+            msg_template = default_template.get('message', None)
+        if not body_template:
+            body_template = default_template.get('body', None)
 
         if msg_template:
             try:
-                notification_subject = env.from_string(msg_template).render(**context)
+                msg = env.from_string(msg_template).render(**context)
             except (TemplateSyntaxError, UndefinedError, SecurityError):
-                notification_subject = ''
-        else:
-            notification_subject = u"{} #{} '{}' {}: {}".format(self.get_notification_friendly_name(),
-                                                                self.id,
-                                                                self.name,
-                                                                status,
-                                                                self.get_ui_url())
-        notification_body = self.notification_data()
-        notification_body['friendly_name'] = self.get_notification_friendly_name()
+                msg = ''
+
         if body_template:
             try:
-                notification_body['body'] = env.from_string(body_template).render(**context)
+                body = env.from_string(body_template).render(**context)
             except (TemplateSyntaxError, UndefinedError, SecurityError):
-                notification_body['body'] = ''
+                body = ''
 
-        return (notification_subject, notification_body)
+        return (msg, body)
 
     def send_notification_templates(self, status):
         from awx.main.tasks import send_notifications  # avoid circular import
@@ -475,16 +476,13 @@ class JobNotificationMixin(object):
             return
 
         for nt in set(notification_templates.get(self.STATUS_TO_TEMPLATE_TYPE[status], [])):
-            try:
-                (notification_subject, notification_body) = self.build_notification_message(nt, status)
-            except AttributeError:
-                raise NotImplementedError("build_notification_message() does not exist" % status)
+            (msg, body) = self.build_notification_message(nt, status)
 
             # Use kwargs to force late-binding
             # https://stackoverflow.com/a/3431699/10669572
-            def send_it(local_nt=nt, local_subject=notification_subject, local_body=notification_body):
+            def send_it(local_nt=nt, local_msg=msg, local_body=body):
                 def _func():
-                    send_notifications.delay([local_nt.generate_notification(local_subject, local_body).id],
+                    send_notifications.delay([local_nt.generate_notification(local_msg, local_body).id],
                                              job_id=self.id)
                 return _func
             connection.on_commit(send_it())

awx/main/models/oauth.py

@@ -1,9 +1,10 @@
 # Python
+import logging
 import re
 
 # Django
 from django.core.validators import RegexValidator
-from django.db import models
+from django.db import models, connection
 from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
@@ -22,6 +23,9 @@ DATA_URI_RE = re.compile(r'.*')  # FIXME
 __all__ = ['OAuth2AccessToken', 'OAuth2Application']
 
 
+logger = logging.getLogger('awx.main.models.oauth')
+
+
 class OAuth2Application(AbstractApplication):
 
     class Meta:
@@ -120,15 +124,27 @@ class OAuth2AccessToken(AbstractAccessToken):
     def is_valid(self, scopes=None):
         valid = super(OAuth2AccessToken, self).is_valid(scopes)
         if valid:
+            try:
+                self.validate_external_users()
+            except oauth2.AccessDeniedError:
+                logger.exception(f'Failed to authenticate {self.user.username}')
+                return False
             self.last_used = now()
-            self.save(update_fields=['last_used'])
+
+            def _update_last_used():
+                if OAuth2AccessToken.objects.filter(pk=self.pk).exists():
+                    self.save(update_fields=['last_used'])
+            connection.on_commit(_update_last_used)
         return valid
 
-    def save(self, *args, **kwargs):
+    def validate_external_users(self):
         if self.user and settings.ALLOW_OAUTH2_FOR_EXTERNAL_USERS is False:
             external_account = get_external_account(self.user)
             if external_account is not None:
                 raise oauth2.AccessDeniedError(_(
                     'OAuth2 Tokens cannot be created by users associated with an external authentication provider ({})'
                 ).format(external_account))
+
+    def save(self, *args, **kwargs):
+        self.validate_external_users()
         super(OAuth2AccessToken, self).save(*args, **kwargs)

awx/main/models/organization.py

@@ -51,6 +51,11 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
         default=0,
         help_text=_('Maximum number of hosts allowed to be managed by this organization.'),
     )
+    notification_templates_approvals = models.ManyToManyField(
+        "NotificationTemplate",
+        blank=True,
+        related_name='%(class)s_notification_templates_for_approvals'
+    )
 
     admin_role = ImplicitRoleField(
         parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,

awx/main/models/projects.py

@@ -329,7 +329,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in ProjectOptions._meta.fields) | set(
-            ['name', 'description', 'schedule']
+            ['name', 'description']
         )
 
     def save(self, *args, **kwargs):
@@ -483,6 +483,12 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         choices=PROJECT_UPDATE_JOB_TYPE_CHOICES,
         default='check',
     )
+    job_tags = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        help_text=_('Parts of the project update playbook that will be run.'),
+    )
     scm_revision = models.CharField(
         max_length=1024,
         blank=True,
@@ -587,3 +593,24 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups
+
+    def save(self, *args, **kwargs):
+        added_update_fields = []
+        if not self.job_tags:
+            job_tags = ['update_{}'.format(self.scm_type)]
+            if self.job_type == 'run':
+                job_tags.append('install_roles')
+                job_tags.append('install_collections')
+            self.job_tags = ','.join(job_tags)
+            added_update_fields.append('job_tags')
+        if self.scm_delete_on_update and 'delete' not in self.job_tags and self.job_type == 'check':
+            self.job_tags = ','.join([self.job_tags, 'delete'])
+            added_update_fields.append('job_tags')
+        elif (not self.scm_delete_on_update) and 'delete' in self.job_tags:
+            job_tags = self.job_tags.split(',')
+            job_tags.remove('delete')
+            self.job_tags = ','.join(job_tags)
+            added_update_fields.append('job_tags')
+        if 'update_fields' in kwargs:
+            kwargs['update_fields'].extend(added_update_fields)
+        return super(ProjectUpdate, self).save(*args, **kwargs)

awx/main/models/schedules.py

@@ -119,10 +119,11 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
                 tzinfo = r._dtstart.tzinfo
                 if tzinfo is utc:
                     return 'UTC'
-                fname = tzinfo._filename
-                for zone in all_zones:
-                    if fname.endswith(zone):
-                        return zone
+                fname = getattr(tzinfo, '_filename', None)
+                if fname:
+                    for zone in all_zones:
+                        if fname.endswith(zone):
+                            return zone
         logger.warn('Could not detect valid zoneinfo for {}'.format(self.rrule))
         return ''
 

awx/main/models/unified_jobs.py

@@ -42,9 +42,9 @@ from awx.main.utils import (
     camelcase_to_underscore, get_model_for_type,
     encrypt_dict, decrypt_field, _inventory_updates,
     copy_model_by_class, copy_m2m_relationships,
-    get_type_for_model, parse_yaml_or_json, getattr_dne
+    get_type_for_model, parse_yaml_or_json, getattr_dne,
+    polymorphic, schedule_task_manager
 )
-from awx.main.utils import polymorphic, schedule_task_manager
 from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
 from awx.main.redact import UriCleaner, REPLACE_STR
 from awx.main.consumers import emit_channel_notification
@@ -532,6 +532,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         ('scheduled', _('Scheduled')),      # Job was started from a schedule.
         ('dependency', _('Dependency')),    # Job was started as a dependency of another job.
         ('workflow', _('Workflow')),        # Job was started from a workflow job.
+        ('webhook', _('Webhook')),          # Job was started from a webhook event.
         ('sync', _('Sync')),                # Job was started from a project sync.
         ('scm', _('SCM Update'))            # Job was created as an Inventory SCM sync.
     ]
@@ -559,11 +560,17 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         related_name='%(class)s_unified_jobs',
         on_delete=polymorphic.SET_NULL,
     )
+    created = models.DateTimeField(
+        default=None,
+        editable=False,
+        db_index=True,  # add an index, this is a commonly queried field
+    )
     launch_type = models.CharField(
         max_length=20,
         choices=LAUNCH_TYPE_CHOICES,
         default='manual',
         editable=False,
+        db_index=True
     )
     schedule = models.ForeignKey( # Which schedule entry was responsible for starting this job.
         'Schedule',
@@ -621,6 +628,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         default=None,
         editable=False,
         help_text=_("The date and time the job finished execution."),
+        db_index=True,
     )
     elapsed = models.DecimalField(
         max_digits=12,
@@ -706,6 +714,10 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
     def supports_isolation(cls):
         return False
 
+    @property
+    def can_run_containerized(self):
+        return False
+
     def _get_parent_field_name(self):
         return 'unified_job_template' # Override in subclasses.
 
@@ -1199,6 +1211,8 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
     def websocket_emit_status(self, status):
         connection.on_commit(lambda: self._websocket_emit_status(status))
+        if hasattr(self, 'update_webhook_status'):
+            connection.on_commit(lambda: self.update_webhook_status(status))
 
     def notification_data(self):
         return dict(id=self.id,
@@ -1379,9 +1393,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
         wj = self.get_workflow_job()
         if wj:
+            schedule = getattr_dne(wj, 'schedule')
             for name in ('awx', 'tower'):
                 r['{}_workflow_job_id'.format(name)] = wj.pk
                 r['{}_workflow_job_name'.format(name)] = wj.name
+                if schedule:
+                    r['{}_parent_job_schedule_id'.format(name)] = schedule.pk
+                    r['{}_parent_job_schedule_name'.format(name)] = schedule.name
 
         if not created_by:
             schedule = getattr_dne(self, 'schedule')
@@ -1411,3 +1429,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
     def is_isolated(self):
         return bool(self.controller_node)
+
+    @property
+    def is_containerized(self):
+        return False

awx/main/models/workflow.py

@@ -2,15 +2,24 @@
 # All Rights Reserved.
 
 # Python
+import json
 import logging
+from copy import copy
+from urllib.parse import urljoin
 
 # Django
-from django.db import models
+from django.db import connection, models
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 from django.core.exceptions import ObjectDoesNotExist
 #from django import settings as tower_settings
 
+# Django-CRUM
+from crum import get_current_user
+
+from jinja2 import sandbox
+from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
+
 # AWX
 from awx.api.versioning import reverse
 from awx.main.models import (prevent_search, accepts_json, UnifiedJobTemplate,
@@ -19,7 +28,7 @@ from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin
 )
-from awx.main.models.base import BaseModel, CreatedModifiedModel, VarsDictProperty
+from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
 from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR
@@ -30,6 +39,8 @@ from awx.main.models.mixins import (
     SurveyJobTemplateMixin,
     SurveyJobMixin,
     RelatedJobsMixin,
+    WebhookMixin,
+    WebhookTemplateMixin,
 )
 from awx.main.models.jobs import LaunchTimeConfigBase, LaunchTimeConfig, JobTemplate
 from awx.main.models.credential import Credential
@@ -38,9 +49,6 @@ from awx.main.fields import JSONField
 from awx.main.utils import schedule_task_manager
 
 
-from copy import copy
-from urllib.parse import urljoin
-
 __all__ = ['WorkflowJobTemplate', 'WorkflowJob', 'WorkflowJobOptions', 'WorkflowJobNode',
            'WorkflowJobTemplateNode', 'WorkflowApprovalTemplate', 'WorkflowApproval']
 
@@ -196,7 +204,7 @@ class WorkflowJobNode(WorkflowNodeBase):
     )
     do_not_run = models.BooleanField(
         default=False,
-        help_text=_("Indidcates that a job will not be created when True. Workflow runtime "
+        help_text=_("Indicates that a job will not be created when True. Workflow runtime "
                     "semantics will mark this True if the node is in a path that will "
                     "decidedly not be ran. A value of False means the node may not run."),
     )
@@ -207,11 +215,14 @@ class WorkflowJobNode(WorkflowNodeBase):
     def prompts_dict(self, *args, **kwargs):
         r = super(WorkflowJobNode, self).prompts_dict(*args, **kwargs)
         # Explanation - WFJT extra_vars still break pattern, so they are not
-        # put through prompts processing, but inventory is only accepted
+        # put through prompts processing, but inventory and others are only accepted
         # if JT prompts for it, so it goes through this mechanism
-        if self.workflow_job and self.workflow_job.inventory_id:
-            # workflow job inventory takes precedence
-            r['inventory'] = self.workflow_job.inventory
+        if self.workflow_job:
+            if self.workflow_job.inventory_id:
+                # workflow job inventory takes precedence
+                r['inventory'] = self.workflow_job.inventory
+            if self.workflow_job.char_prompts:
+                r.update(self.workflow_job.char_prompts)
         return r
 
     def get_job_kwargs(self):
@@ -298,7 +309,7 @@ class WorkflowJobNode(WorkflowNodeBase):
         return data
 
 
-class WorkflowJobOptions(BaseModel):
+class WorkflowJobOptions(LaunchTimeConfigBase):
     class Meta:
         abstract = True
 
@@ -318,10 +329,11 @@ class WorkflowJobOptions(BaseModel):
 
     @classmethod
     def _get_unified_job_field_names(cls):
-        return set(f.name for f in WorkflowJobOptions._meta.fields) | set(
-            # NOTE: if other prompts are added to WFJT, put fields in WJOptions, remove inventory
-            ['name', 'description', 'schedule', 'survey_passwords', 'labels', 'inventory']
+        r = set(f.name for f in WorkflowJobOptions._meta.fields) | set(
+            ['name', 'description', 'survey_passwords', 'labels', 'limit', 'scm_branch']
         )
+        r.remove('char_prompts')  # needed due to copying launch config to launch config
+        return r
 
     def _create_workflow_nodes(self, old_node_list, user=None):
         node_links = {}
@@ -355,7 +367,7 @@ class WorkflowJobOptions(BaseModel):
         return new_workflow_job
 
 
-class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin, RelatedJobsMixin):
+class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin, RelatedJobsMixin, WebhookTemplateMixin):
 
     SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
     FIELDS_TO_PRESERVE_AT_COPY = [
@@ -372,19 +384,24 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
         on_delete=models.SET_NULL,
         related_name='workflows',
     )
-    inventory = models.ForeignKey(
-        'Inventory',
-        related_name='%(class)ss',
-        blank=True,
-        null=True,
-        default=None,
-        on_delete=models.SET_NULL,
-        help_text=_('Inventory applied to all job templates in workflow that prompt for inventory.'),
-    )
     ask_inventory_on_launch = AskForField(
         blank=True,
         default=False,
     )
+    ask_limit_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_scm_branch_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    notification_templates_approvals = models.ManyToManyField(
+        "NotificationTemplate",
+        blank=True,
+        related_name='%(class)s_notification_templates_for_approvals'
+    )
+
     admin_role = ImplicitRoleField(parent_role=[
         'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
         'organization.workflow_admin_role'
@@ -438,9 +455,22 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
                                               .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
         success_notification_templates = list(base_notification_templates
                                               .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
+        approval_notification_templates = list(base_notification_templates
+                                               .filter(workflowjobtemplate_notification_templates_for_approvals__in=[self]))
+        # Get Organization NotificationTemplates
+        if self.organization is not None:
+            error_notification_templates = set(error_notification_templates + list(base_notification_templates.filter(
+                organization_notification_templates_for_errors=self.organization)))
+            started_notification_templates = set(started_notification_templates + list(base_notification_templates.filter(
+                organization_notification_templates_for_started=self.organization)))
+            success_notification_templates = set(success_notification_templates + list(base_notification_templates.filter(
+                organization_notification_templates_for_success=self.organization)))
+            approval_notification_templates = set(approval_notification_templates + list(base_notification_templates.filter(
+                organization_notification_templates_for_approvals=self.organization)))
         return dict(error=list(error_notification_templates),
                     started=list(started_notification_templates),
-                    success=list(success_notification_templates))
+                    success=list(success_notification_templates),
+                    approvals=list(approval_notification_templates))
 
     def create_unified_job(self, **kwargs):
         workflow_job = super(WorkflowJobTemplate, self).create_unified_job(**kwargs)
@@ -515,7 +545,7 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
         return WorkflowJob.objects.filter(workflow_job_template=self)
 
 
-class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificationMixin, LaunchTimeConfigBase):
+class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificationMixin, WebhookMixin):
     class Meta:
         app_label = 'main'
         ordering = ('id',)
@@ -646,7 +676,7 @@ class WorkflowApprovalTemplate(UnifiedJobTemplate):
         return self.workflowjobtemplatenodes.first().workflow_job_template
 
 
-class WorkflowApproval(UnifiedJob):
+class WorkflowApproval(UnifiedJob, JobNotificationMixin):
     class Meta:
         app_label = 'main'
 
@@ -667,6 +697,14 @@ class WorkflowApproval(UnifiedJob):
         default=False,
         help_text=_("Shows when an approval node (with a timeout assigned to it) has timed out.")
     )
+    approved_or_denied_by = models.ForeignKey(
+        'auth.User',
+        related_name='%s(class)s_approved+',
+        default=None,
+        null=True,
+        editable=False,
+        on_delete=models.SET_NULL,
+    )
 
 
     @classmethod
@@ -680,26 +718,101 @@ class WorkflowApproval(UnifiedJob):
     def event_class(self):
         return None
 
+    def get_ui_url(self):
+        return urljoin(settings.TOWER_URL_BASE, '/#/workflows/{}'.format(self.workflow_job.id))
+
     def _get_parent_field_name(self):
         return 'workflow_approval_template'
 
     def approve(self, request=None):
         self.status = 'successful'
+        self.approved_or_denied_by = get_current_user()
         self.save()
+        self.send_approval_notification('approved')
         self.websocket_emit_status(self.status)
         schedule_task_manager()
         return reverse('api:workflow_approval_approve', kwargs={'pk': self.pk}, request=request)
 
     def deny(self, request=None):
         self.status = 'failed'
+        self.approved_or_denied_by = get_current_user()
         self.save()
+        self.send_approval_notification('denied')
         self.websocket_emit_status(self.status)
         schedule_task_manager()
         return reverse('api:workflow_approval_deny', kwargs={'pk': self.pk}, request=request)
 
+    def signal_start(self, **kwargs):
+        can_start = super(WorkflowApproval, self).signal_start(**kwargs)
+        self.send_approval_notification('running')
+        return can_start
+
+    def send_approval_notification(self, approval_status):
+        from awx.main.tasks import send_notifications  # avoid circular import
+        if self.workflow_job_template is None:
+            return
+        for nt in self.workflow_job_template.notification_templates["approvals"]:
+            try:
+                (notification_subject, notification_body) = self.build_approval_notification_message(nt, approval_status)
+            except Exception:
+                raise NotImplementedError("build_approval_notification_message() does not exist")
+
+            # Use kwargs to force late-binding
+            # https://stackoverflow.com/a/3431699/10669572
+            def send_it(local_nt=nt, local_subject=notification_subject, local_body=notification_body):
+                def _func():
+                    send_notifications.delay([local_nt.generate_notification(local_subject, local_body).id],
+                                             job_id=self.id)
+                return _func
+            connection.on_commit(send_it())
+
+    def build_approval_notification_message(self, nt, approval_status):
+        env = sandbox.ImmutableSandboxedEnvironment()
+
+        context = self.context(approval_status)
+
+        msg_template = body_template = None
+        msg = body = ''
+
+        # Use custom template if available
+        if nt.messages and nt.messages.get('workflow_approval', None):
+            template = nt.messages['workflow_approval'].get(approval_status, {})
+            msg_template = template.get('message', None)
+            body_template = template.get('body', None)
+        # If custom template not provided, look up default template
+        default_template = nt.notification_class.default_messages['workflow_approval'][approval_status]
+        if not msg_template:
+            msg_template = default_template.get('message', None)
+        if not body_template:
+            body_template = default_template.get('body', None)
+
+        if msg_template:
+            try:
+                msg = env.from_string(msg_template).render(**context)
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                msg = ''
+
+        if body_template:
+            try:
+                body = env.from_string(body_template).render(**context)
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                body = ''
+
+        return (msg, body)
+
+    def context(self, approval_status):
+        workflow_url = urljoin(settings.TOWER_URL_BASE, '/#/workflows/{}'.format(self.workflow_job.id))
+        return {'approval_status': approval_status,
+                'approval_node_name': self.workflow_approval_template.name,
+                'workflow_url': workflow_url,
+                'job_metadata': json.dumps(self.notification_data(), indent=4)}
+
     @property
     def workflow_job_template(self):
-        return self.unified_job_node.workflow_job.unified_job_template
+        try:
+            return self.unified_job_node.workflow_job.unified_job_template
+        except ObjectDoesNotExist:
+            return None
 
     @property
     def workflow_job(self):

awx/main/notifications/base.py

@@ -1,21 +1,10 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
-import json
-
-from django.utils.encoding import smart_text
 from django.core.mail.backends.base import BaseEmailBackend
-from django.utils.translation import ugettext_lazy as _
 
 
 class AWXBaseEmailBackend(BaseEmailBackend):
 
     def format_body(self, body):
-        if "body" in body:
-            body_actual = body['body']
-        else:
-            body_actual = smart_text(_("{} #{} had status {}, view details at {}\n\n").format(
-                body['friendly_name'], body['id'], body['status'], body['url'])
-            )
-            body_actual += json.dumps(body, indent=4)
-        return body_actual
+        return body

awx/main/notifications/custom_notification_base.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2019 Ansible, Inc.
+# All Rights Reserved.
+
+
+class CustomNotificationBase(object):
+    DEFAULT_MSG = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
+    DEFAULT_BODY = "{{ job_friendly_name }} #{{ job.id }} had status {{ job.status }}, view details at {{ url }}\n\n{{ job_metadata }}"
+
+    DEFAULT_APPROVAL_RUNNING_MSG = 'The approval node "{{ approval_node_name }}" needs review. This node can be viewed at: {{ workflow_url }}'
+    DEFAULT_APPROVAL_RUNNING_BODY = ('The approval node "{{ approval_node_name }}" needs review. '
+                                     'This approval node can be viewed at: {{ workflow_url }}\n\n{{ job_metadata }}')
+
+    DEFAULT_APPROVAL_APPROVED_MSG = 'The approval node "{{ approval_node_name }}" was approved. {{ workflow_url }}'
+    DEFAULT_APPROVAL_APPROVED_BODY = 'The approval node "{{ approval_node_name }}" was approved. {{ workflow_url }}\n\n{{ job_metadata }}'
+
+    DEFAULT_APPROVAL_TIMEOUT_MSG = 'The approval node "{{ approval_node_name }}" has timed out. {{ workflow_url }}'
+    DEFAULT_APPROVAL_TIMEOUT_BODY = 'The approval node "{{ approval_node_name }}" has timed out. {{ workflow_url }}\n\n{{ job_metadata }}'
+
+    DEFAULT_APPROVAL_DENIED_MSG = 'The approval node "{{ approval_node_name }}" was denied. {{ workflow_url }}'
+    DEFAULT_APPROVAL_DENIED_BODY = 'The approval node "{{ approval_node_name }}" was denied. {{ workflow_url }}\n\n{{ job_metadata }}'
+
+
+    default_messages = {"started": {"message": DEFAULT_MSG, "body": None},
+                        "success": {"message": DEFAULT_MSG, "body": None},
+                        "error": {"message": DEFAULT_MSG, "body": None},
+                        "workflow_approval": {"running": {"message": DEFAULT_APPROVAL_RUNNING_MSG, "body": None},
+                                              "approved": {"message": DEFAULT_APPROVAL_APPROVED_MSG, "body": None},
+                                              "timed_out": {"message": DEFAULT_APPROVAL_TIMEOUT_MSG, "body": None},
+                                              "denied": {"message": DEFAULT_APPROVAL_DENIED_MSG, "body": None}}}

awx/main/notifications/email_backend.py

@@ -1,14 +1,27 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
-import json
-
-from django.utils.encoding import smart_text
 from django.core.mail.backends.smtp import EmailBackend
-from django.utils.translation import ugettext_lazy as _
+
+from awx.main.notifications.custom_notification_base import CustomNotificationBase
+
+DEFAULT_MSG = CustomNotificationBase.DEFAULT_MSG
+DEFAULT_BODY = CustomNotificationBase.DEFAULT_BODY
+
+DEFAULT_APPROVAL_RUNNING_MSG = CustomNotificationBase.DEFAULT_APPROVAL_RUNNING_MSG
+DEFAULT_APPROVAL_RUNNING_BODY = CustomNotificationBase.DEFAULT_APPROVAL_RUNNING_BODY
+
+DEFAULT_APPROVAL_APPROVED_MSG = CustomNotificationBase.DEFAULT_APPROVAL_APPROVED_MSG
+DEFAULT_APPROVAL_APPROVED_BODY = CustomNotificationBase.DEFAULT_APPROVAL_APPROVED_BODY
+
+DEFAULT_APPROVAL_TIMEOUT_MSG = CustomNotificationBase.DEFAULT_APPROVAL_TIMEOUT_MSG
+DEFAULT_APPROVAL_TIMEOUT_BODY = CustomNotificationBase.DEFAULT_APPROVAL_TIMEOUT_BODY
+
+DEFAULT_APPROVAL_DENIED_MSG = CustomNotificationBase.DEFAULT_APPROVAL_DENIED_MSG
+DEFAULT_APPROVAL_DENIED_BODY = CustomNotificationBase.DEFAULT_APPROVAL_DENIED_BODY
 
 
-class CustomEmailBackend(EmailBackend):
+class CustomEmailBackend(EmailBackend, CustomNotificationBase):
 
     init_parameters = {"host": {"label": "Host", "type": "string"},
                        "port": {"label": "Port", "type": "int"},
@@ -19,22 +32,17 @@ class CustomEmailBackend(EmailBackend):
                        "sender": {"label": "Sender Email", "type": "string"},
                        "recipients": {"label": "Recipient List", "type": "list"},
                        "timeout": {"label": "Timeout", "type": "int", "default": 30}}
-
-    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
-    DEFAULT_BODY = smart_text(_("{{ job_friendly_name }} #{{ job.id }} had status {{ job.status }}, view details at {{ url }}\n\n{{ job_summary_dict }}"))
-    default_messages = {"started": {"message": DEFAULT_SUBJECT, "body": DEFAULT_BODY},
-                        "success": {"message": DEFAULT_SUBJECT, "body": DEFAULT_BODY},
-                        "error": {"message": DEFAULT_SUBJECT, "body": DEFAULT_BODY}}
     recipient_parameter = "recipients"
     sender_parameter = "sender"
 
+    default_messages = {"started": {"message": DEFAULT_MSG, "body": DEFAULT_BODY},
+                        "success": {"message": DEFAULT_MSG, "body": DEFAULT_BODY},
+                        "error": {"message": DEFAULT_MSG, "body": DEFAULT_BODY},
+                        "workflow_approval": {"running": {"message": DEFAULT_APPROVAL_RUNNING_MSG, "body": DEFAULT_APPROVAL_RUNNING_BODY},
+                                              "approved": {"message": DEFAULT_APPROVAL_APPROVED_MSG, "body": DEFAULT_APPROVAL_APPROVED_BODY},
+                                              "timed_out": {"message": DEFAULT_APPROVAL_TIMEOUT_MSG, "body": DEFAULT_APPROVAL_TIMEOUT_BODY},
+                                              "denied": {"message": DEFAULT_APPROVAL_DENIED_MSG, "body": DEFAULT_APPROVAL_DENIED_BODY}}}
 
     def format_body(self, body):
-        if "body" in body:
-            body_actual = body['body']
-        else:
-            body_actual = smart_text(_("{} #{} had status {}, view details at {}\n\n").format(
-                body['friendly_name'], body['id'], body['status'], body['url'])
-            )
-            body_actual += json.dumps(body, indent=4)
-        return body_actual
+        # leave body unchanged (expect a string)
+        return body

awx/main/notifications/grafana_backend.py

@@ -8,24 +8,21 @@ import dateutil.parser as dp
 
 from django.utils.encoding import smart_text
 from django.utils.translation import ugettext_lazy as _
+
 from awx.main.notifications.base import AWXBaseEmailBackend
+from awx.main.notifications.custom_notification_base import CustomNotificationBase
 
 
 logger = logging.getLogger('awx.main.notifications.grafana_backend')
 
 
-class GrafanaBackend(AWXBaseEmailBackend):
+class GrafanaBackend(AWXBaseEmailBackend, CustomNotificationBase):
 
     init_parameters = {"grafana_url": {"label": "Grafana URL", "type": "string"},
                        "grafana_key": {"label": "Grafana API Key", "type": "password"}}
     recipient_parameter = "grafana_url"
     sender_parameter = None
 
-    DEFAULT_SUBJECT = "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}"
-    default_messages = {"started": {"message": DEFAULT_SUBJECT},
-                        "success": {"message": DEFAULT_SUBJECT},
-                        "error": {"message": DEFAULT_SUBJECT}}
-
     def __init__(self, grafana_key,dashboardId=None, panelId=None, annotation_tags=None, grafana_no_verify_ssl=False, isRegion=True,
                  fail_silently=False, **kwargs):
         super(GrafanaBackend, self).__init__(fail_silently=fail_silently)