Merge pull request #5050 from ryanpetrello/release-8.0.0

Bump VERSION to 8.0.0

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -28,6 +28,9 @@ awx/ui/build_test
 awx/ui/client/languages
 awx/ui/templates/ui/index.html
 awx/ui/templates/ui/installing.html
+awx/ui_next/node_modules/
+awx/ui_next/coverage/
+awx/ui_next/build/locales/_build
 /tower-license
 /tower-license/**
 tools/prometheus/data
@@ -122,6 +125,7 @@ local/
 requirements/vendor
 .i18n_built
 .idea/*
+*credentials*.y*ml*
 
 # AWX python libs populated by requirements.txt
 awx/lib/.deps_built
@@ -129,4 +133,12 @@ awx/lib/site-packages
 venv/*
 use_dev_supervisor.txt
 
+
+# Ansible module tests
+awx_collection_test_venv/
+awx_collection/*.tar.gz
+awx_collection/galaxy.yml
+
 .idea/*
+*.unison.tmp
+*.#

CONTRIBUTING.md

@@ -134,6 +134,8 @@ Run the following to build the AWX UI:
 ```bash
 (host) $ make ui-devel
 ```
+See [the ui development documentation](awx/ui/README.md) for more information on using the frontend development, build, and test tooling.
+
 ### Running the environment
 
 #### Start the containers
@@ -154,8 +156,8 @@ If you start a second terminal session, you can take a look at the running conta
 
 $ docker ps
 CONTAINER ID        IMAGE                                              COMMAND                  CREATED             STATUS              PORTS                                                                                                                                      NAMES
-aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:6899-6999->6899-6999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
-e4c0afeb548c        postgres:9.6                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
+aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:7899-7999->7899-7999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
+e4c0afeb548c        postgres:10                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
 0089699d5afd        tools_logstash                                     "/docker-entrypoin..."   26 seconds ago      Up 25 seconds                                                                                                                                                  tools_logstash_1
 4d4ff0ced266        memcached:alpine                                   "docker-entrypoint..."   26 seconds ago      Up 25 seconds       0.0.0.0:11211->11211/tcp                                                                                                                   tools_memcached_1
 92842acd64cd        rabbitmq:3-management                              "docker-entrypoint..."   26 seconds ago      Up 24 seconds       4369/tcp, 5671-5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp                                                                    tools_rabbitmq_1

INSTALL.md

@@ -36,7 +36,7 @@ This document provides a guide for installing AWX.
       - [PostgreSQL](#postgresql-1)
       - [Proxy settings](#proxy-settings)
   - [Start the build](#start-the-build-2)
-  - [Post build](#post-build-1)
+  - [Post build](#post-build-2)
   - [Accessing AWX](#accessing-awx-2)
 
 ## Getting started
@@ -72,7 +72,7 @@ Before you can run a deployment, you'll need the following installed in your loc
 
 The system that runs the AWX service will need to satisfy the following requirements
 
-- At leasts 4GB of memory
+- At least 4GB of memory
 - At least 2 cpu cores
 - At least 20GB of space
 - Running Docker, Openshift, or Kubernetes
@@ -193,7 +193,7 @@ $ eval $(minishift docker-env)
 
 By default, AWX will deploy a PostgreSQL pod inside of your cluster. You will need to create a [Persistent Volume Claim](https://docs.openshift.org/latest/dev_guide/persistent_volumes.html) which is named `postgresql` by default, and can be overridden by setting the `openshift_pg_pvc_name` variable. For testing and demo purposes, you may set `openshift_pg_emptydir=yes`.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
 
 ### Start the build
 
@@ -503,7 +503,7 @@ If you wish to tag and push built images to a Docker registry, set the following
 
 AWX requires access to a PostgreSQL database, and by default, one will be created and deployed in a container, and data will be persisted to a host volume. In this scenario, you must set the value of `postgres_data_dir` to a path that can be mounted to the container. When the container is stopped, the database files will still exist in the specified path.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information.
 
 ### Start the build
 

Makefile

@@ -18,6 +18,7 @@ COMPOSE_TAG ?= $(GIT_BRANCH)
 COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
+COLLECTION_VENV ?= /awx_devel/awx_collection_test_venv
 SCL_PREFIX ?=
 CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
@@ -59,20 +60,23 @@ UI_RELEASE_FLAG_FILE = awx/ui/.release_built
 I18N_FLAG_FILE = .i18n_built
 
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
-	develop refresh adduser migrate dbchange dbshell runserver \
+	develop refresh adduser migrate dbchange runserver \
 	receiver test test_unit test_coverage coverage_html \
 	dev_build release_build release_clean sdist \
 	ui-docker-machine ui-docker ui-release ui-devel \
 	ui-test ui-deps ui-test-ci VERSION
 
 # remove ui build artifacts
-clean-ui:
+clean-ui: clean-languages
 	rm -rf awx/ui/static/
 	rm -rf awx/ui/node_modules/
 	rm -rf awx/ui/test/unit/reports/
 	rm -rf awx/ui/test/spec/reports/
 	rm -rf awx/ui/test/e2e/reports/
 	rm -rf awx/ui/client/languages/
+	rm -rf awx/ui_next/node_modules/
+	rm -rf awx/ui_next/coverage/
+	rm -rf awx/ui_next/build/locales/_build/
 	rm -f $(UI_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_FLAG_FILE)
@@ -91,21 +95,27 @@ clean-schema:
 	rm -rf schema.json
 	rm -rf reference-schema.json
 
+clean-languages:
+	rm -f $(I18N_FLAG_FILE)
+	find . -type f -regex ".*\.mo$$" -delete
+
 # Remove temporary build files, compiled Python files.
-clean: clean-ui clean-dist
+clean: clean-ui clean-api clean-dist
 	rm -rf awx/public
 	rm -rf awx/lib/site-packages
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3
-	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
 	mkdir tmp
+
+clean-api:
 	rm -rf build $(NAME)-$(VERSION) *.egg-info
 	find . -type f -regex ".*\.py[co]$$" -delete
 	find . -type d -name "__pycache__" -delete
+	rm -f awx/awx_test.sqlite3*
+	rm -rf requirements/vendor
 
 # convenience target to assert environment variables are defined
 guard-%:
@@ -179,7 +189,7 @@ requirements_awx: virtualenv_awx
 	    cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
 	fi
 	echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg
-	#$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
+	$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
 
 requirements_awx_dev:
 	$(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt
@@ -239,10 +249,6 @@ migrate:
 dbchange:
 	$(MANAGEMENT_COMMAND) makemigrations
 
-# access database shell, asks for password
-dbshell:
-	sudo -u postgres psql -d awx-dev
-
 server_noattach:
 	tmux new-session -d -s awx 'exec make uwsgi'
 	tmux rename-window 'AWX'
@@ -369,8 +375,34 @@ test:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
+	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
 	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
+prepare_collection_venv:
+	rm -rf $(COLLECTION_VENV)
+	mkdir $(COLLECTION_VENV)
+	ln -s /usr/lib/python2.7/site-packages/ansible $(COLLECTION_VENV)/ansible
+	$(VENV_BASE)/awx/bin/pip install --target=$(COLLECTION_VENV) git+https://github.com/ansible/tower-cli.git
+
+COLLECTION_TEST_DIRS ?= awx_collection/test/awx
+COLLECTION_PACKAGE ?= awx
+COLLECTION_NAMESPACE ?= awx
+
+test_collection:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	PYTHONPATH=$(COLLECTION_VENV):/awx_devel/awx_collection:$PYTHONPATH py.test $(COLLECTION_TEST_DIRS)
+
+flake8_collection:
+	flake8 awx_collection/  # Different settings, in main exclude list
+
+test_collection_all: prepare_collection_venv test_collection flake8_collection
+
+build_collection:
+	ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION)
+	ansible-galaxy collection build awx_collection --output-path=awx_collection
+
 test_unit:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
@@ -512,9 +544,36 @@ jshint: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) run --prefix awx/ui jshint
 	$(NPM_BIN) run --prefix awx/ui lint
 
+ui-zuul-lint-and-test: $(UI_DEPS_FLAG_FILE)
+	$(NPM_BIN) run --prefix awx/ui jshint
+	$(NPM_BIN) run --prefix awx/ui lint
+	$(NPM_BIN) --prefix awx/ui run test:ci
+	$(NPM_BIN) --prefix awx/ui run unit
+
 # END UI TASKS
 # --------------------------------------
 
+# UI NEXT TASKS
+# --------------------------------------
+
+ui-next-lint:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+
+ui-next-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+ui-next-zuul-lint-and-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+# END UI NEXT TASKS
+# --------------------------------------
+
 # Build a pip-installable package into dist/ with a timestamped version number.
 dev_build:
 	$(PYTHON) setup.py dev_build
@@ -546,8 +605,8 @@ setup-bundle-build:
 	mkdir -p $@
 
 docker-auth:
-	if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
-		docker login -u oauth2accesstoken -p "$(IMAGE_REPOSITORY_AUTH)" $(IMAGE_REPOSITORY_BASE); \
+	@if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
+		echo "$(IMAGE_REPOSITORY_AUTH)" | docker login -u oauth2accesstoken --password-stdin $(IMAGE_REPOSITORY_BASE); \
 	fi;
 
 # Docker isolated rampart
@@ -629,7 +688,7 @@ clean-elk:
 	docker rm tools_kibana_1
 
 psql-container:
-	docker run -it --net tools_default --rm postgres:9.6 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
+	docker run -it --net tools_default --rm postgres:10 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
 
 VERSION:
 	@echo "awx: $(VERSION)"

VERSION

@@ -1 +1 @@
-6.0.0
+8.0.0

awx/__init__.py

@@ -25,9 +25,8 @@ import hashlib
 
 try:
     import django
-    from django.utils.encoding import force_bytes
-    from django.db.backends.base.schema import BaseDatabaseSchemaEditor
     from django.db.backends.base import schema
+    from django.db.backends.utils import names_digest
     HAS_DJANGO = True
 except ImportError:
     HAS_DJANGO = False
@@ -37,30 +36,33 @@ if HAS_DJANGO is True:
     # This line exists to make sure we don't regress on FIPS support if we
     # upgrade Django; if you're upgrading Django and see this error,
     # update the version check below, and confirm that FIPS still works.
-    if django.__version__ != '1.11.20':
-        raise RuntimeError("Django version other than 1.11.20 detected {}. \
-                Subclassing BaseDatabaseSchemaEditor is known to work for Django 1.11.20 \
-                and may not work in newer Django versions.".format(django.__version__))
+    # If operating in a FIPS environment, `hashlib.md5()` will raise a `ValueError`,
+    # but will support the `usedforsecurity` keyword on RHEL and Centos systems.
 
+    # Keep an eye on https://code.djangoproject.com/ticket/28401
+    target_version = '2.2.4'
+    if django.__version__ != target_version:
+        raise RuntimeError(
+            "Django version other than {target} detected: {current}. "
+            "Overriding `names_digest` is known to work for Django {target} "
+            "and may not work in other Django versions.".format(target=target_version,
+                                                                current=django.__version__)
+        )
 
-    class FipsBaseDatabaseSchemaEditor(BaseDatabaseSchemaEditor):
-
-        @classmethod
-        def _digest(cls, *args):
+    try:
+        names_digest('foo', 'bar', 'baz', length=8)
+    except ValueError:
+        def names_digest(*args, length):
             """
-            Generates a 32-bit digest of a set of arguments that can be used to
-            shorten identifying names.
+            Generate a 32-bit digest of a set of arguments that can be used to shorten
+            identifying names.  Support for use in FIPS environments.
             """
-            try:
-                h = hashlib.md5()
-            except ValueError:
-                h = hashlib.md5(usedforsecurity=False)
+            h = hashlib.md5(usedforsecurity=False)
             for arg in args:
-                h.update(force_bytes(arg))
-            return h.hexdigest()[:8]
+                h.update(arg.encode())
+            return h.hexdigest()[:length]
 
-
-    schema.BaseDatabaseSchemaEditor = FipsBaseDatabaseSchemaEditor
+        schema.names_digest = names_digest
 
 
 def find_commands(management_dir):
@@ -80,6 +82,16 @@ def find_commands(management_dir):
     return commands
 
 
+def oauth2_getattribute(self, attr):
+    # Custom method to override
+    # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__
+    from django.conf import settings
+    val = settings.OAUTH2_PROVIDER.get(attr)
+    if val is None:
+        val = object.__getattribute__(self, attr)
+    return val
+
+
 def prepare_env():
     # Update the default settings environment variable based on current mode.
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
@@ -91,6 +103,12 @@ def prepare_env():
     # Monkeypatch Django find_commands to also work with .pyc files.
     import django.core.management
     django.core.management.find_commands = find_commands
+
+    # Monkeypatch Oauth2 toolkit settings class to check for settings
+    # in django.conf settings each time, not just once during import
+    import oauth2_provider.settings
+    oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ = oauth2_getattribute
+
     # Use the AWX_TEST_DATABASE_* environment variables to specify the test
     # database settings to use when management command is run as an external
     # program via unit tests.

awx/api/conf.py

@@ -38,12 +38,15 @@ register(
     'OAUTH2_PROVIDER',
     field_class=OAuth2ProviderField,
     default={'ACCESS_TOKEN_EXPIRE_SECONDS': oauth2_settings.ACCESS_TOKEN_EXPIRE_SECONDS,
-             'AUTHORIZATION_CODE_EXPIRE_SECONDS': 600},
+             'AUTHORIZATION_CODE_EXPIRE_SECONDS': oauth2_settings.AUTHORIZATION_CODE_EXPIRE_SECONDS,
+             'REFRESH_TOKEN_EXPIRE_SECONDS': oauth2_settings.REFRESH_TOKEN_EXPIRE_SECONDS},
     label=_('OAuth 2 Timeout Settings'),
     help_text=_('Dictionary for customizing OAuth 2 timeouts, available items are '
                 '`ACCESS_TOKEN_EXPIRE_SECONDS`, the duration of access tokens in the number '
-                'of seconds, and `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
-                'authorization codes in the number of seconds.'),
+                'of seconds, `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
+                'authorization codes in the number of seconds, and `REFRESH_TOKEN_EXPIRE_SECONDS`, '
+                'the duration of refresh tokens, after expired access tokens, '
+                'in the number of seconds.'),
     category=_('Authentication'),
     category_slug='authentication',
 )

awx/api/fields.py

@@ -80,7 +80,7 @@ class OAuth2ProviderField(fields.DictField):
     default_error_messages = {
         'invalid_key_names': _('Invalid key names: {invalid_key_names}'),
     }
-    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS'}
+    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS', 'REFRESH_TOKEN_EXPIRE_SECONDS'}
     child = fields.IntegerField(min_value=1)
 
     def to_internal_value(self, data):

awx/api/filters.py

@@ -126,7 +126,7 @@ class FieldLookupBackend(BaseFilterBackend):
     '''
 
     RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
-                      'search', 'type', 'host_filter')
+                      'search', 'type', 'host_filter', 'count_disabled', 'no_truncate')
 
     SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                          'startswith', 'istartswith', 'endswith', 'iendswith',

awx/api/generics.py

@@ -34,7 +34,8 @@ from rest_framework.negotiation import DefaultContentNegotiation
 # AWX
 from awx.api.filters import FieldLookupBackend
 from awx.main.models import (
-    UnifiedJob, UnifiedJobTemplate, User, Role, Credential
+    UnifiedJob, UnifiedJobTemplate, User, Role, Credential,
+    WorkflowJobTemplateNode, WorkflowApprovalTemplate
 )
 from awx.main.access import access_registry
 from awx.main.utils import (
@@ -91,7 +92,7 @@ class LoggedLoginView(auth_views.LoginView):
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
         if request.user.is_authenticated:
-            logger.info(smart_text(u"User {} logged in.".format(self.request.user.username)))
+            logger.info(smart_text(u"User {} logged in from {}".format(self.request.user.username,request.META.get('REMOTE_ADDR', None))))
             ret.set_cookie('userLoggedIn', 'true')
             current_user = UserSerializer(self.request.user)
             current_user = smart_text(JSONRenderer().render(current_user.data))
@@ -204,6 +205,9 @@ class APIView(views.APIView):
             response['X-API-Query-Count'] = len(q_times)
             response['X-API-Query-Time'] = '%0.3fs' % sum(q_times)
 
+        if getattr(self, 'deprecated', False):
+            response['Warning'] = '299 awx "This resource has been deprecated and will be removed in a future release."'  # noqa
+
         return response
 
     def get_authenticate_header(self, request):
@@ -401,21 +405,21 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
                 continue
             if getattr(field, 'related_model', None):
                 fields.add('{}__search'.format(field.name))
-        for rel in self.model._meta.related_objects:
-            name = rel.related_name
-            if isinstance(rel, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
+        for related in self.model._meta.related_objects:
+            name = related.related_name
+            if isinstance(related, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
                 # Add underscores for polymorphic subclasses for user utility
-                name = rel.related_model._meta.verbose_name.replace(" ", "_")
+                name = related.related_model._meta.verbose_name.replace(" ", "_")
             if skip_related_name(name) or name.endswith('+'):
                 continue
             fields.add('{}__search'.format(name))
-        m2m_rel = []
-        m2m_rel += self.model._meta.local_many_to_many
+        m2m_related = []
+        m2m_related += self.model._meta.local_many_to_many
         if issubclass(self.model, UnifiedJobTemplate) and self.model != UnifiedJobTemplate:
-            m2m_rel += UnifiedJobTemplate._meta.local_many_to_many
+            m2m_related += UnifiedJobTemplate._meta.local_many_to_many
         if issubclass(self.model, UnifiedJob) and self.model != UnifiedJob:
-            m2m_rel += UnifiedJob._meta.local_many_to_many
-        for relationship in m2m_rel:
+            m2m_related += UnifiedJob._meta.local_many_to_many
+        for relationship in m2m_related:
             if skip_related_name(relationship.name):
                 continue
             if relationship.related_model._meta.app_label != 'main':
@@ -488,9 +492,12 @@ class SubListAPIView(ParentMixin, ListAPIView):
         parent = self.get_parent_object()
         self.check_parent_access(parent)
         qs = self.request.user.get_queryset(self.model).distinct()
-        sublist_qs = getattrd(parent, self.relationship).distinct()
+        sublist_qs = self.get_sublist_queryset(parent)
         return qs & sublist_qs
 
+    def get_sublist_queryset(self, parent):
+        return getattrd(parent, self.relationship).distinct()
+
 
 class DestroyAPIView(generics.DestroyAPIView):
 
@@ -882,6 +889,21 @@ class CopyAPIView(GenericAPIView):
                 create_kwargs[field.name] = CopyAPIView._decrypt_model_field_if_needed(
                     obj, field.name, field_val
                 )
+
+        # WorkflowJobTemplateNodes that represent an approval are *special*;
+        # when we copy them, we actually want to *copy* the UJT they point at
+        # rather than share the template reference between nodes in disparate
+        # workflows
+        if (
+            isinstance(obj, WorkflowJobTemplateNode) and
+            isinstance(getattr(obj, 'unified_job_template'), WorkflowApprovalTemplate)
+        ):
+            new_approval_template, sub_objs = CopyAPIView.copy_model_obj(
+                None, None, WorkflowApprovalTemplate,
+                obj.unified_job_template, creater
+            )
+            create_kwargs['unified_job_template'] = new_approval_template
+
         new_obj = model.objects.create(**create_kwargs)
         logger.debug('Deep copy: Created new object {}({})'.format(
             new_obj, model

awx/api/metadata.py

@@ -5,6 +5,8 @@ from collections import OrderedDict
 
 # Django
 from django.core.exceptions import PermissionDenied
+from django.db.models.fields import PositiveIntegerField, BooleanField
+from django.db.models.fields.related import ForeignKey
 from django.http import Http404
 from django.utils.encoding import force_text, smart_text
 from django.utils.translation import ugettext_lazy as _
@@ -14,10 +16,13 @@ from rest_framework import exceptions
 from rest_framework import metadata
 from rest_framework import serializers
 from rest_framework.relations import RelatedField, ManyRelatedField
+from rest_framework.fields import JSONField as DRFJSONField
 from rest_framework.request import clone_request
 
 # AWX
+from awx.main.fields import JSONField, ImplicitRoleField
 from awx.main.models import InventorySource, NotificationTemplate
+from awx.main.scheduler.kubernetes import PodManager
 
 
 class Metadata(metadata.SimpleMetadata):
@@ -68,6 +73,8 @@ class Metadata(metadata.SimpleMetadata):
             else:
                 for model_field in serializer.Meta.model._meta.fields:
                     if field.field_name == model_field.name:
+                        if getattr(model_field, '__accepts_json__', None):
+                            field_info['type'] = 'json'
                         field_info['filterable'] = True
                         break
                 else:
@@ -114,15 +121,48 @@ class Metadata(metadata.SimpleMetadata):
             for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
                 field_info[notification_type_name] = notification_type_class.init_parameters
 
+        # Special handling of notification messages where the required properties
+        # are conditional on the type selected.
+        try:
+            view_model = field.context['view'].model
+        except (AttributeError, KeyError):
+            view_model = None
+        if view_model == NotificationTemplate and field.field_name == 'messages':
+            for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
+                field_info[notification_type_name] = notification_type_class.default_messages
+
+
         # Update type of fields returned...
+        model_field = None
+        if serializer and hasattr(serializer, 'Meta') and hasattr(serializer.Meta, 'model'):
+            try:
+                model_field = serializer.Meta.model._meta.get_field(field.field_name)
+            except Exception:
+                pass
         if field.field_name == 'type':
             field_info['type'] = 'choice'
-        elif field.field_name == 'url':
+        elif field.field_name in ('url', 'custom_virtualenv', 'token'):
             field_info['type'] = 'string'
         elif field.field_name in ('related', 'summary_fields'):
             field_info['type'] = 'object'
+        elif isinstance(field, PositiveIntegerField):
+            field_info['type'] = 'integer'
         elif field.field_name in ('created', 'modified'):
             field_info['type'] = 'datetime'
+        elif (
+            RelatedField in field.__class__.__bases__ or
+            isinstance(model_field, ForeignKey)
+        ):
+            field_info['type'] = 'id'
+        elif (
+            isinstance(field, JSONField) or
+            isinstance(model_field, JSONField) or
+            isinstance(field, DRFJSONField) or
+            isinstance(getattr(field, 'model_field', None), JSONField)
+        ):
+            field_info['type'] = 'json'
+        elif isinstance(model_field, BooleanField):
+            field_info['type'] = 'boolean'
 
         return field_info
 
@@ -161,6 +201,9 @@ class Metadata(metadata.SimpleMetadata):
                 if not isinstance(meta, dict):
                     continue
 
+                if field == "pod_spec_override":
+                    meta['default'] = PodManager().pod_definition
+
                 # Add type choices if available from the serializer.
                 if field == 'type' and hasattr(serializer, 'get_type_choices'):
                     meta['choices'] = serializer.get_type_choices()
@@ -213,6 +256,16 @@ class Metadata(metadata.SimpleMetadata):
         if getattr(view, 'related_search_fields', None):
             metadata['related_search_fields'] = view.related_search_fields
 
+        # include role names in metadata
+        roles = []
+        model = getattr(view, 'model', None)
+        if model:
+            for field in model._meta.get_fields():
+                if type(field) is ImplicitRoleField:
+                    roles.append(field.name)
+        if len(roles) > 0:
+            metadata['object_roles'] = roles
+
         from rest_framework import generics
         if isinstance(view, generics.ListAPIView) and hasattr(view, 'paginator'):
             metadata['max_page_size'] = view.paginator.max_page_size

awx/api/pagination.py

@@ -3,14 +3,28 @@
 
 # Django REST Framework
 from django.conf import settings
+from django.core.paginator import Paginator as DjangoPaginator
 from rest_framework import pagination
+from rest_framework.response import Response
 from rest_framework.utils.urls import replace_query_param
 
 
+class DisabledPaginator(DjangoPaginator):
+
+    @property
+    def num_pages(self):
+        return 1
+
+    @property
+    def count(self):
+        return 200
+
+
 class Pagination(pagination.PageNumberPagination):
 
     page_size_query_param = 'page_size'
     max_page_size = settings.MAX_PAGE_SIZE
+    count_disabled = False
 
     def get_next_link(self):
         if not self.page.has_next():
@@ -39,3 +53,17 @@ class Pagination(pagination.PageNumberPagination):
                                  for pl in context['page_links']]
 
         return context
+
+    def paginate_queryset(self, queryset, request, **kwargs):
+        self.count_disabled = 'count_disabled' in request.query_params
+        try:
+            if self.count_disabled:
+                self.django_paginator_class = DisabledPaginator
+            return super(Pagination, self).paginate_queryset(queryset, request, **kwargs)
+        finally:
+            self.django_paginator_class = DjangoPaginator
+
+    def get_paginated_response(self, data):
+        if self.count_disabled:
+            return Response({'results': data})
+        return super(Pagination, self).get_paginated_response(data)

awx/api/permissions.py

@@ -17,7 +17,7 @@ logger = logging.getLogger('awx.api.permissions')
 
 __all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission', 'VariableDataPermission',
            'TaskPermission', 'ProjectUpdatePermission', 'InventoryInventorySourcesUpdatePermission',
-           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission',]
+           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission', 'WorkflowApprovalPermission']
 
 
 class ModelAccessPermission(permissions.BasePermission):
@@ -95,7 +95,7 @@ class ModelAccessPermission(permissions.BasePermission):
         '''
 
         # Don't allow anonymous users. 401, not 403, hence no raised exception.
-        if not request.user or request.user.is_anonymous():
+        if not request.user or request.user.is_anonymous:
             return False
 
         # Always allow superusers
@@ -196,6 +196,17 @@ class TaskPermission(ModelAccessPermission):
             return False
 
 
+class WorkflowApprovalPermission(ModelAccessPermission):
+    '''
+    Permission check used by workflow `approval` and `deny` views to determine
+    who has access to approve and deny paused workflow nodes
+    '''
+
+    def check_post_permissions(self, request, view, obj=None):
+        approval = get_object_or_400(view.model, pk=view.kwargs['pk'])
+        return check_user_access(request.user, view.model, 'approve_or_deny', approval)
+
+
 class ProjectUpdatePermission(ModelAccessPermission):
     '''
     Permission check used by ProjectUpdateView to determine who can update projects
@@ -239,3 +250,7 @@ class InstanceGroupTowerPermission(ModelAccessPermission):
             return False
         return super(InstanceGroupTowerPermission, self).has_object_permission(request, view, obj)
 
+
+class WebhookKeyPermission(permissions.BasePermission):
+    def has_object_permission(self, request, view, obj):
+        return request.user.can_access(view.model, 'admin', obj, request.data)

awx/api/renderers.py

@@ -2,6 +2,7 @@
 # All Rights Reserved.
 
 from django.utils.safestring import SafeText
+from prometheus_client.parser import text_string_to_metric_families
 
 # Django REST Framework
 from rest_framework import renderers
@@ -103,3 +104,21 @@ class AnsiTextRenderer(PlainTextRenderer):
 class AnsiDownloadRenderer(PlainTextRenderer):
 
     format = "ansi_download"
+
+
+class PrometheusJSONRenderer(renderers.JSONRenderer):
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if isinstance(data, dict):
+            # HTTP errors are {'detail': ErrorDetail(string='...', code=...)}
+            return super(PrometheusJSONRenderer, self).render(
+                data, accepted_media_type, renderer_context
+            )
+        parsed_metrics = text_string_to_metric_families(data)
+        data = {}
+        for family in parsed_metrics:
+            for sample in family.samples:
+                data[sample[0]] = {"labels": sample[1], "value": sample[2]}
+        return super(PrometheusJSONRenderer, self).render(
+            data, accepted_media_type, renderer_context
+        )

awx/api/templates/api/team_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a Team:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected team.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/user_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a User:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected user.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/webhook_key_view.md

@@ -0,0 +1,12 @@
+Webhook Secret Key:
+
+Make a GET request to this resource to obtain the secret key for a job
+template or workflow job template configured to be triggered by
+webhook events.  The response will include the following fields:
+
+* `webhook_key`: Secret key that needs to be copied and added to the
+  webhook configuration of the service this template will be receiving
+  webhook events from (string, read-only)
+
+Make an empty POST request to this resource to generate a new
+replacement `webhook_key`.

awx/api/urls/job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     JobTemplateList,
@@ -45,6 +45,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/organization.py

@@ -18,6 +18,7 @@ from awx.api.views import (
     OrganizationNotificationTemplatesErrorList,
     OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
+    OrganizationNotificationTemplatesApprovalList,
     OrganizationInstanceGroupsList,
     OrganizationObjectRolesList,
     OrganizationAccessList,
@@ -43,6 +44,8 @@ urls = [
         name='organization_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', OrganizationNotificationTemplatesSuccessList.as_view(),
         name='organization_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', OrganizationNotificationTemplatesApprovalList.as_view(),
+        name='organization_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/instance_groups/$', OrganizationInstanceGroupsList.as_view(), name='organization_instance_groups_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', OrganizationObjectRolesList.as_view(), name='organization_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', OrganizationAccessList.as_view(), name='organization_access_list'),

awx/api/urls/urls.py

@@ -14,6 +14,7 @@ from awx.api.views import (
     ApiV2RootView,
     ApiV2PingView,
     ApiV2ConfigView,
+    ApiV2SubscriptionView,
     AuthView,
     UserMeList,
     DashboardView,
@@ -71,6 +72,8 @@ from .instance import urls as instance_urls
 from .instance_group import urls as instance_group_urls
 from .oauth2 import urls as oauth2_urls
 from .oauth2_root import urls as oauth2_root_urls
+from .workflow_approval_template import urls as workflow_approval_template_urls
+from .workflow_approval import urls as workflow_approval_urls
 
 
 v2_urls = [
@@ -92,6 +95,7 @@ v2_urls = [
     url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
     url(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'),
     url(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'),
+    url(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'),
     url(r'^auth/$', AuthView.as_view()),
     url(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     url(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),
@@ -131,8 +135,11 @@ v2_urls = [
     url(r'^unified_job_templates/$', UnifiedJobTemplateList.as_view(), name='unified_job_template_list'),
     url(r'^unified_jobs/$', UnifiedJobList.as_view(), name='unified_job_list'),
     url(r'^activity_stream/', include(activity_stream_urls)),
+    url(r'^workflow_approval_templates/', include(workflow_approval_template_urls)),
+    url(r'^workflow_approvals/', include(workflow_approval_urls)),
 ]
 
+
 app_name = 'api'
 urlpatterns = [
     url(r'^$', ApiRootView.as_view(), name='api_root_view'),

awx/api/urls/webhooks.py

@@ -0,0 +1,14 @@
+from django.conf.urls import url
+
+from awx.api.views import (
+    WebhookKeyView,
+    GithubWebhookReceiver,
+    GitlabWebhookReceiver,
+)
+
+
+urlpatterns = [
+    url(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
+    url(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
+    url(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
+]

awx/api/urls/workflow_approval.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalList,
+    WorkflowApprovalDetail,
+    WorkflowApprovalApprove,
+    WorkflowApprovalDeny,
+)
+
+
+urls = [
+    url(r'^$', WorkflowApprovalList.as_view(), name='workflow_approval_list'),
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalDetail.as_view(), name='workflow_approval_detail'),
+    url(r'^(?P<pk>[0-9]+)/approve/$', WorkflowApprovalApprove.as_view(), name='workflow_approval_approve'),
+    url(r'^(?P<pk>[0-9]+)/deny/$', WorkflowApprovalDeny.as_view(), name='workflow_approval_deny'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_approval_template.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalTemplateDetail,
+    WorkflowApprovalTemplateJobsList,
+)
+
+
+urls = [
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalTemplateDetail.as_view(), name='workflow_approval_template_detail'),
+    url(r'^(?P<pk>[0-9]+)/approvals/$', WorkflowApprovalTemplateJobsList.as_view(), name='workflow_approval_template_jobs_list'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     WorkflowJobTemplateList,
@@ -16,6 +16,7 @@ from awx.api.views import (
     WorkflowJobTemplateNotificationTemplatesErrorList,
     WorkflowJobTemplateNotificationTemplatesStartedList,
     WorkflowJobTemplateNotificationTemplatesSuccessList,
+    WorkflowJobTemplateNotificationTemplatesApprovalList,
     WorkflowJobTemplateAccessList,
     WorkflowJobTemplateObjectRolesList,
     WorkflowJobTemplateLabelList,
@@ -38,9 +39,12 @@ urls = [
         name='workflow_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', WorkflowJobTemplateNotificationTemplatesSuccessList.as_view(),
         name='workflow_job_template_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', WorkflowJobTemplateNotificationTemplatesApprovalList.as_view(),
+        name='workflow_job_template_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', WorkflowJobTemplateAccessList.as_view(), name='workflow_job_template_access_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', WorkflowJobTemplateObjectRolesList.as_view(), name='workflow_job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobTemplateLabelList.as_view(), name='workflow_job_template_label_list'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'workflow_job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/workflow_job_template_node.py

@@ -10,6 +10,7 @@ from awx.api.views import (
     WorkflowJobTemplateNodeFailureNodesList,
     WorkflowJobTemplateNodeAlwaysNodesList,
     WorkflowJobTemplateNodeCredentialsList,
+    WorkflowJobTemplateNodeCreateApproval,
 )
 
 
@@ -20,6 +21,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/failure_nodes/$', WorkflowJobTemplateNodeFailureNodesList.as_view(), name='workflow_job_template_node_failure_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/always_nodes/$', WorkflowJobTemplateNodeAlwaysNodesList.as_view(), name='workflow_job_template_node_always_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/credentials/$', WorkflowJobTemplateNodeCredentialsList.as_view(), name='workflow_job_template_node_credentials_list'),
+    url(r'^(?P<pk>[0-9]+)/create_approval_template/$', WorkflowJobTemplateNodeCreateApproval.as_view(), name='workflow_job_template_node_create_approval'),
 ]
 
 __all__ = ['urls']

awx/api/views/__init__.py

@@ -91,7 +91,8 @@ from awx.main.redact import UriCleaner
 from awx.api.permissions import (
     JobTemplateCallbackPermission, TaskPermission, ProjectUpdatePermission,
     InventoryInventorySourcesUpdatePermission, UserPermission,
-    InstanceGroupTowerPermission, VariableDataPermission
+    InstanceGroupTowerPermission, VariableDataPermission,
+    WorkflowApprovalPermission
 )
 from awx.api import renderers
 from awx.api import serializers
@@ -118,6 +119,7 @@ from awx.api.views.organization import ( # noqa
     OrganizationNotificationTemplatesErrorList,
     OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
+    OrganizationNotificationTemplatesApprovalList,
     OrganizationInstanceGroupsList,
     OrganizationAccessList,
     OrganizationObjectRolesList,
@@ -146,6 +148,12 @@ from awx.api.views.root import ( # noqa
     ApiV2RootView,
     ApiV2PingView,
     ApiV2ConfigView,
+    ApiV2SubscriptionView,
+)
+from awx.api.views.webhooks import ( # noqa
+    WebhookKeyView,
+    GithubWebhookReceiver,
+    GitlabWebhookReceiver,
 )
 
 
@@ -244,13 +252,6 @@ class DashboardView(APIView):
                                    'total': hg_projects.count(),
                                    'failed': hg_failed_projects.count()}
 
-        user_jobs = get_user_queryset(request.user, models.Job)
-        user_failed_jobs = user_jobs.filter(failed=True)
-        data['jobs'] = {'url': reverse('api:job_list', request=request),
-                        'failure_url': reverse('api:job_list', request=request) + "?failed=True",
-                        'total': user_jobs.count(),
-                        'failed': user_failed_jobs.count()}
-
         user_list = get_user_queryset(request.user, models.User)
         team_list = get_user_queryset(request.user, models.Team)
         credential_list = get_user_queryset(request.user, models.Credential)
@@ -839,8 +840,6 @@ class SystemJobEventsList(SubListAPIView):
         return super(SystemJobEventsList, self).finalize_response(request, response, *args, **kwargs)
 
 
-
-
 class ProjectUpdateCancel(RetrieveAPIView):
 
     model = models.ProjectUpdate
@@ -2569,10 +2568,34 @@ class JobTemplateSurveySpec(GenericAPIView):
                         return Response(dict(error=_(
                             "The {min_or_max} limit in survey question {idx} expected to be integer."
                         ).format(min_or_max=key, **context)))
-            if qtype in ['multiplechoice', 'multiselect'] and 'choices' not in survey_item:
-                return Response(dict(error=_(
-                    "Survey question {idx} of type {survey_item[type]} must specify choices.".format(**context)
-                )))
+            # if it's a multiselect or multiple choice, it must have coices listed
+            # choices and defualts must come in as strings seperated by /n characters.
+            if qtype == 'multiselect' or qtype == 'multiplechoice':
+                if 'choices' in survey_item:
+                    if isinstance(survey_item['choices'], str):
+                        survey_item['choices'] = '\n'.join(choice for choice in survey_item['choices'].splitlines() if choice.strip() != '')
+                else:
+                    return Response(dict(error=_(
+                        "Survey question {idx} of type {survey_item[type]} must specify choices.".format(**context)
+                    )))
+                # If there is a default string split it out removing extra /n characters.
+                # Note: There can still be extra newline characters added in the API, these are sanitized out using .strip()
+                if 'default' in survey_item:
+                    if isinstance(survey_item['default'], str):
+                        survey_item['default'] = '\n'.join(choice for choice in survey_item['default'].splitlines() if choice.strip() != '')
+                        list_of_defaults = survey_item['default'].splitlines()
+                    else:
+                        list_of_defaults = survey_item['default']
+                    if qtype == 'multiplechoice':
+                        # Multiplechoice types should only have 1 default.
+                        if len(list_of_defaults) > 1:
+                            return Response(dict(error=_(
+                                "Multiple Choice (Single Select) can only have one default value.".format(**context)
+                            )))
+                    if any(item not in survey_item['choices'] for item in list_of_defaults):
+                        return Response(dict(error=_(
+                            "Default choice must be answered from the choices listed.".format(**context)
+                        )))
 
             # Process encryption substitution
             if ("default" in survey_item and isinstance(survey_item['default'], str) and
@@ -2997,7 +3020,7 @@ class WorkflowJobTemplateNodeChildrenBaseList(EnforceParentRelationshipMixin, Su
         relationships = ['success_nodes', 'failure_nodes', 'always_nodes']
         relationships.remove(self.relationship)
         qs = functools.reduce(lambda x, y: (x | y),
-                              (Q(**{'{}__in'.format(rel): [sub.id]}) for rel in relationships))
+                              (Q(**{'{}__in'.format(r): [sub.id]}) for r in relationships))
 
         if models.WorkflowJobTemplateNode.objects.filter(Q(pk=parent.id) & qs).exists():
             return {"Error": _("Relationship not allowed.")}
@@ -3013,6 +3036,34 @@ class WorkflowJobTemplateNodeChildrenBaseList(EnforceParentRelationshipMixin, Su
         return None
 
 
+class WorkflowJobTemplateNodeCreateApproval(RetrieveAPIView):
+
+    model = models.WorkflowJobTemplateNode
+    serializer_class = serializers.WorkflowJobTemplateNodeCreateApprovalSerializer
+    permission_classes = []
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        serializer = self.get_serializer(instance=obj, data=request.data)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+        approval_template = obj.create_approval_template(**serializer.validated_data)
+        data = serializers.WorkflowApprovalTemplateSerializer(
+            approval_template,
+            context=self.get_serializer_context()
+        ).data
+        return Response(data, status=status.HTTP_200_OK)
+
+    def check_permissions(self, request):
+        obj = self.get_object().workflow_job_template
+        if request.method == 'POST':
+            if not request.user.can_access(models.WorkflowJobTemplate, 'change', obj, request.data):
+                self.permission_denied(request)
+        else:
+            if not request.user.can_access(models.WorkflowJobTemplate, 'read', obj):
+                self.permission_denied(request)
+
+
 class WorkflowJobTemplateNodeSuccessNodesList(WorkflowJobTemplateNodeChildrenBaseList):
     relationship = 'success_nodes'
 
@@ -3090,6 +3141,17 @@ class WorkflowJobTemplateCopy(CopyAPIView):
             data.update(messages)
         return Response(data)
 
+    def _build_create_dict(self, obj):
+        """Special processing of fields managed by char_prompts
+        """
+        r = super(WorkflowJobTemplateCopy, self)._build_create_dict(obj)
+        field_names = set(f.name for f in obj._meta.get_fields())
+        for field_name, ask_field_name in obj.get_ask_mapping().items():
+            if field_name in r and field_name not in field_names:
+                r.setdefault('char_prompts', {})
+                r['char_prompts'][field_name] = r.pop(field_name)
+        return r
+
     @staticmethod
     def deep_copy_permission_check_func(user, new_objs):
         for obj in new_objs:
@@ -3118,7 +3180,6 @@ class WorkflowJobTemplateLabelList(JobTemplateLabelList):
 
 class WorkflowJobTemplateLaunch(RetrieveAPIView):
 
-
     model = models.WorkflowJobTemplate
     obj_permission_type = 'start'
     serializer_class = serializers.WorkflowJobLaunchSerializer
@@ -3135,10 +3196,15 @@ class WorkflowJobTemplateLaunch(RetrieveAPIView):
                 extra_vars.setdefault(v, u'')
             if extra_vars:
                 data['extra_vars'] = extra_vars
-            if obj.ask_inventory_on_launch:
-                data['inventory'] = obj.inventory_id
-            else:
-                data.pop('inventory', None)
+            modified_ask_mapping = models.WorkflowJobTemplate.get_ask_mapping()
+            modified_ask_mapping.pop('extra_vars')
+            for field_name, ask_field_name in obj.get_ask_mapping().items():
+                if not getattr(obj, ask_field_name):
+                    data.pop(field_name, None)
+                elif field_name == 'inventory':
+                    data[field_name] = getattrd(obj, "%s.%s" % (field_name, 'id'), None)
+                else:
+                    data[field_name] = getattr(obj, field_name)
         return data
 
     def post(self, request, *args, **kwargs):
@@ -3252,6 +3318,11 @@ class WorkflowJobTemplateNotificationTemplatesSuccessList(WorkflowJobTemplateNot
     relationship = 'notification_templates_success'
 
 
+class WorkflowJobTemplateNotificationTemplatesApprovalList(WorkflowJobTemplateNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_approvals'
+
+
 class WorkflowJobTemplateAccessList(ResourceAccessList):
 
     model = models.User # needs to be User for AccessLists's
@@ -3287,7 +3358,7 @@ class WorkflowJobTemplateActivityStreamList(SubListAPIView):
                          Q(workflow_job_template_node__workflow_job_template=parent)).distinct()
 
 
-class WorkflowJobList(ListCreateAPIView):
+class WorkflowJobList(ListAPIView):
 
     model = models.WorkflowJob
     serializer_class = serializers.WorkflowJobListSerializer
@@ -3337,6 +3408,11 @@ class WorkflowJobNotificationsList(SubListAPIView):
     relationship = 'notifications'
     search_fields = ('subject', 'notification_type', 'body',)
 
+    def get_sublist_queryset(self, parent):
+        return self.model.objects.filter(Q(unifiedjob_notifications=parent) |
+                                         Q(unifiedjob_notifications__unified_job_node__workflow_job=parent,
+                                         unifiedjob_notifications__workflowapproval__isnull=False)).distinct()
+
 
 class WorkflowJobActivityStreamList(SubListAPIView):
 
@@ -3692,12 +3768,23 @@ class JobEventList(ListAPIView):
     serializer_class = serializers.JobEventSerializer
     search_fields = ('stdout',)
 
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.query_params.get('no_truncate'):
+            context.update(no_truncate=True)
+        return context
+
 
 class JobEventDetail(RetrieveAPIView):
 
     model = models.JobEvent
     serializer_class = serializers.JobEventSerializer
 
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context.update(no_truncate=True)
+        return context
+
 
 class JobEventChildrenList(SubListAPIView):
 
@@ -3926,12 +4013,23 @@ class AdHocCommandEventList(ListAPIView):
     serializer_class = serializers.AdHocCommandEventSerializer
     search_fields = ('stdout',)
 
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.query_params.get('no_truncate'):
+            context.update(no_truncate=True)
+        return context
+
 
 class AdHocCommandEventDetail(RetrieveAPIView):
 
     model = models.AdHocCommandEvent
     serializer_class = serializers.AdHocCommandEventSerializer
 
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context.update(no_truncate=True)
+        return context
+
 
 class BaseAdHocCommandEventsList(SubListAPIView):
 
@@ -3975,7 +4073,7 @@ class AdHocCommandNotificationsList(SubListAPIView):
     search_fields = ('subject', 'notification_type', 'body',)
 
 
-class SystemJobList(ListCreateAPIView):
+class SystemJobList(ListAPIView):
 
     model = models.SystemJob
     serializer_class = serializers.SystemJobListSerializer
@@ -4405,3 +4503,63 @@ for attr, value in list(locals().items()):
         name = camelcase_to_underscore(attr)
         view = value.as_view()
         setattr(this_module, name, view)
+
+
+class WorkflowApprovalTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
+
+    model = models.WorkflowApprovalTemplate
+    serializer_class = serializers.WorkflowApprovalTemplateSerializer
+
+
+class WorkflowApprovalTemplateJobsList(SubListAPIView):
+
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalListSerializer
+    parent_model = models.WorkflowApprovalTemplate
+    relationship = 'approvals'
+    parent_key = 'workflow_approval_template'
+
+
+class WorkflowApprovalList(ListCreateAPIView):
+
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalListSerializer
+
+    def get(self, request, *args, **kwargs):
+        return super(WorkflowApprovalList, self).get(request, *args, **kwargs)
+
+
+class WorkflowApprovalDetail(UnifiedJobDeletionMixin, RetrieveDestroyAPIView):
+
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalSerializer
+
+
+class WorkflowApprovalApprove(RetrieveAPIView):
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalViewSerializer
+    permission_classes = (WorkflowApprovalPermission,)
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if not request.user.can_access(models.WorkflowApproval, 'approve_or_deny', obj):
+            raise PermissionDenied(detail=_("User does not have permission to approve or deny this workflow."))
+        if obj.status != 'pending':
+            return Response({"error": _("This workflow step has already been approved or denied.")}, status=status.HTTP_400_BAD_REQUEST)
+        obj.approve(request)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class WorkflowApprovalDeny(RetrieveAPIView):
+    model = models.WorkflowApproval
+    serializer_class = serializers.WorkflowApprovalViewSerializer
+    permission_classes = (WorkflowApprovalPermission,)
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if not request.user.can_access(models.WorkflowApproval, 'approve_or_deny', obj):
+            raise PermissionDenied(detail=_("User does not have permission to approve or deny this workflow."))
+        if obj.status != 'pending':
+            return Response({"error": _("This workflow step has already been approved or denied.")}, status=status.HTTP_400_BAD_REQUEST)
+        obj.deny(request)
+        return Response(status=status.HTTP_204_NO_CONTENT)

awx/api/views/inventory.py

@@ -70,12 +70,16 @@ class InventoryUpdateEventsList(SubListAPIView):
 
 class InventoryScriptList(ListCreateAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     serializer_class = CustomInventoryScriptSerializer
 
 
 class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     serializer_class = CustomInventoryScriptSerializer
 
@@ -92,6 +96,8 @@ class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
 
 class InventoryScriptObjectRolesList(SubListAPIView):
 
+    deprecated = True
+
     model = Role
     serializer_class = RoleSerializer
     parent_model = CustomInventoryScript
@@ -105,6 +111,8 @@ class InventoryScriptObjectRolesList(SubListAPIView):
 
 class InventoryScriptCopy(CopyAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     copy_return_serializer_class = CustomInventoryScriptSerializer
 

awx/api/views/metrics.py

@@ -31,9 +31,10 @@ class MetricsView(APIView):
     swagger_topic = 'Metrics'
 
     renderer_classes = [renderers.PlainTextRenderer,
+                        renderers.PrometheusJSONRenderer,
                         renderers.BrowsableAPIRenderer,]
 
-    def get(self, request, format='txt'):
+    def get(self, request):
         ''' Show Metrics Details '''
         if (request.user.is_superuser or request.user.is_system_auditor):
             return Response(metrics().decode('UTF-8'))

awx/api/views/organization.py

@@ -195,6 +195,11 @@ class OrganizationNotificationTemplatesSuccessList(OrganizationNotificationTempl
     relationship = 'notification_templates_success'
 
 
+class OrganizationNotificationTemplatesApprovalList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_approvals'
+
+
 class OrganizationInstanceGroupsList(SubListAttachDetachAPIView):
 
     model = InstanceGroup

awx/api/views/root.py

@@ -17,6 +17,8 @@ from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 from rest_framework import status
 
+import requests
+
 from awx.api.generics import APIView
 from awx.main.ha import is_ha_environment
 from awx.main.utils import (
@@ -124,6 +126,7 @@ class ApiVersionRootView(APIView):
         data['activity_stream'] = reverse('api:activity_stream_list', request=request)
         data['workflow_job_templates'] = reverse('api:workflow_job_template_list', request=request)
         data['workflow_jobs'] = reverse('api:workflow_job_list', request=request)
+        data['workflow_approvals'] = reverse('api:workflow_approval_list', request=request)
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
         return Response(data)
@@ -168,6 +171,45 @@ class ApiV2PingView(APIView):
         return Response(response)
 
 
+class ApiV2SubscriptionView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    name = _('Configuration')
+    swagger_topic = 'System Configuration'
+
+    def check_permissions(self, request):
+        super(ApiV2SubscriptionView, self).check_permissions(request)
+        if not request.user.is_superuser and request.method.lower() not in {'options', 'head'}:
+            self.permission_denied(request)  # Raises PermissionDenied exception.
+
+    def post(self, request):
+        from awx.main.utils.common import get_licenser
+        data = request.data.copy()
+        if data.get('rh_password') == '$encrypted$':
+            data['rh_password'] = settings.REDHAT_PASSWORD
+        try:
+            user, pw = data.get('rh_username'), data.get('rh_password')
+            validated = get_licenser().validate_rh(user, pw)
+            if user:
+                settings.REDHAT_USERNAME = data['rh_username']
+            if pw:
+                settings.REDHAT_PASSWORD = data['rh_password']
+        except Exception as exc:
+            msg = _("Invalid License")
+            if (
+                isinstance(exc, requests.exceptions.HTTPError) and
+                getattr(getattr(exc, 'response', None), 'status_code', None) == 401
+            ):
+                msg = _("The provided credentials are invalid (HTTP 401).")
+            if isinstance(exc, (ValueError, OSError)) and exc.args:
+                msg = exc.args[0]
+            logger.exception(smart_text(u"Invalid license submitted."),
+                             extra=dict(actor=request.user.username))
+            return Response({"error": msg}, status=status.HTTP_400_BAD_REQUEST)
+
+        return Response(validated)
+
+
 class ApiV2ConfigView(APIView):
 
     permission_classes = (IsAuthenticated,)

awx/api/views/webhooks.py

@@ -0,0 +1,247 @@
+from hashlib import sha1
+import hmac
+import json
+import logging
+import urllib.parse
+
+from django.utils.encoding import force_bytes
+from django.utils.translation import ugettext_lazy as _
+from django.views.decorators.csrf import csrf_exempt
+
+from rest_framework import status
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+
+from awx.api import serializers
+from awx.api.generics import APIView, GenericAPIView
+from awx.api.permissions import WebhookKeyPermission
+from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
+
+
+logger = logging.getLogger('awx.api.views.webhooks')
+
+
+class WebhookKeyView(GenericAPIView):
+    serializer_class = serializers.EmptySerializer
+    permission_classes = (WebhookKeyPermission,)
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        self.model = qs_models.get(self.kwargs['model_kwarg'])
+
+        return super().get_queryset()
+
+    def get(self, request, *args, **kwargs):
+        obj = self.get_object()
+
+        return Response({'webhook_key': obj.webhook_key})
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        obj.rotate_webhook_key()
+        obj.save(update_fields=['webhook_key'])
+
+        return Response({'webhook_key': obj.webhook_key}, status=status.HTTP_201_CREATED)
+
+
+class WebhookReceiverBase(APIView):
+    lookup_url_kwarg = None
+    lookup_field = 'pk'
+
+    permission_classes = (AllowAny,)
+    authentication_classes = ()
+
+    ref_keys = {}
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        model = qs_models.get(self.kwargs['model_kwarg'])
+        if model is None:
+            raise PermissionDenied
+
+        return model.objects.filter(webhook_service=self.service).exclude(webhook_key='')
+
+    def get_object(self):
+        queryset = self.get_queryset()
+        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
+        filter_kwargs = {self.lookup_field: self.kwargs[lookup_url_kwarg]}
+
+        obj = queryset.filter(**filter_kwargs).first()
+        if obj is None:
+            raise PermissionDenied
+
+        return obj
+
+    def get_event_type(self):
+        raise NotImplementedError
+
+    def get_event_guid(self):
+        raise NotImplementedError
+
+    def get_event_status_api(self):
+        raise NotImplementedError
+
+    def get_event_ref(self):
+        key = self.ref_keys.get(self.get_event_type(), '')
+        value = self.request.data
+        for element in key.split('.'):
+            try:
+                if element.isdigit():
+                    value = value[int(element)]
+                else:
+                    value = (value or {}).get(element)
+            except Exception:
+                value = None
+        if value == '0000000000000000000000000000000000000000':  # a deleted ref
+            value = None
+        return value
+
+    def get_signature(self):
+        raise NotImplementedError
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        logger.debug("header signature: %s", self.get_signature())
+        logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
+            raise PermissionDenied
+
+    @csrf_exempt
+    def post(self, request, *args, **kwargs):
+        # Ensure that the full contents of the request are captured for multiple uses.
+        request.body
+
+        logger.debug(
+            "headers: {}\n"
+            "data: {}\n".format(request.headers, request.data)
+        )
+        obj = self.get_object()
+        self.check_signature(obj)
+
+        event_type = self.get_event_type()
+        event_guid = self.get_event_guid()
+        event_ref = self.get_event_ref()
+        status_api = self.get_event_status_api()
+
+        kwargs = {
+            'unified_job_template_id': obj.id,
+            'webhook_service': obj.webhook_service,
+            'webhook_guid': event_guid,
+        }
+        if WorkflowJob.objects.filter(**kwargs).exists() or Job.objects.filter(**kwargs).exists():
+            # Short circuit if this webhook has already been received and acted upon.
+            logger.debug("Webhook previously received, returning without action.")
+            return Response({'message': _("Webhook previously received, aborting.")},
+                            status=status.HTTP_202_ACCEPTED)
+
+        kwargs = {
+            '_eager_fields': {
+                'launch_type': 'webhook',
+                'webhook_service': obj.webhook_service,
+                'webhook_credential': obj.webhook_credential,
+                'webhook_guid': event_guid,
+            },
+            'extra_vars': json.dumps({
+                'tower_webhook_event_type': event_type,
+                'tower_webhook_event_guid': event_guid,
+                'tower_webhook_event_ref': event_ref,
+                'tower_webhook_status_api': status_api,
+                'tower_webhook_payload': request.data,
+            })
+        }
+
+        new_job = obj.create_unified_job(**kwargs)
+        new_job.signal_start()
+
+        return Response({'message': "Job queued."}, status=status.HTTP_202_ACCEPTED)
+
+
+class GithubWebhookReceiver(WebhookReceiverBase):
+    service = 'github'
+
+    ref_keys = {
+        'pull_request': 'pull_request.head.sha',
+        'pull_request_review': 'pull_request.head.sha',
+        'pull_request_review_comment': 'pull_request.head.sha',
+        'push': 'after',
+        'release': 'release.tag_name',
+        'commit_comment': 'comment.commit_id',
+        'create': 'ref',
+        'page_build': 'build.commit',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITHUB_EVENT')
+
+    def get_event_guid(self):
+        return self.request.META.get('HTTP_X_GITHUB_DELIVERY')
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'pull_request':
+            return
+        return self.request.data.get('pull_request', {}).get('statuses_url')
+
+    def get_signature(self):
+        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
+        if not header_sig:
+            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
+            raise PermissionDenied
+        hash_alg, signature = header_sig.split('=')
+        if hash_alg != 'sha1':
+            logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
+            raise PermissionDenied
+        return force_bytes(signature)
+
+
+class GitlabWebhookReceiver(WebhookReceiverBase):
+    service = 'gitlab'
+
+    ref_keys = {
+        'Push Hook': 'checkout_sha',
+        'Tag Push Hook': 'checkout_sha',
+        'Merge Request Hook': 'object_attributes.last_commit.id',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITLAB_EVENT')
+
+    def get_event_guid(self):
+        # GitLab does not provide a unique identifier on events, so construct one.
+        h = sha1()
+        h.update(force_bytes(self.request.body))
+        return h.hexdigest()
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'Merge Request Hook':
+            return
+        project = self.request.data.get('project', {})
+        repo_url = project.get('web_url')
+        if not repo_url:
+            return
+        parsed = urllib.parse.urlparse(repo_url)
+
+        return "{}://{}/api/v4/projects/{}/statuses/{}".format(
+            parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
+
+    def get_signature(self):
+        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        # GitLab only returns the secret token, not an hmac hash.  Use
+        # the hmac `compare_digest` helper function to prevent timing
+        # analysis by attackers.
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
+            raise PermissionDenied

awx/conf/fields.py

@@ -10,8 +10,8 @@ from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework.fields import (  # noqa
-    BooleanField, CharField, ChoiceField, DictField, EmailField, IntegerField,
-    ListField, NullBooleanField
+    BooleanField, CharField, ChoiceField, DictField, EmailField,
+    IntegerField, ListField, NullBooleanField
 )
 
 logger = logging.getLogger('awx.conf.fields')
@@ -121,11 +121,14 @@ class URLField(CharField):
 
     def __init__(self, **kwargs):
         schemes = kwargs.pop('schemes', None)
+        regex = kwargs.pop('regex', None)
         self.allow_plain_hostname = kwargs.pop('allow_plain_hostname', False)
         super(URLField, self).__init__(**kwargs)
         validator_kwargs = dict(message=_('Enter a valid URL'))
         if schemes is not None:
             validator_kwargs['schemes'] = schemes
+        if regex is not None:
+            validator_kwargs['regex'] = regex
         self.validators.append(URLValidator(**validator_kwargs))
 
     def to_representation(self, value):

awx/conf/migrations/0001_initial.py

@@ -21,7 +21,8 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('key', models.CharField(max_length=255)),
                 ('value', jsonfield.fields.JSONField(null=True)),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False,
+                                           to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True)),
             ],
             options={
                 'abstract': False,

awx/conf/tests/functional/test_api.py

@@ -8,6 +8,7 @@ from awx.main.utils.encryption import decrypt_field
 from awx.conf import fields
 from awx.conf.registry import settings_registry
 from awx.conf.models import Setting
+from awx.sso import fields as sso_fields
 
 
 @pytest.fixture
@@ -137,7 +138,7 @@ def test_setting_signleton_retrieve_hierachy(api_request, dummy_setting):
 
 
 @pytest.mark.django_db
-def test_setting_signleton_retrieve_readonly(api_request, dummy_setting):
+def test_setting_singleton_retrieve_readonly(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.IntegerField,
@@ -183,6 +184,30 @@ def test_setting_singleton_update(api_request, dummy_setting):
         assert response.data['FOO_BAR'] == 4
 
 
+@pytest.mark.django_db
+def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, dummy_setting):
+    # Some HybridDictField subclasses have a child of _Forbidden,
+    # indicating that only the defined fields can be filled in.  Make
+    # sure that the _Forbidden validator doesn't get used for the
+    # fields.  See also https://github.com/ansible/awx/issues/4099.
+    with dummy_setting(
+        'FOO_BAR',
+        field_class=sso_fields.SAMLOrgAttrField,
+        category='FooBar',
+        category_slug='foobar',
+    ), mock.patch('awx.conf.views.handle_setting_changes'):
+        api_request(
+            'patch',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}),
+            data={'FOO_BAR': {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}}
+        )
+        response = api_request(
+            'get',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'})
+        )
+        assert response.data['FOO_BAR'] == {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}
+
+
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting(
@@ -206,7 +231,7 @@ def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy
 
 
 @pytest.mark.django_db
-def test_setting_singleton_update_dont_change_encripted_mark(api_request, dummy_setting):
+def test_setting_singleton_update_dont_change_encrypted_mark(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.CharField,

awx/main/access.py

@@ -37,6 +37,7 @@ from awx.main.models import (
     ProjectUpdateEvent, Role, Schedule, SystemJob, SystemJobEvent,
     SystemJobTemplate, Team, UnifiedJob, UnifiedJobTemplate, WorkflowJob,
     WorkflowJobNode, WorkflowJobTemplate, WorkflowJobTemplateNode,
+    WorkflowApproval, WorkflowApprovalTemplate,
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
 )
 from awx.main.models.mixins import ResourceMixin
@@ -316,10 +317,19 @@ class BaseAccess(object):
             validation_info['time_remaining'] = 99999999
             validation_info['grace_period_remaining'] = 99999999
 
+        report_violation = lambda message: logger.error(message)
+
+        if (
+            validation_info.get('trial', False) is True or
+            validation_info['instance_count'] == 10  # basic 10 license
+        ):
+            def report_violation(message):
+                raise PermissionDenied(message)
+
         if check_expiration and validation_info.get('time_remaining', None) is None:
             raise PermissionDenied(_("License is missing."))
-        if check_expiration and validation_info.get("grace_period_remaining") <= 0:
-            raise PermissionDenied(_("License has expired."))
+        elif check_expiration and validation_info.get("grace_period_remaining") <= 0:
+            report_violation(_("License has expired."))
 
         free_instances = validation_info.get('free_instances', 0)
         available_instances = validation_info.get('available_instances', 0)
@@ -327,11 +337,11 @@ class BaseAccess(object):
         if add_host_name:
             host_exists = Host.objects.filter(name=add_host_name).exists()
             if not host_exists and free_instances == 0:
-                raise PermissionDenied(_("License count of %s instances has been reached.") % available_instances)
+                report_violation(_("License count of %s instances has been reached.") % available_instances)
             elif not host_exists and free_instances < 0:
-                raise PermissionDenied(_("License count of %s instances has been exceeded.") % available_instances)
+                report_violation(_("License count of %s instances has been exceeded.") % available_instances)
         elif not add_host_name and free_instances < 0:
-            raise PermissionDenied(_("Host count exceeds available instances."))
+            report_violation(_("Host count exceeds available instances."))
 
     def check_org_host_limit(self, data, add_host_name=None):
         validation_info = get_licenser().validate()
@@ -538,7 +548,7 @@ class InstanceGroupAccess(BaseAccess):
 
     def filtered_queryset(self):
         return InstanceGroup.objects.filter(
-            organization__in=Organization.accessible_pk_qs(self.user, 'admin_role'))
+            organization__in=Organization.accessible_pk_qs(self.user, 'admin_role')).distinct()
 
     def can_add(self, data):
         return self.user.is_superuser
@@ -651,7 +661,7 @@ class UserAccess(BaseAccess):
         if obj.is_superuser and super_users.count() == 1:
             # cannot delete the last active superuser
             return False
-        if self.user.is_superuser:
+        if self.can_admin(obj, None, allow_orphans=True):
             return True
         return False
 
@@ -833,10 +843,6 @@ class InventoryAccess(BaseAccess):
     def filtered_queryset(self, allowed=None, ad_hoc=None):
         return self.model.accessible_objects(self.user, 'read_role')
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.read_role
-
     @check_superuser
     def can_use(self, obj):
         return self.user in obj.use_role
@@ -906,9 +912,6 @@ class HostAccess(BaseAccess):
     def filtered_queryset(self):
         return self.model.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role'))
 
-    def can_read(self, obj):
-        return obj and self.user in obj.inventory.read_role
-
     def can_add(self, data):
         if not data:  # So the browseable API will work
             return Inventory.accessible_objects(self.user, 'admin_role').exists()
@@ -970,9 +973,6 @@ class GroupAccess(BaseAccess):
     def filtered_queryset(self):
         return Group.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role'))
 
-    def can_read(self, obj):
-        return obj and self.user in obj.inventory.read_role
-
     def can_add(self, data):
         if not data or 'inventory' not in data:
             return False
@@ -1016,12 +1016,6 @@ class InventorySourceAccess(NotificationAttachMixin, BaseAccess):
     def filtered_queryset(self):
         return self.model.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role'))
 
-    def can_read(self, obj):
-        if obj and obj.inventory:
-            return self.user.can_access(Inventory, 'read', obj.inventory)
-        else:
-            return False
-
     def can_add(self, data):
         if not data or 'inventory' not in data:
             return Organization.accessible_objects(self.user, 'admin_role').exists()
@@ -1114,9 +1108,6 @@ class CredentialTypeAccess(BaseAccess):
     model = CredentialType
     prefetch_related = ('created_by', 'modified_by',)
 
-    def can_read(self, obj):
-        return True
-
     def can_use(self, obj):
         return True
 
@@ -1158,25 +1149,26 @@ class CredentialAccess(BaseAccess):
     def filtered_queryset(self):
         return self.model.accessible_objects(self.user, 'read_role')
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.read_role
-
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
             return True
         if data and data.get('user', None):
             user_obj = get_object_from_data('user', User, data)
-            return bool(self.user == user_obj or UserAccess(self.user).can_admin(user_obj, None, check_setting=False))
+            if not bool(self.user == user_obj or UserAccess(self.user).can_admin(user_obj, None, check_setting=False)):
+                return False
         if data and data.get('team', None):
             team_obj = get_object_from_data('team', Team, data)
-            return check_user_access(self.user, Team, 'change', team_obj, None)
+            if not check_user_access(self.user, Team, 'change', team_obj, None):
+                return False
         if data and data.get('organization', None):
             organization_obj = get_object_from_data('organization', Organization, data)
-            return any([check_user_access(self.user, Organization, 'change', organization_obj, None),
-                        self.user in organization_obj.credential_admin_role])
-        return False
+            if not any([check_user_access(self.user, Organization, 'change', organization_obj, None),
+                        self.user in organization_obj.credential_admin_role]):
+                return False
+        if not any(data.get(key, None) for key in ('user', 'team', 'organization')):
+            return False  # you have to provide 1 owner field
+        return True
 
     @check_superuser
     def can_use(self, obj):
@@ -1219,10 +1211,6 @@ class CredentialInputSourceAccess(BaseAccess):
         return CredentialInputSource.objects.filter(
             target_credential__in=Credential.accessible_pk_qs(self.user, 'read_role'))
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.target_credential.read_role
-
     @check_superuser
     def can_add(self, data):
         return (
@@ -1971,10 +1959,6 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
     def filtered_queryset(self):
         return self.model.accessible_objects(self.user, 'read_role')
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.read_role
-
     @check_superuser
     def can_add(self, data):
         '''
@@ -2372,13 +2356,18 @@ class UnifiedJobTemplateAccess(BaseAccess):
         return self.model.objects.filter(
             Q(pk__in=self.model.accessible_pk_qs(self.user, 'read_role')) |
             Q(inventorysource__inventory__id__in=Inventory._accessible_pk_qs(
-                Inventory, self.user, 'read_role')))
+                Inventory, self.user, 'read_role'))
+        )
 
     def can_start(self, obj, validate_license=True):
         access_class = access_registry[obj.__class__]
         access_instance = access_class(self.user)
         return access_instance.can_start(obj, validate_license=validate_license)
 
+    def get_queryset(self):
+        return super(UnifiedJobTemplateAccess, self).get_queryset().filter(
+            workflowapprovaltemplate__isnull=True)
+
 
 class UnifiedJobAccess(BaseAccess):
     '''
@@ -2425,6 +2414,10 @@ class UnifiedJobAccess(BaseAccess):
         )
         return qs
 
+    def get_queryset(self):
+        return super(UnifiedJobAccess, self).get_queryset().filter(
+            workflowapproval__isnull=True)
+
 
 class ScheduleAccess(BaseAccess):
     '''
@@ -2486,14 +2479,6 @@ class NotificationTemplateAccess(BaseAccess):
             Q(organization__in=self.user.auditor_of_organizations)
         ).distinct()
 
-    def can_read(self, obj):
-        if self.user.is_superuser or self.user.is_system_auditor:
-            return True
-        if obj.organization is not None:
-            if self.user in obj.organization.notification_admin_role or self.user in obj.organization.auditor_role:
-                return True
-        return False
-
     @check_superuser
     def can_add(self, data):
         if not data:
@@ -2533,9 +2518,6 @@ class NotificationAccess(BaseAccess):
             Q(notification_template__organization__in=self.user.auditor_of_organizations)
         ).distinct()
 
-    def can_read(self, obj):
-        return self.user.can_access(NotificationTemplate, 'read', obj.notification_template)
-
     def can_delete(self, obj):
         return self.user.can_access(NotificationTemplate, 'delete', obj.notification_template)
 
@@ -2550,10 +2532,6 @@ class LabelAccess(BaseAccess):
     def filtered_queryset(self):
         return self.model.objects.all()
 
-    @check_superuser
-    def can_read(self, obj):
-        return self.user in obj.organization.read_role
-
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
@@ -2711,15 +2689,6 @@ class RoleAccess(BaseAccess):
             result = result | super_qs
         return result
 
-    def can_read(self, obj):
-        if not obj:
-            return False
-        if self.user.is_superuser or self.user.is_system_auditor:
-            return True
-
-        return Role.filter_visible_roles(
-            self.user, Role.objects.filter(pk=obj.id)).exists()
-
     def can_add(self, obj, data):
         # Unsupported for now
         return False
@@ -2764,5 +2733,80 @@ class RoleAccess(BaseAccess):
         return False
 
 
+class WorkflowApprovalAccess(BaseAccess):
+    '''
+    A user can create a workflow approval if they are a superuser, an org admin
+    of the org connected to the workflow, or if they are assigned as admins to
+    the workflow.
+
+    A user can approve a workflow when they are:
+    - a superuser
+    - a workflow admin
+    - an organization admin
+    - any user who has explicitly been assigned the "approver" role
+
+    A user can see approvals if they have read access to the associated WorkflowJobTemplate.
+    '''
+
+    model = WorkflowApproval
+    prefetch_related = ('created_by', 'modified_by',)
+
+    def can_use(self, obj):
+        return True
+
+    def can_start(self, obj, validate_license=True):
+        return True
+
+    def filtered_queryset(self):
+        return self.model.objects.filter(
+            unified_job_node__workflow_job__unified_job_template__in=WorkflowJobTemplate.accessible_pk_qs(
+                self.user, 'read_role'))
+
+    def can_approve_or_deny(self, obj):
+        if (
+            (obj.workflow_job_template and self.user in obj.workflow_job_template.approval_role) or
+            self.user.is_superuser
+        ):
+            return True
+
+
+class WorkflowApprovalTemplateAccess(BaseAccess):
+    '''
+    A user can create a workflow approval if they are a superuser, an org admin
+    of the org connected to the workflow, or if they are assigned as admins to
+    the workflow.
+
+    A user can approve a workflow when they are:
+    - a superuser
+    - a workflow admin
+    - an organization admin
+    - any user who has explicitly been assigned the "approver" role at the workflow or organization level
+
+    A user can see approval templates if they have read access to the associated WorkflowJobTemplate.
+    '''
+
+    model = WorkflowApprovalTemplate
+    prefetch_related = ('created_by', 'modified_by',)
+
+    @check_superuser
+    def can_add(self, data):
+        if data is None:  # Hide direct creation in API browser
+            return False
+        else:
+            return (self.check_related('workflow_approval_template', UnifiedJobTemplate, role_field='admin_role'))
+
+    def can_start(self, obj, validate_license=False):
+        # for copying WFJTs that contain approval nodes
+        if self.user.is_superuser:
+            return True
+
+        return self.user in obj.workflow_job_template.execute_role
+
+    def filtered_queryset(self):
+        return self.model.objects.filter(
+            workflowjobtemplatenodes__workflow_job_template__in=WorkflowJobTemplate.accessible_pk_qs(
+                self.user, 'read_role'))
+
+
 for cls in BaseAccess.__subclasses__():
     access_registry[cls.model] = cls

awx/main/analytics/__init__.py

@@ -1 +1 @@
-from .core import register, gather, ship  # noqa
+from .core import register, gather, ship, table_version  # noqa

awx/main/analytics/collectors.py

@@ -12,14 +12,14 @@ from awx.main.utils import (get_awx_version, get_ansible_version,
                             get_custom_venv_choices, camelcase_to_underscore)
 from awx.main import models
 from django.contrib.sessions.models import Session
-from awx.main.analytics import register
+from awx.main.analytics import register, table_version
 
 '''
 This module is used to define metrics collected by awx.main.analytics.gather()
 Each function is decorated with a key name, and should return a data
 structure that can be serialized to JSON
 
-@register('something')
+@register('something', '1.0')
 def something(since):
     # the generated archive will contain a `something.json` w/ this JSON
     return {'some': 'json'}
@@ -31,7 +31,7 @@ data _since_ the last report date - i.e., new data in the last 24 hours)
 '''
 
 
-@register('config')
+@register('config', '1.0')
 def config(since):
     license_info = get_license(show_key=False)
     install_type = 'traditional'
@@ -62,7 +62,7 @@ def config(since):
     }
 
 
-@register('counts')
+@register('counts', '1.0')
 def counts(since):
     counts = {}
     for cls in (models.Organization, models.Team, models.User,
@@ -93,10 +93,11 @@ def counts(since):
     counts['active_user_sessions'] = active_user_sessions
     counts['active_anonymous_sessions'] = active_anonymous_sessions
     counts['running_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('running', 'waiting',)).count()
+    counts['pending_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('pending',)).count()
     return counts
 
     
-@register('org_counts')
+@register('org_counts', '1.0')
 def org_counts(since):
     counts = {}
     for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), 
@@ -108,7 +109,7 @@ def org_counts(since):
     return counts
     
     
-@register('cred_type_counts')
+@register('cred_type_counts', '1.0')
 def cred_type_counts(since):
     counts = {}
     for cred_type in models.CredentialType.objects.annotate(num_credentials=Count(
@@ -120,7 +121,7 @@ def cred_type_counts(since):
     return counts
     
     
-@register('inventory_counts')
+@register('inventory_counts', '1.0')
 def inventory_counts(since):
     counts = {}
     for inv in models.Inventory.objects.filter(kind='').annotate(num_sources=Count('inventory_sources', distinct=True), 
@@ -140,7 +141,7 @@ def inventory_counts(since):
     return counts
 
 
-@register('projects_by_scm_type')
+@register('projects_by_scm_type', '1.0')
 def projects_by_scm_type(since):
     counts = dict(
         (t[0] or 'manual', 0)
@@ -159,8 +160,8 @@ def _get_isolated_datetime(last_check):
     return last_check
 
 
-@register('instance_info')
-def instance_info(since):
+@register('instance_info', '1.0')
+def instance_info(since, include_hostnames=False):
     info = {}
     instances = models.Instance.objects.values_list('hostname').values(
         'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled')
@@ -175,11 +176,13 @@ def instance_info(since):
             'last_isolated_check': _get_isolated_datetime(instance['last_isolated_check']),
             'enabled': instance['enabled']
         }
+        if include_hostnames is True:
+            instance_info['hostname'] = instance['hostname']
         info[instance['uuid']] = instance_info
     return info
 
 
-@register('job_counts')
+@register('job_counts', '1.0')
 def job_counts(since):
     counts = {}
     counts['total_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').count()
@@ -189,7 +192,7 @@ def job_counts(since):
     return counts
     
     
-@register('job_instance_counts')
+@register('job_instance_counts', '1.0')
 def job_instance_counts(since):
     counts = {}
     job_types = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
@@ -204,7 +207,19 @@ def job_instance_counts(since):
     return counts
 
 
+@register('query_info', '1.0')
+def query_info(since, collection_type):
+    query_info = {}
+    query_info['last_run'] = str(since)
+    query_info['current_time'] = str(now())
+    query_info['collection_type'] = collection_type
+    return query_info
+
+
 # Copies Job Events from db to a .csv to be shipped
+@table_version('events_table.csv', '1.0')
+@table_version('unified_jobs_table.csv', '1.0')
+@table_version('unified_job_template_table.csv', '1.0')
 def copy_tables(since, full_path):
     def _copy_table(table, query, path):
         file_path = os.path.join(path, table + '_table.csv')
@@ -233,10 +248,12 @@ def copy_tables(since, full_path):
                               WHERE main_jobevent.created > {}
                               ORDER BY main_jobevent.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))
     _copy_table(table='events', query=events_query, path=full_path)
-    
-    unified_job_query = '''COPY (SELECT main_unifiedjob.id, 
+
+    unified_job_query = '''COPY (SELECT main_unifiedjob.id,
                                  main_unifiedjob.polymorphic_ctype_id,
                                  django_content_type.model,
+                                 main_project.organization_id,
+                                 main_organization.name as organization_name,
                                  main_unifiedjob.created,  
                                  main_unifiedjob.name,  
                                  main_unifiedjob.unified_job_template_id, 
@@ -252,13 +269,16 @@ def copy_tables(since, full_path):
                                  main_unifiedjob.elapsed, 
                                  main_unifiedjob.job_explanation, 
                                  main_unifiedjob.instance_group_id
-                                 FROM main_unifiedjob, django_content_type
-                                 WHERE main_unifiedjob.created > {} AND 
-                                 main_unifiedjob.polymorphic_ctype_id = django_content_type.id AND 
-                                 main_unifiedjob.launch_type != 'sync'
+                                 FROM main_unifiedjob
+                                 JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id
+                                 JOIN django_content_type ON main_unifiedjob.polymorphic_ctype_id = django_content_type.id
+                                 JOIN main_project ON main_project.unifiedjobtemplate_ptr_id = main_job.project_id
+                                 JOIN main_organization ON main_organization.id = main_project.organization_id
+                                 WHERE main_unifiedjob.created > {} 
+                                 AND main_unifiedjob.launch_type != 'sync'
                                  ORDER BY main_unifiedjob.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))    
     _copy_table(table='unified_jobs', query=unified_job_query, path=full_path)
-    
+
     unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id, 
                                  main_unifiedjobtemplate.polymorphic_ctype_id,
                                  django_content_type.model,
@@ -279,4 +299,3 @@ def copy_tables(since, full_path):
                                  ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))    
     _copy_table(table='unified_job_template', query=unified_job_template_query, path=full_path)
     return
-

awx/main/analytics/core.py

@@ -5,10 +5,9 @@ import os
 import os.path
 import tempfile
 import shutil
-import subprocess
+import requests
 
 from django.conf import settings
-from django.utils.encoding import smart_str
 from django.utils.timezone import now, timedelta
 from rest_framework.exceptions import PermissionDenied
 
@@ -18,11 +17,13 @@ from awx.main.access import access_registry
 from awx.main.models.ha import TowerAnalyticsState
 
 
-__all__ = ['register', 'gather', 'ship']
+__all__ = ['register', 'gather', 'ship', 'table_version']
 
 
 logger = logging.getLogger('awx.main.analytics')
 
+manifest = dict()
+
 
 def _valid_license():
     try:
@@ -35,25 +36,37 @@ def _valid_license():
     return True
 
 
-def register(key):
+def register(key, version):
     """
     A decorator used to register a function as a metric collector.
 
     Decorated functions should return JSON-serializable objects.
 
-    @register('projects_by_scm_type')
+    @register('projects_by_scm_type', 1)
     def projects_by_scm_type():
         return {'git': 5, 'svn': 1, 'hg': 0}
     """
 
     def decorate(f):
         f.__awx_analytics_key__ = key
+        f.__awx_analytics_version__ = version
         return f
 
     return decorate
 
 
-def gather(dest=None, module=None):
+def table_version(file_name, version):
+
+    global manifest
+    manifest[file_name] = version
+
+    def decorate(f):
+        return f
+
+    return decorate
+
+
+def gather(dest=None, module=None, collection_type='scheduled'):
     """
     Gather all defined metrics and write them as JSON files in a .tgz
 
@@ -67,35 +80,49 @@ def gather(dest=None, module=None):
     last_run = state.last_run
     logger.debug("Last analytics run was: {}".format(last_run))
     
-    max_interval = now() - timedelta(days=7)
+    max_interval = now() - timedelta(weeks=4)
     if last_run < max_interval or not last_run:
         last_run = max_interval
 
-
     if _valid_license() is False:
         logger.exception("Invalid License provided, or No License Provided")
         return "Error: Invalid License provided, or No License Provided"
-    
-    if not settings.INSIGHTS_TRACKING_STATE:
-        logger.error("Insights analytics not enabled")
+
+    if collection_type != 'dry-run' and not settings.INSIGHTS_TRACKING_STATE:
+        logger.error("Automation Analytics not enabled. Use --dry-run to gather locally without sending.")
         return
 
     if module is None:
         from awx.main.analytics import collectors
         module = collectors
 
+
     dest = dest or tempfile.mkdtemp(prefix='awx_analytics')
     for name, func in inspect.getmembers(module):
         if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__'):
             key = func.__awx_analytics_key__
+            manifest['{}.json'.format(key)] = func.__awx_analytics_version__
             path = '{}.json'.format(os.path.join(dest, key))
             with open(path, 'w', encoding='utf-8') as f:
                 try:
-                    json.dump(func(last_run), f)
+                    if func.__name__ == 'query_info':
+                        json.dump(func(last_run, collection_type=collection_type), f)
+                    else:
+                        json.dump(func(last_run), f)
                 except Exception:
                     logger.exception("Could not generate metric {}.json".format(key))
                     f.close()
                     os.remove(f.name)
+    
+    path = os.path.join(dest, 'manifest.json')
+    with open(path, 'w', encoding='utf-8') as f:
+        try:
+            json.dump(manifest, f)
+        except Exception:
+            logger.exception("Could not generate manifest.json")
+            f.close()
+            os.remove(f.name)
+
     try:
         collectors.copy_tables(since=last_run, full_path=dest)
     except Exception:
@@ -117,30 +144,39 @@ def gather(dest=None, module=None):
 
 def ship(path):
     """
-    Ship gathered metrics via the Insights agent
+    Ship gathered metrics to the Insights API
     """
+    if not path:
+        logger.error('Automation Analytics TAR not found')
+        return
+    if "Error:" in str(path):
+        return
     try:
-        agent = 'insights-client'
-        if shutil.which(agent) is None:
-            logger.error('could not find {} on PATH'.format(agent))
-            return
         logger.debug('shipping analytics file: {}'.format(path))
-        try:
-            cmd = [
-                agent, '--payload', path, '--content-type', settings.INSIGHTS_AGENT_MIME
-            ]
-            output = smart_str(subprocess.check_output(cmd, timeout=60 * 5))
-            logger.debug(output)
-            # reset the `last_run` when data is shipped
-            run_now = now()
-            state = TowerAnalyticsState.get_solo()
-            state.last_run = run_now
-            state.save()
-
-        except subprocess.CalledProcessError:
-            logger.exception('{} failure:'.format(cmd))
-        except subprocess.TimeoutExpired:
-            logger.exception('{} timeout:'.format(cmd))
+        url = getattr(settings, 'AUTOMATION_ANALYTICS_URL', None)
+        if not url:
+            logger.error('AUTOMATION_ANALYTICS_URL is not set')
+            return
+        rh_user = getattr(settings, 'REDHAT_USERNAME', None)
+        rh_password = getattr(settings, 'REDHAT_PASSWORD', None)
+        if not rh_user:
+            return logger.error('REDHAT_USERNAME is not set')
+        if not rh_password:
+            return logger.error('REDHAT_PASSWORD is not set')
+        with open(path, 'rb') as f:
+            files = {'file': (os.path.basename(path), f, settings.INSIGHTS_AGENT_MIME)}
+            response = requests.post(url, 
+                                     files=files,
+                                     verify="/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
+                                     auth=(rh_user, rh_password),
+                                     timeout=(31, 31))
+            if response.status_code != 202:
+                return logger.exception('Upload failed with status {}, {}'.format(response.status_code,
+                                                                                  response.text))
+        run_now = now()
+        state = TowerAnalyticsState.get_solo()
+        state.last_run = run_now
+        state.save()
     finally:
         # cleanup tar.gz
         os.remove(path)

awx/main/analytics/metrics.py

@@ -15,6 +15,7 @@ from awx.main.analytics.collectors import (
     counts, 
     instance_info,
     job_instance_counts,
+    job_counts,
 )
 
 
@@ -36,11 +37,13 @@ INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scri
 USER_SESSIONS = Gauge('awx_sessions_total', 'Number of sessions', ['type',])
 CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs')
 RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system')
+PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the Tower system')
+STATUS = Gauge('awx_status_total', 'Status of Job launched', ['status',])
 
-INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['instance_uuid',])
-INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['instance_uuid',])
-INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower system', ['instance_uuid',])
-INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['instance_uuid',])
+INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['hostname', 'instance_uuid',])
 INSTANCE_LAUNCH_TYPE = Gauge('awx_instance_launch_type_total', 'Type of Job launched', ['node', 'launch_type',])
 INSTANCE_STATUS = Gauge('awx_instance_status_total', 'Status of Job launched', ['node', 'status',])
 
@@ -87,15 +90,21 @@ def metrics():
     USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions'])
     USER_SESSIONS.labels(type='anonymous').set(current_counts['active_anonymous_sessions'])
 
+    all_job_data = job_counts(None)
+    statuses = all_job_data.get('status', {})
+    for status, value in statuses.items():
+        STATUS.labels(status=status).set(value)
+
     RUNNING_JOBS.set(current_counts['running_jobs'])
+    PENDING_JOBS.set(current_counts['pending_jobs'])
 
-
-    instance_data = instance_info(None)
-    for uuid in instance_data:
-        INSTANCE_CAPACITY.labels(instance_uuid=uuid).set(instance_data[uuid]['capacity'])
-        INSTANCE_CPU.labels(instance_uuid=uuid).set(instance_data[uuid]['cpu'])
-        INSTANCE_MEMORY.labels(instance_uuid=uuid).set(instance_data[uuid]['memory'])
-        INSTANCE_INFO.labels(instance_uuid=uuid).info({
+    instance_data = instance_info(None, include_hostnames=True)
+    for uuid, info in instance_data.items():
+        hostname = info['hostname']
+        INSTANCE_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['capacity'])
+        INSTANCE_CPU.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['cpu'])
+        INSTANCE_MEMORY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['memory'])
+        INSTANCE_INFO.labels(hostname=hostname, instance_uuid=uuid).info({
             'enabled': str(instance_data[uuid]['enabled']),
             'last_isolated_check': getattr(instance_data[uuid], 'last_isolated_check', 'None'),
             'managed_by_policy': str(instance_data[uuid]['managed_by_policy']),

awx/main/conf.py

@@ -2,12 +2,14 @@
 import json
 import logging
 import os
+from distutils.version import LooseVersion as Version
 
 # Django
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework import serializers
+from rest_framework.fields import FloatField
 
 # Tower
 from awx.conf import fields, register, register_validate
@@ -124,6 +126,44 @@ register(
     category_slug='system',
 )
 
+register(
+    'REDHAT_USERNAME',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=False,
+    read_only=False,
+    label=_('Red Hat customer username'),
+    help_text=_('This username is used to retrieve license information and to send Automation Analytics'),  # noqa
+    category=_('System'),
+    category_slug='system',
+)
+
+register(
+    'REDHAT_PASSWORD',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=True,
+    read_only=False,
+    label=_('Red Hat customer password'),
+    help_text=_('This password is used to retrieve license information and to send Automation Analytics'),  # noqa
+    category=_('System'),
+    category_slug='system',
+)
+
+register(
+    'AUTOMATION_ANALYTICS_URL',
+    field_class=fields.URLField,
+    default='https://example.com',
+    schemes=('http', 'https'),
+    allow_plain_hostname=True,  # Allow hostname only without TLD.
+    label=_('Automation Analytics upload URL.'),
+    help_text=_('This setting is used to to configure data collection for the Automation Analytics dashboard'),
+    category=_('System'),
+    category_slug='system',
+)
+
 register(
     'INSTALL_UUID',
     field_class=fields.CharField,
@@ -260,6 +300,16 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_ISOLATED_HOST_KEY_CHECKING',
+    field_class=fields.BooleanField,
+    label=_('Isolated host key checking'),
+    help_text=_('When set to True, AWX will enforce strict host key checking for communication with isolated nodes.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    default=False
+)
+
 register(
     'AWX_ISOLATED_KEY_GENERATION',
     field_class=fields.BooleanField,
@@ -297,6 +347,53 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_RESOURCE_PROFILING_ENABLED',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Enable detailed resource profiling on all playbook runs'),
+    help_text=_('If set, detailed resource profiling data will be collected on all jobs. '
+                'This data can be gathered with `sosreport`.'),  # noqa
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
+register(
+    'AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL',
+    field_class=FloatField,
+    default='0.25',
+    label=_('Interval (in seconds) between polls for cpu usage.'),
+    help_text=_('Interval (in seconds) between polls for cpu usage. '
+                'Setting this lower than the default will affect playbook performance.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    required=False,
+)
+
+register(
+    'AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL',
+    field_class=FloatField,
+    default='0.25',
+    label=_('Interval (in seconds) between polls for memory usage.'),
+    help_text=_('Interval (in seconds) between polls for memory usage. '
+                'Setting this lower than the default will affect playbook performance.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    required=False,
+)
+
+register(
+    'AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL',
+    field_class=FloatField,
+    default='0.25',
+    label=_('Interval (in seconds) between polls for PID count.'),
+    help_text=_('Interval (in seconds) between polls for PID count. '
+                'Setting this lower than the default will affect playbook performance.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    required=False,
+)
+
 register(
     'AWX_TASK_ENV',
     field_class=fields.KeyValueField,
@@ -312,12 +409,21 @@ register(
     'INSIGHTS_TRACKING_STATE',
     field_class=fields.BooleanField,
     default=False,
-    label=_('Gather data for Automation Insights'),
-    help_text=_('Enables Tower to gather data on automation and send it to Red Hat Insights.'),
+    label=_('Gather data for Automation Analytics'),
+    help_text=_('Enables Tower to gather data on automation and send it to Red Hat.'),
     category=_('System'),
     category_slug='system',
 )
 
+register(
+    'PROJECT_UPDATE_VVV',
+    field_class=fields.BooleanField,
+    label=_('Run Project Updates With Higher Verbosity'),
+    help_text=_('Adds the CLI -vvv flag to ansible-playbook runs of project_update.yml used for project updates.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'AWX_ROLES_ENABLED',
     field_class=fields.BooleanField,
@@ -328,6 +434,85 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_COLLECTIONS_ENABLED',
+    field_class=fields.BooleanField,
+    default=True,
+    label=_('Enable Collection(s) Download'),
+    help_text=_('Allows collections to be dynamically downloaded from a requirements.yml file for SCM projects.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
+register(
+    'PRIMARY_GALAXY_URL',
+    field_class=fields.URLField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server URL'),
+    help_text=_(
+        'For organizations that run their own Galaxy service, this gives the option to specify a '
+        'host as the primary galaxy server. Requirements will be downloaded from the primary if the '
+        'specific role or collection is available there. If the content is not avilable in the primary, '
+        'or if this field is left blank, it will default to galaxy.ansible.com.'
+    ),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_USERNAME',
+    field_class=fields.CharField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Username'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The username to use for basic authentication against the Galaxy instance, '
+                'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_PASSWORD',
+    field_class=fields.CharField,
+    encrypted=True,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Password'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The password to use for basic authentication against the Galaxy instance, '
+                'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_TOKEN',
+    field_class=fields.CharField,
+    encrypted=True,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Token'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The token to use for connecting with the Galaxy instance, '
+                'this is mutually exclusive with corresponding username and password settings.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_AUTH_URL',
+    field_class=fields.CharField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Authentication URL'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The token_endpoint of a Keycloak server.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
 register(
     'STDOUT_MAX_BYTES_DISPLAY',
     field_class=fields.IntegerField,
@@ -568,6 +753,16 @@ register(
     category=_('Logging'),
     category_slug='logging',
 )
+register(
+    'LOG_AGGREGATOR_AUDIT',
+    field_class=fields.BooleanField,
+    allow_null=True,
+    default=False,
+    label=_('Enabled external log aggregation auditing'),
+    help_text=_('When enabled, all external logs emitted by Tower will also be written to /var/log/tower/external.log.  This is an experimental setting intended to be used for debugging external log aggregation issues (and may be subject to change in the future).'),  # noqa
+    category=_('Logging'),
+    category_slug='logging',
+)
 
 
 register(
@@ -598,4 +793,75 @@ def logging_validate(serializer, attrs):
     return attrs
 
 
+def galaxy_validate(serializer, attrs):
+    """Ansible Galaxy config options have mutual exclusivity rules, these rules
+    are enforced here on serializer validation so that users will not be able
+    to save settings which obviously break all project updates.
+    """
+    prefix = 'PRIMARY_GALAXY_'
+
+    from awx.main.constants import GALAXY_SERVER_FIELDS
+    if not any('{}{}'.format(prefix, subfield.upper()) in attrs for subfield in GALAXY_SERVER_FIELDS):
+        return attrs
+
+    def _new_value(setting_name):
+        if setting_name in attrs:
+            return attrs[setting_name]
+        elif not serializer.instance:
+            return ''
+        return getattr(serializer.instance, setting_name, '')
+
+    galaxy_data = {}
+    for subfield in GALAXY_SERVER_FIELDS:
+        galaxy_data[subfield] = _new_value('{}{}'.format(prefix, subfield.upper()))
+    errors = {}
+    if not galaxy_data['url']:
+        for k, v in galaxy_data.items():
+            if v:
+                setting_name = '{}{}'.format(prefix, k.upper())
+                errors.setdefault(setting_name, [])
+                errors[setting_name].append(_(
+                    'Cannot provide field if PRIMARY_GALAXY_URL is not set.'
+                ))
+    for k in GALAXY_SERVER_FIELDS:
+        if galaxy_data[k]:
+            setting_name = '{}{}'.format(prefix, k.upper())
+            if (not serializer.instance) or (not getattr(serializer.instance, setting_name, '')):
+                # new auth is applied, so check if compatible with version
+                from awx.main.utils import get_ansible_version
+                current_version = get_ansible_version()
+                min_version = '2.9'
+                if Version(current_version) < Version(min_version):
+                    errors.setdefault(setting_name, [])
+                    errors[setting_name].append(_(
+                        'Galaxy server settings are not available until Ansible {min_version}, '
+                        'you are running {current_version}.'
+                    ).format(min_version=min_version, current_version=current_version))
+    if (galaxy_data['password'] or galaxy_data['username']) and (galaxy_data['token'] or galaxy_data['auth_url']):
+        for k in ('password', 'username', 'token', 'auth_url'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            if setting_name in attrs:
+                errors.setdefault(setting_name, [])
+                errors[setting_name].append(_(
+                    'Setting Galaxy token and authentication URL is mutually exclusive with username and password.'
+                ))
+    if bool(galaxy_data['username']) != bool(galaxy_data['password']):
+        msg = _('If authenticating via username and password, both must be provided.')
+        for k in ('username', 'password'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            errors.setdefault(setting_name, [])
+            errors[setting_name].append(msg)
+    if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']):
+        msg = _('If authenticating via token, both token and authentication URL must be provided.')
+        for k in ('token', 'auth_url'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            errors.setdefault(setting_name, [])
+            errors[setting_name].append(msg)
+
+    if errors:
+        raise serializers.ValidationError(errors)
+    return attrs
+
+
 register_validate('logging', logging_validate)
+register_validate('jobs', galaxy_validate)

awx/main/constants.py

@@ -51,3 +51,7 @@ LOGGER_BLACKLIST = (
     # loggers that may be called getting logging settings
     'awx.conf'
 )
+
+# these correspond to both AWX and Ansible settings to keep naming consistent
+# for instance, settings.PRIMARY_GALAXY_AUTH_URL vs env var ANSIBLE_GALAXY_SERVER_FOO_AUTH_URL
+GALAXY_SERVER_FIELDS = ('url', 'username', 'password', 'token', 'auth_url')

awx/main/consumers.py

@@ -24,7 +24,7 @@ def ws_connect(message):
     headers = dict(message.content.get('headers', ''))
     message.reply_channel.send({"accept": True})
     message.content['method'] = 'FAKE'
-    if message.user.is_authenticated():
+    if message.user.is_authenticated:
         message.reply_channel.send(
             {"text": json.dumps({"accept": True, "user": message.user.id})}
         )

awx/main/credential_plugins/aim.py

@@ -1,14 +1,14 @@
 from .plugin import CredentialPlugin
 
-import os
-import stat
-import tempfile
-import threading
 from urllib.parse import quote, urlencode, urljoin
 
 from django.utils.translation import ugettext_lazy as _
 import requests
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
 
 aim_inputs = {
     'fields': [{
@@ -60,24 +60,6 @@ aim_inputs = {
 }
 
 
-def create_temporary_fifo(data):
-    """Open fifo named pipe in a new thread using a temporary file path. The
-    thread blocks until data is read from the pipe.
-
-    Returns the path to the fifo.
-
-    :param data(bytes): Data to write to the pipe.
-    """
-    path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
-    os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    threading.Thread(
-        target=lambda p, d: open(p, 'wb').write(d),
-        args=(path, data)
-    ).start()
-    return path
-
-
 def aim_backend(**kwargs):
     url = kwargs['url']
     client_cert = kwargs.get('client_cert', None)
@@ -119,7 +101,7 @@ def aim_backend(**kwargs):
 
 
 aim_plugin = CredentialPlugin(
-    'CyberArk AIM Secret Lookup',
+    'CyberArk AIM Central Credential Provider Lookup',
     inputs=aim_inputs,
     backend=aim_backend
 )

awx/main/credential_plugins/conjur.py

@@ -1,15 +1,16 @@
 from .plugin import CredentialPlugin
 
 import base64
-import os
-import stat
-import tempfile
-import threading
 from urllib.parse import urljoin, quote_plus
 
 from django.utils.translation import ugettext_lazy as _
 import requests
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
+
 
 conjur_inputs = {
     'fields': [{
@@ -51,24 +52,6 @@ conjur_inputs = {
 }
 
 
-def create_temporary_fifo(data):
-    """Open fifo named pipe in a new thread using a temporary file path. The
-    thread blocks until data is read from the pipe.
-
-    Returns the path to the fifo.
-
-    :param data(bytes): Data to write to the pipe.
-    """
-    path = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names()))
-    os.mkfifo(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    threading.Thread(
-        target=lambda p, d: open(p, 'wb').write(d),
-        args=(path, data)
-    ).start()
-    return path
-
-
 def conjur_backend(**kwargs):
     url = kwargs['url']
     api_key = kwargs['api_key']

awx/main/credential_plugins/hashivault.py

@@ -8,6 +8,10 @@ from .plugin import CredentialPlugin
 import requests
 from django.utils.translation import ugettext_lazy as _
 
+# AWX
+from awx.main.utils import (
+    create_temporary_fifo,
+)
 
 base_inputs = {
     'fields': [{
@@ -22,12 +26,18 @@ base_inputs = {
         'type': 'string',
         'secret': True,
         'help_text': _('The access token used to authenticate to the Vault server'),
+    }, {
+        'id': 'cacert',
+        'label': _('CA Certificate'),
+        'type': 'string',
+        'multiline': True,
+        'help_text': _('The CA certificate used to verify the SSL certificate of the Vault server')
     }],
     'metadata': [{
         'id': 'secret_path',
         'label': _('Path to Secret'),
         'type': 'string',
-        'help_text': _('The path to the secret e.g., /some-engine/some-secret/'),
+        'help_text': _('The path to the secret stored in the secret backend e.g, /some/secret/')
     }],
     'required': ['url', 'token', 'secret_path'],
 }
@@ -40,7 +50,12 @@ hashi_kv_inputs['fields'].append({
     'help_text': _('API v1 is for static key/value lookups.  API v2 is for versioned key/value lookups.'),
     'default': 'v1',
 })
-hashi_kv_inputs['metadata'].extend([{
+hashi_kv_inputs['metadata'] = [{
+    'id': 'secret_backend',
+    'label': _('Name of Secret Backend'),
+    'type': 'string',
+    'help_text': _('The name of the kv secret backend (if left empty, the first segment of the secret path will be used).')
+}] + hashi_kv_inputs['metadata'] + [{
     'id': 'secret_key',
     'label': _('Key Name'),
     'type': 'string',
@@ -50,7 +65,7 @@ hashi_kv_inputs['metadata'].extend([{
     'label': _('Secret Version (v2 only)'),
     'type': 'string',
     'help_text': _('Used to specify a specific secret version (if left empty, the latest version will be used).'),
-}])
+}]
 hashi_kv_inputs['required'].extend(['api_version', 'secret_key'])
 
 hashi_ssh_inputs = copy.deepcopy(base_inputs)
@@ -75,36 +90,48 @@ hashi_ssh_inputs['required'].extend(['public_key', 'role'])
 
 def kv_backend(**kwargs):
     token = kwargs['token']
-    url = urljoin(kwargs['url'], 'v1')
+    url = kwargs['url']
     secret_path = kwargs['secret_path']
+    secret_backend = kwargs.get('secret_backend', None)
     secret_key = kwargs.get('secret_key', None)
-
+    cacert = kwargs.get('cacert', None)
     api_version = kwargs['api_version']
 
+    request_kwargs = {'timeout': 30}
+    if cacert:
+        request_kwargs['verify'] = create_temporary_fifo(cacert.encode())
+
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
+    # Compatability header for older installs of Hashicorp Vault
+    sess.headers['X-Vault-Token'] = token
+
     if api_version == 'v2':
-        params = {}
         if kwargs.get('secret_version'):
-            params['version'] = kwargs['secret_version']
-        try:
-            mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts
-            '/'.join(*path)
-        except Exception:
-            mount_point, path = secret_path, []
-        # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
-        response = sess.get(
-            '/'.join([url, mount_point, 'data'] + path).rstrip('/'),
-            params=params,
-            timeout=30
-        )
-        response.raise_for_status()
-        json = response.json()['data']
+            request_kwargs['params'] = {'version': kwargs['secret_version']}
+        if secret_backend:
+            path_segments = [secret_backend, 'data', secret_path]
+        else:
+            try:
+                mount_point, *path = pathlib.Path(secret_path.lstrip(os.sep)).parts
+                '/'.join(path)
+            except Exception:
+                mount_point, path = secret_path, []
+            # https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version
+            path_segments = [mount_point, 'data'] + path
     else:
-        # https://www.vaultproject.io/api/secret/kv/kv-v1.html#read-secret
-        response = sess.get('/'.join([url, secret_path]).rstrip('/'), timeout=30)
-        response.raise_for_status()
-        json = response.json()
+        if secret_backend:
+            path_segments = [secret_backend, secret_path]
+        else:
+            path_segments = [secret_path]
+
+    request_url = urljoin(url, '/'.join(['v1'] + path_segments)).rstrip('/')
+    response = sess.get(request_url, **request_kwargs)
+    response.raise_for_status()
+
+    json = response.json()
+    if api_version == 'v2':
+        json = json['data']
 
     if secret_key:
         try:
@@ -121,20 +148,24 @@ def ssh_backend(**kwargs):
     url = urljoin(kwargs['url'], 'v1')
     secret_path = kwargs['secret_path']
     role = kwargs['role']
+    cacert = kwargs.get('cacert', None)
+
+    request_kwargs = {'timeout': 30}
+    if cacert:
+        request_kwargs['verify'] = create_temporary_fifo(cacert.encode())
+
+    request_kwargs['json'] = {'public_key': kwargs['public_key']}
+    if kwargs.get('valid_principals'):
+        request_kwargs['json']['valid_principals'] = kwargs['valid_principals']
 
     sess = requests.Session()
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
-    json = {
-        'public_key': kwargs['public_key']
-    }
-    if kwargs.get('valid_principals'):
-        json['valid_principals'] = kwargs['valid_principals']
+    # Compatability header for older installs of Hashicorp Vault
+    sess.headers['X-Vault-Token'] = token
     # https://www.vaultproject.io/api/secret/ssh/index.html#sign-ssh-key
-    resp = sess.post(
-        '/'.join([url, secret_path, 'sign', role]).rstrip('/'),
-        json=json,
-        timeout=30
-    )
+    request_url = '/'.join([url, secret_path, 'sign', role]).rstrip('/')
+    resp = sess.post(request_url, **request_kwargs)
+
     resp.raise_for_status()
     return resp.json()['data']['signed_key']
 

awx/main/dispatch/reaper.py

@@ -33,7 +33,11 @@ def reap(instance=None, status='failed', excluded_uuids=[]):
     '''
     Reap all jobs in waiting|running for this instance.
     '''
-    me = instance or Instance.objects.me()
+    me = instance
+    if me is None:
+        (changed, me) = Instance.objects.get_or_register()
+        if changed:
+            logger.info("Registered tower node '{}'".format(me.hostname))
     now = tz_now()
     workflow_ctype_id = ContentType.objects.get_for_model(WorkflowJob).id
     jobs = UnifiedJob.objects.filter(

awx/main/fields.py

@@ -11,8 +11,9 @@ from jinja2 import Environment, StrictUndefined
 from jinja2.exceptions import UndefinedError, TemplateSyntaxError
 
 # Django
-import django
+from django.contrib.postgres.fields import JSONField as upstream_JSONBField
 from django.core import exceptions as django_exceptions
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models.signals import (
     post_save,
     post_delete,
@@ -37,7 +38,6 @@ import jsonschema.exceptions
 
 # Django-JSONField
 from jsonfield import JSONField as upstream_JSONField
-from jsonbfield.fields import JSONField as upstream_JSONBField
 
 # DRF
 from rest_framework import serializers
@@ -76,10 +76,10 @@ class JSONField(upstream_JSONField):
     def db_type(self, connection):
         return 'text'
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if value in {'', None} and not self.null:
             return {}
-        return super(JSONField, self).from_db_value(value, expression, connection, context)
+        return super(JSONField, self).from_db_value(value, expression, connection)
 
 
 class JSONBField(upstream_JSONBField):
@@ -91,12 +91,12 @@ class JSONBField(upstream_JSONBField):
     def get_db_prep_value(self, value, connection, prepared=False):
         if connection.vendor == 'sqlite':
             # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value)
+            return json.dumps(value, cls=DjangoJSONEncoder)
         return super(JSONBField, self).get_db_prep_value(
             value, connection, prepared
         )
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         # Work around a bug in django-jsonfield
         # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
         if isinstance(value, str):
@@ -112,14 +112,9 @@ class AutoSingleRelatedObjectDescriptor(ReverseOneToOneDescriptor):
 
     def __get__(self, instance, instance_type=None):
         try:
-            return super(AutoSingleRelatedObjectDescriptor,
-                         self).__get__(instance, instance_type)
+            return super(AutoSingleRelatedObjectDescriptor, self).__get__(instance, instance_type)
         except self.related.related_model.DoesNotExist:
             obj = self.related.related_model(**{self.related.field.name: instance})
-            if self.related.field.rel.parent_link:
-                raise NotImplementedError('not supported with polymorphic!')
-                for f in instance._meta.local_fields:
-                    setattr(obj, f.name, getattr(instance, f.name))
             obj.save()
             return obj
 
@@ -169,7 +164,7 @@ def is_implicit_parent(parent_role, child_role):
         # The only singleton implicit parent is the system admin being
         # a parent of the system auditor role
         return bool(
-            child_role.singleton_name == ROLE_SINGLETON_SYSTEM_AUDITOR and 
+            child_role.singleton_name == ROLE_SINGLETON_SYSTEM_AUDITOR and
             parent_role.singleton_name == ROLE_SINGLETON_SYSTEM_ADMINISTRATOR
         )
     # Get the list of implicit parents that were defined at the class level.
@@ -453,21 +448,6 @@ class JSONSchemaField(JSONBField):
                 params={'value': value},
             )
 
-    def get_db_prep_value(self, value, connection, prepared=False):
-        if connection.vendor == 'sqlite':
-            # sqlite (which we use for tests) does not support jsonb;
-            return json.dumps(value)
-        return super(JSONSchemaField, self).get_db_prep_value(
-            value, connection, prepared
-        )
-
-    def from_db_value(self, value, expression, connection, context):
-        # Work around a bug in django-jsonfield
-        # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos
-        if isinstance(value, str):
-            return json.loads(value)
-        return value
-
 
 @JSONSchemaField.format_checker.checks('vault_id')
 def format_vault_id(value):
@@ -711,10 +691,12 @@ class CredentialInputField(JSONSchemaField):
 
             if model_instance.has_encrypted_ssh_key_data and not value.get('ssh_key_unlock'):
                 errors['ssh_key_unlock'] = [_('must be set when SSH key is encrypted.')]
+            
             if all([
                 model_instance.inputs.get('ssh_key_data'),
                 value.get('ssh_key_unlock'),
-                not model_instance.has_encrypted_ssh_key_data
+                not model_instance.has_encrypted_ssh_key_data,
+                'ssh_key_data' not in errors
             ]):
                 errors['ssh_key_unlock'] = [_('should not be set when SSH key is not encrypted.')]
 
@@ -986,7 +968,7 @@ class OAuth2ClientSecretField(models.CharField):
             encrypt_value(value), connection, prepared
         )
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection):
         if value and value.startswith('$encrypted$'):
             return decrypt_value(get_encryption_key('value', pk=None), value)
         return value
@@ -1022,38 +1004,6 @@ class OrderedManyToManyDescriptor(ManyToManyDescriptor):
                         '%s__position' % self.through._meta.model_name
                     )
 
-                def add(self, *objs):
-                    # Django < 2 doesn't support this method on
-                    # ManyToManyFields w/ an intermediary model
-                    # We should be able to remove this code snippet when we
-                    # upgrade Django.
-                    # see: https://github.com/django/django/blob/stable/1.11.x/django/db/models/fields/related_descriptors.py#L926
-                    if not django.__version__.startswith('1.'):
-                        raise RuntimeError(
-                            'This method is no longer necessary in Django>=2'
-                        )
-                    try:
-                        self.through._meta.auto_created = True
-                        super(OrderedManyRelatedManager, self).add(*objs)
-                    finally:
-                        self.through._meta.auto_created = False
-
-                def remove(self, *objs):
-                    # Django < 2 doesn't support this method on
-                    # ManyToManyFields w/ an intermediary model
-                    # We should be able to remove this code snippet when we
-                    # upgrade Django.
-                    # see: https://github.com/django/django/blob/stable/1.11.x/django/db/models/fields/related_descriptors.py#L944
-                    if not django.__version__.startswith('1.'):
-                        raise RuntimeError(
-                            'This method is no longer necessary in Django>=2'
-                        )
-                    try:
-                        self.through._meta.auto_created = True
-                        super(OrderedManyRelatedManager, self).remove(*objs)
-                    finally:
-                        self.through._meta.auto_created = False
-
             return OrderedManyRelatedManager
 
         return add_custom_queryset_to_many_related_manager(

awx/main/isolated/manager.py

@@ -11,7 +11,9 @@ from django.conf import settings
 import ansible_runner
 
 import awx
-from awx.main.utils import get_system_task_capacity
+from awx.main.utils import (
+    get_system_task_capacity
+)
 from awx.main.queue import CallbackQueueDispatcher
 
 logger = logging.getLogger('awx.isolated.manager')
@@ -29,7 +31,7 @@ def set_pythonpath(venv_libdir, env):
 
 class IsolatedManager(object):
 
-    def __init__(self, cancelled_callback=None, check_callback=None):
+    def __init__(self, cancelled_callback=None, check_callback=None, pod_manager=None):
         """
         :param cancelled_callback:  a callable - which returns `True` or `False`
                                     - signifying if the job has been prematurely
@@ -40,11 +42,29 @@ class IsolatedManager(object):
         self.idle_timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT)
         self.started_at = None
         self.captured_command_artifact = False
+        self.instance = None
+        self.pod_manager = pod_manager
+
+    def build_inventory(self, hosts):
+        if self.instance and self.instance.is_containerized:
+            inventory = {'all': {'hosts': {}}}
+            for host in hosts:
+                inventory['all']['hosts'][host] = {
+                    "ansible_connection": "kubectl",
+                    "ansible_kubectl_config": self.pod_manager.kube_config
+                }
+        else:
+            inventory = '\n'.join([
+                '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME)
+                for host in hosts
+            ])
+
+        return inventory
 
     def build_runner_params(self, hosts, verbosity=1):
         env = dict(os.environ.items())
         env['ANSIBLE_RETRY_FILES_ENABLED'] = 'False'
-        env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        env['ANSIBLE_HOST_KEY_CHECKING'] = str(settings.AWX_ISOLATED_HOST_KEY_CHECKING)
         env['ANSIBLE_LIBRARY'] = os.path.join(os.path.dirname(awx.__file__), 'plugins', 'isolated')
         set_pythonpath(os.path.join(settings.ANSIBLE_VENV_PATH, 'lib'), env)
 
@@ -69,17 +89,12 @@ class IsolatedManager(object):
             else:
                 playbook_logger.info(runner_obj.stdout.read())
 
-        inventory = '\n'.join([
-            '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME)
-            for host in hosts
-        ])
-
         return {
             'project_dir': os.path.abspath(os.path.join(
                 os.path.dirname(awx.__file__),
                 'playbooks'
             )),
-            'inventory': inventory,
+            'inventory': self.build_inventory(hosts),
             'envvars': env,
             'finished_callback': finished_callback,
             'verbosity': verbosity,
@@ -153,6 +168,12 @@ class IsolatedManager(object):
         runner_obj = self.run_management_playbook('run_isolated.yml',
                                                   self.private_data_dir,
                                                   extravars=extravars)
+
+        if runner_obj.status == 'failed':
+            self.instance.result_traceback = runner_obj.stdout.read()
+            self.instance.save(update_fields=['result_traceback'])
+            return 'error', runner_obj.rc
+
         return runner_obj.status, runner_obj.rc
 
     def check(self, interval=None):
@@ -175,6 +196,7 @@ class IsolatedManager(object):
         rc = None
         last_check = time.time()
         dispatcher = CallbackQueueDispatcher()
+
         while status == 'failed':
             canceled = self.cancelled_callback() if self.cancelled_callback else False
             if not canceled and time.time() - last_check < interval:
@@ -279,7 +301,6 @@ class IsolatedManager(object):
 
 
     def cleanup(self):
-        # If the job failed for any reason, make a last-ditch effort at cleanup
         extravars = {
             'private_data_dir': self.private_data_dir,
             'cleanup_dirs': [
@@ -393,6 +414,7 @@ class IsolatedManager(object):
             [instance.execution_node],
             verbosity=min(5, self.instance.verbosity)
         )
+
         status, rc = self.dispatch(playbook, module, module_args)
         if status == 'successful':
             status, rc = self.check()

awx/main/management/commands/check_db.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+class Command(BaseCommand):
+    """Checks connection to the database, and prints out connection info if not connected"""
+
+    def handle(self, *args, **options):
+
+        with connection.cursor() as cursor:
+            cursor.execute("SELECT version()")
+            version = str(cursor.fetchone()[0])
+
+        return "Database Version: {}".format(version)

awx/main/management/commands/check_license.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved
 
+import json
+
 from awx.main.utils import get_licenser
 from django.core.management.base import BaseCommand
 
@@ -8,6 +10,15 @@ from django.core.management.base import BaseCommand
 class Command(BaseCommand):
     """Returns license type, e.g., 'enterprise', 'open', 'none'"""
 
+    def add_arguments(self, parser):
+        parser.add_argument('--data', dest='data', action='store_true',
+                            help='verbose, prints the actual (sanitized) license')
+
     def handle(self, *args, **options):
         super(Command, self).__init__()
-        return get_licenser().validate().get('license_type', 'none')
+        license = get_licenser().validate()
+        if options.get('data'):
+            if license.get('license_key', '') != 'UNLICENSED':
+                license['license_key'] = '********'
+            return json.dumps(license)
+        return license.get('license_type', 'none')

awx/main/management/commands/gather_analytics.py

@@ -11,8 +11,10 @@ class Command(BaseCommand):
     help = 'Gather AWX analytics data'
 
     def add_arguments(self, parser):
+        parser.add_argument('--dry-run', dest='dry-run', action='store_true',
+                            help='Gather analytics without shipping. Works even if analytics are disabled in settings.')
         parser.add_argument('--ship', dest='ship', action='store_true',
-                            help='Enable to ship metrics via insights-client')
+                            help='Enable to ship metrics to the Red Hat Cloud')
 
     def init_logging(self):
         self.logger = logging.getLogger('awx.main.analytics')
@@ -23,9 +25,14 @@ class Command(BaseCommand):
         self.logger.propagate = False
 
     def handle(self, *args, **options):
-        tgz = gather()
         self.init_logging()
+        opt_ship = options.get('ship')
+        opt_dry_run = options.get('dry-run')
+        if opt_ship and opt_dry_run:
+            self.logger.error('Both --ship and --dry-run cannot be processed at the same time.')
+            return
+        tgz = gather(collection_type='manual' if not opt_dry_run else 'dry-run')
         if tgz:
             self.logger.debug(tgz)
-        if options.get('ship'):
+        if opt_ship:
             ship(tgz)

awx/main/management/commands/inventory_import.py

@@ -919,7 +919,8 @@ class Command(BaseCommand):
         new_count = Host.objects.active_count()
         if time_remaining <= 0 and not license_info.get('demo', False):
             logger.error(LICENSE_EXPIRED_MESSAGE)
-            raise CommandError("License has expired!")
+            if license_info.get('trial', False) is True:
+                raise CommandError("License has expired!")
         # special check for tower-type inventory sources
         # but only if running the plugin
         TOWER_SOURCE_FILES = ['tower.yml', 'tower.yaml']
@@ -936,7 +937,11 @@ class Command(BaseCommand):
                 logger.error(DEMO_LICENSE_MESSAGE % d)
             else:
                 logger.error(LICENSE_MESSAGE % d)
-            raise CommandError('License count exceeded!')
+            if (
+                license_info.get('trial', False) is True or
+                license_info['instance_count'] == 10  # basic 10 license
+            ):
+                raise CommandError('License count exceeded!')
 
     def check_org_host_limit(self):
         license_info = get_licenser().validate()

awx/main/management/commands/test_isolated_connection.py

@@ -33,6 +33,7 @@ class Command(BaseCommand):
             ]):
                 ssh_key = settings.AWX_ISOLATED_PRIVATE_KEY
             env = dict(os.environ.items())
+            env['ANSIBLE_HOST_KEY_CHECKING'] = str(settings.AWX_ISOLATED_HOST_KEY_CHECKING)
             set_pythonpath(os.path.join(settings.ANSIBLE_VENV_PATH, 'lib'), env)
             res = ansible_runner.interface.run(
                 private_data_dir=path,

awx/main/managers.py

@@ -221,8 +221,9 @@ class InstanceGroupManager(models.Manager):
             elif t.status == 'running':
                 # Subtract capacity from all groups that contain the instance
                 if t.execution_node not in instance_ig_mapping:
-                    logger.warning('Detected %s running inside lost instance, '
-                                   'may still be waiting for reaper.', t.log_format)
+                    if not t.is_containerized:
+                        logger.warning('Detected %s running inside lost instance, '
+                                       'may still be waiting for reaper.', t.log_format)
                     if t.instance_group:
                         impacted_groups = [t.instance_group.name]
                     else:

awx/main/middleware.py

@@ -73,7 +73,7 @@ class ActivityStreamMiddleware(threading.local, MiddlewareMixin):
         super().__init__(get_response)
 
     def process_request(self, request):
-        if hasattr(request, 'user') and hasattr(request.user, 'is_authenticated') and request.user.is_authenticated():
+        if hasattr(request, 'user') and request.user.is_authenticated:
             user = request.user
         else:
             user = None

awx/main/migrations/0001_initial.py

@@ -44,7 +44,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
                 ('event', models.CharField(max_length=100, choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_skipped', 'Host Skipped')])),
-                ('event_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('counter', models.PositiveIntegerField(default=0)),
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
                 ('expires', models.DateTimeField(default=django.utils.timezone.now)),
                 ('request_hash', models.CharField(default='', max_length=40, blank=True)),
                 ('reason', models.CharField(default='', help_text='Reason the auth token was invalidated.', max_length=1024, blank=True)),
-                ('user', models.ForeignKey(related_name='auth_tokens', to=settings.AUTH_USER_MODEL)),
+                ('user', models.ForeignKey(related_name='auth_tokens', on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),
         migrations.CreateModel(
@@ -198,7 +198,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('event', models.CharField(max_length=100, choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete')])),
-                ('event_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('event_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('host_name', models.CharField(default='', max_length=1024, editable=False)),
@@ -241,7 +241,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('modified', models.DateTimeField(auto_now=True)),
-                ('instance', models.ForeignKey(to='main.Instance')),
+                ('instance', models.ForeignKey(on_delete=models.CASCADE, to='main.Instance')),
             ],
         ),
         migrations.CreateModel(
@@ -287,7 +287,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('ldap_dn', models.CharField(default='', max_length=1024)),
-                ('user', awx.main.fields.AutoOneToOneField(related_name='profile', editable=False, to=settings.AUTH_USER_MODEL)),
+                ('user', awx.main.fields.AutoOneToOneField(related_name='profile', editable=False, on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),
         migrations.CreateModel(
@@ -304,7 +304,7 @@ class Migration(migrations.Migration):
                 ('dtend', models.DateTimeField(default=None, null=True, editable=False)),
                 ('rrule', models.CharField(max_length=255)),
                 ('next_run', models.DateTimeField(default=None, null=True, editable=False)),
-                ('extra_data', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('extra_data', jsonfield.fields.JSONField(default=dict, blank=True)),
                 ('created_by', models.ForeignKey(related_name="{u'class': 'schedule', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('modified_by', models.ForeignKey(related_name="{u'class': 'schedule', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
@@ -343,7 +343,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('old_pk', models.PositiveIntegerField(default=None, null=True, editable=False)),
                 ('launch_type', models.CharField(default='manual', max_length=20, editable=False, choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency')])),
-                ('cancel_flag', models.BooleanField(default=False, editable=False)),
+                ('cancel_flag', models.BooleanField(blank=True, default=False, editable=False)),
                 ('status', models.CharField(default='new', max_length=20, editable=False, choices=[('new', 'New'), ('pending', 'Pending'), ('waiting', 'Waiting'), ('running', 'Running'), ('successful', 'Successful'), ('failed', 'Failed'), ('error', 'Error'), ('canceled', 'Canceled')])),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('started', models.DateTimeField(default=None, null=True, editable=False)),
@@ -351,7 +351,7 @@ class Migration(migrations.Migration):
                 ('elapsed', models.DecimalField(editable=False, max_digits=12, decimal_places=3)),
                 ('job_args', models.TextField(default='', editable=False, blank=True)),
                 ('job_cwd', models.CharField(default='', max_length=1024, editable=False, blank=True)),
-                ('job_env', jsonfield.fields.JSONField(default={}, editable=False, blank=True)),
+                ('job_env', jsonfield.fields.JSONField(default=dict, editable=False, blank=True)),
                 ('job_explanation', models.TextField(default='', editable=False, blank=True)),
                 ('start_args', models.TextField(default='', editable=False, blank=True)),
                 ('result_stdout_text', models.TextField(default='', editable=False, blank=True)),
@@ -380,7 +380,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='AdHocCommand',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check')])),
                 ('limit', models.CharField(default='', max_length=1024, blank=True)),
                 ('module_name', models.CharField(default='', max_length=1024, blank=True)),
@@ -394,7 +394,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='InventorySource',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('source', models.CharField(default='', max_length=32, blank=True, choices=[('', 'Manual'), ('file', 'Local File, Directory or Script'), ('rax', 'Rackspace Cloud Servers'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure', 'Microsoft Azure'), ('vmware', 'VMware vCenter'), ('openstack', 'OpenStack'), ('custom', 'Custom Script')])),
                 ('source_path', models.CharField(default='', max_length=1024, editable=False, blank=True)),
                 ('source_vars', models.TextField(default='', help_text='Inventory source variables in YAML or JSON format.', blank=True)),
@@ -411,7 +411,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='InventoryUpdate',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('source', models.CharField(default='', max_length=32, blank=True, choices=[('', 'Manual'), ('file', 'Local File, Directory or Script'), ('rax', 'Rackspace Cloud Servers'), ('ec2', 'Amazon EC2'), ('gce', 'Google Compute Engine'), ('azure', 'Microsoft Azure'), ('vmware', 'VMware vCenter'), ('openstack', 'OpenStack'), ('custom', 'Custom Script')])),
                 ('source_path', models.CharField(default='', max_length=1024, editable=False, blank=True)),
                 ('source_vars', models.TextField(default='', help_text='Inventory source variables in YAML or JSON format.', blank=True)),
@@ -427,7 +427,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Job',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check'), ('scan', 'Scan')])),
                 ('playbook', models.CharField(default='', max_length=1024, blank=True)),
                 ('forks', models.PositiveIntegerField(default=0, blank=True)),
@@ -435,7 +435,7 @@ class Migration(migrations.Migration):
                 ('verbosity', models.PositiveIntegerField(default=0, blank=True, choices=[(0, '0 (Normal)'), (1, '1 (Verbose)'), (2, '2 (More Verbose)'), (3, '3 (Debug)'), (4, '4 (Connection Debug)'), (5, '5 (WinRM Debug)')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('job_tags', models.CharField(default='', max_length=1024, blank=True)),
-                ('force_handlers', models.BooleanField(default=False)),
+                ('force_handlers', models.BooleanField(blank=True, default=False)),
                 ('skip_tags', models.CharField(default='', max_length=1024, blank=True)),
                 ('start_at_task', models.CharField(default='', max_length=1024, blank=True)),
                 ('become_enabled', models.BooleanField(default=False)),
@@ -448,7 +448,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='JobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('job_type', models.CharField(default='run', max_length=64, choices=[('run', 'Run'), ('check', 'Check'), ('scan', 'Scan')])),
                 ('playbook', models.CharField(default='', max_length=1024, blank=True)),
                 ('forks', models.PositiveIntegerField(default=0, blank=True)),
@@ -456,14 +456,14 @@ class Migration(migrations.Migration):
                 ('verbosity', models.PositiveIntegerField(default=0, blank=True, choices=[(0, '0 (Normal)'), (1, '1 (Verbose)'), (2, '2 (More Verbose)'), (3, '3 (Debug)'), (4, '4 (Connection Debug)'), (5, '5 (WinRM Debug)')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('job_tags', models.CharField(default='', max_length=1024, blank=True)),
-                ('force_handlers', models.BooleanField(default=False)),
+                ('force_handlers', models.BooleanField(blank=True, default=False)),
                 ('skip_tags', models.CharField(default='', max_length=1024, blank=True)),
                 ('start_at_task', models.CharField(default='', max_length=1024, blank=True)),
                 ('become_enabled', models.BooleanField(default=False)),
                 ('host_config_key', models.CharField(default='', max_length=1024, blank=True)),
                 ('ask_variables_on_launch', models.BooleanField(default=False)),
                 ('survey_enabled', models.BooleanField(default=False)),
-                ('survey_spec', jsonfield.fields.JSONField(default={}, blank=True)),
+                ('survey_spec', jsonfield.fields.JSONField(default=dict, blank=True)),
             ],
             options={
                 'ordering': ('name',),
@@ -473,7 +473,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Project',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('local_path', models.CharField(help_text='Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.', max_length=1024, blank=True)),
                 ('scm_type', models.CharField(default='', max_length=8, verbose_name='SCM Type', blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion')])),
                 ('scm_url', models.CharField(default='', max_length=1024, verbose_name='SCM URL', blank=True)),
@@ -492,7 +492,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='ProjectUpdate',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('local_path', models.CharField(help_text='Local path (relative to PROJECTS_ROOT) containing playbooks and related files for this project.', max_length=1024, blank=True)),
                 ('scm_type', models.CharField(default='', max_length=8, verbose_name='SCM Type', blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion')])),
                 ('scm_url', models.CharField(default='', max_length=1024, verbose_name='SCM URL', blank=True)),
@@ -505,7 +505,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SystemJob',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJob')),
                 ('job_type', models.CharField(default='', max_length=32, blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_deleted', 'Purge previously deleted items from the database'), ('cleanup_facts', 'Purge and/or reduce the granularity of system tracking data')])),
                 ('extra_vars', models.TextField(default='', blank=True)),
             ],
@@ -517,7 +517,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='SystemJobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=django.db.models.deletion.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('job_type', models.CharField(default='', max_length=32, blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('cleanup_deleted', 'Purge previously deleted items from the database'), ('cleanup_facts', 'Purge and/or reduce the granularity of system tracking data')])),
             ],
             bases=('main.unifiedjobtemplate', models.Model),
@@ -550,7 +550,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjobtemplate',
             name='polymorphic_ctype',
-            field=models.ForeignKey(related_name='polymorphic_main.unifiedjobtemplate_set+', editable=False, to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(related_name='polymorphic_main.unifiedjobtemplate_set+', editable=False, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjobtemplate',
@@ -575,7 +575,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='polymorphic_ctype',
-            field=models.ForeignKey(related_name='polymorphic_main.unifiedjob_set+', editable=False, to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(related_name='polymorphic_main.unifiedjob_set+', editable=False, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjob',
@@ -595,7 +595,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='unified_job_template',
-            field=models.ForeignKey(related_name='schedules', to='main.UnifiedJobTemplate'),
+            field=models.ForeignKey(related_name='schedules', on_delete=django.db.models.deletion.CASCADE, to='main.UnifiedJobTemplate'),
         ),
         migrations.AddField(
             model_name='permission',
@@ -610,12 +610,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='joborigin',
             name='unified_job',
-            field=models.OneToOneField(related_name='job_origin', to='main.UnifiedJob'),
+            field=models.OneToOneField(related_name='job_origin', on_delete=django.db.models.deletion.CASCADE, to='main.UnifiedJob'),
         ),
         migrations.AddField(
             model_name='inventory',
             name='organization',
-            field=models.ForeignKey(related_name='inventories', to='main.Organization', help_text='Organization containing this inventory.'),
+            field=models.ForeignKey(related_name='inventories', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization containing this inventory.'),
         ),
         migrations.AddField(
             model_name='inventory',
@@ -625,7 +625,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='host',
             name='inventory',
-            field=models.ForeignKey(related_name='hosts', to='main.Inventory'),
+            field=models.ForeignKey(related_name='hosts', on_delete=django.db.models.deletion.CASCADE, to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='host',
@@ -650,7 +650,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='group',
             name='inventory',
-            field=models.ForeignKey(related_name='groups', to='main.Inventory'),
+            field=models.ForeignKey(related_name='groups', on_delete=django.db.models.deletion.CASCADE, to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='group',
@@ -680,12 +680,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='team',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to='main.Team', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Team', null=True),
         ),
         migrations.AddField(
             model_name='credential',
             name='user',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
         ),
         migrations.AddField(
             model_name='adhoccommandevent',
@@ -774,7 +774,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='projectupdate',
             name='project',
-            field=models.ForeignKey(related_name='project_updates', editable=False, to='main.Project'),
+            field=models.ForeignKey(related_name='project_updates', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Project'),
         ),
         migrations.AddField(
             model_name='project',
@@ -814,12 +814,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='jobhostsummary',
             name='job',
-            field=models.ForeignKey(related_name='job_host_summaries', editable=False, to='main.Job'),
+            field=models.ForeignKey(related_name='job_host_summaries', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Job'),
         ),
         migrations.AddField(
             model_name='jobevent',
             name='job',
-            field=models.ForeignKey(related_name='job_events', editable=False, to='main.Job'),
+            field=models.ForeignKey(related_name='job_events', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.Job'),
         ),
         migrations.AddField(
             model_name='job',
@@ -859,7 +859,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventoryupdate',
             name='inventory_source',
-            field=models.ForeignKey(related_name='inventory_updates', editable=False, to='main.InventorySource'),
+            field=models.ForeignKey(related_name='inventory_updates', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.InventorySource'),
         ),
         migrations.AddField(
             model_name='inventoryupdate',
@@ -874,12 +874,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventorysource',
             name='group',
-            field=awx.main.fields.AutoOneToOneField(related_name='inventory_source', null=True, default=None, editable=False, to='main.Group'),
+            field=awx.main.fields.AutoOneToOneField(related_name='inventory_source', on_delete=django.db.models.deletion.SET_NULL, null=True, default=None, editable=False, to='main.Group'),
         ),
         migrations.AddField(
             model_name='inventorysource',
             name='inventory',
-            field=models.ForeignKey(related_name='inventory_sources', default=None, editable=False, to='main.Inventory', null=True),
+            field=models.ForeignKey(related_name='inventory_sources', on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to='main.Inventory', null=True),
         ),
         migrations.AddField(
             model_name='inventorysource',
@@ -916,7 +916,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='adhoccommandevent',
             name='ad_hoc_command',
-            field=models.ForeignKey(related_name='ad_hoc_command_events', editable=False, to='main.AdHocCommand'),
+            field=models.ForeignKey(related_name='ad_hoc_command_events', on_delete=django.db.models.deletion.CASCADE, editable=False, to='main.AdHocCommand'),
         ),
         migrations.AddField(
             model_name='adhoccommand',

awx/main/migrations/0002_squashed_v300_release.py

@@ -13,7 +13,6 @@ from django.conf import settings
 from django.utils.timezone import now
 
 import jsonfield.fields
-import jsonbfield.fields
 import taggit.managers
 
 
@@ -144,7 +143,7 @@ class Migration(migrations.Migration):
                 ('category', models.CharField(max_length=128)),
                 ('value', models.TextField(blank=True)),
                 ('value_type', models.CharField(max_length=12, choices=[('string', 'String'), ('int', 'Integer'), ('float', 'Decimal'), ('json', 'JSON'), ('bool', 'Boolean'), ('password', 'Password'), ('list', 'List')])),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True)),
             ],
         ),
         # Notification changes
@@ -185,7 +184,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='notification',
             name='notification_template',
-            field=models.ForeignKey(related_name='notifications', editable=False, to='main.NotificationTemplate'),
+            field=models.ForeignKey(related_name='notifications', editable=False, on_delete=models.CASCADE, to='main.NotificationTemplate'),
         ),
         migrations.AddField(
             model_name='activitystream',
@@ -239,8 +238,8 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)),
                 ('module', models.CharField(max_length=128)),
-                ('facts', jsonbfield.fields.JSONField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)),
-                ('host', models.ForeignKey(related_name='facts', to='main.Host', help_text='Host for the facts that the fact scan captured.')),
+                ('facts', awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)),
+                ('host', models.ForeignKey(related_name='facts', to='main.Host', on_delete=models.CASCADE, help_text='Host for the facts that the fact scan captured.')),
             ],
         ),
         migrations.AlterIndexTogether(
@@ -318,7 +317,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='project',
             name='organization',
-            field=models.ForeignKey(related_name='projects', to='main.Organization', blank=True, null=True),
+            field=models.ForeignKey(related_name='projects', to='main.Organization', on_delete=models.CASCADE, blank=True, null=True),
         ),
         migrations.AlterField(
             model_name='team',
@@ -367,7 +366,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='organization',
-            field=models.ForeignKey(related_name='credentials', default=None, blank=True, to='main.Organization', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=models.CASCADE, default=None, blank=True, to='main.Organization', null=True),
         ),
 
         #
@@ -382,7 +381,7 @@ class Migration(migrations.Migration):
                 ('members', models.ManyToManyField(related_name='roles', to=settings.AUTH_USER_MODEL)),
                 ('parents', models.ManyToManyField(related_name='children', to='main.Role')),
                 ('implicit_parents', models.TextField(default='[]')),
-                ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', null=True)),
+                ('content_type', models.ForeignKey(default=None, to='contenttypes.ContentType', on_delete=models.CASCADE, null=True)),
                 ('object_id', models.PositiveIntegerField(default=None, null=True)),
 
             ],
@@ -398,8 +397,8 @@ class Migration(migrations.Migration):
                 ('role_field', models.TextField()),
                 ('content_type_id', models.PositiveIntegerField()),
                 ('object_id', models.PositiveIntegerField()),
-                ('ancestor', models.ForeignKey(related_name='+', to='main.Role')),
-                ('descendent', models.ForeignKey(related_name='+', to='main.Role')),
+                ('ancestor', models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Role')),
+                ('descendent', models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Role')),
             ],
             options={
                 'db_table': 'main_rbac_role_ancestors',
@@ -569,7 +568,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('created_by', models.ForeignKey(related_name="{u'class': 'label', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('modified_by', models.ForeignKey(related_name="{u'class': 'label', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
-                ('organization', models.ForeignKey(related_name='labels', to='main.Organization', help_text='Organization this label belongs to.')),
+                ('organization', models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization this label belongs to.')),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
             ],
             options={
@@ -599,12 +598,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='label',
             name='organization',
-            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Organization', help_text='Organization this label belongs to.', null=True),
+            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, default=None, blank=True, to='main.Organization', help_text='Organization this label belongs to.', null=True),
         ),
         migrations.AlterField(
             model_name='label',
             name='organization',
-            field=models.ForeignKey(related_name='labels', to='main.Organization', help_text='Organization this label belongs to.'),
+            field=models.ForeignKey(related_name='labels', on_delete=django.db.models.deletion.CASCADE, to='main.Organization', help_text='Organization this label belongs to.'),
         ),
         # InventorySource Credential
         migrations.AddField(
@@ -630,12 +629,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='credential',
             name='deprecated_team',
-            field=models.ForeignKey(related_name='deprecated_credentials', default=None, blank=True, to='main.Team', null=True),
+            field=models.ForeignKey(related_name='deprecated_credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.Team', null=True),
         ),
         migrations.AlterField(
             model_name='credential',
             name='deprecated_user',
-            field=models.ForeignKey(related_name='deprecated_credentials', default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
+            field=models.ForeignKey(related_name='deprecated_credentials', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True),
         ),
         migrations.AlterField(
             model_name='credential',

awx/main/migrations/0003_squashed_v300_v303_updates.py

@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='team',
             name='organization',
-            field=models.ForeignKey(related_name='teams', to='main.Organization'),
+            field=models.ForeignKey(related_name='teams', on_delete=models.CASCADE, to='main.Organization'),
             preserve_default=False,
         ),
     ] + _squashed.operations(SQUASHED_30, applied=True)

awx/main/migrations/0004_squashed_v310_release.py

@@ -74,7 +74,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='WorkflowJob',
             fields=[
-                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('unifiedjob_ptr', models.OneToOneField(parent_link=True, auto_created=True, on_delete=models.CASCADE, primary_key=True, serialize=False, to='main.UnifiedJob')),
                 ('extra_vars', models.TextField(default='', blank=True)),
             ],
             options={
@@ -100,7 +100,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='WorkflowJobTemplate',
             fields=[
-                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('unifiedjobtemplate_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, on_delete=models.CASCADE, serialize=False, to='main.UnifiedJobTemplate')),
                 ('extra_vars', models.TextField(default='', blank=True)),
                 ('admin_role', awx.main.fields.ImplicitRoleField(related_name='+', parent_role='singleton:system_administrator', to='main.Role', null='True')),
             ],
@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
                 ('failure_nodes', models.ManyToManyField(related_name='workflowjobtemplatenodes_failure', to='main.WorkflowJobTemplateNode', blank=True)),
                 ('success_nodes', models.ManyToManyField(related_name='workflowjobtemplatenodes_success', to='main.WorkflowJobTemplateNode', blank=True)),
                 ('unified_job_template', models.ForeignKey(related_name='workflowjobtemplatenodes', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJobTemplate', null=True)),
-                ('workflow_job_template', models.ForeignKey(related_name='workflow_job_template_nodes', default=None, blank=True, to='main.WorkflowJobTemplate', null=True)),
+                ('workflow_job_template', models.ForeignKey(related_name='workflow_job_template_nodes', on_delete=models.SET_NULL, default=None, blank=True, to='main.WorkflowJobTemplate', null=True)),
             ],
             options={
                 'abstract': False,
@@ -161,7 +161,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -191,7 +191,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -211,7 +211,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='workflowjobnode',
             name='workflow_job',
-            field=models.ForeignKey(related_name='workflow_job_nodes', default=None, blank=True, to='main.WorkflowJob', null=True),
+            field=models.ForeignKey(related_name='workflow_job_nodes', on_delete=django.db.models.deletion.CASCADE, default=None, blank=True, to='main.WorkflowJob', null=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
@@ -227,12 +227,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='job',
             name='artifacts',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         # Job timeout settings
         migrations.AddField(
@@ -397,7 +397,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',
@@ -407,33 +407,33 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=jsonfield.fields.JSONField(default={}, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, blank=True),
         ),
         # JSON field changes
         migrations.AlterField(
             model_name='adhoccommandevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='artifacts',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='job',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='jobevent',
             name='event_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='notification',
@@ -453,37 +453,37 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='schedule',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='unifiedjob',
             name='job_env',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjob',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='ancestor_artifacts',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobnode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
             name='survey_spec',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AlterField(
             model_name='workflowjobtemplatenode',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         # Job Project Update
         migrations.AddField(

awx/main/migrations/0006_v320_release.py

@@ -55,12 +55,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='inventorysource',
             name='deprecated_group',
-            field=models.OneToOneField(related_name='deprecated_inventory_source', null=True, default=None, to='main.Group'),
+            field=models.OneToOneField(related_name='deprecated_inventory_source', on_delete=models.CASCADE, null=True, default=None, to='main.Group'),
         ),
         migrations.AlterField(
             model_name='inventorysource',
             name='inventory',
-            field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True),
+            field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', on_delete=models.CASCADE, null=True),
         ),
 
         # Smart Inventory
@@ -78,13 +78,13 @@ class Migration(migrations.Migration):
             name='SmartInventoryMembership',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('host', models.ForeignKey(related_name='+', to='main.Host')),
+                ('host', models.ForeignKey(related_name='+', on_delete=models.CASCADE, to='main.Host')),
             ],
         ),
         migrations.AddField(
             model_name='smartinventorymembership',
             name='inventory',
-            field=models.ForeignKey(related_name='+', to='main.Inventory'),
+            field=models.ForeignKey(on_delete=models.CASCADE, related_name='+', to='main.Inventory'),
         ),
         migrations.AddField(
             model_name='host',
@@ -105,19 +105,19 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='inventory',
             name='organization',
-            field=models.ForeignKey(related_name='inventories', on_delete=models.deletion.SET_NULL, to='main.Organization', help_text='Organization containing this inventory.', null=True),
+            field=models.ForeignKey(related_name='inventories', on_delete=models.SET_NULL, to='main.Organization', help_text='Organization containing this inventory.', null=True),
         ),
 
         # Facts
         migrations.AlterField(
             model_name='fact',
             name='facts',
-            field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
+            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
             name='ansible_facts',
-            field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
+            field=awx.main.fields.JSONBField(default=dict, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True),
         ),
         migrations.AddField(
             model_name='host',
@@ -148,12 +148,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventorysource',
             name='source_project',
-            field=models.ForeignKey(related_name='scm_inventory_sources', default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
+            field=models.ForeignKey(related_name='scm_inventory_sources', on_delete=models.CASCADE, default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
         ),
         migrations.AddField(
             model_name='inventoryupdate',
             name='source_project_update',
-            field=models.ForeignKey(related_name='scm_inventory_updates', default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
+            field=models.ForeignKey(related_name='scm_inventory_updates', on_delete=models.CASCADE, default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
         ),
         migrations.AddField(
             model_name='project',
@@ -200,7 +200,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='notificationtemplate',
             name='organization',
-            field=models.ForeignKey(related_name='notification_templates', to='main.Organization', null=True),
+            field=models.ForeignKey(related_name='notification_templates', on_delete=models.CASCADE, to='main.Organization', null=True),
         ),
         migrations.AlterUniqueTogether(
             name='notificationtemplate',
@@ -312,7 +312,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='inventory',
             name='insights_credential',
-            field=models.ForeignKey(related_name='insights_inventories', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
+            field=models.ForeignKey(related_name='insights_inventories', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
         ),
         migrations.AlterField(
             model_name='inventory',
@@ -382,10 +382,10 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=512)),
                 ('kind', models.CharField(max_length=32, choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('insights', 'Insights')])),
                 ('managed_by_tower', models.BooleanField(default=False, editable=False)),
-                ('inputs', awx.main.fields.CredentialTypeInputField(default={}, blank=True, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
-                ('injectors', awx.main.fields.CredentialTypeInjectorField(default={}, blank=True, help_text='Enter injectors using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
-                ('created_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_created+", on_delete=models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
-                ('modified_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_modified+", on_delete=models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('inputs', awx.main.fields.CredentialTypeInputField(default=dict, blank=True, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
+                ('injectors', awx.main.fields.CredentialTypeInjectorField(default=dict, blank=True, help_text='Enter injectors using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.')),
+                ('created_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_created+", on_delete=models.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('modified_by', models.ForeignKey(related_name="{u'class': 'credentialtype', u'app_label': 'main'}(class)s_modified+", on_delete=models.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')),
             ],
             options={
@@ -399,23 +399,23 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='credential',
             name='inputs',
-            field=awx.main.fields.CredentialInputField(default={}, blank=True),
+            field=awx.main.fields.CredentialInputField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='credential',
             name='credential_type',
-            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', null=True),
+            field=models.ForeignKey(related_name='credentials', on_delete=models.CASCADE, to='main.CredentialType', null=True),
             preserve_default=False,
         ),
         migrations.AddField(
             model_name='job',
             name='vault_credential',
-            field=models.ForeignKey(related_name='jobs_as_vault_credential+', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
+            field=models.ForeignKey(related_name='jobs_as_vault_credential+', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
         ),
         migrations.AddField(
             model_name='jobtemplate',
             name='vault_credential',
-            field=models.ForeignKey(related_name='jobtemplates_as_vault_credential+', on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
+            field=models.ForeignKey(related_name='jobtemplates_as_vault_credential+', on_delete=models.SET_NULL, default=None, blank=True, to='main.Credential', null=True),
         ),
         migrations.AddField(
             model_name='job',
@@ -452,7 +452,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(unique=True, max_length=250)),
                 ('created', models.DateTimeField(auto_now_add=True)),
                 ('modified', models.DateTimeField(auto_now=True)),
-                ('controller', models.ForeignKey(related_name='controlled_groups', default=None, editable=False, to='main.InstanceGroup', help_text='Instance Group to remotely control this group.', null=True)),
+                ('controller', models.ForeignKey(related_name='controlled_groups', on_delete=models.CASCADE, default=None, editable=False, to='main.InstanceGroup', help_text='Instance Group to remotely control this group.', null=True)),
                 ('instances', models.ManyToManyField(help_text='Instances that are members of this InstanceGroup', related_name='rampart_groups', editable=False, to='main.Instance')),
             ],
         ),
@@ -464,7 +464,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='unifiedjob',
             name='instance_group',
-            field=models.ForeignKey(on_delete=models.deletion.SET_NULL, default=None, blank=True, to='main.InstanceGroup', help_text='The Rampart/Instance group the job was run under', null=True),
+            field=models.ForeignKey(on_delete=models.SET_NULL, default=None, blank=True, to='main.InstanceGroup', help_text='The Rampart/Instance group the job was run under', null=True),
         ),
         migrations.AddField(
             model_name='unifiedjobtemplate',

awx/main/migrations/0008_v320_drop_v1_credential_fields.py

@@ -103,12 +103,12 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='credential',
             name='credential_type',
-            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', null=False, help_text='Specify the type of credential you want to create. Refer to the Ansible Tower documentation for details on each type.')
+            field=models.ForeignKey(related_name='credentials', to='main.CredentialType', on_delete=models.CASCADE, null=False, help_text='Specify the type of credential you want to create. Refer to the Ansible Tower documentation for details on each type.')
         ),
         migrations.AlterField(
             model_name='credential',
             name='inputs',
-            field=awx.main.fields.CredentialInputField(default={}, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.', blank=True),
+            field=awx.main.fields.CredentialInputField(default=dict, help_text='Enter inputs using either JSON or YAML syntax. Use the radio button to toggle between the two. Refer to the Ansible Tower documentation for example syntax.', blank=True),
         ),
         migrations.RemoveField(
             model_name='job',

awx/main/migrations/0014_v330_saved_launchtime_configs.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='char_prompts',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='schedule',
@@ -35,7 +35,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='schedule',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
@@ -45,12 +45,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobnode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobnode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
@@ -60,12 +60,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='extra_data',
-            field=awx.main.fields.JSONField(default={}, blank=True),
+            field=awx.main.fields.JSONField(default=dict, blank=True),
         ),
         migrations.AddField(
             model_name='workflowjobtemplatenode',
             name='survey_passwords',
-            field=awx.main.fields.JSONField(default={}, editable=False, blank=True),
+            field=awx.main.fields.JSONField(default=dict, editable=False, blank=True),
         ),
         # Run data migration before removing the old credential field
         migrations.RunPython(migration_utils.set_current_apps_for_migrations, migrations.RunPython.noop),
@@ -83,9 +83,9 @@ class Migration(migrations.Migration):
             name='JobLaunchConfig',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('extra_data', awx.main.fields.JSONField(blank=True, default={})),
-                ('survey_passwords', awx.main.fields.JSONField(blank=True, default={}, editable=False)),
-                ('char_prompts', awx.main.fields.JSONField(blank=True, default={})),
+                ('extra_data', awx.main.fields.JSONField(blank=True, default=dict)),
+                ('survey_passwords', awx.main.fields.JSONField(blank=True, default=dict, editable=False)),
+                ('char_prompts', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('credentials', models.ManyToManyField(related_name='joblaunchconfigs', to='main.Credential')),
                 ('inventory', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='joblaunchconfigs', to='main.Inventory')),
                 ('job', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='launch_config', to='main.UnifiedJob')),
@@ -94,51 +94,51 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='ask_variables_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_credential_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_diff_mode_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_inventory_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_job_type_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_limit_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_skip_tags_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_tags_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_variables_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='ask_verbosity_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
     ]

awx/main/migrations/0018_v330_add_additional_stdout_events.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),
@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('event', models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('failed', models.BooleanField(default=False, editable=False)),
                 ('changed', models.BooleanField(default=False, editable=False)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
@@ -65,7 +65,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('created', models.DateTimeField(default=None, editable=False)),
                 ('modified', models.DateTimeField(default=None, editable=False)),
-                ('event_data', awx.main.fields.JSONField(blank=True, default={})),
+                ('event_data', awx.main.fields.JSONField(blank=True, default=dict)),
                 ('uuid', models.CharField(default='', editable=False, max_length=1024)),
                 ('counter', models.PositiveIntegerField(default=0, editable=False)),
                 ('stdout', models.TextField(default='', editable=False)),

awx/main/migrations/0053_v340_workflow_inventory.py

@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjob',
             name='char_prompts',
-            field=awx.main.fields.JSONField(blank=True, default={}),
+            field=awx.main.fields.JSONField(blank=True, default=dict),
         ),
         migrations.AddField(
             model_name='workflowjob',
@@ -27,7 +27,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='workflowjobtemplate',
             name='ask_inventory_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AddField(
             model_name='workflowjobtemplate',

awx/main/migrations/0067_v350_credential_plugins.py

@@ -34,7 +34,7 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('description', models.TextField(blank=True, default='')),
                 ('input_field_name', models.CharField(max_length=1024)),
-                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default={})),
+                ('metadata', awx.main.fields.DynamicCredentialInputField(blank=True, default=dict)),
                 ('created_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_created+", to=settings.AUTH_USER_MODEL)),
                 ('modified_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'credentialinputsource', 'model_name': 'credentialinputsource', 'app_label': 'main'}(class)s_modified+", to=settings.AUTH_USER_MODEL)),
                 ('source_credential', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_input_sources', to='main.Credential')),

awx/main/migrations/0082_v360_webhook_http_method.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def add_webhook_notification_template_fields(apps, schema_editor):
+    # loop over all existing webhook notification templates and make
+    # sure they have the new "http_method" field filled in with "POST"
+    NotificationTemplate = apps.get_model('main', 'notificationtemplate')
+    webhooks = NotificationTemplate.objects.filter(notification_type='webhook')
+    for w in webhooks:
+        w.notification_configuration['http_method'] = 'POST'
+        w.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0081_v360_notify_on_start'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_webhook_notification_template_fields, migrations.RunPython.noop),
+    ]

awx/main/migrations/0083_v360_job_branch_override.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-14 15:08
+from __future__ import unicode_literals
+
+import awx.main.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0082_v360_webhook_http_method'),
+    ]
+
+    operations = [
+        # Add fields for user-provided project refspec
+        migrations.AddField(
+            model_name='project',
+            name='scm_refspec',
+            field=models.CharField(blank=True, default='', help_text='For git projects, an additional refspec to fetch.', max_length=1024, verbose_name='SCM refspec'),
+        ),
+        migrations.AddField(
+            model_name='projectupdate',
+            name='scm_refspec',
+            field=models.CharField(blank=True, default='', help_text='For git projects, an additional refspec to fetch.', max_length=1024, verbose_name='SCM refspec'),
+        ),
+        # Add fields for job specification of project branch
+        migrations.AddField(
+            model_name='job',
+            name='scm_branch',
+            field=models.CharField(blank=True, default='', help_text='Branch to use in job run. Project default used if blank. Only allowed if project allow_override field is set to true.', max_length=1024),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_scm_branch_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='scm_branch',
+            field=models.CharField(blank=True, default='', help_text='Branch to use in job run. Project default used if blank. Only allowed if project allow_override field is set to true.', max_length=1024),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='allow_override',
+            field=models.BooleanField(default=False, help_text='Allow changing the SCM branch or revision in a job template that uses this project.'),
+        ),
+        # Fix typo in help_text
+        migrations.AlterField(
+            model_name='project',
+            name='scm_update_cache_timeout',
+            field=models.PositiveIntegerField(blank=True, default=0, help_text='The number of seconds after the last project update ran that a new project update will be launched as a job dependency.'),
+        ),
+        # Start tracking the fetched revision on project update model
+        migrations.AddField(
+            model_name='projectupdate',
+            name='scm_revision',
+            field=models.CharField(blank=True, default='', editable=False, help_text='The SCM Revision discovered by this update for the given project and branch.', max_length=1024, verbose_name='SCM Revision'),
+        ),
+    ]

awx/main/migrations/0084_v360_token_description.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.4 on 2019-08-16 13:22
+
+from django.db import migrations, models
+
+import awx
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0083_v360_job_branch_override'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='oauth2accesstoken',
+            name='description',
+            field=models.TextField(blank=True, default=''),
+        ),
+    ]

awx/main/migrations/0085_v360_add_notificationtemplate_messages.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-06-10 16:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+import awx.main.fields
+import awx.main.models.notifications
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0084_v360_token_description'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='notificationtemplate',
+            name='messages',
+            field=awx.main.fields.JSONField(default=awx.main.models.notifications.NotificationTemplate.default_messages,
+                                            help_text='Optional custom messages for notification template.',
+                                            null=True,
+                                            blank=True),
+        ),
+        migrations.AlterField(
+            model_name='notification',
+            name='notification_type',
+            field=models.CharField(choices=[('email', 'Email'), ('grafana', 'Grafana'), ('hipchat', 'HipChat'), ('irc', 'IRC'), ('mattermost', 'Mattermost'), ('pagerduty', 'Pagerduty'), ('rocketchat', 'Rocket.Chat'), ('slack', 'Slack'), ('twilio', 'Twilio'), ('webhook', 'Webhook')], max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='notificationtemplate',
+            name='notification_type',
+            field=models.CharField(choices=[('email', 'Email'), ('grafana', 'Grafana'), ('hipchat', 'HipChat'), ('irc', 'IRC'), ('mattermost', 'Mattermost'), ('pagerduty', 'Pagerduty'), ('rocketchat', 'Rocket.Chat'), ('slack', 'Slack'), ('twilio', 'Twilio'), ('webhook', 'Webhook')], max_length=32),
+        ),
+    ]

awx/main/migrations/0086_v360_workflow_approval.py

@@ -0,0 +1,83 @@
+# Generated by Django 2.2.4 on 2019-08-02 17:51
+
+import awx.main.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0085_v360_add_notificationtemplate_messages'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WorkflowApprovalTemplate',
+            fields=[
+                ('unifiedjobtemplate_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='main.UnifiedJobTemplate')),
+                ('timeout', models.IntegerField(blank=True, default=0, help_text='The amount of time (in seconds) before the approval node expires and fails.')),
+            ],
+            bases=('main.unifiedjobtemplate',),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='approval_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role='admin_role', related_name='+', to='main.Role'),
+            preserve_default='True',
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='approval_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['organization.approval_role', 'admin_role'], related_name='+', to='main.Role'),
+            preserve_default='True',
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='unified_job_template',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobnodes', to='main.UnifiedJobTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='unified_job_template',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobtemplatenodes', to='main.UnifiedJobTemplate'),
+        ),
+        migrations.CreateModel(
+            name='WorkflowApproval',
+            fields=[
+                ('unifiedjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='main.UnifiedJob')),
+                ('workflow_approval_template', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='approvals', to='main.WorkflowApprovalTemplate')),
+            ],
+            bases=('main.unifiedjob',),
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='workflow_approval',
+            field=models.ManyToManyField(blank=True, to='main.WorkflowApproval'),
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='workflow_approval_template',
+            field=models.ManyToManyField(blank=True, to='main.WorkflowApprovalTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='read_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['member_role', 'auditor_role', 'execute_role', 'project_admin_role', 'inventory_admin_role', 'workflow_admin_role', 'notification_admin_role', 'credential_admin_role', 'job_template_admin_role', 'approval_role'], related_name='+', to='main.Role'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplate',
+            name='read_role',
+            field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['singleton:system_auditor', 'organization.auditor_role', 'execute_role', 'admin_role', 'approval_role'], related_name='+', to='main.Role'),
+        ),
+        migrations.AddField(
+            model_name='workflowapproval',
+            name='timeout',
+            field=models.IntegerField(blank=True, default=0, help_text='The amount of time (in seconds) before the approval node expires and fails.'),
+        ),
+        migrations.AddField(
+            model_name='workflowapproval',
+            name='timed_out',
+            field=models.BooleanField(default=False, help_text='Shows when an approval node (with a timeout assigned to it) has timed out.'),
+        ),
+    ]

awx/main/migrations/0087_v360_update_credential_injector_help_text.py

@@ -0,0 +1,29 @@
+# Generated by Django 2.2.4 on 2019-08-27 21:50
+
+import awx.main.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0086_v360_workflow_approval'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='credential',
+            name='inputs',
+            field=awx.main.fields.CredentialInputField(blank=True, default=dict, help_text='Enter inputs using either JSON or YAML syntax. Refer to the Ansible Tower documentation for example syntax.'),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='injectors',
+            field=awx.main.fields.CredentialTypeInjectorField(blank=True, default=dict, help_text='Enter injectors using either JSON or YAML syntax. Refer to the Ansible Tower documentation for example syntax.'),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='inputs',
+            field=awx.main.fields.CredentialTypeInputField(blank=True, default=dict, help_text='Enter inputs using either JSON or YAML syntax. Refer to the Ansible Tower documentation for example syntax.'),
+        ),
+    ]

awx/main/migrations/0088_v360_dashboard_optimizations.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.4 on 2019-09-10 21:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0087_v360_update_credential_injector_help_text'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='finished',
+            field=models.DateTimeField(db_index=True, default=None, editable=False, help_text='The date and time the job finished execution.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='launch_type',
+            field=models.CharField(choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency'), ('workflow', 'Workflow'), ('sync', 'Sync'), ('scm', 'SCM Update')], db_index=True, default='manual', editable=False, max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='created',
+            field=models.DateTimeField(db_index=True, default=None, editable=False),
+        ),
+    ]

awx/main/migrations/0089_v360_new_job_event_types.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.4 on 2019-09-12 13:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0088_v360_dashboard_optimizations'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='jobevent',
+            name='event',
+            field=models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_start', 'Host Started'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='projectupdateevent',
+            name='event',
+            field=models.CharField(choices=[('runner_on_failed', 'Host Failed'), ('runner_on_start', 'Host Started'), ('runner_on_ok', 'Host OK'), ('runner_on_error', 'Host Failure'), ('runner_on_skipped', 'Host Skipped'), ('runner_on_unreachable', 'Host Unreachable'), ('runner_on_no_hosts', 'No Hosts Remaining'), ('runner_on_async_poll', 'Host Polling'), ('runner_on_async_ok', 'Host Async OK'), ('runner_on_async_failed', 'Host Async Failure'), ('runner_item_on_ok', 'Item OK'), ('runner_item_on_failed', 'Item Failed'), ('runner_item_on_skipped', 'Item Skipped'), ('runner_retry', 'Host Retry'), ('runner_on_file_diff', 'File Difference'), ('playbook_on_start', 'Playbook Started'), ('playbook_on_notify', 'Running Handlers'), ('playbook_on_include', 'Including File'), ('playbook_on_no_hosts_matched', 'No Hosts Matched'), ('playbook_on_no_hosts_remaining', 'No Hosts Remaining'), ('playbook_on_task_start', 'Task Started'), ('playbook_on_vars_prompt', 'Variables Prompted'), ('playbook_on_setup', 'Gathering Facts'), ('playbook_on_import_for_host', 'internal: on Import for Host'), ('playbook_on_not_import_for_host', 'internal: on Not Import for Host'), ('playbook_on_play_start', 'Play Started'), ('playbook_on_stats', 'Playbook Complete'), ('debug', 'Debug'), ('verbose', 'Verbose'), ('deprecated', 'Deprecated'), ('warning', 'Warning'), ('system_warning', 'System Warning'), ('error', 'Error')], max_length=100),
+        ),
+    ]

awx/main/migrations/0090_v360_WFJT_prompts.py

@@ -0,0 +1,59 @@
+# Generated by Django 2.2.2 on 2019-07-23 17:56
+
+import awx.main.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0089_v360_new_job_event_types'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_limit_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_scm_branch_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='char_prompts',
+            field=awx.main.fields.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='joblaunchconfig',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='joblaunchconfigs', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='schedule',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='schedules', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjob',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobs', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobnodes', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplate',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobtemplates', to='main.Inventory'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobtemplatenode',
+            name='inventory',
+            field=models.ForeignKey(blank=True, default=None, help_text='Inventory applied as a prompt, assuming job template prompts for inventory', null=True, on_delete=models.deletion.SET_NULL, related_name='workflowjobtemplatenodes', to='main.Inventory'),
+        ),
+    ]

awx/main/migrations/0091_v360_approval_node_notifications.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.4 on 2019-09-11 13:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0090_v360_WFJT_prompts'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organization',
+            name='notification_templates_approvals',
+            field=models.ManyToManyField(blank=True, related_name='organization_notification_templates_for_approvals', to='main.NotificationTemplate'),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='notification_templates_approvals',
+            field=models.ManyToManyField(blank=True, related_name='workflowjobtemplate_notification_templates_for_approvals', to='main.NotificationTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='workflowjobnode',
+            name='do_not_run',
+            field=models.BooleanField(default=False, help_text='Indicates that a job will not be created when True. Workflow runtime semantics will mark this True if the node is in a path that will decidedly not be ran. A value of False means the node may not run.'),
+        ),
+    ]

awx/main/migrations/0092_v360_webhook_mixin.py

@@ -0,0 +1,49 @@
+# Generated by Django 2.2.4 on 2019-09-12 14:49
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0091_v360_approval_node_notifications'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='jobtemplates', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='webhook_key',
+            field=models.CharField(blank=True, help_text='Shared secret that the webhook service will use to sign requests', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobtemplates', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='webhook_key',
+            field=models.CharField(blank=True, help_text='Shared secret that the webhook service will use to sign requests', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='launch_type',
+            field=models.CharField(choices=[('manual', 'Manual'), ('relaunch', 'Relaunch'), ('callback', 'Callback'), ('scheduled', 'Scheduled'), ('dependency', 'Dependency'), ('workflow', 'Workflow'), ('webhook', 'Webhook'), ('sync', 'Sync'), ('scm', 'SCM Update')], db_index=True, default='manual', editable=False, max_length=20),
+        ),
+    ]

awx/main/migrations/0093_v360_personal_access_tokens.py

@@ -0,0 +1,27 @@
+# Generated by Django 2.2.4 on 2019-09-12 14:50
+
+from django.db import migrations, models
+
+from awx.main.models import CredentialType
+from awx.main.utils.common import set_current_apps
+
+
+def setup_tower_managed_defaults(apps, schema_editor):
+    set_current_apps(apps)
+    CredentialType.setup_tower_managed_defaults()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0092_v360_webhook_mixin'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='kind',
+            field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External')], max_length=32),
+        ),
+        migrations.RunPython(setup_tower_managed_defaults),
+    ]

awx/main/migrations/0094_v360_webhook_mixin2.py

@@ -0,0 +1,44 @@
+# Generated by Django 2.2.4 on 2019-09-12 14:52
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0093_v360_personal_access_tokens'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='job',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='jobs', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='webhook_guid',
+            field=models.CharField(blank=True, help_text='Unique identifier of the event that triggered this webhook', max_length=128),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='webhook_credential',
+            field=models.ForeignKey(blank=True, help_text='Personal Access Token for posting back the status to the service API', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='workflowjobs', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='webhook_guid',
+            field=models.CharField(blank=True, help_text='Unique identifier of the event that triggered this webhook', max_length=128),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='webhook_service',
+            field=models.CharField(blank=True, choices=[('github', 'GitHub'), ('gitlab', 'GitLab')], help_text='Service that webhook requests will be accepted from', max_length=16),
+        ),
+    ]

awx/main/migrations/0095_v360_increase_instance_version_length.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-04 00:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0094_v360_webhook_mixin2'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='version',
+            field=models.CharField(blank=True, max_length=120),
+        ),
+    ]

awx/main/migrations/0096_v360_container_groups.py

@@ -0,0 +1,38 @@
+# Generated by Django 2.2.4 on 2019-09-16 23:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+from awx.main.models import CredentialType
+from awx.main.utils.common import set_current_apps
+
+
+def create_new_credential_types(apps, schema_editor):
+    set_current_apps(apps)
+    CredentialType.setup_tower_managed_defaults()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0095_v360_increase_instance_version_length'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instancegroup',
+            name='credential',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instancegroups', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='pod_spec_override',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='kind',
+            field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External'), ('kubernetes', 'Kubernetes')], max_length=32),
+        ),
+        migrations.RunPython(create_new_credential_types)
+    ]

awx/main/migrations/0097_v360_workflowapproval_approved_or_denied_by.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.4 on 2019-10-11 15:40
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('main', '0096_v360_container_groups'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='workflowapproval',
+            name='approved_or_denied_by',
+            field=models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'workflowapproval', 'model_name': 'workflowapproval', 'app_label': 'main'}(class)s_approved+", to=settings.AUTH_USER_MODEL),
+        ),
+    ]

awx/main/migrations/0098_v360_rename_cyberark_aim_credential_type.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.4 on 2019-10-16 19:51
+
+from django.db import migrations
+from awx.main.models import CredentialType
+
+
+def update_cyberark_aim_name(apps, schema_editor):
+    CredentialType.setup_tower_managed_defaults()
+    aim_types = apps.get_model('main', 'CredentialType').objects.filter(
+        namespace='aim'
+    ).order_by('id')
+
+    if aim_types.count() == 2:
+        original, renamed = aim_types.all()
+        apps.get_model('main', 'Credential').objects.filter(
+            credential_type_id=original.id
+        ).update(
+            credential_type_id=renamed.id
+        )
+        original.delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0097_v360_workflowapproval_approved_or_denied_by'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_cyberark_aim_name)
+    ]

awx/main/migrations/_rbac.py

@@ -1,17 +1,11 @@
 import logging
 from time import time
 
-from django.utils.encoding import smart_text
-from django.db import transaction
-from django.db.models import Q
-from django.db.utils import IntegrityError
-
-from collections import defaultdict
-from awx.main.utils import getattrd
 from awx.main.models.rbac import Role, batch_role_ancestor_rebuilding
 
 logger = logging.getLogger('rbac_migrations')
 
+
 def create_roles(apps, schema_editor):
     '''
     Implicit role creation happens in our post_save hook for all of our
@@ -41,465 +35,19 @@ def create_roles(apps, schema_editor):
                 obj.save()
 
 
-def migrate_users(apps, schema_editor):
-    User = apps.get_model('auth', "User")
-    Role = apps.get_model('main', "Role")
-    ContentType = apps.get_model('contenttypes', "ContentType")
-    user_content_type = ContentType.objects.get_for_model(User)
-
-    for user in User.objects.iterator():
-        user.save()
-        try:
-            Role.objects.get(content_type=user_content_type, object_id=user.id)
-            logger.info(smart_text(u"found existing role for user: {}".format(user.username)))
-        except Role.DoesNotExist:
-            role = Role.objects.create(
-                role_field='admin_role',
-                content_type = user_content_type,
-                object_id = user.id
-            )
-            role.members.add(user)
-            logger.info(smart_text(u"migrating to new role for user: {}".format(user.username)))
-
-        if user.is_superuser:
-            if Role.objects.filter(singleton_name='system_administrator').exists():
-                sa_role = Role.objects.get(singleton_name='system_administrator')
-            else:
-                sa_role = Role.objects.create(
-                    singleton_name='system_administrator',
-                    role_field='system_administrator'
-                )
-
-            sa_role.members.add(user)
-            logger.warning(smart_text(u"added superuser: {}".format(user.username)))
-
-def migrate_organization(apps, schema_editor):
-    Organization = apps.get_model('main', "Organization")
-    for org in Organization.objects.iterator():
-        for admin in org.deprecated_admins.all():
-            org.admin_role.members.add(admin)
-            logger.info(smart_text(u"added admin: {}, {}".format(org.name, admin.username)))
-        for user in org.deprecated_users.all():
-            org.member_role.members.add(user)
-            logger.info(smart_text(u"added member: {}, {}".format(org.name, user.username)))
-
-def migrate_team(apps, schema_editor):
-    Team = apps.get_model('main', 'Team')
-    for t in Team.objects.iterator():
-        for user in t.deprecated_users.all():
-            t.member_role.members.add(user)
-            logger.info(smart_text(u"team: {}, added user: {}".format(t.name, user.username)))
-
-def attrfunc(attr_path):
-    '''attrfunc returns a function that will
-    attempt to use the attr_path to access the attribute
-    of an instance that is passed in to the returned function.
-
-    Example:
-        get_org = attrfunc('inventory.organization')
-        org = get_org(JobTemplateInstance)
-    '''
-    def attr(inst):
-        return getattrd(inst, attr_path)
-    return attr
-
-def _update_credential_parents(org, cred):
-    cred.organization = org
-    cred.save()
-
-def _discover_credentials(instances, cred, orgfunc):
-    '''_discover_credentials will find shared credentials across
-    organizations. If a shared credential is found, it will duplicate
-    the credential, ensure the proper role permissions are added to the new
-    credential, and update any references from the old to the newly created
-    credential.
-
-    instances is a list of all objects that were matched when filtered
-    with cred.
-
-    orgfunc is a function that when called with an instance from instances
-    will produce an Organization object.
-    '''
-    orgs = defaultdict(list)
-    for inst in instances:
-        try:
-            orgs[orgfunc(inst)].append(inst)
-        except AttributeError:
-            # JobTemplate.inventory can be NULL sometimes, eg when an inventory
-            # has been deleted. This protects against that.
-            pass
-
-    if len(orgs) == 1:
-        try:
-            _update_credential_parents(orgfunc(instances[0]), cred)
-        except AttributeError:
-            # JobTemplate.inventory can be NULL sometimes, eg when an inventory
-            # has been deleted. This protects against that.
-            pass
-    else:
-        for pos, org in enumerate(orgs):
-            if pos == 0:
-                _update_credential_parents(org, cred)
-            else:
-                # Create a new credential
-                cred.pk = None
-                cred.organization = None
-                cred.save()
-
-                cred.admin_role, cred.use_role = None, None
-
-                for i in orgs[org]:
-                    i.credential = cred
-                    i.save()
-
-                _update_credential_parents(org, cred)
-
-def migrate_credential(apps, schema_editor):
-    Credential = apps.get_model('main', "Credential")
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Project = apps.get_model('main', 'Project')
-    InventorySource = apps.get_model('main', 'InventorySource')
-
-    for cred in Credential.objects.iterator():
-        results = [x for x in JobTemplate.objects.filter(Q(credential=cred) | Q(cloud_credential=cred), inventory__isnull=False).all()] + \
-                  [x for x in InventorySource.objects.filter(credential=cred).all()]
-        if cred.deprecated_team is not None and results:
-            if len(results) == 1:
-                _update_credential_parents(results[0].inventory.organization, cred)
-            else:
-                _discover_credentials(results, cred, attrfunc('inventory.organization'))
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at organization level".format(cred.name, cred.kind, cred.host)))
-
-        projs = Project.objects.filter(credential=cred).all()
-        if cred.deprecated_team is not None and projs:
-            if len(projs) == 1:
-                _update_credential_parents(projs[0].organization, cred)
-            else:
-                _discover_credentials(projs, cred, attrfunc('organization'))
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at organization level".format(cred.name, cred.kind, cred.host)))
-
-        if cred.deprecated_team is not None:
-            cred.deprecated_team.admin_role.children.add(cred.admin_role)
-            cred.deprecated_team.member_role.children.add(cred.use_role)
-            cred.save()
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at user level".format(cred.name, cred.kind, cred.host)))
-        elif cred.deprecated_user is not None:
-            cred.admin_role.members.add(cred.deprecated_user)
-            cred.save()
-            logger.info(smart_text(u"added Credential(name={}, kind={}, host={}) at user level".format(cred.name, cred.kind, cred.host, )))
-        else:
-            logger.warning(smart_text(u"orphaned credential found Credential(name={}, kind={}, host={}), superuser only".format(cred.name, cred.kind, cred.host, )))
-
-
-def migrate_inventory(apps, schema_editor):
-    Inventory = apps.get_model('main', 'Inventory')
-    Permission = apps.get_model('main', 'Permission')
-
-    def role_from_permission(perm):
-        if perm.permission_type == 'admin':
-            return inventory.admin_role
-        elif perm.permission_type == 'read':
-            return inventory.read_role
-        elif perm.permission_type == 'write':
-            return inventory.update_role
-        elif perm.permission_type == 'check' or perm.permission_type == 'run' or perm.permission_type == 'create':
-            # These permission types are handled differntly in RBAC now, nothing to migrate.
-            return False
-        else:
-            return None
-
-    for inventory in Inventory.objects.iterator():
-        for perm in Permission.objects.filter(inventory=inventory):
-            role = None
-            execrole = None
-
-            role = role_from_permission(perm)
-            if role is None:
-                raise Exception(smart_text(u'Unhandled permission type for inventory: {}'.format( perm.permission_type)))
-
-            if perm.run_ad_hoc_commands:
-                execrole = inventory.use_role
-
-            if perm.team:
-                if role:
-                    perm.team.member_role.children.add(role)
-                if execrole:
-                    perm.team.member_role.children.add(execrole)
-                logger.info(smart_text(u'added Team({}) access to Inventory({})'.format(perm.team.name, inventory.name)))
-
-            if perm.user:
-                if role:
-                    role.members.add(perm.user)
-                if execrole:
-                    execrole.members.add(perm.user)
-                logger.info(smart_text(u'added User({}) access to Inventory({})'.format(perm.user.username, inventory.name)))
-
-def migrate_projects(apps, schema_editor):
-    '''
-    I can see projects when:
-     X I am a superuser.
-     X I am an admin in an organization associated with the project.
-     X I am a user in an organization associated with the project.
-     X I am on a team associated with the project.
-     X I have been explicitly granted permission to run/check jobs using the
-       project.
-     X I created the project but it isn't associated with an organization
-    I can change/delete when:
-     X I am a superuser.
-     X I am an admin in an organization associated with the project.
-     X I created the project but it isn't associated with an organization
-    '''
-    Project = apps.get_model('main', 'Project')
-    Permission = apps.get_model('main', 'Permission')
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-
-    # Migrate projects to single organizations, duplicating as necessary
-    for project in Project.objects.iterator():
-        original_project_name = project.name
-        project_orgs = project.deprecated_organizations.distinct().all()
-
-        if len(project_orgs) >= 1:
-            first_org = None
-            for org in project_orgs:
-                if first_org is None:
-                    # For the first org, re-use our existing Project object, so don't do the below duplication effort
-                    first_org = org
-                    if len(project_orgs) > 1:
-                        project.name = smart_text(u'{} - {}'.format(first_org.name, original_project_name))
-                    project.organization = first_org
-                    project.save()
-                else:
-                    new_prj = Project.objects.create(
-                        created                   = project.created,
-                        modified                  = project.modified,
-                        polymorphic_ctype_id      = project.polymorphic_ctype_id,
-                        description               = project.description,
-                        name                      = smart_text(u'{} - {}'.format(org.name, original_project_name)),
-                        old_pk                    = project.old_pk,
-                        created_by_id             = project.created_by_id,
-                        modified_by_id            = project.modified_by_id,
-                        scm_type                  = project.scm_type,
-                        scm_url                   = project.scm_url,
-                        scm_branch                = project.scm_branch,
-                        scm_clean                 = project.scm_clean,
-                        scm_delete_on_update      = project.scm_delete_on_update,
-                        scm_delete_on_next_update = project.scm_delete_on_next_update,
-                        scm_update_on_launch      = project.scm_update_on_launch,
-                        scm_update_cache_timeout  = project.scm_update_cache_timeout,
-                        credential                = project.credential,
-                        organization              = org
-                    )
-                    if project.scm_type == "":
-                        new_prj.local_path = project.local_path
-                        new_prj.save()
-                    for team in project.deprecated_teams.iterator():
-                        new_prj.deprecated_teams.add(team)
-                    logger.warning(smart_text(u'cloning Project({}) onto {} as Project({})'.format(original_project_name, org, new_prj)))
-                    job_templates = JobTemplate.objects.filter(project=project, inventory__organization=org).all()
-                    for jt in job_templates:
-                        jt.project = new_prj
-                        jt.save()
-                    for perm in Permission.objects.filter(project=project):
-                        Permission.objects.create(
-                            created                   = perm.created,
-                            modified                  = perm.modified,
-                            created_by                = perm.created_by,
-                            modified_by               = perm.modified_by,
-                            description               = perm.description,
-                            name                      = perm.name,
-                            user                      = perm.user,
-                            team                      = perm.team,
-                            project                   = new_prj,
-                            inventory                 = perm.inventory,
-                            permission_type           = perm.permission_type,
-                            run_ad_hoc_commands       = perm.run_ad_hoc_commands,
-                        )
-
-    # Migrate permissions
-    for project in Project.objects.iterator():
-        if project.organization is None and project.created_by is not None:
-            project.admin_role.members.add(project.created_by)
-            logger.warn(smart_text(u'adding Project({}) admin: {}'.format(project.name, project.created_by.username)))
-
-        for team in project.deprecated_teams.all():
-            team.member_role.children.add(project.read_role)
-            logger.info(smart_text(u'adding Team({}) access for Project({})'.format(team.name, project.name)))
-
-        for perm in Permission.objects.filter(project=project):
-            if perm.permission_type == 'create':
-                role = project.use_role
-            else:
-                role = project.read_role
-
-            if perm.team:
-                perm.team.member_role.children.add(role)
-                logger.info(smart_text(u'adding Team({}) access for Project({})'.format(perm.team.name, project.name)))
-
-            if perm.user:
-                role.members.add(perm.user)
-                logger.info(smart_text(u'adding User({}) access for Project({})'.format(perm.user.username, project.name)))
-
-        if project.organization is not None:
-            for user in project.organization.deprecated_users.all():
-                if not (project.use_role.members.filter(pk=user.id).exists() or project.admin_role.members.filter(pk=user.id).exists()):
-                    project.read_role.members.add(user)
-                    logger.info(smart_text(u'adding Organization({}) member access to Project({})'.format(project.organization.name, project.name)))
-
-
-
-def migrate_job_templates(apps, schema_editor):
-    '''
-    NOTE: This must be run after orgs, inventory, projects, credential, and
-    users have been migrated
-    '''
-
-
-    '''
-    I can see job templates when:
-     X I am a superuser.
-     - I can read the inventory, project and credential (which means I am an
-       org admin or member of a team with access to all of the above).
-     - I have permission explicitly granted to check/deploy with the inventory
-       and project.
-
-
-    #This does not mean I would be able to launch a job from the template or
-    #edit the template.
-     - access.py can_read for JobTemplate enforces that you can only
-       see it if you can launch it, so the above imply launch too
-    '''
-
-
-    '''
-    Tower administrators, organization administrators, and project
-    administrators, within a project under their purview, may create and modify
-    new job templates for that project.
-
-    When editing a job template, they may select among the inventory groups and
-    credentials in the organization for which they have usage permissions, or
-    they may leave either blank to be selected at runtime.
-
-    Additionally, they may specify one or more users/teams that have execution
-    permission for that job template, among the users/teams that are a member
-    of that project.
-
-    That execution permission is valid irrespective of any explicit permissions
-    the user has or has not been granted to the inventory group or credential
-    specified in the job template.
-
-    '''
-
-    User = apps.get_model('auth', 'User')
-    JobTemplate = apps.get_model('main', 'JobTemplate')
-    Team = apps.get_model('main', 'Team')
-    Permission = apps.get_model('main', 'Permission')
-    Credential = apps.get_model('main', 'Credential')
-
-    jt_queryset = JobTemplate.objects.select_related('inventory', 'project', 'inventory__organization', 'execute_role')
-
-    for jt in jt_queryset.iterator():
-        if jt.inventory is None:
-            # If inventory is None, then only system admins and org admins can
-            # do anything with the JT in 2.4
-            continue
-
-        jt_permission_qs = Permission.objects.filter(
-            inventory=jt.inventory,
-            project=jt.project,
-        )
-
-        inventory_permission_qs = Permission.objects.filter(
-            inventory=jt.inventory,
-            project__isnull=True,
-        )
-
-        team_create_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['create'])
-            .values_list('team__id', flat=True)
-        )
-        team_run_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['check', 'run'] if jt.job_type == 'check' else ['run'])
-            .values_list('team__id', flat=True)
-        )
-        user_create_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['create'])
-            .values_list('user__id', flat=True)
-        )
-        user_run_permissions = set(
-            jt_permission_qs
-            .filter(permission_type__in=['check', 'run'] if jt.job_type == 'check' else ['run'])
-            .values_list('user__id', flat=True)
-        )
-
-        team_inv_permissions = defaultdict(set)
-        user_inv_permissions = defaultdict(set)
-
-        for user_id, team_id, inventory_id in inventory_permission_qs.values_list('user_id', 'team_id', 'inventory_id'):
-            if user_id:
-                user_inv_permissions[user_id].add(inventory_id)
-            if team_id:
-                team_inv_permissions[team_id].add(inventory_id)
-
-
-        for team in Team.objects.filter(id__in=team_create_permissions).iterator():
-            if jt.inventory.id in team_inv_permissions[team.id] and \
-                ((not jt.credential and not jt.cloud_credential) or
-                    Credential.objects.filter(deprecated_team=team, jobtemplates=jt).exists()):
-                team.member_role.children.add(jt.admin_role)
-                logger.info(smart_text(u'transfering admin access on JobTemplate({}) to Team({})'.format(jt.name, team.name)))
-        for team in Team.objects.filter(id__in=team_run_permissions).iterator():
-            if jt.inventory.id in team_inv_permissions[team.id] and \
-               ((not jt.credential and not jt.cloud_credential) or
-                    Credential.objects.filter(deprecated_team=team, jobtemplates=jt).exists()):
-                team.member_role.children.add(jt.execute_role)
-                logger.info(smart_text(u'transfering execute access on JobTemplate({}) to Team({})'.format(jt.name, team.name)))
-
-        for user in User.objects.filter(id__in=user_create_permissions).iterator():
-            cred = jt.credential or jt.cloud_credential
-            if (jt.inventory.id in user_inv_permissions[user.id] or
-                    any([jt.inventory.id in team_inv_permissions[team.id] for team in user.deprecated_teams.all()])) and \
-                    (not cred or cred.deprecated_user == user or
-                        (cred.deprecated_team and cred.deprecated_team.deprecated_users.filter(pk=user.id).exists())):
-                jt.admin_role.members.add(user)
-                logger.info(smart_text(u'transfering admin access on JobTemplate({}) to User({})'.format(jt.name, user.username)))
-        for user in User.objects.filter(id__in=user_run_permissions).iterator():
-            cred = jt.credential or jt.cloud_credential
-
-            if (jt.inventory.id in user_inv_permissions[user.id] or
-                    any([jt.inventory.id in team_inv_permissions[team.id] for team in user.deprecated_teams.all()])) and \
-                    (not cred or cred.deprecated_user == user or
-                        (cred.deprecated_team and cred.deprecated_team.deprecated_users.filter(pk=user.id).exists())):
-                jt.execute_role.members.add(user)
-                logger.info(smart_text(u'transfering execute access on JobTemplate({}) to User({})'.format(jt.name, user.username)))
-
-
-
 def rebuild_role_hierarchy(apps, schema_editor):
-        logger.info('Computing role roots..')
-        start = time()
-        roots = Role.objects \
-                    .all() \
-                    .values_list('id', flat=True)
-        stop = time()
-        logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start))
-        start = time()
-        Role.rebuild_role_ancestor_list(roots, [])
-        stop = time()
-        logger.info('Rebuild completed in %f seconds' % (stop - start))
-        logger.info('Done.')
-
-
-def infer_credential_org_from_team(apps, schema_editor):
-    Credential = apps.get_model('main', "Credential")
-    for cred in Credential.objects.exclude(deprecated_team__isnull=True):
-        try:
-            with transaction.atomic():
-                _update_credential_parents(cred.deprecated_team.organization, cred)
-        except IntegrityError:
-            logger.info("Organization<{}> credential for old Team<{}> credential already created".format(cred.deprecated_team.organization.pk, cred.pk))
+    logger.info('Computing role roots..')
+    start = time()
+    roots = Role.objects \
+                .all() \
+                .values_list('id', flat=True)
+    stop = time()
+    logger.info('Found %d roots in %f seconds, rebuilding ancestry map' % (len(roots), stop - start))
+    start = time()
+    Role.rebuild_role_ancestor_list(roots, [])
+    stop = time()
+    logger.info('Rebuild completed in %f seconds' % (stop - start))
+    logger.info('Done.')
 
 
 def delete_all_user_roles(apps, schema_editor):

awx/main/migrations/_squashed_30.py

@@ -30,7 +30,7 @@ SQUASHED_30 = {
         migrations.AddField(
             model_name='job',
             name='survey_passwords',
-            field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
+            field=jsonfield.fields.JSONField(default=dict, editable=False, blank=True),
         ),
     ],
     '0031_v302_migrate_survey_passwords': [

awx/main/models/__init__.py

@@ -7,7 +7,8 @@ from django.db.models.signals import pre_delete  # noqa
 
 # AWX
 from awx.main.models.base import (  # noqa
-    BaseModel, PrimordialModel, prevent_search, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES
+    BaseModel, PrimordialModel, prevent_search, accepts_json,
+    CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES
 )
 from awx.main.models.unified_jobs import (  # noqa
     UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded
@@ -48,11 +49,14 @@ from awx.main.models.mixins import (  # noqa
     TaskManagerJobMixin, TaskManagerProjectUpdateMixin,
     TaskManagerUnifiedJobMixin,
 )
-from awx.main.models.notifications import Notification, NotificationTemplate # noqa
+from awx.main.models.notifications import (  # noqa
+    Notification, NotificationTemplate,
+    JobNotificationMixin
+)
 from awx.main.models.label import Label # noqa
 from awx.main.models.workflow import (  # noqa
     WorkflowJob, WorkflowJobNode, WorkflowJobOptions, WorkflowJobTemplate,
-    WorkflowJobTemplateNode,
+    WorkflowJobTemplateNode, WorkflowApproval, WorkflowApprovalTemplate,
 )
 from awx.main.models.channels import ChannelGroup # noqa
 from awx.api.versioning import reverse
@@ -62,24 +66,6 @@ from awx.main.models.oauth import ( # noqa
 from oauth2_provider.models import Grant, RefreshToken # noqa -- needed django-oauth-toolkit model migrations
 
 
-
-# Monkeypatch Django serializer to ignore django-taggit fields (which break
-# the dumpdata command; see https://github.com/alex/django-taggit/issues/155).
-from django.core.serializers.python import Serializer as _PythonSerializer
-_original_handle_m2m_field = _PythonSerializer.handle_m2m_field
-
-
-def _new_handle_m2m_field(self, obj, field):
-    try:
-        field.rel.through._meta
-    except AttributeError:
-        return
-    return _original_handle_m2m_field(self, obj, field)
-
-
-_PythonSerializer.handle_m2m_field = _new_handle_m2m_field
-
-
 # Add custom methods to User model for permissions checks.
 from django.contrib.auth.models import User  # noqa
 from awx.main.access import (  # noqa
@@ -140,25 +126,29 @@ def user_is_system_auditor(user):
 
 @user_is_system_auditor.setter
 def user_is_system_auditor(user, tf):
-    if user.id:
-        if tf:
-            role = Role.singleton('system_auditor')
-            # must check if member to not duplicate activity stream
-            if user not in role.members.all():
-                role.members.add(user)
-            user._is_system_auditor = True
-        else:
-            role = Role.singleton('system_auditor')
-            if user in role.members.all():
-                role.members.remove(user)
-            user._is_system_auditor = False
+    if not user.id:
+        # If the user doesn't have a primary key yet (i.e., this is the *first*
+        # time they've logged in, and we've just created the new User in this
+        # request), we need one to set up the system auditor role
+        user.save()
+    if tf:
+        role = Role.singleton('system_auditor')
+        # must check if member to not duplicate activity stream
+        if user not in role.members.all():
+            role.members.add(user)
+        user._is_system_auditor = True
+    else:
+        role = Role.singleton('system_auditor')
+        if user in role.members.all():
+            role.members.remove(user)
+        user._is_system_auditor = False
 
 
 User.add_to_class('is_system_auditor', user_is_system_auditor)
 
 
 def user_is_in_enterprise_category(user, category):
-    ret = (category,) in user.enterprise_auth.all().values_list('provider') and not user.has_usable_password()
+    ret = (category,) in user.enterprise_auth.values_list('provider') and not user.has_usable_password()
     # NOTE: this if-else block ensures existing enterprise users are still able to
     # log in. Remove it in a future release
     if category == 'radius':
@@ -213,6 +203,8 @@ activity_stream_registrar.connect(User)
 activity_stream_registrar.connect(WorkflowJobTemplate)
 activity_stream_registrar.connect(WorkflowJobTemplateNode)
 activity_stream_registrar.connect(WorkflowJob)
+activity_stream_registrar.connect(WorkflowApproval)
+activity_stream_registrar.connect(WorkflowApprovalTemplate)
 activity_stream_registrar.connect(OAuth2Application)
 activity_stream_registrar.connect(OAuth2AccessToken)
 
@@ -223,4 +215,3 @@ prevent_search(RefreshToken._meta.get_field('token'))
 prevent_search(OAuth2Application._meta.get_field('client_secret'))
 prevent_search(OAuth2Application._meta.get_field('client_id'))
 prevent_search(Grant._meta.get_field('code'))
-

awx/main/models/activity_stream.py

@@ -4,6 +4,7 @@
 # Tower
 from awx.api.versioning import reverse
 from awx.main.fields import JSONField
+from awx.main.models.base import accepts_json
 
 # Django
 from django.db import models
@@ -34,7 +35,7 @@ class ActivityStream(models.Model):
     actor = models.ForeignKey('auth.User', null=True, on_delete=models.SET_NULL, related_name='activity_stream')
     operation = models.CharField(max_length=13, choices=OPERATION_CHOICES)
     timestamp = models.DateTimeField(auto_now_add=True)
-    changes = models.TextField(blank=True)
+    changes = accepts_json(models.TextField(blank=True))
     deleted_actor = JSONField(null=True)
     action_node = models.CharField(
         blank=True,
@@ -66,6 +67,8 @@ class ActivityStream(models.Model):
     workflow_job_node = models.ManyToManyField("WorkflowJobNode", blank=True)
     workflow_job_template = models.ManyToManyField("WorkflowJobTemplate", blank=True)
     workflow_job = models.ManyToManyField("WorkflowJob", blank=True)
+    workflow_approval_template = models.ManyToManyField("WorkflowApprovalTemplate", blank=True)
+    workflow_approval = models.ManyToManyField("WorkflowApproval", blank=True)
     unified_job_template = models.ManyToManyField("UnifiedJobTemplate", blank=True, related_name='activity_stream_as_unified_job_template+')
     unified_job = models.ManyToManyField("UnifiedJob", blank=True, related_name='activity_stream_as_unified_job+')
     ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True)

awx/main/models/ad_hoc_commands.py

@@ -150,6 +150,14 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
     def supports_isolation(cls):
         return True
 
+    @property
+    def is_containerized(self):
+        return bool(self.instance_group and self.instance_group.is_containerized)
+
+    @property
+    def can_run_containerized(self):
+        return True
+
     def get_absolute_url(self, request=None):
         return reverse('api:ad_hoc_command_detail', kwargs={'pk': self.pk}, request=request)
 

awx/main/models/base.py

@@ -408,3 +408,14 @@ def prevent_search(relation):
     """
     setattr(relation, '__prevent_search__', True)
     return relation
+
+
+def accepts_json(relation):
+    """
+    Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars
+    This is *mostly* used as a way to provide type hints for certain fields
+    so that HTTP OPTIONS reports the type data we need for the CLI to allow
+    JSON/YAML input.
+    """
+    setattr(relation, '__accepts_json__', True)
+    return relation

awx/main/models/credential/__init__.py

@@ -64,7 +64,7 @@ def build_safe_env(env):
     for k, v in safe_env.items():
         if k == 'AWS_ACCESS_KEY_ID':
             continue
-        elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET'):
+        elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET') and not k.startswith('ANSIBLE_GALAXY_SERVER'):
             continue
         elif hidden_re.search(k):
             safe_env[k] = HIDDEN_PASSWORD
@@ -105,10 +105,9 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     )
     inputs = CredentialInputField(
         blank=True,
-        default={},
-        help_text=_('Enter inputs using either JSON or YAML syntax. Use the '
-                    'radio button to toggle between the two. Refer to the '
-                    'Ansible Tower documentation for example syntax.')
+        default=dict,
+        help_text=_('Enter inputs using either JSON or YAML syntax. '
+                    'Refer to the Ansible Tower documentation for example syntax.')
     )
     admin_role = ImplicitRoleField(
         parent_role=[
@@ -136,6 +135,10 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     def cloud(self):
         return self.credential_type.kind == 'cloud'
 
+    @property
+    def kubernetes(self):
+        return self.credential_type.kind == 'kubernetes'
+
     def get_absolute_url(self, request=None):
         return reverse('api:credential_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -152,7 +155,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     @property
     def has_encrypted_ssh_key_data(self):
         try:
-            ssh_key_data = decrypt_field(self, 'ssh_key_data')
+            ssh_key_data = self.get_input('ssh_key_data')
         except AttributeError:
             return False
 
@@ -323,8 +326,10 @@ class CredentialType(CommonModelNameNotUnique):
         ('net', _('Network')),
         ('scm', _('Source Control')),
         ('cloud', _('Cloud')),
+        ('token', _('Personal Access Token')),
         ('insights', _('Insights')),
         ('external', _('External')),
+        ('kubernetes', _('Kubernetes')),
     )
 
     kind = models.CharField(
@@ -343,17 +348,15 @@ class CredentialType(CommonModelNameNotUnique):
     )
     inputs = CredentialTypeInputField(
         blank=True,
-        default={},
-        help_text=_('Enter inputs using either JSON or YAML syntax. Use the '
-                    'radio button to toggle between the two. Refer to the '
-                    'Ansible Tower documentation for example syntax.')
+        default=dict,
+        help_text=_('Enter inputs using either JSON or YAML syntax. '
+                    'Refer to the Ansible Tower documentation for example syntax.')
     )
     injectors = CredentialTypeInjectorField(
         blank=True,
-        default={},
-        help_text=_('Enter injectors using either JSON or YAML syntax. Use the '
-                    'radio button to toggle between the two. Refer to the '
-                    'Ansible Tower documentation for example syntax.')
+        default=dict,
+        help_text=_('Enter injectors using either JSON or YAML syntax. '
+                    'Refer to the Ansible Tower documentation for example syntax.')
     )
 
     @classmethod
@@ -636,9 +639,6 @@ ManagedCredentialType(
             'secret': True,
             'ask_at_runtime': True
         }],
-        'dependencies': {
-            'ssh_key_unlock': ['ssh_key_data'],
-        }
     }
 )
 
@@ -670,9 +670,6 @@ ManagedCredentialType(
             'type': 'string',
             'secret': True
         }],
-        'dependencies': {
-            'ssh_key_unlock': ['ssh_key_data'],
-        }
     }
 )
 
@@ -741,7 +738,6 @@ ManagedCredentialType(
             'secret': True,
         }],
         'dependencies': {
-            'ssh_key_unlock': ['ssh_key_data'],
             'authorize_password': ['authorize'],
         },
         'required': ['username'],
@@ -978,6 +974,40 @@ ManagedCredentialType(
     }
 )
 
+ManagedCredentialType(
+    namespace='github_token',
+    kind='token',
+    name=ugettext_noop('GitHub Personal Access Token'),
+    managed_by_tower=True,
+    inputs={
+        'fields': [{
+            'id': 'token',
+            'label': ugettext_noop('Token'),
+            'type': 'string',
+            'secret': True,
+            'help_text': ugettext_noop('This token needs to come from your profile settings in GitHub')
+        }],
+        'required': ['token'],
+    },
+)
+
+ManagedCredentialType(
+    namespace='gitlab_token',
+    kind='token',
+    name=ugettext_noop('GitLab Personal Access Token'),
+    managed_by_tower=True,
+    inputs={
+        'fields': [{
+            'id': 'token',
+            'label': ugettext_noop('Token'),
+            'type': 'string',
+            'secret': True,
+            'help_text': ugettext_noop('This token needs to come from your profile settings in GitLab')
+        }],
+        'required': ['token'],
+    },
+)
+
 ManagedCredentialType(
     namespace='insights',
     kind='insights',
@@ -1093,6 +1123,38 @@ ManagedCredentialType(
 )
 
 
+ManagedCredentialType(
+    namespace='kubernetes_bearer_token',
+    kind='kubernetes',
+    name=ugettext_noop('OpenShift or Kubernetes API Bearer Token'),
+    inputs={
+        'fields': [{
+            'id': 'host',
+            'label': ugettext_noop('OpenShift or Kubernetes API Endpoint'),
+            'type': 'string',
+            'help_text': ugettext_noop('The OpenShift or Kubernetes API Endpoint to authenticate with.')
+        },{
+            'id': 'bearer_token',
+            'label': ugettext_noop('API authentication bearer token.'),
+            'type': 'string',
+            'secret': True,
+        },{
+            'id': 'verify_ssl',
+            'label': ugettext_noop('Verify SSL'),
+            'type': 'boolean',
+            'default': True,
+        },{
+            'id': 'ssl_ca_cert',
+            'label': ugettext_noop('Certificate Authority data'),
+            'type': 'string',
+            'secret': True,
+            'multiline': True,
+        }],
+        'required': ['host', 'bearer_token'],
+    }
+)
+
+
 class CredentialInputSource(PrimordialModel):
 
     class Meta:
@@ -1117,7 +1179,7 @@ class CredentialInputSource(PrimordialModel):
     )
     metadata = DynamicCredentialInputField(
         blank=True,
-        default={}
+        default=dict
     )
 
     def clean_target_credential(self):

awx/main/models/credential/injectors.py

@@ -28,10 +28,7 @@ def gce(cred, env, private_data_dir):
     if 'INVENTORY_UPDATE_ID' not in env:
         env['GCE_EMAIL'] = username
         env['GCE_PROJECT'] = project
-    else:
-        # gcp_compute inventory plugin requires token_uri
-        # although it probably should not, since gce_modules do not
-        json_cred['token_uri'] = 'https://oauth2.googleapis.com/token'
+    json_cred['token_uri'] = 'https://oauth2.googleapis.com/token'
 
     handle, path = tempfile.mkstemp(dir=private_data_dir)
     f = os.fdopen(handle, 'w')
@@ -39,6 +36,14 @@ def gce(cred, env, private_data_dir):
     f.close()
     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
     env['GCE_CREDENTIALS_FILE_PATH'] = path
+    env['GCP_SERVICE_ACCOUNT_FILE'] = path
+
+    # Handle env variables for new module types.
+    # This includes gcp_compute inventory plugin and
+    # all new gcp_* modules.
+    env['GCP_AUTH_KIND'] = 'serviceaccount'
+    env['GCP_PROJECT'] = project
+    env['GCP_ENV_TYPE'] = 'tower'
     return path
 
 

awx/main/models/events.py

@@ -83,6 +83,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     #       - runner_on*
     #     - playbook_on_task_start (once for each task within a play)
     #       - runner_on_failed
+    #       - runner_on_start
     #       - runner_on_ok
     #       - runner_on_error (not used for v2)
     #       - runner_on_skipped
@@ -102,6 +103,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     EVENT_TYPES = [
         # (level, event, verbose name, failed)
         (3, 'runner_on_failed', _('Host Failed'), True),
+        (3, 'runner_on_start', _('Host Started'), False),
         (3, 'runner_on_ok', _('Host OK'), False),
         (3, 'runner_on_error', _('Host Failure'), True),
         (3, 'runner_on_skipped', _('Host Skipped'), False),
@@ -149,7 +151,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
     )
     event_data = JSONField(
         blank=True,
-        default={},
+        default=dict,
     )
     failed = models.BooleanField(
         default=False,
@@ -322,7 +324,10 @@ class BasePlaybookEvent(CreatedModifiedModel):
             kwargs.pop('created', None)
 
         sanitize_event_keys(kwargs, cls.VALID_KEYS)
+        workflow_job_id = kwargs.pop('workflow_job_id', None)
         job_event = cls.objects.create(**kwargs)
+        if workflow_job_id:
+            setattr(job_event, 'workflow_job_id', workflow_job_id)
         analytics_logger.info('Event data saved.', extra=dict(python_objects=dict(job_event=job_event)))
         return job_event
 
@@ -394,7 +399,7 @@ class JobEvent(BasePlaybookEvent):
     An event/message logged from the callback when running a job.
     '''
 
-    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id']
+    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'
@@ -528,7 +533,7 @@ class JobEvent(BasePlaybookEvent):
 
 class ProjectUpdateEvent(BasePlaybookEvent):
 
-    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['project_update_id']
+    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['project_update_id', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'
@@ -567,7 +572,7 @@ class BaseCommandEvent(CreatedModifiedModel):
 
     event_data = JSONField(
         blank=True,
-        default={},
+        default=dict,
     )
     uuid = models.CharField(
         max_length=1024,
@@ -614,7 +619,14 @@ class BaseCommandEvent(CreatedModifiedModel):
             kwargs.pop('created', None)
 
         sanitize_event_keys(kwargs, cls.VALID_KEYS)
-        return cls.objects.create(**kwargs)
+        kwargs.pop('workflow_job_id', None)
+        event = cls.objects.create(**kwargs)
+        if isinstance(event, AdHocCommandEvent):
+            analytics_logger.info(
+                'Event data saved.',
+                extra=dict(python_objects=dict(job_event=event))
+            )
+        return event
 
     def get_event_display(self):
         '''
@@ -622,13 +634,16 @@ class BaseCommandEvent(CreatedModifiedModel):
         '''
         return self.event
 
+    def get_event_display2(self):
+        return self.get_event_display()
+
     def get_host_status_counts(self):
         return create_host_status_counts(getattr(self, 'event_data', {}))
 
 
 class AdHocCommandEvent(BaseCommandEvent):
 
-    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['ad_hoc_command_id', 'event']
+    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['ad_hoc_command_id', 'event', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'
@@ -736,7 +751,7 @@ class AdHocCommandEvent(BaseCommandEvent):
 
 class InventoryUpdateEvent(BaseCommandEvent):
 
-    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['inventory_update_id']
+    VALID_KEYS = BaseCommandEvent.VALID_KEYS + ['inventory_update_id', 'workflow_job_id']
 
     class Meta:
         app_label = 'main'

awx/main/models/ha.py

@@ -18,7 +18,7 @@ from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
 from awx.main.managers import InstanceManager, InstanceGroupManager
 from awx.main.fields import JSONField
-from awx.main.models.base import BaseModel, HasEditsMixin
+from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.utils import get_cpu_capacity, get_mem_capacity, get_system_task_capacity
 from awx.main.models.mixins import RelatedJobsMixin
@@ -59,7 +59,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         null=True,
         editable=False,
     )
-    version = models.CharField(max_length=24, blank=True)
+    version = models.CharField(max_length=120, blank=True)
     capacity = models.PositiveIntegerField(
         default=100,
         editable=False,
@@ -176,6 +176,18 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
         null=True,
         on_delete=models.CASCADE
     )
+    credential = models.ForeignKey(
+        'Credential',
+        related_name='%(class)ss',
+        blank=True,
+        null=True,
+        default=None,
+        on_delete=models.SET_NULL,
+    )
+    pod_spec_override = prevent_search(models.TextField(
+        blank=True,
+        default='',
+    ))
     policy_instance_percentage = models.IntegerField(
         default=0,
         help_text=_("Percentage of Instances to automatically assign to this group")
@@ -218,6 +230,10 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
     def is_isolated(self):
         return bool(self.controller)
 
+    @property
+    def is_containerized(self):
+        return bool(self.credential and self.credential.kubernetes)
+
     '''
     RelatedJobsMixin
     '''
@@ -271,7 +287,8 @@ def schedule_policy_task():
 @receiver(post_save, sender=InstanceGroup)
 def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs):
     if created or instance.has_policy_changes():
-        schedule_policy_task()
+        if not instance.is_containerized:
+            schedule_policy_task()
 
 
 @receiver(post_save, sender=Instance)
@@ -282,7 +299,8 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
 
 @receiver(post_delete, sender=InstanceGroup)
 def on_instance_group_deleted(sender, instance, using, **kwargs):
-    schedule_policy_task()
+    if not instance.is_containerized:
+        schedule_policy_task()
 
 
 @receiver(post_delete, sender=Instance)

awx/main/models/inventory.py

@@ -45,7 +45,7 @@ from awx.main.models.base import (
     CommonModelNameNotUnique,
     VarsDictProperty,
     CLOUD_INVENTORY_SOURCES,
-    prevent_search
+    prevent_search, accepts_json
 )
 from awx.main.models.events import InventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
@@ -93,11 +93,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         on_delete=models.SET_NULL,
         null=True,
     )
-    variables = models.TextField(
+    variables = accepts_json(models.TextField(
         blank=True,
         default='',
         help_text=_('Inventory variables in JSON or YAML format.'),
-    )
+    ))
     has_active_failures = models.BooleanField(
         default=False,
         editable=False,
@@ -309,7 +309,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
 
             # Now use in-memory maps to build up group info.
             all_group_names = []
-            for group in self.groups.only('name', 'id', 'variables'):
+            for group in self.groups.only('name', 'id', 'variables', 'inventory_id'):
                 group_info = dict()
                 if group.id in group_hosts_map:
                     group_info['hosts'] = group_hosts_map[group.id]
@@ -608,11 +608,11 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
         default='',
         help_text=_('The value used by the remote inventory source to uniquely identify the host'),
     )
-    variables = models.TextField(
+    variables = accepts_json(models.TextField(
         blank=True,
         default='',
         help_text=_('Host variables in JSON or YAML format.'),
-    )
+    ))
     last_job = models.ForeignKey(
         'Job',
         related_name='hosts_as_last_job+',
@@ -650,7 +650,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
     )
     ansible_facts = JSONBField(
         blank=True,
-        default={},
+        default=dict,
         help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'),
     )
     ansible_facts_modified = models.DateTimeField(
@@ -796,11 +796,11 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
         related_name='children',
         blank=True,
     )
-    variables = models.TextField(
+    variables = accepts_json(models.TextField(
         blank=True,
         default='',
         help_text=_('Group variables in JSON or YAML format.'),
-    )
+    ))
     hosts = models.ManyToManyField(
         'Host',
         related_name='groups',
@@ -1501,7 +1501,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in InventorySourceOptions._meta.fields) | set(
-            ['name', 'description', 'schedule', 'credentials', 'inventory']
+            ['name', 'description', 'credentials', 'inventory']
         )
 
     def save(self, *args, **kwargs):
@@ -1991,6 +1991,8 @@ class azure_rm(PluginFileInjector):
 
         source_vars = inventory_update.source_vars_dict
 
+        ret['fail_on_template_errors'] = False
+
         group_by_hostvar = {
             'location': {'prefix': '', 'separator': '', 'key': 'location'},
             'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'},
@@ -2046,8 +2048,10 @@ class azure_rm(PluginFileInjector):
             'provisioning_state': 'provisioning_state | title',
             'computer_name': 'name',
             'type': 'resource_type',
-            'private_ip': 'private_ipv4_addresses[0]',
-            'public_ip': 'public_ipv4_addresses[0]',
+            'private_ip': 'private_ipv4_addresses[0] if private_ipv4_addresses else None',
+            'public_ip': 'public_ipv4_addresses[0] if public_ipv4_addresses else None',
+            'public_ip_name': 'public_ip_name if public_ip_name is defined else None',
+            'public_ip_id': 'public_ip_id if public_ip_id is defined else None',
             'tags': 'tags if tags else None'
         }
         # Special functionality from script
@@ -2330,6 +2334,12 @@ class gce(PluginFileInjector):
     ini_env_reference = 'GCE_INI_PATH'
     base_injector = 'managed'
 
+    def get_plugin_env(self, *args, **kwargs):
+        ret = super(gce, self).get_plugin_env(*args, **kwargs)
+        # We need native jinja2 types so that ip addresses can give JSON null value
+        ret['ANSIBLE_JINJA2_NATIVE'] = str(True)
+        return ret
+
     def get_script_env(self, inventory_update, private_data_dir, private_data_files):
         env = super(gce, self).get_script_env(inventory_update, private_data_dir, private_data_files)
         cred = inventory_update.get_cloud_credential()
@@ -2350,7 +2360,7 @@ class gce(PluginFileInjector):
             'gce_name': 'name',
             'gce_network': 'networkInterfaces[0].network.name',
             'gce_private_ip': 'networkInterfaces[0].networkIP',
-            'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP',
+            'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP | default(None)',
             'gce_status': 'status',
             'gce_subnetwork': 'networkInterfaces[0].subnetwork.name',
             'gce_tags': 'tags.get("items", [])',
@@ -2360,7 +2370,7 @@ class gce(PluginFileInjector):
             'gce_image': 'image',
             # We need this as long as hostnames is non-default, otherwise hosts
             # will not be addressed correctly, was returned in script
-            'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP'
+            'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)'
         }
 
     def inventory_as_dict(self, inventory_update, private_data_dir):

awx/main/models/jobs.py

@@ -27,7 +27,7 @@ from rest_framework.exceptions import ParseError
 from awx.api.versioning import reverse
 from awx.main.models.base import (
     BaseModel, CreatedModifiedModel,
-    prevent_search,
+    prevent_search, accepts_json,
     JOB_TYPE_CHOICES, VERBOSITY_CHOICES,
     VarsDictProperty
 )
@@ -39,7 +39,7 @@ from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin,
 )
-from awx.main.utils import parse_yaml_or_json, getattr_dne
+from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
 from awx.main.fields import ImplicitRoleField, JSONField, AskForField
 from awx.main.models.mixins import (
     ResourceMixin,
@@ -48,6 +48,8 @@ from awx.main.models.mixins import (
     TaskManagerJobMixin,
     CustomVirtualEnvMixin,
     RelatedJobsMixin,
+    WebhookMixin,
+    WebhookTemplateMixin,
 )
 
 
@@ -96,6 +98,13 @@ class JobOptions(BaseModel):
         default='',
         blank=True,
     )
+    scm_branch = models.CharField(
+        max_length=1024,
+        default='',
+        blank=True,
+        help_text=_('Branch to use in job run. Project default used if blank. '
+                    'Only allowed if project allow_override field is set to true.'),
+    )
     forks = models.PositiveIntegerField(
         blank=True,
         default=0,
@@ -109,10 +118,10 @@ class JobOptions(BaseModel):
         blank=True,
         default=0,
     )
-    extra_vars = prevent_search(models.TextField(
+    extra_vars = prevent_search(accepts_json(models.TextField(
         blank=True,
         default='',
-    ))
+    )))
     job_tags = models.CharField(
         max_length=1024,
         blank=True,
@@ -180,7 +189,7 @@ class JobOptions(BaseModel):
         return needed
 
 
-class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin):
+class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin, WebhookTemplateMixin):
     '''
     A job template is a reusable job definition for applying a project (with
     playbook) to an inventory source with a given credential.
@@ -234,6 +243,11 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         default=False,
         allows_field='credentials'
     )
+    ask_scm_branch_on_launch = AskForField(
+        blank=True,
+        default=False,
+        allows_field='scm_branch'
+    )
     job_slice_count = models.PositiveIntegerField(
         blank=True,
         default=1,
@@ -259,7 +273,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in JobOptions._meta.fields) | set(
-            ['name', 'description', 'schedule', 'survey_passwords', 'labels', 'credentials',
+            ['name', 'description', 'survey_passwords', 'labels', 'credentials',
              'job_slice_number', 'job_slice_count']
         )
 
@@ -387,7 +401,21 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                 # no-op case: Fields the same as template's value
                 # counted as neither accepted or ignored
                 continue
+            elif field_name == 'scm_branch' and old_value == '' and self.project and new_value == self.project.scm_branch:
+                # special case of "not provided" for branches
+                # job template does not provide branch, runs with default branch
+                continue
             elif getattr(self, ask_field_name):
+                # Special case where prompts can be rejected based on project setting
+                if field_name == 'scm_branch':
+                    if not self.project:
+                        rejected_data[field_name] = new_value
+                        errors_dict[field_name] = _('Project is missing.')
+                        continue
+                    if kwargs['scm_branch'] != self.project.scm_branch and not self.project.allow_override:
+                        rejected_data[field_name] = new_value
+                        errors_dict[field_name] = _('Project does not allow override of branch.')
+                        continue
                 # accepted prompt
                 prompted_data[field_name] = new_value
             else:
@@ -396,7 +424,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                 # Not considered an error for manual launch, to support old
                 # behavior of putting them in ignored_fields and launching anyway
                 if 'prompts' not in exclude_errors:
-                    errors_dict[field_name] = _('Field is not configured to prompt on launch.').format(field_name=field_name)
+                    errors_dict[field_name] = _('Field is not configured to prompt on launch.')
 
         if ('prompts' not in exclude_errors and
                 (not getattr(self, 'ask_credential_on_launch', False)) and
@@ -458,7 +486,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         return UnifiedJob.objects.filter(unified_job_template=self)
 
 
-class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin):
+class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin, WebhookMixin):
     '''
     A job applies a project (with playbook) to an inventory source with a given
     credential.  It represents a single invocation of ansible-playbook with the
@@ -485,7 +513,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     )
     artifacts = JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     )
     scm_revision = models.CharField(
@@ -601,15 +629,17 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
 
     @property
     def task_impact(self):
-        # NOTE: We sorta have to assume the host count matches and that forks default to 5
-        from awx.main.models.inventory import Host
         if self.launch_type == 'callback':
             count_hosts = 2
         else:
-            count_hosts = Host.objects.filter(inventory__jobs__pk=self.pk).count()
-            if self.job_slice_count > 1:
-                # Integer division intentional
-                count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count
+            # If for some reason we can't count the hosts then lets assume the impact as forks
+            if self.inventory is not None:
+                count_hosts = self.inventory.hosts.count()
+                if self.job_slice_count > 1:
+                    # Integer division intentional
+                    count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count
+            else:
+                count_hosts = 5 if self.forks == 0 else self.forks
         return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1
 
     @property
@@ -640,11 +670,19 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     def processed_hosts(self):
         return self._get_hosts(job_host_summaries__processed__gt=0)
 
+    @property
+    def ignored_hosts(self):
+        return self._get_hosts(job_host_summaries__ignored__gt=0)
+
+    @property
+    def rescued_hosts(self):
+        return self._get_hosts(job_host_summaries__rescued__gt=0)
+
     def notification_data(self, block=5):
         data = super(Job, self).notification_data()
         all_hosts = {}
         # NOTE: Probably related to job event slowness, remove at some point -matburt
-        if block:
+        if block and self.status != 'running':
             summaries = self.job_host_summaries.all()
             while block > 0 and not len(summaries):
                 time.sleep(1)
@@ -658,7 +696,9 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
                                           failures=h.failures,
                                           ok=h.ok,
                                           processed=h.processed,
-                                          skipped=h.skipped)
+                                          skipped=h.skipped,
+                                          rescued=h.rescued,
+                                          ignored=h.ignored)
         data.update(dict(inventory=self.inventory.name if self.inventory else None,
                          project=self.project.name if self.project else None,
                          playbook=self.playbook,
@@ -680,6 +720,14 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
             return "$hidden due to Ansible no_log flag$"
         return artifacts
 
+    @property
+    def can_run_containerized(self):
+        return any([ig for ig in self.preferred_instance_groups if ig.is_containerized])
+
+    @property
+    def is_containerized(self):
+        return bool(self.instance_group and self.instance_group.is_containerized)
+
     @property
     def preferred_instance_groups(self):
         if self.project is not None and self.project.organization is not None:
@@ -803,25 +851,6 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
                 host.save()
 
 
-# Add on aliases for the non-related-model fields
-class NullablePromptPsuedoField(object):
-    """
-    Interface for psuedo-property stored in `char_prompts` dict
-    Used in LaunchTimeConfig and submodels
-    """
-    def __init__(self, field_name):
-        self.field_name = field_name
-
-    def __get__(self, instance, type=None):
-        return instance.char_prompts.get(self.field_name, None)
-
-    def __set__(self, instance, value):
-        if value in (None, {}):
-            instance.char_prompts.pop(self.field_name, None)
-        else:
-            instance.char_prompts[self.field_name] = value
-
-
 class LaunchTimeConfigBase(BaseModel):
     '''
     Needed as separate class from LaunchTimeConfig because some models
@@ -842,12 +871,13 @@ class LaunchTimeConfigBase(BaseModel):
         null=True,
         default=None,
         on_delete=models.SET_NULL,
+        help_text=_('Inventory applied as a prompt, assuming job template prompts for inventory')
     )
     # All standard fields are stored in this dictionary field
     # This is a solution to the nullable CharField problem, specific to prompting
     char_prompts = JSONField(
         blank=True,
-        default={}
+        default=dict
     )
 
     def prompts_dict(self, display=False):
@@ -878,42 +908,14 @@ class LaunchTimeConfigBase(BaseModel):
                     data[prompt_name] = prompt_val
         return data
 
-    def display_extra_vars(self):
-        '''
-        Hides fields marked as passwords in survey.
-        '''
-        if self.survey_passwords:
-            extra_vars = parse_yaml_or_json(self.extra_vars).copy()
-            for key, value in self.survey_passwords.items():
-                if key in extra_vars:
-                    extra_vars[key] = value
-            return extra_vars
-        else:
-            return self.extra_vars
 
-    def display_extra_data(self):
-        return self.display_extra_vars()
-
-    @property
-    def _credential(self):
-        '''
-        Only used for workflow nodes to support backward compatibility.
-        '''
-        try:
-            return [cred for cred in self.credentials.all() if cred.credential_type.kind == 'ssh'][0]
-        except IndexError:
-            return None
-
-    @property
-    def credential(self):
-        '''
-        Returns an integer so it can be used as IntegerField in serializer
-        '''
-        cred = self._credential
-        if cred is not None:
-            return cred.pk
-        else:
-            return None
+for field_name in JobTemplate.get_ask_mapping().keys():
+    if field_name == 'extra_vars':
+        continue
+    try:
+        LaunchTimeConfigBase._meta.get_field(field_name)
+    except FieldDoesNotExist:
+        setattr(LaunchTimeConfigBase, field_name, NullablePromptPseudoField(field_name))
 
 
 class LaunchTimeConfig(LaunchTimeConfigBase):
@@ -927,11 +929,11 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
     # Special case prompting fields, even more special than the other ones
     extra_data = JSONField(
         blank=True,
-        default={}
+        default=dict
     )
     survey_passwords = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
     # Credentials needed for non-unified job / unified JT models
@@ -948,14 +950,21 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
     def extra_vars(self, extra_vars):
         self.extra_data = extra_vars
 
+    def display_extra_vars(self):
+        '''
+        Hides fields marked as passwords in survey.
+        '''
+        if hasattr(self, 'survey_passwords') and self.survey_passwords:
+            extra_vars = parse_yaml_or_json(self.extra_vars).copy()
+            for key, value in self.survey_passwords.items():
+                if key in extra_vars:
+                    extra_vars[key] = value
+            return extra_vars
+        else:
+            return self.extra_vars
 
-for field_name in JobTemplate.get_ask_mapping().keys():
-    if field_name == 'extra_vars':
-        continue
-    try:
-        LaunchTimeConfig._meta.get_field(field_name)
-    except FieldDoesNotExist:
-        setattr(LaunchTimeConfig, field_name, NullablePromptPsuedoField(field_name))
+    def display_extra_data(self):
+        return self.display_extra_vars()
 
 
 class JobLaunchConfig(LaunchTimeConfig):

awx/main/models/mixins.py

@@ -1,31 +1,37 @@
 # Python
-import os
-import json
 from copy import copy, deepcopy
+import json
+import logging
+import os
 
+import requests
 
 # Django
 from django.apps import apps
 from django.conf import settings
-from django.db import models
-from django.contrib.contenttypes.models import ContentType
 from django.contrib.auth.models import User # noqa
-from django.utils.translation import ugettext_lazy as _
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
+from django.db import models
 from django.db.models.query import QuerySet
+from django.utils.crypto import get_random_string
+from django.utils.translation import ugettext_lazy as _
 
 # AWX
 from awx.main.models.base import prevent_search
 from awx.main.models.rbac import (
     Role, RoleAncestorEntry, get_roles_on_resource
 )
-from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices
+from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
 from awx.main.fields import JSONField, AskForField
 from awx.main.constants import ACTIVE_STATES
 
 
+logger = logging.getLogger('awx.main.models.mixins')
+
+
 __all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin',
            'TaskManagerUnifiedJobMixin', 'TaskManagerJobMixin', 'TaskManagerProjectUpdateMixin',
            'TaskManagerInventoryUpdateMixin', 'CustomVirtualEnvMixin']
@@ -100,7 +106,7 @@ class SurveyJobTemplateMixin(models.Model):
     )
     survey_spec = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
     ))
     ask_variables_on_launch = AskForField(
         blank=True,
@@ -247,7 +253,7 @@ class SurveyJobTemplateMixin(models.Model):
                 else:
                     choice_list = copy(survey_element['choices'])
                     if isinstance(choice_list, str):
-                        choice_list = choice_list.split('\n')
+                        choice_list = [choice for choice in choice_list.splitlines() if choice.strip() != '']
                     for val in data[survey_element['variable']]:
                         if val not in choice_list:
                             errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'],
@@ -255,7 +261,7 @@ class SurveyJobTemplateMixin(models.Model):
         elif survey_element['type'] == 'multiplechoice':
             choice_list = copy(survey_element['choices'])
             if isinstance(choice_list, str):
-                choice_list = choice_list.split('\n')
+                choice_list = [choice for choice in choice_list.splitlines() if choice.strip() != '']
             if survey_element['variable'] in data:
                 if data[survey_element['variable']] not in choice_list:
                     errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']],
@@ -360,7 +366,7 @@ class SurveyJobMixin(models.Model):
 
     survey_passwords = prevent_search(JSONField(
         blank=True,
-        default={},
+        default=dict,
         editable=False,
     ))
 
@@ -484,3 +490,138 @@ class RelatedJobsMixin(object):
 
         return [dict(id=t[0], type=mapping[t[1]]) for t in jobs.values_list('id', 'polymorphic_ctype_id')]
 
+
+class WebhookTemplateMixin(models.Model):
+    class Meta:
+        abstract = True
+
+    SERVICES = [
+        ('github', "GitHub"),
+        ('gitlab', "GitLab"),
+    ]
+
+    webhook_service = models.CharField(
+        max_length=16,
+        choices=SERVICES,
+        blank=True,
+        help_text=_('Service that webhook requests will be accepted from')
+    )
+    webhook_key = prevent_search(models.CharField(
+        max_length=64,
+        blank=True,
+        help_text=_('Shared secret that the webhook service will use to sign requests')
+    ))
+    webhook_credential = models.ForeignKey(
+        'Credential',
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name='%(class)ss',
+        help_text=_('Personal Access Token for posting back the status to the service API')
+    )
+
+    def rotate_webhook_key(self):
+        self.webhook_key = get_random_string(length=50)
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get('update_fields')
+
+        if not self.pk or self._values_have_edits({'webhook_service': self.webhook_service}):
+            if self.webhook_service:
+                self.rotate_webhook_key()
+            else:
+                self.webhook_key = ''
+
+            if update_fields and 'webhook_service' in update_fields:
+                update_fields.add('webhook_key')
+
+        super().save(*args, **kwargs)
+
+
+class WebhookMixin(models.Model):
+    class Meta:
+        abstract = True
+
+    SERVICES = WebhookTemplateMixin.SERVICES
+
+    webhook_service = models.CharField(
+        max_length=16,
+        choices=SERVICES,
+        blank=True,
+        help_text=_('Service that webhook requests will be accepted from')
+    )
+    webhook_credential = models.ForeignKey(
+        'Credential',
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name='%(class)ss',
+        help_text=_('Personal Access Token for posting back the status to the service API')
+    )
+    webhook_guid = models.CharField(
+        blank=True,
+        max_length=128,
+        help_text=_('Unique identifier of the event that triggered this webhook')
+    )
+
+    def update_webhook_status(self, status):
+        if not self.webhook_credential:
+            logger.debug("No credential configured to post back webhook status, skipping.")
+            return
+
+        status_api = self.extra_vars_dict.get('tower_webhook_status_api')
+        if not status_api:
+            logger.debug("Webhook event did not have a status API endpoint associated, skipping.")
+            return
+
+        service_header = {
+            'github': ('Authorization', 'token {}'),
+            'gitlab': ('PRIVATE-TOKEN', '{}'),
+        }
+        service_statuses = {
+            'github': {
+                'pending': 'pending',
+                'successful': 'success',
+                'failed': 'failure',
+                'canceled': 'failure',  # GitHub doesn't have a 'canceled' status :(
+                'error': 'error',
+            },
+            'gitlab': {
+                'pending': 'pending',
+                'running': 'running',
+                'successful': 'success',
+                'failed': 'failed',
+                'error': 'failed',  # GitLab doesn't have an 'error' status distinct from 'failed' :(
+                'canceled': 'canceled',
+            },
+        }
+
+        statuses = service_statuses[self.webhook_service]
+        if status not in statuses:
+            logger.debug("Skipping webhook job status change: '{}'".format(status))
+            return
+        try:
+            license_type = get_licenser().validate().get('license_type')
+            data = {
+                'state': statuses[status],
+                'context': 'ansible/awx' if license_type == 'open' else 'ansible/tower',
+                'target_url': self.get_ui_url(),
+            }
+            k, v = service_header[self.webhook_service]
+            headers = {
+                k: v.format(self.webhook_credential.get_input('token')),
+                'Content-Type': 'application/json'
+            }
+            response = requests.post(status_api, data=json.dumps(data), headers=headers, timeout=30)
+        except Exception:
+            logger.exception("Posting webhook status caused an error.")
+            return
+
+        if response.status_code < 400:
+            logger.debug("Webhook status update sent.")
+        else:
+            logger.error(
+                "Posting webhook status failed, code: {}\n"
+                "{}\n"
+                "Payload sent: {}".format(response.status_code, response.text, json.dumps(data))
+            )

awx/main/models/notifications.py

@@ -2,7 +2,9 @@
 # All Rights Reserved.
 
 from copy import deepcopy
+import datetime
 import logging
+import json
 
 from django.db import models
 from django.conf import settings
@@ -10,10 +12,12 @@ from django.core.mail.message import EmailMessage
 from django.db import connection
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_str, force_text
+from jinja2 import sandbox
+from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel
+from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
 from awx.main.utils import encrypt_field, decrypt_field, set_environ
 from awx.main.notifications.email_backend import CustomEmailBackend
 from awx.main.notifications.slack_backend import SlackBackend
@@ -45,7 +49,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
                           ('mattermost', _('Mattermost'), MattermostBackend),
                           ('rocketchat', _('Rocket.Chat'), RocketChatBackend),
                           ('irc', _('IRC'), IrcBackend)]
-    NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES]
+    NOTIFICATION_TYPE_CHOICES = sorted([(x[0], x[1]) for x in NOTIFICATION_TYPES])
     CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES])
 
     class Meta:
@@ -66,7 +70,46 @@ class NotificationTemplate(CommonModelNameNotUnique):
         choices=NOTIFICATION_TYPE_CHOICES,
     )
 
-    notification_configuration = JSONField(blank=False)
+    notification_configuration = prevent_search(JSONField(blank=False))
+
+    def default_messages():
+        return {'started': None, 'success': None, 'error': None}
+
+    messages = JSONField(
+        null=True,
+        blank=True,
+        default=default_messages,
+        help_text=_('Optional custom messages for notification template.'))
+
+    def has_message(self, condition):
+        potential_template = self.messages.get(condition, {})
+        if potential_template == {}:
+            return False
+        if potential_template.get('message', {}) == {}:
+            return False
+        return True
+
+    def get_message(self, condition):
+        return self.messages.get(condition, {})
+
+    def build_notification_message(self, event_type, context):
+        env = sandbox.ImmutableSandboxedEnvironment()
+        templates = self.get_message(event_type)
+        msg_template = templates.get('message', {})
+
+        try:
+            notification_subject = env.from_string(msg_template).render(**context)
+        except (TemplateSyntaxError, UndefinedError, SecurityError):
+            notification_subject = ''
+
+
+        msg_body = templates.get('body', {})
+        try:
+            notification_body = env.from_string(msg_body).render(**context)
+        except (TemplateSyntaxError, UndefinedError, SecurityError):
+            notification_body = ''
+
+        return (notification_subject, notification_body)
 
     def get_absolute_url(self, request=None):
         return reverse('api:notification_template_detail', kwargs={'pk': self.pk}, request=request)
@@ -78,6 +121,26 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def save(self, *args, **kwargs):
         new_instance = not bool(self.pk)
         update_fields = kwargs.get('update_fields', [])
+
+        # preserve existing notification messages if not overwritten by new messages
+        if not new_instance:
+            old_nt = NotificationTemplate.objects.get(pk=self.id)
+            old_messages = old_nt.messages
+            new_messages = self.messages
+
+            if old_messages is not None and new_messages is not None:
+                for event in ['started', 'success', 'error']:
+                    if not new_messages.get(event, {}) and old_messages.get(event, {}):
+                        new_messages[event] = old_messages[event]
+                        continue
+                    if new_messages.get(event, {}) and old_messages.get(event, {}):
+                        old_event_msgs = old_messages[event]
+                        new_event_msgs = new_messages[event]
+                        for msg_type in ['message', 'body']:
+                            if msg_type not in new_event_msgs and old_event_msgs.get(msg_type, None):
+                                new_event_msgs[msg_type] = old_event_msgs[msg_type]
+                    new_messages.setdefault(event, None)
+
         for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
                             self.notification_class.init_parameters):
             if self.notification_configuration[field].startswith("$encrypted$"):
@@ -118,9 +181,10 @@ class NotificationTemplate(CommonModelNameNotUnique):
     def send(self, subject, body):
         for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password",
                             self.notification_class.init_parameters):
-            self.notification_configuration[field] = decrypt_field(self,
-                                                                   'notification_configuration',
-                                                                   subfield=field)
+            if field in self.notification_configuration:
+                self.notification_configuration[field] = decrypt_field(self,
+                                                                       'notification_configuration',
+                                                                       subfield=field)
         recipients = self.notification_configuration.pop(self.notification_class.recipient_parameter)
         if not isinstance(recipients, list):
             recipients = [recipients]
@@ -200,56 +264,227 @@ class Notification(CreatedModifiedModel):
 
 
 class JobNotificationMixin(object):
+    STATUS_TO_TEMPLATE_TYPE = {'succeeded': 'success',
+                               'running': 'started',
+                               'failed': 'error'}
+    # Tree of fields that can be safely referenced in a notification message
+    JOB_FIELDS_WHITELIST = ['id', 'type', 'url', 'created', 'modified', 'name', 'description', 'job_type', 'playbook',
+                            'forks', 'limit', 'verbosity', 'job_tags', 'force_handlers', 'skip_tags', 'start_at_task',
+                            'timeout', 'use_fact_cache', 'launch_type', 'status', 'failed', 'started', 'finished',
+                            'elapsed', 'job_explanation', 'execution_node', 'controller_node', 'allow_simultaneous',
+                            'scm_revision', 'diff_mode', 'job_slice_number', 'job_slice_count', 'custom_virtualenv',
+                            {'host_status_counts': ['skipped', 'ok', 'changed', 'failures', 'dark']},
+                            {'playbook_counts': ['play_count', 'task_count']},
+                            {'summary_fields': [{'inventory': ['id', 'name', 'description', 'has_active_failures',
+                                                               'total_hosts', 'hosts_with_active_failures', 'total_groups',
+                                                               'groups_with_active_failures', 'has_inventory_sources',
+                                                               'total_inventory_sources', 'inventory_sources_with_failures',
+                                                               'organization_id', 'kind']},
+                                                {'project': ['id', 'name', 'description', 'status', 'scm_type']},
+                                                {'project_update': ['id', 'name', 'description', 'status', 'failed']},
+                                                {'job_template': ['id', 'name', 'description']},
+                                                {'unified_job_template': ['id', 'name', 'description', 'unified_job_type']},
+                                                {'instance_group': ['name', 'id']},
+                                                {'created_by': ['id', 'username', 'first_name', 'last_name']},
+                                                {'labels': ['count', 'results']},
+                                                {'source_workflow_job': ['description', 'elapsed', 'failed', 'id', 'name', 'status']}]}]
+
+    @classmethod
+    def context_stub(cls):
+        """Returns a stub context that can be used for validating notification messages.
+        Context has the same structure as the context that will actually be used to render
+        a notification message."""
+        context = {'job': {'allow_simultaneous': False,
+                           'controller_node': 'foo_controller',
+                           'created': datetime.datetime(2018, 11, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
+                           'custom_virtualenv': 'my_venv',
+                           'description': 'Sample job description',
+                           'diff_mode': False,
+                           'elapsed': 0.403018,
+                           'execution_node': 'awx',
+                           'failed': False,
+                           'finished': False,
+                           'force_handlers': False,
+                           'forks': 0,
+                           'host_status_counts': {'skipped': 1, 'ok': 5, 'changed': 3, 'failures': 0, 'dark': 0},
+                           'id': 42,
+                           'job_explanation': 'Sample job explanation',
+                           'job_slice_count': 1,
+                           'job_slice_number': 0,
+                           'job_tags': '',
+                           'job_type': 'run',
+                           'launch_type': 'workflow',
+                           'limit': 'bar_limit',
+                           'modified': datetime.datetime(2018, 12, 13, 6, 4, 0, 0, tzinfo=datetime.timezone.utc),
+                           'name': 'Stub JobTemplate',
+                           'playbook_counts': {'play_count': 5, 'task_count': 10},
+                           'playbook': 'ping.yml',
+                           'scm_revision': '',
+                           'skip_tags': '',
+                           'start_at_task': '',
+                           'started': '2019-07-29T17:38:14.137461Z',
+                           'status': 'running',
+                           'summary_fields': {'created_by': {'first_name': '',
+                                                             'id': 1,
+                                                             'last_name': '',
+                                                             'username': 'admin'},
+                                              'instance_group': {'id': 1, 'name': 'tower'},
+                                              'inventory': {'description': 'Sample inventory description',
+                                                            'groups_with_active_failures': 0,
+                                                            'has_active_failures': False,
+                                                            'has_inventory_sources': False,
+                                                            'hosts_with_active_failures': 0,
+                                                            'id': 17,
+                                                            'inventory_sources_with_failures': 0,
+                                                            'kind': '',
+                                                            'name': 'Stub Inventory',
+                                                            'organization_id': 121,
+                                                            'total_groups': 0,
+                                                            'total_hosts': 1,
+                                                            'total_inventory_sources': 0},
+                                              'job_template': {'description': 'Sample job template description',
+                                                               'id': 39,
+                                                               'name': 'Stub JobTemplate'},
+                                              'labels': {'count': 0, 'results': []},
+                                              'project': {'description': 'Sample project description',
+                                                          'id': 38,
+                                                          'name': 'Stub project',
+                                                          'scm_type': 'git',
+                                                          'status': 'successful'},
+                                              'project_update': {'id': 5, 'name': 'Stub Project Update', 'description': 'Project Update',
+                                                                 'status': 'running', 'failed': False},
+                                              'unified_job_template': {'description': 'Sample unified job template description',
+                                                                       'id': 39,
+                                                                       'name': 'Stub Job Template',
+                                                                       'unified_job_type': 'job'},
+                                              'source_workflow_job': {'description': 'Sample workflow job description',
+                                                                      'elapsed': 0.000,
+                                                                      'failed': False,
+                                                                      'id': 88,
+                                                                      'name': 'Stub WorkflowJobTemplate',
+                                                                      'status': 'running'}},
+                           'timeout': 0,
+                           'type': 'job',
+                           'url': '/api/v2/jobs/13/',
+                           'use_fact_cache': False,
+                           'verbosity': 0},
+                   'job_friendly_name': 'Job',
+                   'url': 'https://towerhost/#/jobs/playbook/1010',
+                   'job_summary_dict': """{'url': 'https://towerhost/$/jobs/playbook/13',
+ 'traceback': '',
+ 'status': 'running',
+ 'started': '2019-08-07T21:46:38.362630+00:00',
+ 'project': 'Stub project',
+ 'playbook': 'ping.yml',
+ 'name': 'Stub Job Template',
+ 'limit': '',
+ 'inventory': 'Stub Inventory',
+ 'id': 42,
+ 'hosts': {},
+ 'friendly_name': 'Job',
+ 'finished': False,
+ 'credential': 'Stub credential',
+ 'created_by': 'admin'}"""}
+
+        return context
+
+    def context(self, serialized_job):
+        """Returns a context that can be used for rendering notification messages.
+        Context contains whitelisted content retrieved from a serialized job object
+        (see JobNotificationMixin.JOB_FIELDS_WHITELIST), the job's friendly name,
+        and a url to the job run."""
+        context = {'job': {},
+                   'job_friendly_name': self.get_notification_friendly_name(),
+                   'url': self.get_ui_url(),
+                   'job_summary_dict': json.dumps(self.notification_data(), indent=4)}
+
+        def build_context(node, fields, whitelisted_fields):
+            for safe_field in whitelisted_fields:
+                if type(safe_field) is dict:
+                    field, whitelist_subnode = safe_field.copy().popitem()
+                    # ensure content present in job serialization
+                    if field not in fields:
+                        continue
+                    subnode = fields[field]
+                    node[field] = {}
+                    build_context(node[field], subnode, whitelist_subnode)
+                else:
+                    # ensure content present in job serialization
+                    if safe_field not in fields:
+                        continue
+                    node[safe_field] = fields[safe_field]
+        build_context(context['job'], serialized_job, self.JOB_FIELDS_WHITELIST)
+
+        return context
+
     def get_notification_templates(self):
         raise RuntimeError("Define me")
 
     def get_notification_friendly_name(self):
         raise RuntimeError("Define me")
 
-    def _build_notification_message(self, status_str):
+    def notification_data(self):
+        raise RuntimeError("Define me")
+
+    def build_notification_message(self, nt, status):
+        env = sandbox.ImmutableSandboxedEnvironment()
+
+        from awx.api.serializers import UnifiedJobSerializer
+        job_serialization = UnifiedJobSerializer(self).to_representation(self)
+        context = self.context(job_serialization)
+
+        msg_template = body_template = None
+
+        if nt.messages:
+            templates = nt.messages.get(self.STATUS_TO_TEMPLATE_TYPE[status], {}) or {}
+            msg_template = templates.get('message', {})
+            body_template = templates.get('body', {})
+
+        if msg_template:
+            try:
+                notification_subject = env.from_string(msg_template).render(**context)
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                notification_subject = ''
+        else:
+            notification_subject = u"{} #{} '{}' {}: {}".format(self.get_notification_friendly_name(),
+                                                                self.id,
+                                                                self.name,
+                                                                status,
+                                                                self.get_ui_url())
         notification_body = self.notification_data()
-        notification_subject = u"{} #{} '{}' {}: {}".format(self.get_notification_friendly_name(),
-                                                            self.id,
-                                                            self.name,
-                                                            status_str,
-                                                            notification_body['url'])
         notification_body['friendly_name'] = self.get_notification_friendly_name()
+        if body_template:
+            try:
+                notification_body['body'] = env.from_string(body_template).render(**context)
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                notification_body['body'] = ''
+
         return (notification_subject, notification_body)
 
-    def build_notification_succeeded_message(self):
-        return self._build_notification_message('succeeded')
-
-    def build_notification_failed_message(self):
-        return self._build_notification_message('failed')
-
-    def build_notification_running_message(self):
-        return self._build_notification_message('running')
-
-    def send_notification_templates(self, status_str):
+    def send_notification_templates(self, status):
         from awx.main.tasks import send_notifications  # avoid circular import
-        if status_str not in ['succeeded', 'failed', 'running']:
-            raise ValueError(_("status_str must be either running, succeeded or failed"))
+        if status not in ['running', 'succeeded', 'failed']:
+            raise ValueError(_("status must be either running, succeeded or failed"))
         try:
             notification_templates = self.get_notification_templates()
         except Exception:
             logger.warn("No notification template defined for emitting notification")
-            notification_templates = None
-        if notification_templates:
-            if status_str == 'succeeded':
-                notification_template_type = 'success'
-            elif status_str == 'running':
-                notification_template_type = 'started'
-            else:
-                notification_template_type = 'error'
-            all_notification_templates = set(notification_templates.get(notification_template_type, []))
-            if len(all_notification_templates):
-                try:
-                    (notification_subject, notification_body) = getattr(self, 'build_notification_%s_message' % status_str)()
-                except AttributeError:
-                    raise NotImplementedError("build_notification_%s_message() does not exist" % status_str)
+            return
 
-                def send_it():
-                    send_notifications.delay([n.generate_notification(notification_subject, notification_body).id
-                                              for n in all_notification_templates],
+        if not notification_templates:
+            return
+
+        for nt in set(notification_templates.get(self.STATUS_TO_TEMPLATE_TYPE[status], [])):
+            try:
+                (notification_subject, notification_body) = self.build_notification_message(nt, status)
+            except AttributeError:
+                raise NotImplementedError("build_notification_message() does not exist" % status)
+
+            # Use kwargs to force late-binding
+            # https://stackoverflow.com/a/3431699/10669572
+            def send_it(local_nt=nt, local_subject=notification_subject, local_body=notification_body):
+                def _func():
+                    send_notifications.delay([local_nt.generate_notification(local_subject, local_body).id],
                                              job_id=self.id)
-                connection.on_commit(send_it)
+                return _func
+            connection.on_commit(send_it())

awx/main/models/oauth.py

@@ -3,7 +3,7 @@ import re
 
 # Django
 from django.core.validators import RegexValidator
-from django.db import models
+from django.db import models, connection
 from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
@@ -98,8 +98,7 @@ class OAuth2AccessToken(AbstractAccessToken):
         related_name="%(app_label)s_%(class)s",
         help_text=_('The user representing the token owner')
     )
-    description = models.CharField(
-        max_length=200,
+    description = models.TextField(
         default='',
         blank=True,
     )
@@ -122,7 +121,7 @@ class OAuth2AccessToken(AbstractAccessToken):
         valid = super(OAuth2AccessToken, self).is_valid(scopes)
         if valid:
             self.last_used = now()
-            self.save(update_fields=['last_used'])
+            connection.on_commit(lambda: self.save(update_fields=['last_used']))
         return valid
 
     def save(self, *args, **kwargs):

awx/main/models/organization.py

@@ -51,6 +51,11 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
         default=0,
         help_text=_('Maximum number of hosts allowed to be managed by this organization.'),
     )
+    notification_templates_approvals = models.ManyToManyField(
+        "NotificationTemplate",
+        blank=True,
+        related_name='%(class)s_notification_templates_for_approvals'
+    )
 
     admin_role = ImplicitRoleField(
         parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
@@ -87,7 +92,10 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
                      'execute_role', 'project_admin_role',
                      'inventory_admin_role', 'workflow_admin_role',
                      'notification_admin_role', 'credential_admin_role',
-                     'job_template_admin_role',],
+                     'job_template_admin_role', 'approval_role',],
+    )
+    approval_role = ImplicitRoleField(
+        parent_role='admin_role',
     )
 
 

awx/main/models/projects.py

@@ -106,6 +106,13 @@ class ProjectOptions(models.Model):
         verbose_name=_('SCM Branch'),
         help_text=_('Specific branch, tag or commit to checkout.'),
     )
+    scm_refspec = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        verbose_name=_('SCM refspec'),
+        help_text=_('For git projects, an additional refspec to fetch.'),
+    )
     scm_clean = models.BooleanField(
         default=False,
         help_text=_('Discard any local changes before syncing the project.'),
@@ -241,7 +248,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
     FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials']
     FIELDS_TO_DISCARD_AT_COPY = ['local_path']
-    FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type'])
+    FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type', 'scm_refspec'])
 
     class Meta:
         app_label = 'main'
@@ -261,9 +268,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     scm_update_cache_timeout = models.PositiveIntegerField(
         default=0,
         blank=True,
-        help_text=_('The number of seconds after the last project update ran that a new'
+        help_text=_('The number of seconds after the last project update ran that a new '
                     'project update will be launched as a job dependency.'),
     )
+    allow_override = models.BooleanField(
+        default=False,
+        help_text=_('Allow changing the SCM branch or revision in a job template '
+                    'that uses this project.'),
+    )
 
     scm_revision = models.CharField(
         max_length=1024,
@@ -317,7 +329,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in ProjectOptions._meta.fields) | set(
-            ['name', 'description', 'schedule']
+            ['name', 'description']
         )
 
     def save(self, *args, **kwargs):
@@ -471,6 +483,14 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         choices=PROJECT_UPDATE_JOB_TYPE_CHOICES,
         default='check',
     )
+    scm_revision = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        editable=False,
+        verbose_name=_('SCM Revision'),
+        help_text=_('The SCM Revision discovered by this update for the given project and branch.'),
+    )
 
     def _get_parent_field_name(self):
         return 'project'