Bump VERSION to 9.1.0

Don't reload template panel on tab change

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.dput.cf

@@ -1,6 +0,0 @@
-[mini_dinstall]
-fqdn = localhost
-method = local
-incoming = FIXME/deb-repo/mini-dinstall/incoming
-run_dinstall = 0
-post_upload_command = mini-dinstall -b -v

.editorconfig

@@ -1,20 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = true
-
-[Makefile]
-indent_style = tab
-
-[**.py]
-indent_style = space
-indent_size = 4
-
-[**.{js,less,html}]
-indent_style = space
-indent_size = 4
-
-[**.{json}]
-indent_style = space
-indent_size = 2

.env

@@ -1,2 +1,3 @@
 PYTHONUNBUFFERED=true
+SELENIUM_DOCKER_TAG=latest
 

.github/BOTMETA.yml

@@ -1,3 +1,4 @@
+---
 files:
     awx/ui/:
         labels: component:ui

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: "\U0001F41B Bug report"
+about: Create a report to help us improve
+
+---
+
+##### ISSUE TYPE
+ - Bug Report
+
+##### SUMMARY
+<!-- Briefly describe the problem. -->
+
+##### ENVIRONMENT
+* AWX version: X.Y.Z
+* AWX install method: openshift, minishift, docker on linux, docker for mac, boot2docker
+* Ansible version:  X.Y.Z
+* Operating System:
+* Web Browser:
+
+##### STEPS TO REPRODUCE
+
+<!-- Please describe exactly how to reproduce the problem. -->
+
+##### EXPECTED RESULTS
+
+<!-- What did you expect to happen when running the steps above? -->
+
+##### ACTUAL RESULTS
+
+<!-- What actually happened? -->
+
+##### ADDITIONAL INFORMATION
+
+<!-- Include any links to sosreport, database dumps, screenshots or other
+information. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,11 @@
+---
+name: "✨ Feature request"
+about: Suggest an idea for this project
+
+---
+
+##### ISSUE TYPE
+ - Feature Idea
+
+##### SUMMARY
+<!-- Briefly describe the problem or desired enhancement. -->

.github/ISSUE_TEMPLATE/security_bug_report.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F525 Security bug report"
+about: How to report security vulnerabilities
+
+---
+
+For all security related bugs, email security@ansible.com instead of using this issue tracker and you will receive a prompt response.
+
+For more information on the Ansible community's practices regarding responsible disclosure, see https://www.ansible.com/security

.gitignore

@@ -1,8 +1,14 @@
+# Ignore generated schema
+swagger.json
+schema.json
+reference-schema.json
+
 # Tags
 .tags
 .tags1
 
 # Tower
+awx-dev
 awx/settings/local_*.py*
 awx/*.sqlite3
 awx/*.sqlite3_*
@@ -22,8 +28,12 @@ awx/ui/build_test
 awx/ui/client/languages
 awx/ui/templates/ui/index.html
 awx/ui/templates/ui/installing.html
+awx/ui_next/node_modules/
+awx/ui_next/coverage/
+awx/ui_next/build/locales/_build
 /tower-license
 /tower-license/**
+tools/prometheus/data
 
 # Tower setup playbook testing
 setup/test/roles/postgresql
@@ -52,10 +62,12 @@ __pycache__
 **/node_modules/**
 /tmp
 **/npm-debug.log*
+**/package-lock.json
 
 # UI build flag files
 awx/ui/.deps_built
 awx/ui/.release_built
+awx/ui/.release_deps_built
 
 # Testing
 .cache
@@ -67,6 +79,7 @@ pep8.txt
 scratch
 testem.log
 awx/awx_test.sqlite3-journal
+.pytest_cache/
 
 # Mac OS X
 *.DS_Store
@@ -111,8 +124,8 @@ local/
 *.mo
 requirements/vendor
 .i18n_built
-VERSION
 .idea/*
+*credentials*.y*ml*
 
 # AWX python libs populated by requirements.txt
 awx/lib/.deps_built
@@ -120,4 +133,13 @@ awx/lib/site-packages
 venv/*
 use_dev_supervisor.txt
 
+
+# Ansible module tests
+/awx_collection_test_venv/
+/awx_collection/*.tar.gz
+/awx_collection/galaxy.yml
+/sanity/
+
 .idea/*
+*.unison.tmp
+*.#

.mini-dinstall.cf

@@ -1,16 +0,0 @@
-[DEFAULT]
-archivedir = FIXME/deb-repo
-mail_to =
-verify_sigs = false
-architectures = all, amd64
-archive_style = flat
-generate_release = true
-mail_on_success = false
-release_codename = ansible-tower
-release_description = Ansible Tower
-release_label = ansible-tower
-release_origin = ansible-tower
-
-[trusty]
-
-[precise]

.pylintrc

@@ -1,6 +0,0 @@
-[MASTER]
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=site-packages,ui,migrations,data
-

.yamllint

@@ -0,0 +1,12 @@
+---
+ignore: |
+  .tox
+  awx/main/tests/data/inventory/plugins/**
+  # vault files
+  awx/main/tests/data/ansible_utils/playbooks/valid/vault.yml
+  awx/ui/test/e2e/tests/smoke-vars.yml
+
+extends: default
+
+rules:
+  line-length: disable

CONTRIBUTING.md

@@ -2,29 +2,29 @@
 
 Hi there! We're excited to have you as a contributor.
 
-Have questions about this document or anything not covered here? Come chat with us at `#ansible-awx` on irc.freenode.net, or submit your question to the [mailing list](https://groups.google.com/forum/#!forum/awx-project) .
+Have questions about this document or anything not covered here? Come chat with us at `#ansible-awx` on irc.freenode.net, or submit your question to the [mailing list](https://groups.google.com/forum/#!forum/awx-project).
 
 ## Table of contents
 
-* [Things to know prior to submitting code](#things-to-know-before-contributing-code)
+* [Things to know prior to submitting code](#things-to-know-prior-to-submitting-code)
 * [Setting up your development environment](#setting-up-your-development-environment)
   * [Prerequisites](#prerequisites)
     * [Docker](#docker)
     * [Docker compose](#docker-compose)
     * [Node and npm](#node-and-npm)
-  * [Building the environment](#building-the-environment)
-    * [Clone the AWX repo](#clone-the-awx-repo)
+  * [Build the environment](#build-the-environment)
+    * [Fork and clone the AWX repo](#fork-and-clone-the-awx-repo)
     * [Create local settings](#create-local-settings)
     * [Build the base image](#build-the-base-image)
     * [Build the user interface](#build-the-user-interface)
-  # [Running the environment](#running-the-environment)
+  * [Running the environment](#running-the-environment)
     * [Start the containers](#start-the-containers)
     * [Start from the container shell](#start-from-the-container-shell)
   * [Post Build Steps](#post-build-steps)
-     * [Start a shell](#start-the-shell)
-     * [Create a superuser](#create-a-superuser)
-     * [Load the data](#load-the-data)
-     * [Building API Documentation](#build-documentation)
+    * [Start a shell](#start-a-shell)
+    * [Create a superuser](#create-a-superuser)
+    * [Load the data](#load-the-data)
+    * [Building API Documentation](#build-api-documentation)
   * [Accessing the AWX web interface](#accessing-the-awx-web-interface)
   * [Purging containers and images](#purging-containers-and-images)
 * [What should I work on?](#what-should-i-work-on)
@@ -34,10 +34,11 @@ Have questions about this document or anything not covered here? Come chat with
 ## Things to know prior to submitting code
 
 - All code submissions are done through pull requests against the `devel` branch.
-- You must use `git commit --signoff` for any commit to be merged, and agree that usage of --signoff constitutes agreement with the terms of [DCO 1.1](./DCO_1_1.md).  
+- You must use `git commit --signoff` for any commit to be merged, and agree that usage of --signoff constitutes agreement with the terms of [DCO 1.1](./DCO_1_1.md).
 - Take care to make sure no merge commits are in the submission, and use `git rebase` vs `git merge` for this reason.
+  - If collaborating with someone else on the same branch, consider using `--force-with-lease` instead of `--force`. This will prevent you from accidentally overwriting commits pushed by someone else. For more information, see https://git-scm.com/docs/git-push#git-push---force-with-leaseltrefnamegt
 - If submitting a large code change, it's a good idea to join the `#ansible-awx` channel on irc.freenode.net, and talk about what you would like to do or add first. This not only helps everyone know what's going on, it also helps save time and effort, if the community decides some changes are needed.
-- We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com)   
+- We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com)
 
 ## Setting up your development environment
 
@@ -49,7 +50,7 @@ The AWX development environment workflow and toolchain is based on Docker, and t
 
 Prior to starting the development services, you'll need `docker` and `docker-compose`. On Linux, you can generally find these in your distro's packaging, but you may find that Docker themselves maintain a separate repo that tracks more closely to the latest releases.
 
-For macOS and Windows, we recommend [Docker for Mac](https://www.docker.com/docker-mac) and [Docker for Windows](https://www.docker.com/docker-windows) 
+For macOS and Windows, we recommend [Docker for Mac](https://www.docker.com/docker-mac) and [Docker for Windows](https://www.docker.com/docker-windows)
 respectively.
 
 For Linux platforms, refer to the following from Docker:
@@ -82,12 +83,10 @@ If you're not using Docker for Mac, or Docker for Windows, you may need, or choo
 (host)$ pip install docker-compose
 ```
 
-#### Node and npm
+#### Frontend Development
 
-The AWX UI requires the following:
+See [the ui development documentation](awx/ui/README.md).
 
-- Node 6.x LTS version
-- NPM 3.x LTS
 
 ### Build the environment
 
@@ -135,28 +134,30 @@ Run the following to build the AWX UI:
 ```bash
 (host) $ make ui-devel
 ```
+See [the ui development documentation](awx/ui/README.md) for more information on using the frontend development, build, and test tooling.
+
 ### Running the environment
 
-#### Start the containers 
+#### Start the containers
 
 Start the development containers by running the following:
 
 ```bash
 (host)$ make docker-compose
 ```
-    
-The above utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, your session will be attached to the *awx* container, and you'll be able to watch log messages and events in real time. You will see messages from Django, celery, and the front end build process.
+
+The above utilizes the image built in the previous step, and will automatically start all required services and dependent containers. Once the containers launch, your session will be attached to the *awx* container, and you'll be able to watch log messages and events in real time. You will see messages from Django and the front end build process.
 
 If you start a second terminal session, you can take a look at the running containers using the `docker ps` command. For example:
 
 ```bash
 # List running containers
-(host)$ docker ps 
+(host)$ docker ps
 
 $ docker ps
 CONTAINER ID        IMAGE                                              COMMAND                  CREATED             STATUS              PORTS                                                                                                                                      NAMES
-aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:6899-6999->6899-6999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
-e4c0afeb548c        postgres:9.6                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
+aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:7899-7999->7899-7999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
+e4c0afeb548c        postgres:10                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
 0089699d5afd        tools_logstash                                     "/docker-entrypoin..."   26 seconds ago      Up 25 seconds                                                                                                                                                  tools_logstash_1
 4d4ff0ced266        memcached:alpine                                   "docker-entrypoint..."   26 seconds ago      Up 25 seconds       0.0.0.0:11211->11211/tcp                                                                                                                   tools_memcached_1
 92842acd64cd        rabbitmq:3-management                              "docker-entrypoint..."   26 seconds ago      Up 24 seconds       4369/tcp, 5671-5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp                                                                    tools_rabbitmq_1
@@ -174,7 +175,7 @@ The first time you start the environment, database migrations need to run in ord
 ```bash
 awx_1        | Operations to perform:
 awx_1        |   Synchronize unmigrated apps: solo, api, staticfiles, debug_toolbar, messages, channels, django_extensions, ui, rest_framework, polymorphic
-awx_1        |   Apply all migrations: sso, taggit, sessions, djcelery, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
+awx_1        |   Apply all migrations: sso, taggit, sessions, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
 awx_1        | Synchronizing apps without migrations:
 awx_1        |   Creating tables...
 awx_1        |     Running deferred SQL...
@@ -219,7 +220,7 @@ If you want to start and use the development environment, you'll first need to b
 ```
 
 The above will do all the setup tasks, including running database migrations, so it may take a couple minutes.
-    
+
 Now you can start each service individually, or start all services in a pre-configured tmux session like so:
 
 ```bash
@@ -248,9 +249,9 @@ Before you can log into AWX, you need to create an admin user. With this user yo
 (container)# awx-manage createsuperuser
 ```
 You will be prompted for a username, an email address, and a password, and you will be asked to confirm the password. The email address is not important, so just enter something that looks like an email address. Remember the username and password, as you will use them to log into the web interface for the first time.
-    
+
 ##### Load demo data
-  
+
 You can optionally load some demo data. This will create a demo project, inventory, and job template. From within the container shell, run the following to load the data:
 
 ```bash
@@ -276,7 +277,7 @@ in OpenAPI format.  A variety of online tools are available for translating
 this data into more consumable formats (such as HTML). http://editor.swagger.io
 is an example of one such service.
 
-### Accessing the AWX web interface 
+### Accessing the AWX web interface
 
 You can now log into the AWX web interface at [https://localhost:8043](https://localhost:8043), and access the API directly at [https://localhost:8043/api/](https://localhost:8043/api/).
 
@@ -289,7 +290,7 @@ When necessary, remove any AWX containers and images by running the following:
 ```bash
 (host)$ make docker-clean
 ```
-        
+
 ## What should I work on?
 
 For feature work, take a look at the current [Enhancements](https://github.com/ansible/awx/issues?q=is%3Aissue+is%3Aopen+label%3Atype%3Aenhancement).
@@ -329,7 +330,24 @@ We like to keep our commit history clean, and will require resubmission of pull
 
 Sometimes it might take us a while to fully review your PR. We try to keep the `devel` branch in good working order, and so we review requests carefully. Please be patient.
 
-All submitted PRs will have the linter and unit tests run against them, and the status reported in the PR.
+All submitted PRs will have the linter and unit tests run against them via Zuul, and the status reported in the PR.
+
+## PR Checks ran by Zuul
+Zuul jobs for awx are defined in the [zuul-jobs](https://github.com/ansible/zuul-jobs) repo.
+
+Zuul runs the following checks that must pass:
+1) `tox-awx-api-lint`
+2) `tox-awx-ui-lint`
+3) `tox-awx-api`
+4) `tox-awx-ui`
+5) `tox-awx-swagger`
+
+Zuul runs the following checks that are non-voting (can not pass but serve to inform PR reviewers):
+1) `tox-awx-detect-schema-change`
+    This check generates the schema and diffs it against a reference copy of the `devel` version of the schema.
+    Reviewers should inspect the `job-output.txt.gz` related to the check if their is a failure (grep for `diff -u -b` to find beginning of diff).
+    If the schema change is expected and makes sense in relation to the changes made by the PR, then you are good to go!
+    If not, the schema changes should be fixed, but this decision must be enforced by reviewers.
 
 ## Reporting Issues
 

DATA_MIGRATION.md

@@ -0,0 +1,97 @@
+# Migrating Data Between AWX Installations
+
+## Introduction
+
+Upgrades using Django migrations are not expected to work in AWX.  As a result, to upgrade to a new version, it is necessary to export resources from the old AWX node and import them into a freshly-installed node with the new version.  The recommended way to do this is to use the tower-cli send/receive feature.
+
+This tool does __not__ support export/import of the following:
+* Logs/history
+* Credential passwords
+* LDAP/AWX config
+
+### Install & Configure Tower-CLI
+
+In terminal, pip install tower-cli (if you do not have pip already, install [here](https://pip.pypa.io/en/stable/installing/)):
+```
+$ pip install --upgrade ansible-tower-cli
+```
+
+The AWX host URL, user, and password must be set for the AWX instance to be exported:
+```
+$ tower-cli config host http://<old-awx-host.example.com>
+$ tower-cli config username <user>
+$ tower-cli config password <pass>
+```
+
+For more information on installing tower-cli look [here](http://tower-cli.readthedocs.io/en/latest/quickstart.html).
+
+
+### Export Resources
+
+Export all objects
+
+```$ tower-cli receive --all > assets.json```
+
+
+
+### Teardown Old AWX
+
+Clean up remnants of the old AWX install:
+
+```docker rm -f $(docker ps -aq)```     # remove all old awx containers
+
+```make clean-ui```              # clean up ui artifacts
+
+
+### Install New AWX version
+
+If you are installing AWX as a dev container, pull down the latest code or version you want from GitHub, build
+the image locally, then start the container
+
+```
+git pull                                        # retrieve latest AWX changes from repository
+make docker-compose-build                       # build AWX image
+make docker-compose                             # run container
+```
+For other install methods, refer to the [Install.md](https://github.com/ansible/awx/blob/devel/INSTALL.md). 
+ 
+
+### Import Resources
+
+
+Configure tower-cli for your new AWX host as shown earlier.  Import from a JSON file named assets.json
+
+```
+$ tower-cli config host http://<new-awx-host.example.com>
+$ tower-cli config username <user>
+$ tower-cli config password <pass>
+$ tower-cli send assets.json
+```
+
+--------------------------------------------------------------------------------
+
+## Additional Info
+
+If you have two running AWX hosts, it is possible to copy all assets from one instance to another
+
+```$ tower-cli receive --tower-host old-awx-host.example.com --all | tower-cli send --tower-host new-awx-host.example.com```
+
+
+
+#### More Granular Exports:
+
+Export all credentials
+
+```$ tower-cli receive --credential all > credentials.json```
+> Note: This exports the credentials with blank strings for passwords and secrets
+
+Export a credential named "My Credential"
+
+```$ tower-cli receive --credential "My Credential"```
+
+#### More Granular Imports:
+
+
+You could import anything except an organization defined in a JSON file named assets.json
+
+```$ tower-cli send --prevent organization assets.json```

INSTALL.md

@@ -4,41 +4,45 @@ This document provides a guide for installing AWX.
 
 ## Table of contents
 
-- [Getting started](#getting-started)
-  - [Clone the repo](#clone-the-repo)
-  - [AWX branding](#awx-branding)
-  - [Prerequisites](#prerequisites)
-  - [System Requirements](#system-requirements)
-  - [AWX Tunables](#awx-tunables)
-  - [Choose a deployment platform](#choose-a-deployment-platform)
-  - [Official vs Building Images](#official-vs-building-images)
-- [OpenShift](#openshift)
-  - [Prerequisites](#prerequisites-1)
-    - [Deploying to Minishift](#deploying-to-minishift)
-  - [Pre-build steps](#pre-build-steps)
-  - [PostgreSQL](#postgresql)
-  - [Start the build](#start-the-build)
-  - [Post build](#post-build)
-  - [Accessing AWX](#accessing-awx)
-- [Kubernetes](#kubernetes)
-  - [Prerequisites](#prerequisites-2)
-  - [Pre-build steps](#pre-build-steps-1)
-  - [Configuring Helm](#configuring-helm)
-  - [Start the build](#start-the-build-1)
-  - [Accessing AWX](#accessing-awx-1)
-  - [SSL Termination](#ssl-termination)
-- [Docker or Docker Compose](#docker-or-docker-compose)
-  - [Prerequisites](#prerequisites-3)
-  - [Pre-build steps](#pre-build-steps-2)
-    - [Deploying to a remote host](#deploying-to-a-remote-host)
-    - [Inventory variables](#inventory-variables)
+- [Installing AWX](#installing-awx)
+  * [Getting started](#getting-started)
+    + [Clone the repo](#clone-the-repo)
+    + [AWX branding](#awx-branding)
+    + [Prerequisites](#prerequisites)
+    + [System Requirements](#system-requirements)
+    + [AWX Tunables](#awx-tunables)
+    + [Choose a deployment platform](#choose-a-deployment-platform)
+    + [Official vs Building Images](#official-vs-building-images)
+  * [Upgrading from previous versions](#upgrading-from-previous-versions)
+  * [OpenShift](#openshift)
+    + [Prerequisites](#prerequisites-1)
+    + [Pre-install steps](#pre-install-steps)
+      - [Deploying to Minishift](#deploying-to-minishift)
+      - [PostgreSQL](#postgresql)
+    + [Run the installer](#run-the-installer)
+    + [Post-install](#post-install)
+    + [Accessing AWX](#accessing-awx)
+  * [Kubernetes](#kubernetes)
+    + [Prerequisites](#prerequisites-2)
+    + [Pre-install steps](#pre-install-steps-1)
+    + [Configuring Helm](#configuring-helm)
+    + [Run the installer](#run-the-installer-1)
+    + [Post-install](#post-install-1)
+    + [Accessing AWX](#accessing-awx-1)
+    + [SSL Termination](#ssl-termination)
+  * [Docker-Compose](#docker-compose)
+    + [Prerequisites](#prerequisites-3)
+    + [Pre-install steps](#pre-install-steps-2)
+      - [Deploying to a remote host](#deploying-to-a-remote-host)
+      - [Inventory variables](#inventory-variables)
       - [Docker registry](#docker-registry)
-      - [PostgreSQL](#postgresql-1)
       - [Proxy settings](#proxy-settings)
-  - [Start the build](#start-the-build-2)
-  - [Post build](#post-build-1)
-  - [Accessing AWX](#accessing-awx-2)
+      - [PostgreSQL](#postgresql-1)
+    + [Run the installer](#run-the-installer-2)
+    + [Post-install](#post-install-2)
+    + [Accessing AWX](#accessing-awx-2)
 
+    
 ## Getting started
 
 ### Clone the repo
@@ -57,23 +61,26 @@ To install the assets, clone the `awx-logos` repo so that it is next to your `aw
 
 Before you can run a deployment, you'll need the following installed in your local environment:
 
-- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) Requires Version 2.4+
+- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) Requires Version 2.8+
 - [Docker](https://docs.docker.com/engine/installation/)
-- [docker-py](https://github.com/docker/docker-py) Python module
+    + A recent version
+- [docker](https://pypi.org/project/docker/) Python module
+    + This is incompatible with `docker-py`. If you have previously installed `docker-py`, please uninstall it.
+    + We use this module instead of `docker-py` because it is what the `docker-compose` Python module requires.
 - [GNU Make](https://www.gnu.org/software/make/)
 - [Git](https://git-scm.com/) Requires Version 1.8.4+
-- [Node 6.x LTS version](https://nodejs.org/en/download/)
-- [NPM 3.x LTS](https://docs.npmjs.com/)
+- [Node 10.x LTS version](https://nodejs.org/en/download/)
+- [NPM 6.x LTS](https://docs.npmjs.com/)
 
 ### System Requirements
 
 The system that runs the AWX service will need to satisfy the following requirements
 
-- At leasts 4GB of memory
+- At least 4GB of memory
 - At least 2 cpu cores
 - At least 20GB of space
 - Running Docker, Openshift, or Kubernetes
-- If you choose to use an external PostgreSQL database, please note that the minimum version is 9.4.
+- If you choose to use an external PostgreSQL database, please note that the minimum version is 9.6+.
 
 ### AWX Tunables
 
@@ -81,14 +88,14 @@ The system that runs the AWX service will need to satisfy the following requirem
 
 ### Choose a deployment platform
 
-We currently support running AWX as a containerized application using Docker images deployed to either an OpenShift cluster, docker-compose or a standalone Docker daemon. The remainder of this document will walk you through the process of building the images, and deploying them to either platform.
+We currently support running AWX as a containerized application using Docker images deployed to either an OpenShift cluster, a Kubernetes cluster, or docker-compose. The remainder of this document will walk you through the process of building the images, and deploying them to either platform.
 
 The [installer](./installer) directory contains an [inventory](./installer/inventory) file, and a playbook, [install.yml](./installer/install.yml). You'll begin by setting variables in the inventory file according to the platform you wish to use, and then you'll start the image build and deployment process by running the playbook.
 
 In the sections below, you'll find deployment details and instructions for each platform:
 - [OpenShift](#openshift)
 - [Kubernetes](#kubernetes)
-- [Docker or Docker Compose](#docker-or-docker-compose).
+- [Docker Compose](#docker-compose).
 
 ### Official vs Building Images
 
@@ -111,47 +118,59 @@ If these variables are present then all deployments will use these hosted images
 
 > Multiple versions are provided. `latest` always pulls the most recent. You may also select version numbers at different granularities: 1, 1.0, 1.0.1, 1.0.0.123
 
+
+## Upgrading from previous versions
+
+Upgrading AWX involves rerunning the install playbook. Download a newer release from [https://github.com/ansible/awx/releases](https://github.com/ansible/awx/releases) and re-populate the inventory file with your customized variables.
+
+For convenience, you can create a file called `vars.yml`:
+
+```
+admin_password: 'adminpass'
+pg_password: 'pgpass'
+rabbitmq_password: 'rabbitpass'
+secret_key: 'mysupersecret'
+```
+
+And pass it to the installer:
+
+```
+$ ansible-playbook -i inventory install.yml -e @vars.yml
+```
+
 ## OpenShift
 
 ### Prerequisites
 
 To complete a deployment to OpenShift, you will obviously need access to an OpenShift cluster. For demo and testing purposes, you can use [Minishift](https://github.com/minishift/minishift) to create a single node cluster running inside a virtual machine.
 
+When using OpenShift for deploying AWX make sure you have correct privileges to add the security context 'privileged', otherwise the installation will fail. The privileged context is needed because of the use of [the bubblewrap tool](https://github.com/containers/bubblewrap) to add an additional layer of security when using containers.
+
 You will also need to have the `oc` command in your PATH. The `install.yml` playbook will call out to `oc` when logging into, and creating objects on the cluster.
 
-The default resource requests per-pod requires:
+The default resource requests per-deployment requires:
 
 > Memory: 6GB
 > CPU: 3 cores
 
-This can be tuned by overriding the variables found in [/installer/openshift/defaults/main.yml](/installer/openshift/defaults/main.yml). Special care should be taken when doing this as undersized instances will experience crashes and resource exhaustion.
+This can be tuned by overriding the variables found in [/installer/roles/kubernetes/defaults/main.yml](/installer/roles/kubernetes/defaults/main.yml). Special care should be taken when doing this as undersized instances will experience crashes and resource exhaustion.
 
 For more detail on how resource requests are formed see: [https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources](https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources)
 
-#### Deploying to Minishift
+### Pre-install steps
 
-Install Minishift by following the [installation guide](https://docs.openshift.org/latest/minishift/getting-started/installing.html).
-
-The Minishift VM contains a Docker daemon, which you can use to build the AWX images. This is generally the approach you should take, and we recommend doing so. To use this instance, run the following command to setup your environment:
-
-```bash
-# Set DOCKER environment variable to point to the Minishift VM
-$ eval $(minishift docker-env)
-```
-
-**Note**
-
-> If you choose to not use the Docker instance running inside the VM, and build the images externally, you will have to enable the OpenShift cluster to access the images. This involves pushing the images to an external Docker registry, and granting the cluster access to it, or exposing the internal registry, and pushing the images into it.
-
-### Pre-build steps
-
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
+Before starting the install, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
 
 *openshift_host*
 
 > IP address or hostname of the OpenShift cluster. If you're using Minishift, this will be the value returned by `minishift ip`.
 
-*awx_openshift_project*
+
+*openshift_skip_tls_verify*
+
+> Boolean. Set to True if using self-signed certs.
+
+*openshift_project*
 
 > Name of the OpenShift project that will be created, and used as the namespace for the AWX app. Defaults to *awx*.
 
@@ -159,6 +178,10 @@ Before starting the build process, review the [inventory](./installer/inventory)
 
 > Username of the OpenShift user that will create the project, and deploy the application. Defaults to *developer*.
 
+*openshift_pg_emptydir*
+
+> Boolean. Set to True to use an emptyDir volume when deploying the PostgreSQL pod. Note: This should only be used for demo and testing purposes.
+
 *docker_registry*
 
 > IP address and port, or URL, for accessing a registry that the OpenShift cluster can access. Defaults to *172.30.1.1:5000*, the internal registry delivered with Minishift. This is not needed if you are using official hosted images.
@@ -171,28 +194,47 @@ Before starting the build process, review the [inventory](./installer/inventory)
 
 > Username of the user that will push images to the registry. Will generally match the *openshift_user* value. Defaults to *developer*. This is not needed if you are using official hosted images.
 
+#### Deploying to Minishift
+
+Install Minishift by following the [installation guide](https://docs.openshift.org/latest/minishift/getting-started/installing.html).
+
+The recommended minimum resources for your Minishift VM:
+
+```bash
+$ minishift start --cpus=4 --memory=8GB
+```
+
+The Minishift VM contains a Docker daemon, which you can use to build the AWX images. This is generally the approach you should take, and we recommend doing so. To use this instance, run the following command to setup your environment:
+
+```bash
+# Set DOCKER environment variable to point to the Minishift VM
+$ eval $(minishift docker-env)
+```
+
+**Note**
+
+> If you choose to not use the Docker instance running inside the VM, and build the images externally, you will have to enable the OpenShift cluster to access the images. This involves pushing the images to an external Docker registry, and granting the cluster access to it, or exposing the internal registry, and pushing the images into it.
+
 #### PostgreSQL
 
-AWX requires access to a PostgreSQL database, and by default, one will be created and deployed in a pod. The database is configured for persistence and will create a persistent volume claim named `postgresql`. By default it will claim 5GB from the available persistent volume pool. This can be tuned by setting a variable in the inventory file or on the command line during the `ansible-playbook` run.
+By default, AWX will deploy a PostgreSQL pod inside of your cluster. You will need to create a [Persistent Volume Claim](https://docs.openshift.org/latest/dev_guide/persistent_volumes.html) which is named `postgresql` by default, and can be overridden by setting the `openshift_pg_pvc_name` variable. For testing and demo purposes, you may set `openshift_pg_emptydir=yes`.
 
-    ansible-playbook ... -e pg_volume_capacity=n
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
+### Run the installer
 
-### Start the build
-
-To start the build, you will pass two *extra* variables on the command line. The first is *openshift_password*, which is the password for the *openshift_user*, and the second is *docker_registry_password*, which is the password associated with *docker_registry_username*.
+To start the install, you will pass two *extra* variables on the command line. The first is *openshift_password*, which is the password for the *openshift_user*, and the second is *docker_registry_password*, which is the password associated with *docker_registry_username*.
 
 If you're using the OpenShift internal registry, then you'll pass an access token for the *docker_registry_password* value, rather than a password. The `oc whoami -t` command will generate the required token, as long as you're logged into the cluster via `oc cluster login`.
 
-To start the build and deployment, run the following (docker_registry_password is optional if using official images):
+Run the following command (docker_registry_password is optional if using official images):
 
 ```bash
-# Start the build and deployment
+# Start the install
 $ ansible-playbook -i inventory install.yml -e openshift_password=developer  -e docker_registry_password=$(oc whoami -t)
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, check the status of the deployment by running `oc get pods`:
 
@@ -223,7 +265,7 @@ Using /etc/ansible/ansible.cfg as config file
 }
 Operations to perform:
   Synchronize unmigrated apps: solo, api, staticfiles, messages, channels, django_extensions, ui, rest_framework, polymorphic
-  Apply all migrations: sso, taggit, sessions, djcelery, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
+  Apply all migrations: sso, taggit, sessions, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
 Synchronizing apps without migrations:
   Creating tables...
     Running deferred SQL...
@@ -305,19 +347,19 @@ The default resource requests per-pod requires:
 > Memory: 6GB
 > CPU: 3 cores
 
-This can be tuned by overriding the variables found in [/installer/kubernetes/defaults/main.yml](/installer/kubernetes[/defaults/main.yml). Special care should be taken when doing this as undersized instances will experience crashes and resource exhaustion.
+This can be tuned by overriding the variables found in [/installer/roles/kubernetes/defaults/main.yml](/installer/roles/kubernetes/defaults/main.yml). Special care should be taken when doing this as undersized instances will experience crashes and resource exhaustion.
 
 For more detail on how resource requests are formed see: [https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)
 
-### Pre-build steps
+### Pre-install steps
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section uncommenting when necessary. Make sure the openshift and standalone docker sections are commented out:
+Before starting the install process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section uncommenting when necessary. Make sure the openshift and standalone docker sections are commented out:
 
 *kubernetes_context*
 
 > Prior to running the installer, make sure you've configured the context for the cluster you'll be installing to. This is how the installer knows which cluster to connect to and what authentication to use
 
-*awx_kubernetes_namespace*
+*kubernetes_namespace*
 
 > Name of the Kubernetes namespace where the AWX resources will be installed. This will be created if it doesn't exist
 
@@ -331,7 +373,7 @@ If you want the AWX installer to manage creating the database pod (rather than i
 
 Newer Kubernetes clusters with RBAC enabled will need to make sure a service account is created, make sure to follow the instructions here [https://docs.helm.sh/using_helm/#role-based-access-control](https://docs.helm.sh/using_helm/#role-based-access-control)
 
-### Start the build
+### Run the installer
 
 After making changes to the `inventory` file use `ansible-playbook` to begin the install
 
@@ -339,7 +381,7 @@ After making changes to the `inventory` file use `ansible-playbook` to begin the
 $ ansible-playbook -i inventory install.yml
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, check the status of the deployment by running `kubectl get pods --namespace awx` (replace awx with the namespace you used):
 
@@ -378,20 +420,20 @@ If your provider is able to allocate an IP Address from the Ingress controller t
 Unlike Openshift's `Route` the Kubernetes `Ingress` doesn't yet handle SSL termination. As such the default configuration will only expose AWX through HTTP on port 80. You are responsible for configuring SSL support until support is added (either to Kubernetes or AWX itself).
 
 
-## Docker or Docker-Compose
+## Docker-Compose
 
 ### Prerequisites
 
 - [Docker](https://docs.docker.com/engine/installation/) on the host where AWX will be deployed. After installing Docker, the Docker service must be started (depending on your OS, you may have to add the local user that uses Docker to the ``docker`` group, refer to the documentation for details)
-- [docker-py](https://github.com/docker/docker-py) Python module.
+- [docker-compose](https://pypi.org/project/docker-compose/) Python module.
+    + This also installs the `docker` Python module, which is incompatible with `docker-py`. If you have previously installed `docker-py`, please uninstall it.
+- [Docker Compose](https://docs.docker.com/compose/install/).
 
-If you're installing using Docker Compose, you'll need [Docker Compose](https://docs.docker.com/compose/install/).
-
-### Pre-build steps
+### Pre-install steps
 
 #### Deploying to a remote host
 
-By default, the delivered [installer/inventory](./installer/inventory) file will deploy AWX to the local host. It is possible; however, to deploy to a remote host. The [installer/install.yml](./installer/install.yml) playbook can be used to build images on the local host, and ship the built images to, and run deployment tasks on, a remote host. To do this, modify the [installer/inventory](./installer/inventory) file, by commenting out `localhost`, and adding the remote host.
+By default, the delivered [installer/inventory](./installer/inventory) file will deploy AWX to the local host. It is possible, however, to deploy to a remote host. The [installer/install.yml](./installer/install.yml) playbook can be used to build images on the local host, and ship the built images to, and run deployment tasks on, a remote host. To do this, modify the [installer/inventory](./installer/inventory) file, by commenting out `localhost`, and adding the remote host.
 
 For example, suppose you wish to build images locally on your CI/CD host, and deploy them to a remote host named *awx-server*. To do this, add *awx-server* to the [installer/inventory](./installer/inventory) file, and comment out or remove `localhost`, as demonstrated by the following:
 
@@ -413,12 +455,12 @@ If you choose to use the official images then the remote host will be the one to
 
 > As mentioned above, in [Prerequisites](#prerequisites-1), the prerequisites are required on the remote host.
 
-> When deploying to a remote host, the playook does not execute tasks with the `become` option. For this reason, make sure the user that connects to the remote host has privileges to run the `docker` command. This typically means that non-privileged users need to be part of the `docker` group.
+> When deploying to a remote host, the playbook does not execute tasks with the `become` option. For this reason, make sure the user that connects to the remote host has privileges to run the `docker` command. This typically means that non-privileged users need to be part of the `docker` group.
 
 
 #### Inventory variables
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
+Before starting the install process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
 
 *postgres_data_dir*
 
@@ -428,13 +470,25 @@ Before starting the build process, review the [inventory](./installer/inventory)
 
 > Provide a port number that can be mapped from the Docker daemon host to the web server running inside the AWX container. Defaults to *80*.
 
-*use_docker_compose*
+*host_port_ssl*
 
-> Switch to ``true`` to use Docker Compose instead of the standalone Docker install.
+> Provide a port number that can be mapped from the Docker daemon host to the web server running inside the AWX container for SSL support. Defaults to *443*, only works if you also set `ssl_certificate` (see below).
+
+*ssl_certificate*
+
+> Optionally, provide the path to a file that contains a certificate and its private key.
 
 *docker_compose_dir*
 
-When using docker-compose, the `docker-compose.yml` file will be created there (default `/var/lib/awx`).
+> When using docker-compose, the `docker-compose.yml` file will be created there (default `/tmp/awxcompose`).
+
+*custom_venv_dir*
+
+> Adds the custom venv environments from the local host to be passed into the containers at install.
+
+*ca_trust_dir*
+
+> If you're using a non trusted CA, provide a path where the untrusted Certs are stored on your Host.
 
 #### Docker registry
 
@@ -479,11 +533,11 @@ If you wish to tag and push built images to a Docker registry, set the following
 
 AWX requires access to a PostgreSQL database, and by default, one will be created and deployed in a container, and data will be persisted to a host volume. In this scenario, you must set the value of `postgres_data_dir` to a path that can be mounted to the container. When the container is stopped, the database files will still exist in the specified path.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information.
 
-### Start the build
+### Run the installer
 
-If you are not pushing images to a Docker registry, start the build by running the following:
+If you are not pushing images to a Docker registry, start the install by running the following:
 
 ```bash
 # Set the working directory to installer
@@ -503,14 +557,14 @@ $ cd installer
 $ ansible-playbook -i inventory -e docker_registry_password=password install.yml
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, Docker will report up to 5 running containers. If you chose to use an existing PostgresSQL database, then it will report 4. You can view the running containers using the `docker ps` command, as follows:
 
 ```bash
 CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                NAMES
 e240ed8209cd        awx_task:1.0.0.8    "/tini -- /bin/sh ..."   2 minutes ago       Up About a minute   8052/tcp                             awx_task
-1cfd02601690        awx_web:1.0.0.8     "/tini -- /bin/sh ..."   2 minutes ago       Up About a minute   0.0.0.0:80->8052/tcp                 awx_web
+1cfd02601690        awx_web:1.0.0.8     "/tini -- /bin/sh ..."   2 minutes ago       Up About a minute   0.0.0.0:443->8052/tcp                 awx_web
 55a552142bcd        memcached:alpine    "docker-entrypoint..."   2 minutes ago       Up 2 minutes        11211/tcp                            memcached
 84011c072aad        rabbitmq:3          "docker-entrypoint..."   2 minutes ago       Up 2 minutes        4369/tcp, 5671-5672/tcp, 25672/tcp   rabbitmq
 97e196120ab3        postgres:9.6        "docker-entrypoint..."   2 minutes ago       Up 2 minutes        5432/tcp                             postgres
@@ -535,7 +589,7 @@ Using /etc/ansible/ansible.cfg as config file
 }
 Operations to perform:
   Synchronize unmigrated apps: solo, api, staticfiles, messages, channels, django_extensions, ui, rest_framework, polymorphic
-  Apply all migrations: sso, taggit, sessions, djcelery, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
+  Apply all migrations: sso, taggit, sessions, sites, kombu_transport_django, social_auth, contenttypes, auth, conf, main
 Synchronizing apps without migrations:
   Creating tables...
     Running deferred SQL...
@@ -580,14 +634,3 @@ Added instance awx to tower
 The AWX web server is accessible on the deployment host, using the *host_port* value set in the *inventory* file. The default URL is [http://localhost](http://localhost).
 
 You will prompted with a login dialog. The default administrator username is `admin`, and the password is `password`.
-
-### Maintenance using docker-compose
-
-After the installation, maintenance operations with docker-compose can be done by using the  `docker-compose.yml` file created at the location pointed by `docker_compose_dir`.
-
-Among the possible operations, you may:
-
-- Stop AWX : `docker-compose stop`
-- Upgrade AWX : `docker-compose pull && docker-compose up --force-recreate`
-
-See the [docker-compose documentation](https://docs.docker.com/compose/) for details.

Makefile

@@ -1,4 +1,4 @@
-PYTHON ?= python
+PYTHON ?= python3
 PYTHON_VERSION = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_version; print(get_python_version())")
 SITELIB=$(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 OFFICIAL ?= no
@@ -11,17 +11,14 @@ GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 MANAGEMENT_COMMAND ?= awx-manage
 IMAGE_REPOSITORY_AUTH ?=
 IMAGE_REPOSITORY_BASE ?= https://gcr.io
-
-VERSION=$(shell git describe --long --first-parent)
-VERSION3=$(shell git describe --long --first-parent | sed 's/\-g.*//')
-VERSION3DOT=$(shell git describe --long --first-parent | sed 's/\-g.*//' | sed 's/\-/\./')
-RELEASE_VERSION=$(shell git describe --long --first-parent | sed 's@\([0-9.]\{1,\}\).*@\1@')
+VERSION := $(shell cat VERSION)
 
 # NOTE: This defaults the container image version to the branch that's active
 COMPOSE_TAG ?= $(GIT_BRANCH)
 COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
+COLLECTION_VENV ?= /awx_devel/awx_collection_test_venv
 SCL_PREFIX ?=
 CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
@@ -46,20 +43,9 @@ DATE := $(shell date -u +%Y%m%d%H%M)
 NAME ?= awx
 GIT_REMOTE_URL = $(shell git config --get remote.origin.url)
 
-ifeq ($(OFFICIAL),yes)
-    VERSION_TARGET ?= $(RELEASE_VERSION)
-else
-    VERSION_TARGET ?= $(VERSION3DOT)
-endif
-
 # TAR build parameters
-ifeq ($(OFFICIAL),yes)
-    SDIST_TAR_NAME=$(NAME)-$(RELEASE_VERSION)
-    WHEEL_NAME=$(NAME)-$(RELEASE_VERSION)
-else
-    SDIST_TAR_NAME=$(NAME)-$(VERSION3DOT)
-    WHEEL_NAME=$(NAME)-$(VERSION3DOT)
-endif
+SDIST_TAR_NAME=$(NAME)-$(VERSION)
+WHEEL_NAME=$(NAME)-$(VERSION)
 
 SDIST_COMMAND ?= sdist
 WHEEL_COMMAND ?= bdist_wheel
@@ -68,26 +54,31 @@ WHEEL_FILE ?= $(WHEEL_NAME)-py2-none-any.whl
 
 # UI flag files
 UI_DEPS_FLAG_FILE = awx/ui/.deps_built
+UI_RELEASE_DEPS_FLAG_FILE = awx/ui/.release_deps_built
 UI_RELEASE_FLAG_FILE = awx/ui/.release_built
 
 I18N_FLAG_FILE = .i18n_built
 
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
-	develop refresh adduser migrate dbchange dbshell runserver celeryd \
-	receiver test test_unit test_ansible test_coverage coverage_html \
+	develop refresh adduser migrate dbchange runserver \
+	receiver test test_unit test_coverage coverage_html \
 	dev_build release_build release_clean sdist \
 	ui-docker-machine ui-docker ui-release ui-devel \
 	ui-test ui-deps ui-test-ci VERSION
 
 # remove ui build artifacts
-clean-ui:
+clean-ui: clean-languages
 	rm -rf awx/ui/static/
 	rm -rf awx/ui/node_modules/
 	rm -rf awx/ui/test/unit/reports/
 	rm -rf awx/ui/test/spec/reports/
 	rm -rf awx/ui/test/e2e/reports/
 	rm -rf awx/ui/client/languages/
+	rm -rf awx/ui_next/node_modules/
+	rm -rf awx/ui_next/coverage/
+	rm -rf awx/ui_next/build/locales/_build/
 	rm -f $(UI_DEPS_FLAG_FILE)
+	rm -f $(UI_RELEASE_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_FLAG_FILE)
 
 clean-tmp:
@@ -99,22 +90,36 @@ clean-venv:
 clean-dist:
 	rm -rf dist
 
+clean-schema:
+	rm -rf swagger.json
+	rm -rf schema.json
+	rm -rf reference-schema.json
+
+clean-languages:
+	rm -f $(I18N_FLAG_FILE)
+	find . -type f -regex ".*\.mo$$" -delete
+
 # Remove temporary build files, compiled Python files.
-clean: clean-ui clean-dist
+clean: clean-ui clean-api clean-awxkit clean-dist
 	rm -rf awx/public
 	rm -rf awx/lib/site-packages
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3
-	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
-	rm -f VERSION
 	mkdir tmp
+
+clean-api:
 	rm -rf build $(NAME)-$(VERSION) *.egg-info
 	find . -type f -regex ".*\.py[co]$$" -delete
 	find . -type d -name "__pycache__" -delete
+	rm -f awx/awx_test.sqlite3*
+	rm -rf requirements/vendor
+	rm -rf awx/projects
+
+clean-awxkit:
+	rm -rf awxkit/*.egg-info awxkit/.tox
 
 # convenience target to assert environment variables are defined
 guard-%:
@@ -131,23 +136,31 @@ virtualenv_ansible:
 			mkdir $(VENV_BASE); \
 		fi; \
 		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
-			virtualenv --system-site-packages $(VENV_BASE)/ansible && \
+			virtualenv -p python --system-site-packages $(VENV_BASE)/ansible && \
 			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed six packaging appdirs && \
 			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==36.0.1 && \
 			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==9.0.1; \
 		fi; \
 	fi
 
+virtualenv_ansible_py3:
+	if [ "$(VENV_BASE)" ]; then \
+		if [ ! -d "$(VENV_BASE)" ]; then \
+			mkdir $(VENV_BASE); \
+		fi; \
+		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
+			$(PYTHON) -m venv --system-site-packages $(VENV_BASE)/ansible; \
+		fi; \
+	fi
+
 virtualenv_awx:
 	if [ "$(VENV_BASE)" ]; then \
 		if [ ! -d "$(VENV_BASE)" ]; then \
 			mkdir $(VENV_BASE); \
 		fi; \
 		if [ ! -d "$(VENV_BASE)/awx" ]; then \
-			virtualenv --system-site-packages $(VENV_BASE)/awx && \
-			$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed six packaging appdirs && \
-			$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==36.0.1 && \
-			$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==9.0.1; \
+			$(PYTHON) -m venv --system-site-packages $(VENV_BASE)/awx; \
+			$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed docutils==0.14; \
 		fi; \
 	fi
 
@@ -159,20 +172,19 @@ requirements_ansible: virtualenv_ansible
 	fi
 	$(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt
 
+requirements_ansible_py3: virtualenv_ansible_py3
+	if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
+	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --ignore-installed -r /dev/stdin ; \
+	else \
+	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
+	fi
+	$(VENV_BASE)/ansible/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt
+
 requirements_ansible_dev:
 	if [ "$(VENV_BASE)" ]; then \
 		$(VENV_BASE)/ansible/bin/pip install pytest mock; \
 	fi
 
-requirements_isolated:
-	if [ ! -d "$(VENV_BASE)/awx" ]; then \
-		virtualenv --system-site-packages $(VENV_BASE)/awx && \
-		$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed six packaging appdirs && \
-		$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==35.0.2 && \
-		$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==9.0.1; \
-	fi;
-	$(VENV_BASE)/awx/bin/pip install -r requirements/requirements_isolated.txt
-
 # Install third-party requirements needed for AWX's environment.
 requirements_awx: virtualenv_awx
 	if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
@@ -180,6 +192,7 @@ requirements_awx: virtualenv_awx
 	else \
 	    cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
 	fi
+	echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg
 	$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
 
 requirements_awx_dev:
@@ -187,7 +200,7 @@ requirements_awx_dev:
 
 requirements: requirements_ansible requirements_awx
 
-requirements_dev: requirements requirements_awx_dev requirements_ansible_dev
+requirements_dev: requirements_awx requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev
 
 requirements_test: requirements
 
@@ -206,7 +219,7 @@ version_file:
 	if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	python -c "import awx as awx; print awx.__version__" > /var/lib/awx/.awx_version; \
+	python -c "import awx; print(awx.__version__)" > /var/lib/awx/.awx_version; \
 
 # Do any one-time init tasks.
 comma := ,
@@ -219,7 +232,7 @@ init:
 	if [ "$(AWX_GROUP_QUEUES)" == "tower,thepentagon" ]; then \
 		$(MANAGEMENT_COMMAND) provision_instance --hostname=isolated; \
 		$(MANAGEMENT_COMMAND) register_queue --queuename='thepentagon' --hostnames=isolated --controller=tower; \
-		$(MANAGEMENT_COMMAND) generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat > /root/.ssh/authorized_keys'; \
+		$(MANAGEMENT_COMMAND) generate_isolated_key > /awx_devel/awx/main/isolated/authorized_keys; \
 	fi;
 
 # Refresh development environment after pulling new code.
@@ -240,15 +253,11 @@ migrate:
 dbchange:
 	$(MANAGEMENT_COMMAND) makemigrations
 
-# access database shell, asks for password
-dbshell:
-	sudo -u postgres psql -d awx-dev
-
 server_noattach:
 	tmux new-session -d -s awx 'exec make uwsgi'
 	tmux rename-window 'AWX'
 	tmux select-window -t awx:0
-	tmux split-window -v 'exec make celeryd'
+	tmux split-window -v 'exec make dispatcher'
 	tmux new-window 'exec make daphne'
 	tmux select-window -t awx:1
 	tmux rename-window 'WebSockets'
@@ -270,21 +279,7 @@ supervisor:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	supervisord --configuration /supervisor.conf --pidfile=/tmp/supervisor_pid
-
-# Alternate approach to tmux to run all development tasks specified in
-# Procfile.  https://youtu.be/OPMgaibszjk
-honcho:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	honcho start -f tools/docker-compose/Procfile
-
-flower:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	celery flower --address=0.0.0.0 --port=5555 --broker=amqp://guest:guest@$(RABBITMQ_HOST):5672//
+	supervisord --pidfile=/tmp/supervisor_pid -n
 
 collectstatic:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -296,7 +291,7 @@ uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --master-fifo=/awxfifo --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1-once="exec:/bin/sh -c '[ -f /tmp/celery_pid ] && kill -1 `cat /tmp/celery_pid` || true'"
+    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1="exec:supervisorctl restart tower-processes:awx-dispatcher tower-processes:awx-receiver"
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -317,13 +312,13 @@ runserver:
 	fi; \
 	$(PYTHON) manage.py runserver
 
-# Run to start the background celery worker for development.
-celeryd:
-	rm -f /tmp/celery_pid
+# Run to start the background task dispatcher for development.
+dispatcher:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 --schedule=$(CELERY_SCHEDULE_FILE) -n celery@$(COMPOSE_HOST) --pidfile /tmp/celery_pid
+	$(PYTHON) manage.py run_dispatcher
+
 
 # Run to start the zeromq callback receiver
 receiver:
@@ -359,28 +354,64 @@ pyflakes: reports
 pylint: reports
 	@(set -o pipefail && $@ | reports/$@.report)
 
+genschema: reports
+	$(MAKE) swagger PYTEST_ARGS="--genschema --create-db "
+	mv swagger.json schema.json
+
 swagger: reports
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	(set -o pipefail && py.test awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs --release=$(VERSION_TARGET) | tee reports/$@.report)
+	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs --release=$(VERSION_TARGET) | tee reports/$@.report)
 
 check: flake8 pep8 # pyflakes pylint
 
 awx-link:
 	cp -R /tmp/awx.egg-info /awx_devel/ || true
-	sed -i "s/placeholder/$(shell git describe --long | sed 's/\./\\./g')/" /awx_devel/awx.egg-info/PKG-INFO
-	cp /tmp/awx.egg-link /venv/awx/lib/python2.7/site-packages/awx.egg-link
+	sed -i "s/placeholder/$(shell cat VERSION)/" /awx_devel/awx.egg-info/PKG-INFO
+	cp -f /tmp/awx.egg-link /venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.egg-link
 
 TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests
+
 # Run all API unit tests.
 test:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	py.test $(TEST_DIRS)
+	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
+	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
+	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
-test_combined: test_ansible test
+prepare_collection_venv:
+	rm -rf $(COLLECTION_VENV)
+	mkdir $(COLLECTION_VENV)
+	$(VENV_BASE)/awx/bin/pip install --target=$(COLLECTION_VENV) git+https://github.com/ansible/tower-cli.git
+
+COLLECTION_TEST_DIRS ?= awx_collection/test/awx
+COLLECTION_PACKAGE ?= awx
+COLLECTION_NAMESPACE ?= awx
+
+test_collection:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	PYTHONPATH=$(COLLECTION_VENV):/awx_devel/awx_collection:$PYTHONPATH py.test $(COLLECTION_TEST_DIRS)
+
+flake8_collection:
+	flake8 awx_collection/  # Different settings, in main exclude list
+
+test_collection_all: prepare_collection_venv test_collection flake8_collection
+
+test_collection_sanity:
+	rm -rf sanity
+	mkdir -p sanity/ansible_collections/awx
+	cp -Ra awx_collection sanity/ansible_collections/awx/awx  # symlinks do not work
+	cd sanity/ansible_collections/awx/awx && git init && git add . # requires both this file structure and a git repo, so there you go
+	cd sanity/ansible_collections/awx/awx && ansible-test sanity
+
+build_collection:
+	ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION)
+	ansible-galaxy collection build awx_collection --output-path=awx_collection
 
 test_unit:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -388,12 +419,6 @@ test_unit:
 	fi; \
 	py.test awx/main/tests/unit awx/conf/tests/unit awx/sso/tests/unit
 
-test_ansible:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/ansible/bin/activate; \
-	fi; \
-	py.test awx/lib/tests -c awx/lib/tests/pytest.ini
-
 # Run all API unit tests with coverage enabled.
 test_coverage:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -465,7 +490,7 @@ messages:
 # generate l10n .json .mo
 languages: $(I18N_FLAG_FILE)
 
-$(I18N_FLAG_FILE): $(UI_DEPS_FLAG_FILE)
+$(I18N_FLAG_FILE): $(UI_RELEASE_DEPS_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui run languages
 	$(PYTHON) tools/scripts/compilemessages.py
 	touch $(I18N_FLAG_FILE)
@@ -473,13 +498,31 @@ $(I18N_FLAG_FILE): $(UI_DEPS_FLAG_FILE)
 # End l10n TASKS
 # --------------------------------------
 
-# UI TASKS
+# UI RELEASE TASKS
+# --------------------------------------
+ui-release: $(UI_RELEASE_FLAG_FILE)
+
+$(UI_RELEASE_FLAG_FILE): $(I18N_FLAG_FILE) $(UI_RELEASE_DEPS_FLAG_FILE)
+	$(NPM_BIN) --prefix awx/ui run build-release
+	touch $(UI_RELEASE_FLAG_FILE)
+
+$(UI_RELEASE_DEPS_FLAG_FILE):
+	PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 $(NPM_BIN) --unsafe-perm --prefix awx/ui ci --no-save awx/ui
+	touch $(UI_RELEASE_DEPS_FLAG_FILE)
+
+# END UI RELEASE TASKS
 # --------------------------------------
 
+# UI TASKS
+# --------------------------------------
 ui-deps: $(UI_DEPS_FLAG_FILE)
 
 $(UI_DEPS_FLAG_FILE):
-	$(NPM_BIN) --unsafe-perm --prefix awx/ui install awx/ui
+	@if [ -f ${UI_RELEASE_DEPS_FLAG_FILE} ]; then \
+		rm -rf awx/ui/node_modules; \
+		rm -f ${UI_RELEASE_DEPS_FLAG_FILE}; \
+	fi; \
+	$(NPM_BIN) --unsafe-perm --prefix awx/ui ci --no-save awx/ui
 	touch $(UI_DEPS_FLAG_FILE)
 
 ui-docker-machine: $(UI_DEPS_FLAG_FILE)
@@ -493,15 +536,13 @@ ui-docker: $(UI_DEPS_FLAG_FILE)
 ui-devel: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui run build-devel -- $(MAKEFLAGS)
 
-ui-release: $(UI_RELEASE_FLAG_FILE)
-
-$(UI_RELEASE_FLAG_FILE): $(I18N_FLAG_FILE) $(UI_DEPS_FLAG_FILE)
-	$(NPM_BIN) --prefix awx/ui run build-release
-	touch $(UI_RELEASE_FLAG_FILE)
-
 ui-test: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui run test
 
+ui-lint: $(UI_DEPS_FLAG_FILE)
+	$(NPM_BIN) run --prefix awx/ui jshint
+	$(NPM_BIN) run --prefix awx/ui lint
+
 # A standard go-to target for API developers to use building the frontend
 ui: clean-ui ui-devel
 
@@ -509,16 +550,40 @@ ui-test-ci: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui run test:ci
 	$(NPM_BIN) --prefix awx/ui run unit
 
-testjs_ci:
-	echo "Update UI unittests later" #ui-test-ci
-
 jshint: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) run --prefix awx/ui jshint
 	$(NPM_BIN) run --prefix awx/ui lint
 
+ui-zuul-lint-and-test: $(UI_DEPS_FLAG_FILE)
+	$(NPM_BIN) run --prefix awx/ui jshint
+	$(NPM_BIN) run --prefix awx/ui lint
+	$(NPM_BIN) --prefix awx/ui run test:ci
+	$(NPM_BIN) --prefix awx/ui run unit
+
 # END UI TASKS
 # --------------------------------------
 
+# UI NEXT TASKS
+# --------------------------------------
+
+ui-next-lint:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+
+ui-next-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+ui-next-zuul-lint-and-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+# END UI NEXT TASKS
+# --------------------------------------
+
 # Build a pip-installable package into dist/ with a timestamped version number.
 dev_build:
 	$(PYTHON) setup.py dev_build
@@ -550,43 +615,60 @@ setup-bundle-build:
 	mkdir -p $@
 
 docker-auth:
-	if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
-		docker login -u oauth2accesstoken -p "$(IMAGE_REPOSITORY_AUTH)" $(IMAGE_REPOSITORY_BASE); \
+	@if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
+		echo "$(IMAGE_REPOSITORY_AUTH)" | docker login -u oauth2accesstoken --password-stdin $(IMAGE_REPOSITORY_BASE); \
 	fi;
 
+# This directory is bind-mounted inside of the development container and
+# needs to be pre-created for permissions to be set correctly. Otherwise,
+# Docker will create this directory as root.
+awx/projects:
+	@mkdir -p $@
+
 # Docker isolated rampart
-docker-isolated:
-	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml create
-	docker start tools_awx_1
-	docker start tools_isolated_1
-	echo "__version__ = '`python setup.py --version`'" | docker exec -i tools_isolated_1 /bin/bash -c "cat > /venv/awx/lib/python2.7/site-packages/awx.py"
-	if [ "`docker exec -i -t tools_isolated_1 cat /root/.ssh/authorized_keys`" == "`docker exec -t tools_awx_1 cat /root/.ssh/id_rsa.pub`" ]; then \
-		echo "SSH keys already copied to isolated instance"; \
-	else \
-		docker exec "tools_isolated_1" bash -c "mkdir -p /root/.ssh && rm -f /root/.ssh/authorized_keys && echo $$(docker exec -t tools_awx_1 cat /root/.ssh/id_rsa.pub) >> /root/.ssh/authorized_keys"; \
-	fi
-	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml up
+docker-compose-isolated: awx/projects
+	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml up
 
 # Docker Compose Development environment
-docker-compose: docker-auth
-	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml up --no-recreate awx
+docker-compose: docker-auth awx/projects
+	CURRENT_UID=$(shell id -u) OS="$(shell docker info | grep 'Operating System')" TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml up --no-recreate awx
 
-docker-compose-cluster: docker-auth
-	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml up
+docker-compose-cluster: docker-auth awx/projects
+	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml up
 
-docker-compose-test: docker-auth
-	cd tools && TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /bin/bash
+docker-compose-credential-plugins: docker-auth awx/projects
+	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
+	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx
+
+docker-compose-test: docker-auth awx/projects
+	cd tools && CURRENT_UID=$(shell id -u) OS="$(shell docker info | grep 'Operating System')" TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /bin/bash
+
+docker-compose-runtest: awx/projects
+	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh
+
+docker-compose-build-swagger: awx/projects
+	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh swagger
+
+detect-schema-change: genschema
+	curl https://s3.amazonaws.com/awx-public-ci-files/schema.json -o reference-schema.json
+	# Ignore differences in whitespace with -b
+	diff -u -b reference-schema.json schema.json
+
+docker-compose-clean: awx/projects
+	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm -w /awx_devel --service-ports awx make clean
+	cd tools && TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose rm -sf
 
 docker-compose-build: awx-devel-build
 
 # Base development image build
 awx-devel-build:
-	docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile .
+	docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile \
+		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 	docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 
 # For use when developing on "isolated" AWX deployments
-awx-isolated-build:
+docker-compose-isolated-build: awx-devel-build
 	docker build -t ansible/awx_isolated -f tools/docker-isolated/Dockerfile .
 	docker tag ansible/awx_isolated $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG)
@@ -600,12 +682,18 @@ docker-clean:
 docker-refresh: docker-clean docker-compose
 
 # Docker Development Environment with Elastic Stack Connected
-docker-compose-elk: docker-auth
-	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
+docker-compose-elk: docker-auth awx/projects
+	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
-docker-compose-cluster-elk: docker-auth
+docker-compose-cluster-elk: docker-auth awx/projects
 	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
+prometheus:
+	docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml
+
+minishift-dev:
+	ansible-playbook -i localhost, -e devtree_directory=$(CURDIR) tools/clusterdevel/start_minishift_dev.yml
+
 clean-elk:
 	docker stop tools_kibana_1
 	docker stop tools_logstash_1
@@ -615,8 +703,7 @@ clean-elk:
 	docker rm tools_kibana_1
 
 psql-container:
-	docker run -it --net tools_default --rm postgres:9.6 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
+	docker run -it --net tools_default --rm postgres:10 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
 
 VERSION:
-	@echo $(VERSION_TARGET) > $@
-	@echo "awx: $(VERSION_TARGET)"
+	@echo "awx: $(VERSION)"

README.md

@@ -1,7 +1,6 @@
-[![Run Status](https://api.shippable.com/projects/591c82a22f895107009e8b35/badge?branch=devel)](https://app.shippable.com/github/ansible/awx)
+[![Gated by Zuul](https://zuul-ci.org/gated.svg)](https://ansible.softwarefactory-project.io/zuul/status)
 
-AWX
-===
+<img src="https://raw.githubusercontent.com/ansible/awx-logos/master/awx/ui/client/assets/logo-login.svg?sanitize=true" width=200 alt="AWX" />
 
 AWX provides a web-based user interface, REST API, and task engine built on top of [Ansible](https://github.com/ansible/ansible). It is the upstream project for [Tower](https://www.ansible.com/tower), a commercial derivative of AWX.  
 
@@ -11,6 +10,8 @@ To learn more about using AWX, and Tower, view the [Tower docs site](http://docs
 
 The AWX Project Frequently Asked Questions can be found [here](https://www.ansible.com/awx-project-faq).
 
+The AWX logos and branding assets are covered by [our trademark guidelines](https://github.com/ansible/awx-logos/blob/master/TRADEMARKS.md).
+
 Contributing
 ------------
 
@@ -23,7 +24,7 @@ Contributing
 Reporting Issues
 ----------------
 
-If you're experiencing a problem, we encourage you to open an issue, and share your feedback. But before opening a new issue, we ask that you please take a look at our [Issues guide](./ISSUES.md).
+If you're experiencing a problem that you feel is a bug in AWX, or have ideas for how to improve AWX, we encourage you to open an issue, and share your feedback. But before opening a new issue, we ask that you please take a look at our [Issues guide](./ISSUES.md).
 
 Code of Conduct
 ---------------
@@ -33,11 +34,10 @@ We ask all of our community members and contributors to adhere to the [Ansible c
 Get Involved
 ------------
 
-We welcome your feedback and ideas. Here's how to reach us:
+We welcome your feedback and ideas. Here's how to reach us with feedback and questions:
 
 - Join the `#ansible-awx` channel on irc.freenode.net
 - Join the [mailing list](https://groups.google.com/forum/#!forum/awx-project) 
-- [Open an Issue](https://github.com/ansible/awx/issues)
 
 License
 -------

VERSION

@@ -0,0 +1 @@
+9.1.0

awx/__init__.py

@@ -7,11 +7,10 @@ import sys
 import warnings
 
 from pkg_resources import get_distribution
-from .celery import app as celery_app # noqa
 
 __version__ = get_distribution('awx').version
+__all__ = ['__version__']
 
-__all__ = ['__version__', 'celery_app']
 
 # Check for the presence/absence of "devonly" module to determine if running
 # from a source code checkout or release packaage.
@@ -22,6 +21,50 @@ except ImportError: # pragma: no cover
     MODE = 'production'
 
 
+import hashlib
+
+try:
+    import django
+    from django.db.backends.base import schema
+    from django.db.backends.utils import names_digest
+    HAS_DJANGO = True
+except ImportError:
+    HAS_DJANGO = False
+
+
+if HAS_DJANGO is True:
+    # This line exists to make sure we don't regress on FIPS support if we
+    # upgrade Django; if you're upgrading Django and see this error,
+    # update the version check below, and confirm that FIPS still works.
+    # If operating in a FIPS environment, `hashlib.md5()` will raise a `ValueError`,
+    # but will support the `usedforsecurity` keyword on RHEL and Centos systems.
+
+    # Keep an eye on https://code.djangoproject.com/ticket/28401
+    target_version = '2.2.4'
+    if django.__version__ != target_version:
+        raise RuntimeError(
+            "Django version other than {target} detected: {current}. "
+            "Overriding `names_digest` is known to work for Django {target} "
+            "and may not work in other Django versions.".format(target=target_version,
+                                                                current=django.__version__)
+        )
+
+    try:
+        names_digest('foo', 'bar', 'baz', length=8)
+    except ValueError:
+        def names_digest(*args, length):
+            """
+            Generate a 32-bit digest of a set of arguments that can be used to shorten
+            identifying names.  Support for use in FIPS environments.
+            """
+            h = hashlib.md5(usedforsecurity=False)
+            for arg in args:
+                h.update(arg.encode())
+            return h.hexdigest()[:length]
+
+        schema.names_digest = names_digest
+
+
 def find_commands(management_dir):
     # Modified version of function from django/core/management/__init__.py.
     command_dir = os.path.join(management_dir, 'commands')
@@ -39,6 +82,23 @@ def find_commands(management_dir):
     return commands
 
 
+def oauth2_getattribute(self, attr):
+    # Custom method to override
+    # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__
+    from django.conf import settings
+    val = None
+    if 'migrate' not in sys.argv:
+        # certain Django OAuth Toolkit migrations actually reference
+        # setting lookups for references to model classes (e.g.,
+        # oauth2_settings.REFRESH_TOKEN_MODEL)
+        # If we're doing an OAuth2 setting lookup *while running* a migration,
+        # don't do our usual "Configure Tower in Tower" database setting lookup
+        val = settings.OAUTH2_PROVIDER.get(attr)
+    if val is None:
+        val = object.__getattribute__(self, attr)
+    return val
+
+
 def prepare_env():
     # Update the default settings environment variable based on current mode.
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
@@ -50,16 +110,12 @@ def prepare_env():
     # Monkeypatch Django find_commands to also work with .pyc files.
     import django.core.management
     django.core.management.find_commands = find_commands
-    # Fixup sys.modules reference to django.utils.six to allow jsonfield to
-    # work when using Django 1.4.
-    import django.utils
-    try:
-        import django.utils.six
-    except ImportError: # pragma: no cover
-        import six
-        sys.modules['django.utils.six'] = sys.modules['six']
-        django.utils.six = sys.modules['django.utils.six']
-        from django.utils import six # noqa
+
+    # Monkeypatch Oauth2 toolkit settings class to check for settings
+    # in django.conf settings each time, not just once during import
+    import oauth2_provider.settings
+    oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ = oauth2_getattribute
+
     # Use the AWX_TEST_DATABASE_* environment variables to specify the test
     # database settings to use when management command is run as an external
     # program via unit tests.

awx/api/authentication.py

@@ -11,7 +11,7 @@ from django.utils.encoding import smart_text
 # Django REST Framework
 from rest_framework import authentication
 
-# Django OAuth Toolkit
+# Django-OAuth-Toolkit
 from oauth2_provider.contrib.rest_framework import OAuth2Authentication
 
 logger = logging.getLogger('awx.api.authentication')
@@ -25,7 +25,7 @@ class LoggedBasicAuthentication(authentication.BasicAuthentication):
         ret = super(LoggedBasicAuthentication, self).authenticate(request)
         if ret:
             username = ret[0].username if ret[0] else '<none>'
-            logger.debug(smart_text(u"User {} performed a {} to {} through the API".format(username, request.method, request.path)))
+            logger.info(smart_text(u"User {} performed a {} to {} through the API".format(username, request.method, request.path)))
         return ret
 
     def authenticate_header(self, request):
@@ -39,9 +39,6 @@ class SessionAuthentication(authentication.SessionAuthentication):
     def authenticate_header(self, request):
         return 'Session'
 
-    def enforce_csrf(self, request):
-        return None
-
 
 class LoggedOAuth2Authentication(OAuth2Authentication):
 
@@ -50,8 +47,8 @@ class LoggedOAuth2Authentication(OAuth2Authentication):
         if ret:
             user, token = ret
             username = user.username if user else '<none>'
-            logger.debug(smart_text(
-                u"User {} performed a {} to {} through the API using OAuth token {}.".format(
+            logger.info(smart_text(
+                u"User {} performed a {} to {} through the API using OAuth 2 token {}.".format(
                     username, request.method, request.path, token.pk
                 )
             ))

awx/api/conf.py

@@ -4,6 +4,7 @@ from django.utils.translation import ugettext_lazy as _
 # AWX
 from awx.conf import fields, register
 from awx.api.fields import OAuth2ProviderField
+from oauth2_provider.settings import oauth2_settings
 
 
 register(
@@ -36,13 +37,39 @@ register(
 register(
     'OAUTH2_PROVIDER',
     field_class=OAuth2ProviderField,
-    default={'ACCESS_TOKEN_EXPIRE_SECONDS': 315360000000,
-             'AUTHORIZATION_CODE_EXPIRE_SECONDS': 600},
+    default={'ACCESS_TOKEN_EXPIRE_SECONDS': oauth2_settings.ACCESS_TOKEN_EXPIRE_SECONDS,
+             'AUTHORIZATION_CODE_EXPIRE_SECONDS': oauth2_settings.AUTHORIZATION_CODE_EXPIRE_SECONDS,
+             'REFRESH_TOKEN_EXPIRE_SECONDS': oauth2_settings.REFRESH_TOKEN_EXPIRE_SECONDS},
     label=_('OAuth 2 Timeout Settings'),
     help_text=_('Dictionary for customizing OAuth 2 timeouts, available items are '
                 '`ACCESS_TOKEN_EXPIRE_SECONDS`, the duration of access tokens in the number '
-                'of seconds, and `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
-                'authorization grants in the number of seconds.'),
+                'of seconds, `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
+                'authorization codes in the number of seconds, and `REFRESH_TOKEN_EXPIRE_SECONDS`, '
+                'the duration of refresh tokens, after expired access tokens, '
+                'in the number of seconds.'),
+    category=_('Authentication'),
+    category_slug='authentication',
+)
+register(
+    'ALLOW_OAUTH2_FOR_EXTERNAL_USERS',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Allow External Users to Create OAuth2 Tokens'),
+    help_text=_('For security reasons, users from external auth providers (LDAP, SAML, '
+                'SSO, Radius, and others) are not allowed to create OAuth2 tokens. '
+                'To change this behavior, enable this setting. Existing tokens will '
+                'not be deleted when this setting is toggled off.'),
+    category=_('Authentication'),
+    category_slug='authentication',
+)
+register(
+    'LOGIN_REDIRECT_OVERRIDE',
+    field_class=fields.CharField,
+    allow_blank=True,
+    required=False,
+    label=_('Login redirect override URL'),
+    help_text=_('URL to which unauthorized users will be redirected to log in. '
+                'If blank, users will be sent to the Tower login page.'),
     category=_('Authentication'),
     category_slug='authentication',
 )

awx/api/exceptions.py

@@ -12,7 +12,11 @@ class ActiveJobConflict(ValidationError):
     status_code = 409
 
     def __init__(self, active_jobs):
-        super(ActiveJobConflict, self).__init__({
+        # During APIException.__init__(), Django Rest Framework
+        # turn everything in self.detail into string by using force_text.
+        # Declare detail afterwards circumvent this behavior.
+        super(ActiveJobConflict, self).__init__()
+        self.detail = {
             "error": _("Resource is being used by running jobs."),
             "active_jobs": active_jobs
-        })
+        }

awx/api/fields.py

@@ -80,7 +80,7 @@ class OAuth2ProviderField(fields.DictField):
     default_error_messages = {
         'invalid_key_names': _('Invalid key names: {invalid_key_names}'),
     }
-    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS'}
+    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS', 'REFRESH_TOKEN_EXPIRE_SECONDS'}
     child = fields.IntegerField(min_value=1)
 
     def to_internal_value(self, data):
@@ -97,10 +97,14 @@ class DeprecatedCredentialField(serializers.IntegerField):
         kwargs['allow_null'] = True
         kwargs['default'] = None
         kwargs['min_value'] = 1
-        kwargs['help_text'] = 'This resource has been deprecated and will be removed in a future release'
+        kwargs.setdefault('help_text', 'This resource has been deprecated and will be removed in a future release')
         super(DeprecatedCredentialField, self).__init__(**kwargs)
 
     def to_internal_value(self, pk):
+        try:
+            pk = int(pk)
+        except ValueError:
+            self.fail('invalid')
         try:
             Credential.objects.get(pk=pk)
         except ObjectDoesNotExist:

awx/api/filters.py

@@ -4,6 +4,7 @@
 # Python
 import re
 import json
+from functools import reduce
 
 # Django
 from django.core.exceptions import FieldError, ValidationError
@@ -23,21 +24,6 @@ from rest_framework.filters import BaseFilterBackend
 # AWX
 from awx.main.utils import get_type_for_model, to_python_boolean
 from awx.main.utils.db import get_all_field_names
-from awx.main.models.credential import CredentialType
-from awx.main.models.rbac import RoleAncestorEntry
-
-
-class V1CredentialFilterBackend(BaseFilterBackend):
-    '''
-    For /api/v1/ requests, filter out v2 (custom) credentials
-    '''
-
-    def filter_queryset(self, request, queryset, view):
-        # TODO: remove in 3.3
-        from awx.api.versioning import get_request_version
-        if get_request_version(request) == 1:
-            queryset = queryset.filter(credential_type__managed_by_tower=True)
-        return queryset
 
 
 class TypeFilterBackend(BaseFilterBackend):
@@ -65,7 +51,7 @@ class TypeFilterBackend(BaseFilterBackend):
                 model = queryset.model
                 model_type = get_type_for_model(model)
                 if 'polymorphic_ctype' in get_all_field_names(model):
-                    types_pks = set([v for k,v in types_map.items() if k in types])
+                    types_pks = set([v for k, v in types_map.items() if k in types])
                     queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
                 elif model_type in types:
                     queryset = queryset
@@ -77,13 +63,70 @@ class TypeFilterBackend(BaseFilterBackend):
             raise ParseError(*e.args)
 
 
+def get_field_from_path(model, path):
+    '''
+    Given a Django ORM lookup path (possibly over multiple models)
+    Returns the last field in the line, and also the revised lookup path
+    ex., given
+        model=Organization
+        path='project__timeout'
+    returns tuple of field at the end of the line as well as a corrected
+    path, for special cases we do substitutions
+        (<IntegerField for timeout>, 'project__timeout')
+    '''
+    # Store of all the fields used to detect repeats
+    field_set = set([])
+    new_parts = []
+    for name in path.split('__'):
+        if model is None:
+            raise ParseError(_('No related model for field {}.').format(name))
+        # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
+        if model._meta.object_name in ('Project', 'InventorySource'):
+            name = {
+                'current_update': 'current_job',
+                'last_update': 'last_job',
+                'last_update_failed': 'last_job_failed',
+                'last_updated': 'last_job_run',
+            }.get(name, name)
+
+        if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model):
+            name = 'polymorphic_ctype'
+            new_parts.append('polymorphic_ctype__model')
+        else:
+            new_parts.append(name)
+
+        if name in getattr(model, 'PASSWORD_FIELDS', ()):
+            raise PermissionDenied(_('Filtering on password fields is not allowed.'))
+        elif name == 'pk':
+            field = model._meta.pk
+        else:
+            name_alt = name.replace("_", "")
+            if name_alt in model._meta.fields_map.keys():
+                field = model._meta.fields_map[name_alt]
+                new_parts.pop()
+                new_parts.append(name_alt)
+            else:
+                field = model._meta.get_field(name)
+            if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
+                raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
+            elif getattr(field, '__prevent_search__', False):
+                raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
+        if field in field_set:
+            # Field traversed twice, could create infinite JOINs, DoSing Tower
+            raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
+        field_set.add(field)
+        model = getattr(field, 'related_model', None)
+
+    return field, '__'.join(new_parts)
+
+
 class FieldLookupBackend(BaseFilterBackend):
     '''
     Filter using field lookups provided via query string parameters.
     '''
 
     RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
-                      'search', 'type', 'host_filter')
+                      'search', 'type', 'host_filter', 'count_disabled', 'no_truncate')
 
     SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                          'startswith', 'istartswith', 'endswith', 'iendswith',
@@ -91,61 +134,23 @@ class FieldLookupBackend(BaseFilterBackend):
                          'isnull', 'search')
 
     def get_field_from_lookup(self, model, lookup):
-        field = None
-        parts = lookup.split('__')
-        if parts and parts[-1] not in self.SUPPORTED_LOOKUPS:
-            parts.append('exact')
+
+        if '__' in lookup and lookup.rsplit('__', 1)[-1] in self.SUPPORTED_LOOKUPS:
+            path, suffix = lookup.rsplit('__', 1)
+        else:
+            path = lookup
+            suffix = 'exact'
+
+        if not path:
+            raise ParseError(_('Query string field name not provided.'))
+
         # FIXME: Could build up a list of models used across relationships, use
         # those lookups combined with request.user.get_queryset(Model) to make
         # sure user cannot query using objects he could not view.
-        new_parts = []
+        field, new_path = get_field_from_path(model, path)
 
-        # Store of all the fields used to detect repeats
-        field_set = set([])
-
-        for name in parts[:-1]:
-            # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
-            if model._meta.object_name in ('Project', 'InventorySource'):
-                name = {
-                    'current_update': 'current_job',
-                    'last_update': 'last_job',
-                    'last_update_failed': 'last_job_failed',
-                    'last_updated': 'last_job_run',
-                }.get(name, name)
-
-            if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model):
-                name = 'polymorphic_ctype'
-                new_parts.append('polymorphic_ctype__model')
-            else:
-                new_parts.append(name)
-
-            if name in getattr(model, 'PASSWORD_FIELDS', ()):
-                raise PermissionDenied(_('Filtering on password fields is not allowed.'))
-            elif name == 'pk':
-                field = model._meta.pk
-            else:
-                name_alt = name.replace("_", "")
-                if name_alt in model._meta.fields_map.keys():
-                    field = model._meta.fields_map[name_alt]
-                    new_parts.pop()
-                    new_parts.append(name_alt)
-                else:
-                    field = model._meta.get_field(name)
-                if 'auth' in name or 'token' in name:
-                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
-                if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
-                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
-                elif getattr(field, '__prevent_search__', False):
-                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
-            if field in field_set:
-                # Field traversed twice, could create infinite JOINs, DoSing Tower
-                raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
-            field_set.add(field)
-            model = getattr(field, 'related_model', None) or field.model
-
-        if parts:
-            new_parts.append(parts[-1])
-        new_lookup = '__'.join(new_parts)
+        new_lookup = new_path
+        new_lookup = '__'.join([new_path, suffix])
         return field, new_lookup
 
     def to_python_related(self, value):
@@ -173,7 +178,7 @@ class FieldLookupBackend(BaseFilterBackend):
 
     def value_to_python(self, model, lookup, value):
         try:
-            lookup = lookup.encode("ascii")
+            lookup.encode("ascii")
         except UnicodeEncodeError:
             raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
 
@@ -204,7 +209,7 @@ class FieldLookupBackend(BaseFilterBackend):
                 raise ValueError('%s is not searchable' % new_lookup[:-8])
             new_lookups = []
             for rm_field in related_model._meta.fields:
-                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description'):
+                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'):
                     new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
             return value, new_lookups
         else:
@@ -219,7 +224,11 @@ class FieldLookupBackend(BaseFilterBackend):
             or_filters = []
             chain_filters = []
             role_filters = []
-            search_filters = []
+            search_filters = {}
+            # Can only have two values: 'AND', 'OR'
+            # If 'AND' is used, an iterm must satisfy all condition to show up in the results.
+            # If 'OR' is used, an item just need to satisfy one condition to appear in results.
+            search_filter_relation = 'OR'
             for key, values in request.query_params.lists():
                 if key in self.RESERVED_NAMES:
                     continue
@@ -243,11 +252,13 @@ class FieldLookupBackend(BaseFilterBackend):
 
                 # Search across related objects.
                 if key.endswith('__search'):
+                    if values and ',' in values[0]:
+                        search_filter_relation = 'AND'
+                        values = reduce(lambda list1, list2: list1 + list2, [i.split(',') for i in values])
                     for value in values:
                         search_value, new_keys = self.value_to_python(queryset.model, key, force_text(value))
                         assert isinstance(new_keys, list)
-                        for new_key in new_keys:
-                            search_filters.append((new_key, search_value))
+                        search_filters[search_value] = new_keys
                     continue
 
                 # Custom chain__ and or__ filters, mutually exclusive (both can
@@ -267,39 +278,6 @@ class FieldLookupBackend(BaseFilterBackend):
                     key = key[5:]
                     q_not = True
 
-                # Make legacy v1 Job/Template fields work for backwards compatability
-                # TODO: remove after API v1 deprecation period
-                if queryset.model._meta.object_name in ('JobTemplate', 'Job') and key in (
-                        'credential', 'vault_credential', 'cloud_credential', 'network_credential'
-                ) or queryset.model._meta.object_name in ('InventorySource', 'InventoryUpdate') and key == 'credential':
-                    key = 'credentials'
-
-                # Make legacy v1 Credential fields work for backwards compatability
-                # TODO: remove after API v1 deprecation period
-                #
-                # convert v1 `Credential.kind` queries to `Credential.credential_type__pk`
-                if queryset.model._meta.object_name == 'Credential' and key == 'kind':
-                    key = key.replace('kind', 'credential_type')
-
-                    if 'ssh' in values:
-                        # In 3.2, SSH and Vault became separate credential types, but in the v1 API,
-                        # they're both still "kind=ssh"
-                        # under the hood, convert `/api/v1/credentials/?kind=ssh` to
-                        # `/api/v1/credentials/?or__credential_type=<ssh_pk>&or__credential_type=<vault_pk>`
-                        values = set(values)
-                        values.add('vault')
-                        values = list(values)
-                        q_or = True
-
-                    for i, kind in enumerate(values):
-                        if kind == 'vault':
-                            type_ = CredentialType.objects.get(kind=kind)
-                        else:
-                            type_ = CredentialType.from_v1_kind(kind)
-                        if type_ is None:
-                            raise ParseError(_('cannot filter on kind %s') % kind)
-                        values[i] = type_.pk
-
                 # Convert value(s) to python and add to the appropriate list.
                 for value in values:
                     if q_int:
@@ -321,12 +299,12 @@ class FieldLookupBackend(BaseFilterBackend):
                     else:
                         args.append(Q(**{k:v}))
                 for role_name in role_filters:
+                    if not hasattr(queryset.model, 'accessible_pk_qs'):
+                        raise ParseError(_(
+                            'Cannot apply role_level filter to this list because its model '
+                            'does not use roles for access control.'))
                     args.append(
-                        Q(pk__in=RoleAncestorEntry.objects.filter(
-                            ancestor__in=request.user.roles.all(),
-                            content_type_id=ContentType.objects.get_for_model(queryset.model).id,
-                            role_field=role_name
-                        ).values_list('object_id').distinct())
+                        Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name))
                     )
                 if or_filters:
                     q = Q()
@@ -336,11 +314,18 @@ class FieldLookupBackend(BaseFilterBackend):
                         else:
                             q |= Q(**{k:v})
                     args.append(q)
-                if search_filters:
+                if search_filters and search_filter_relation == 'OR':
                     q = Q()
-                    for k,v in search_filters:
-                        q |= Q(**{k:v})
+                    for term, constrains in search_filters.items():
+                        for constrain in constrains:
+                            q |= Q(**{constrain: term})
                     args.append(q)
+                elif search_filters and search_filter_relation == 'AND':
+                    for term, constrains in search_filters.items():
+                        q_chain = Q()
+                        for constrain in constrains:
+                            q_chain |= Q(**{constrain: term})
+                        queryset = queryset.filter(q_chain)
                 for n,k,v in chain_filters:
                     if n:
                         q = ~Q(**{k:v})
@@ -370,8 +355,10 @@ class OrderByBackend(BaseFilterBackend):
                         order_by = value.split(',')
                     else:
                         order_by = (value,)
+            if order_by is None:
+                order_by = self.get_default_ordering(view)
             if order_by:
-                order_by = self._strip_sensitive_model_fields(queryset.model, order_by)
+                order_by = self._validate_ordering_fields(queryset.model, order_by)
 
                 # Special handling of the type field for ordering. In this
                 # case, we're not sorting exactly on the type field, but
@@ -396,15 +383,23 @@ class OrderByBackend(BaseFilterBackend):
             # Return a 400 for invalid field names.
             raise ParseError(*e.args)
 
-    def _strip_sensitive_model_fields(self, model, order_by):
+    def get_default_ordering(self, view):
+        ordering = getattr(view, 'ordering', None)
+        if isinstance(ordering, str):
+            return (ordering,)
+        return ordering
+
+    def _validate_ordering_fields(self, model, order_by):
         for field_name in order_by:
             # strip off the negation prefix `-` if it exists
-            _field_name = field_name.split('-')[-1]
+            prefix = ''
+            path = field_name
+            if field_name[0] == '-':
+                prefix = field_name[0]
+                path = field_name[1:]
             try:
-                # if the field name is encrypted/sensitive, don't sort on it
-                if _field_name in getattr(model, 'PASSWORD_FIELDS', ()) or \
-                        getattr(model._meta.get_field(_field_name), '__prevent_search__', False):
-                    raise ParseError(_('cannot order by field %s') % _field_name)
-            except FieldDoesNotExist:
-                pass
-            yield field_name
+                field, new_path = get_field_from_path(model, path)
+                new_path = '{}{}'.format(prefix, new_path)
+            except (FieldError, FieldDoesNotExist) as e:
+                raise ParseError(e.args[0])
+            yield new_path

awx/api/generics.py

@@ -5,7 +5,7 @@
 import inspect
 import logging
 import time
-import six
+import urllib.parse
 
 # Django
 from django.conf import settings
@@ -22,25 +22,32 @@ from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth import views as auth_views
 
 # Django REST Framework
-from rest_framework.authentication import get_authorization_header
-from rest_framework.exceptions import PermissionDenied, AuthenticationFailed, ParseError
+from rest_framework.exceptions import PermissionDenied, AuthenticationFailed, ParseError, NotAcceptable, UnsupportedMediaType
 from rest_framework import generics
 from rest_framework.response import Response
 from rest_framework import status
 from rest_framework import views
 from rest_framework.permissions import AllowAny
-
-# cryptography
-from cryptography.fernet import InvalidToken
+from rest_framework.renderers import StaticHTMLRenderer, JSONRenderer
+from rest_framework.negotiation import DefaultContentNegotiation
 
 # AWX
 from awx.api.filters import FieldLookupBackend
-from awx.main.models import *  # noqa
+from awx.main.models import (
+    UnifiedJob, UnifiedJobTemplate, User, Role, Credential,
+    WorkflowJobTemplateNode, WorkflowApprovalTemplate
+)
 from awx.main.access import access_registry
-from awx.main.utils import * # noqa
+from awx.main.utils import (
+    camelcase_to_underscore,
+    get_search_fields,
+    getattrd,
+    get_object_or_400,
+    decrypt_field
+)
 from awx.main.utils.db import get_all_field_names
-from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer
-from awx.api.versioning import URLPathVersioning, get_request_version
+from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer, UserSerializer
+from awx.api.versioning import URLPathVersioning
 from awx.api.metadata import SublistAttachDetatchMetadata, Metadata
 
 __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView',
@@ -54,7 +61,7 @@ __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView',
            'ParentMixin',
            'DeleteLastUnattachLabelMixin',
            'SubListAttachDetachAPIView',
-           'CopyAPIView']
+           'CopyAPIView', 'BaseUsersList',]
 
 logger = logging.getLogger('awx.api.generics')
 analytics_logger = logging.getLogger('awx.analytics.performance')
@@ -62,16 +69,40 @@ analytics_logger = logging.getLogger('awx.analytics.performance')
 
 class LoggedLoginView(auth_views.LoginView):
 
+    def get(self, request, *args, **kwargs):
+        # The django.auth.contrib login form doesn't perform the content
+        # negotiation we've come to expect from DRF; add in code to catch
+        # situations where Accept != text/html (or */*) and reply with
+        # an HTTP 406
+        try:
+            DefaultContentNegotiation().select_renderer(
+                request,
+                [StaticHTMLRenderer],
+                'html'
+            )
+        except NotAcceptable:
+            resp = Response(status=status.HTTP_406_NOT_ACCEPTABLE)
+            resp.accepted_renderer = StaticHTMLRenderer()
+            resp.accepted_media_type = 'text/plain'
+            resp.renderer_context = {}
+            return resp
+        return super(LoggedLoginView, self).get(request, *args, **kwargs)
+
     def post(self, request, *args, **kwargs):
-        original_user = getattr(request, 'user', None)
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
-
-        if current_user and getattr(current_user, 'pk', None) and current_user != original_user:
-            logger.info("User {} logged in.".format(current_user.username))
         if request.user.is_authenticated:
+            logger.info(smart_text(u"User {} logged in from {}".format(self.request.user.username,request.META.get('REMOTE_ADDR', None))))
+            ret.set_cookie('userLoggedIn', 'true')
+            current_user = UserSerializer(self.request.user)
+            current_user = smart_text(JSONRenderer().render(current_user.data))
+            current_user = urllib.parse.quote('%s' % current_user, '')
+            ret.set_cookie('current_user', current_user, secure=settings.SESSION_COOKIE_SECURE or None)
+
             return ret
         else:
+            if 'username' in self.request.POST:
+                logger.warn(smart_text(u"Login failed for user {} from {}".format(self.request.POST.get('username'),request.META.get('REMOTE_ADDR', None))))
             ret.status_code = 401
             return ret
 
@@ -82,45 +113,19 @@ class LoggedLogoutView(auth_views.LogoutView):
         original_user = getattr(request, 'user', None)
         ret = super(LoggedLogoutView, self).dispatch(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
+        ret.set_cookie('userLoggedIn', 'false')
         if (not current_user or not getattr(current_user, 'pk', True)) \
                 and current_user != original_user:
             logger.info("User {} logged out.".format(original_user.username))
         return ret
 
 
-def get_view_name(cls, suffix=None):
-    '''
-    Wrapper around REST framework get_view_name() to support get_name() method
-    and view_name property on a view class.
-    '''
-    name = ''
-    if hasattr(cls, 'get_name') and callable(cls.get_name):
-        name = cls().get_name()
-    elif hasattr(cls, 'view_name'):
-        if callable(cls.view_name):
-            name = cls.view_name()
-        else:
-            name = cls.view_name
-    if name:
-        return ('%s %s' % (name, suffix)) if suffix else name
-    return views.get_view_name(cls, suffix=None)
+def get_view_description(view, html=False):
+    '''Wrapper around REST framework get_view_description() to continue
+    to support our historical div.
 
-
-def get_view_description(cls, request, html=False):
     '''
-    Wrapper around REST framework get_view_description() to support
-    get_description() method and view_description property on a view class.
-    '''
-    if hasattr(cls, 'get_description') and callable(cls.get_description):
-        desc = cls().get_description(request, html=html)
-        cls = type(cls.__name__, (object,), {'__doc__': desc})
-    elif hasattr(cls, 'view_description'):
-        if callable(cls.view_description):
-            view_desc = cls.view_description()
-        else:
-            view_desc = cls.view_description
-        cls = type(cls.__name__, (object,), {'__doc__': view_desc})
-    desc = views.get_view_description(cls, html=html)
+    desc = views.get_view_description(view, html=html)
     if html:
         desc = '<div class="description">%s</div>' % desc
     return mark_safe(desc)
@@ -165,9 +170,13 @@ class APIView(views.APIView):
             request.drf_request_user = getattr(drf_request, 'user', False)
         except AuthenticationFailed:
             request.drf_request_user = None
-        except ParseError as exc:
+        except (PermissionDenied, ParseError) as exc:
             request.drf_request_user = None
             self.__init_request_error__ = exc
+        except UnsupportedMediaType as exc:
+            exc.detail = _('You did not use correct Content-Type in your HTTP request. '
+                           'If you are using our REST API, the Content-Type must be application/json')
+            self.__init_request_error__ = exc
         return drf_request
 
     def finalize_response(self, request, response, *args, **kwargs):
@@ -180,6 +189,7 @@ class APIView(views.APIView):
             if hasattr(self, '__init_request_error__'):
                 response = self.handle_exception(self.__init_request_error__)
             if response.status_code == 401:
+                response.data['detail'] += ' To establish a login session, visit /api/login/.'
                 logger.info(status_msg)
             else:
                 logger.warn(status_msg)
@@ -195,37 +205,41 @@ class APIView(views.APIView):
             response['X-API-Query-Count'] = len(q_times)
             response['X-API-Query-Time'] = '%0.3fs' % sum(q_times)
 
+        if getattr(self, 'deprecated', False):
+            response['Warning'] = '299 awx "This resource has been deprecated and will be removed in a future release."'  # noqa
+
         return response
 
     def get_authenticate_header(self, request):
-        """
-        Determine the WWW-Authenticate header to use for 401 responses.  Try to
-        use the request header as an indication for which authentication method
-        was attempted.
-        """
-        for authenticator in self.get_authenticators():
-            resp_hdr = authenticator.authenticate_header(request)
-            if not resp_hdr:
-                continue
-            req_hdr = get_authorization_header(request)
-            if not req_hdr:
-                continue
-            if resp_hdr.split()[0] and resp_hdr.split()[0] == req_hdr.split()[0]:
-                return resp_hdr
-        # If it can't be determined from the request, use the last
-        # authenticator (should be Basic).
-        try:
-            return authenticator.authenticate_header(request)
-        except NameError:
-            pass
-
-    def get_view_description(self, html=False):
-        """
-        Return some descriptive text for the view, as used in OPTIONS responses
-        and in the browsable API.
-        """
-        func = self.settings.VIEW_DESCRIPTION_FUNCTION
-        return func(self.__class__, getattr(self, '_request', None), html)
+        # HTTP Basic auth is insecure by default, because the basic auth
+        # backend does not provide CSRF protection.
+        #
+        # If you visit `/api/v2/job_templates/` and we return
+        # `WWW-Authenticate: Basic ...`, your browser will prompt you for an
+        # HTTP basic auth username+password and will store it _in the browser_
+        # for subsequent requests.  Because basic auth does not require CSRF
+        # validation (because it's commonly used with e.g., tower-cli and other
+        # non-browser clients), browsers that save basic auth in this way are
+        # vulnerable to cross-site request forgery:
+        #
+        # 1. Visit `/api/v2/job_templates/` and specify a user+pass for basic auth.
+        # 2. Visit a nefarious website and submit a
+        #    `<form action='POST' method='https://tower.example.org/api/v2/job_templates/N/launch/'>`
+        # 3. The browser will use your persisted user+pass and your login
+        #    session is effectively hijacked.
+        #
+        # To prevent this, we will _no longer_ send `WWW-Authenticate: Basic ...`
+        # headers in responses; this means that unauthenticated /api/v2/... requests
+        # will now return HTTP 401 in-browser, rather than popping up an auth dialog.
+        #
+        # This means that people who wish to use the interactive API browser
+        # must _first_ login in via `/api/login/` to establish a session (which
+        # _does_ enforce CSRF).
+        #
+        # CLI users can _still_ specify basic auth credentials explicitly via
+        # a header or in the URL e.g.,
+        # `curl https://user:pass@tower.example.org/api/v2/job_templates/N/launch/`
+        return 'Bearer realm=api authorization_url=/api/o/authorize/'
 
     def get_description_context(self):
         return {
@@ -235,20 +249,14 @@ class APIView(views.APIView):
             'swagger_method': getattr(self.request, 'swagger_method', None),
         }
 
-    def get_description(self, request, html=False):
-        self.request = request
+    @property
+    def description(self):
         template_list = []
         for klass in inspect.getmro(type(self)):
             template_basename = camelcase_to_underscore(klass.__name__)
             template_list.append('api/%s.md' % template_basename)
         context = self.get_description_context()
 
-        # "v2" -> 2
-        default_version = int(settings.REST_FRAMEWORK['DEFAULT_VERSION'].lstrip('v'))
-        request_version = get_request_version(self.request)
-        if request_version is not None and request_version < default_version:
-            context['deprecated'] = True
-
         description = render_to_string(template_list, context)
         if context.get('deprecated') and context.get('swagger_method') is None:
             # render deprecation messages at the very top
@@ -266,7 +274,7 @@ class APIView(views.APIView):
         # submitted data was rejected.
         request_method = getattr(self, '_raw_data_request_method', None)
         response_status = getattr(self, '_raw_data_response_status', 0)
-        if request_method in ('POST', 'PUT', 'PATCH') and response_status in xrange(400, 500):
+        if request_method in ('POST', 'PUT', 'PATCH') and response_status in range(400, 500):
             return self.request.data.copy()
 
         return data
@@ -288,6 +296,12 @@ class APIView(views.APIView):
                 kwargs.pop('version')
         return super(APIView, self).dispatch(request, *args, **kwargs)
 
+    def check_permissions(self, request):
+        if request.method not in ('GET', 'OPTIONS', 'HEAD'):
+            if 'write' not in getattr(request.user, 'oauth_scopes', ['write']):
+                raise PermissionDenied()
+        return super(APIView, self).check_permissions(request)
+
 
 class GenericAPIView(generics.GenericAPIView, APIView):
     # Base class for all model-based views.
@@ -303,7 +317,7 @@ class GenericAPIView(generics.GenericAPIView, APIView):
         # form.
         if hasattr(self, '_raw_data_form_marker'):
             # Always remove read only fields from serializer.
-            for name, field in serializer.fields.items():
+            for name, field in list(serializer.fields.items()):
                 if getattr(field, 'read_only', None):
                     del serializer.fields[name]
             serializer._data = self.update_raw_data(serializer.data)
@@ -338,14 +352,15 @@ class GenericAPIView(generics.GenericAPIView, APIView):
                     'model_verbose_name_plural': smart_text(self.model._meta.verbose_name_plural),
                 })
             serializer = self.get_serializer()
+            metadata = self.metadata_class()
+            metadata.request = self.request
             for method, key in [
                 ('GET', 'serializer_fields'),
                 ('POST', 'serializer_create_fields'),
                 ('PUT', 'serializer_update_fields')
             ]:
-                d[key] = self.metadata_class().get_serializer_info(serializer, method=method)
+                d[key] = metadata.get_serializer_info(serializer, method=method)
         d['settings'] = settings
-        d['has_named_url'] = self.model in settings.NAMED_URL_GRAPH
         return d
 
 
@@ -390,21 +405,21 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
                 continue
             if getattr(field, 'related_model', None):
                 fields.add('{}__search'.format(field.name))
-        for rel in self.model._meta.related_objects:
-            name = rel.related_name
-            if isinstance(rel, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
+        for related in self.model._meta.related_objects:
+            name = related.related_name
+            if isinstance(related, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
                 # Add underscores for polymorphic subclasses for user utility
-                name = rel.related_model._meta.verbose_name.replace(" ", "_")
+                name = related.related_model._meta.verbose_name.replace(" ", "_")
             if skip_related_name(name) or name.endswith('+'):
                 continue
             fields.add('{}__search'.format(name))
-        m2m_rel = []
-        m2m_rel += self.model._meta.local_many_to_many
+        m2m_related = []
+        m2m_related += self.model._meta.local_many_to_many
         if issubclass(self.model, UnifiedJobTemplate) and self.model != UnifiedJobTemplate:
-            m2m_rel += UnifiedJobTemplate._meta.local_many_to_many
+            m2m_related += UnifiedJobTemplate._meta.local_many_to_many
         if issubclass(self.model, UnifiedJob) and self.model != UnifiedJob:
-            m2m_rel += UnifiedJob._meta.local_many_to_many
-        for relationship in m2m_rel:
+            m2m_related += UnifiedJob._meta.local_many_to_many
+        for relationship in m2m_related:
             if skip_related_name(relationship.name):
                 continue
             if relationship.related_model._meta.app_label != 'main':
@@ -477,9 +492,12 @@ class SubListAPIView(ParentMixin, ListAPIView):
         parent = self.get_parent_object()
         self.check_parent_access(parent)
         qs = self.request.user.get_queryset(self.model).distinct()
-        sublist_qs = getattrd(parent, self.relationship).distinct()
+        sublist_qs = self.get_sublist_queryset(parent)
         return qs & sublist_qs
 
+    def get_sublist_queryset(self, parent):
+        return getattrd(parent, self.relationship).distinct()
+
 
 class DestroyAPIView(generics.DestroyAPIView):
 
@@ -508,9 +526,8 @@ class SubListDestroyAPIView(DestroyAPIView, SubListAPIView):
 
     def perform_list_destroy(self, instance_list):
         if self.check_sub_obj_permission:
-            # Check permissions for all before deleting, avoiding half-deleted lists
             for instance in instance_list:
-                if self.has_delete_permission(instance):
+                if not self.has_delete_permission(instance):
                     raise PermissionDenied()
         for instance in instance_list:
             self.perform_destroy(instance, check_permission=False)
@@ -557,7 +574,7 @@ class SubListCreateAPIView(SubListAPIView, ListCreateAPIView):
                             status=status.HTTP_400_BAD_REQUEST)
 
         # Verify we have permission to add the object as given.
-        if not request.user.can_access(self.model, 'add', serializer.initial_data):
+        if not request.user.can_access(self.model, 'add', serializer.validated_data):
             raise PermissionDenied()
 
         # save the object through the serializer, reload and returned the saved
@@ -705,7 +722,7 @@ class SubListAttachDetachAPIView(SubListCreateAttachDetachAPIView):
     def update_raw_data(self, data):
         request_method = getattr(self, '_raw_data_request_method', None)
         response_status = getattr(self, '_raw_data_response_status', 0)
-        if request_method == 'POST' and response_status in xrange(400, 500):
+        if request_method == 'POST' and response_status in range(400, 500):
             return super(SubListAttachDetachAPIView, self).update_raw_data(data)
         return {'id': None}
 
@@ -716,6 +733,7 @@ class DeleteLastUnattachLabelMixin(object):
     when the last disassociate is called should inherit from this class. Further,
     the model should implement is_detached()
     '''
+
     def unattach(self, request, *args, **kwargs):
         (sub_id, res) = super(DeleteLastUnattachLabelMixin, self).unattach_validate(request)
         if res:
@@ -765,6 +783,7 @@ class RetrieveUpdateDestroyAPIView(RetrieveUpdateAPIView, DestroyAPIView):
 class ResourceAccessList(ParentMixin, ListAPIView):
 
     serializer_class = ResourceAccessListElementSerializer
+    ordering = ('username',)
 
     def get_queryset(self):
         obj = self.get_parent_object()
@@ -804,17 +823,20 @@ class CopyAPIView(GenericAPIView):
     def _decrypt_model_field_if_needed(obj, field_name, field_val):
         if field_name in getattr(type(obj), 'REENCRYPTION_BLACKLIST_AT_COPY', []):
             return field_val
-        if isinstance(field_val, dict):
+        if isinstance(obj, Credential) and field_name == 'inputs':
+            for secret in obj.credential_type.secret_fields:
+                if secret in field_val:
+                    field_val[secret] = decrypt_field(obj, secret)
+        elif isinstance(field_val, dict):
             for sub_field in field_val:
-                if isinstance(sub_field, six.string_types) \
-                        and isinstance(field_val[sub_field], six.string_types):
-                    try:
-                        field_val[sub_field] = decrypt_field(obj, field_name, sub_field)
-                    except InvalidToken:
-                        # Catching the corner case with v1 credential fields
-                        field_val[sub_field] = decrypt_field(obj, sub_field)
-        elif isinstance(field_val, six.string_types):
-            field_val = decrypt_field(obj, field_name)
+                if isinstance(sub_field, str) \
+                        and isinstance(field_val[sub_field], str):
+                    field_val[sub_field] = decrypt_field(obj, field_name, sub_field)
+        elif isinstance(field_val, str):
+            try:
+                field_val = decrypt_field(obj, field_name)
+            except AttributeError:
+                return field_val
         return field_val
 
     def _build_create_dict(self, obj):
@@ -867,14 +889,34 @@ class CopyAPIView(GenericAPIView):
                 create_kwargs[field.name] = CopyAPIView._decrypt_model_field_if_needed(
                     obj, field.name, field_val
                 )
+
+        # WorkflowJobTemplateNodes that represent an approval are *special*;
+        # when we copy them, we actually want to *copy* the UJT they point at
+        # rather than share the template reference between nodes in disparate
+        # workflows
+        if (
+            isinstance(obj, WorkflowJobTemplateNode) and
+            isinstance(getattr(obj, 'unified_job_template'), WorkflowApprovalTemplate)
+        ):
+            new_approval_template, sub_objs = CopyAPIView.copy_model_obj(
+                None, None, WorkflowApprovalTemplate,
+                obj.unified_job_template, creater
+            )
+            create_kwargs['unified_job_template'] = new_approval_template
+
         new_obj = model.objects.create(**create_kwargs)
+        logger.debug('Deep copy: Created new object {}({})'.format(
+            new_obj, model
+        ))
         # Need to save separatedly because Djang-crum get_current_user would
         # not work properly in non-request-response-cycle context.
         new_obj.created_by = creater
         new_obj.save()
-        for m2m in m2m_to_preserve:
-            for related_obj in m2m_to_preserve[m2m].all():
-                getattr(new_obj, m2m).add(related_obj)
+        from awx.main.signals import disable_activity_stream
+        with disable_activity_stream():
+            for m2m in m2m_to_preserve:
+                for related_obj in m2m_to_preserve[m2m].all():
+                    getattr(new_obj, m2m).add(related_obj)
         if not old_parent:
             sub_objects = []
             for o2m in o2m_to_preserve:
@@ -890,10 +932,17 @@ class CopyAPIView(GenericAPIView):
 
     def get(self, request, *args, **kwargs):
         obj = self.get_object()
+        if not request.user.can_access(obj.__class__, 'read', obj):
+            raise PermissionDenied()
         create_kwargs = self._build_create_dict(obj)
         for key in create_kwargs:
             create_kwargs[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key]
-        return Response({'can_copy': request.user.can_access(self.model, 'add', create_kwargs)})
+        try:
+            can_copy = request.user.can_access(self.model, 'add', create_kwargs) and \
+                request.user.can_access(self.model, 'copy_related', obj)
+        except PermissionDenied:
+            return Response({'can_copy': False})
+        return Response({'can_copy': can_copy})
 
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
@@ -903,6 +952,8 @@ class CopyAPIView(GenericAPIView):
             create_kwargs_check[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key]
         if not request.user.can_access(self.model, 'add', create_kwargs_check):
             raise PermissionDenied()
+        if not request.user.can_access(self.model, 'copy_related', obj):
+            raise PermissionDenied()
         serializer = self.get_serializer(data=request.data)
         if not serializer.is_valid():
             return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
@@ -910,7 +961,7 @@ class CopyAPIView(GenericAPIView):
             None, None, self.model, obj, request.user, create_kwargs=create_kwargs,
             copy_name=serializer.validated_data.get('name', '')
         )
-        if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role:
+        if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role.members.all():
             new_obj.admin_role.members.add(request.user)
         if sub_objs:
             permission_check_func = None
@@ -924,4 +975,24 @@ class CopyAPIView(GenericAPIView):
                 permission_check_func=permission_check_func
             )
         serializer = self._get_copy_return_serializer(new_obj)
-        return Response(serializer.data, status=status.HTTP_201_CREATED)
+        headers = {'Location': new_obj.get_absolute_url(request=request)}
+        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
+
+
+class BaseUsersList(SubListCreateAttachDetachAPIView):
+    def post(self, request, *args, **kwargs):
+        ret = super(BaseUsersList, self).post( request, *args, **kwargs)
+        if ret.status_code != 201:
+            return ret
+        try:
+            if ret.data is not None and request.data.get('is_system_auditor', False):
+                # This is a faux-field that just maps to checking the system
+                # auditor role member list.. unfortunately this means we can't
+                # set it on creation, and thus needs to be set here.
+                user = User.objects.get(id=ret.data['id'])
+                user.is_system_auditor = request.data['is_system_auditor']
+                ret.data['is_system_auditor'] = request.data['is_system_auditor']
+        except AttributeError as exc:
+            print(exc)
+            pass
+        return ret

awx/api/metadata.py

@@ -5,6 +5,8 @@ from collections import OrderedDict
 
 # Django
 from django.core.exceptions import PermissionDenied
+from django.db.models.fields import PositiveIntegerField, BooleanField
+from django.db.models.fields.related import ForeignKey
 from django.http import Http404
 from django.utils.encoding import force_text, smart_text
 from django.utils.translation import ugettext_lazy as _
@@ -14,10 +16,13 @@ from rest_framework import exceptions
 from rest_framework import metadata
 from rest_framework import serializers
 from rest_framework.relations import RelatedField, ManyRelatedField
+from rest_framework.fields import JSONField as DRFJSONField
 from rest_framework.request import clone_request
 
 # AWX
+from awx.main.fields import JSONField, ImplicitRoleField
 from awx.main.models import InventorySource, NotificationTemplate
+from awx.main.scheduler.kubernetes import PodManager
 
 
 class Metadata(metadata.SimpleMetadata):
@@ -62,14 +67,17 @@ class Metadata(metadata.SimpleMetadata):
                 opts = serializer.Meta.model._meta.concrete_model._meta
                 verbose_name = smart_text(opts.verbose_name)
                 field_info['help_text'] = field_help_text[field.field_name].format(verbose_name)
-            # If field is not part of the model, then show it as non-filterable
+
+            if field.field_name == 'type':
+                field_info['filterable'] = True
             else:
-                is_model_field = False
                 for model_field in serializer.Meta.model._meta.fields:
                     if field.field_name == model_field.name:
-                        is_model_field = True
+                        if getattr(model_field, '__accepts_json__', None):
+                            field_info['type'] = 'json'
+                        field_info['filterable'] = True
                         break
-                if not is_model_field:
+                else:
                     field_info['filterable'] = False
 
         # Indicate if a field has a default value.
@@ -113,15 +121,55 @@ class Metadata(metadata.SimpleMetadata):
             for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
                 field_info[notification_type_name] = notification_type_class.init_parameters
 
+        # Special handling of notification messages where the required properties
+        # are conditional on the type selected.
+        try:
+            view_model = field.context['view'].model
+        except (AttributeError, KeyError):
+            view_model = None
+        if view_model == NotificationTemplate and field.field_name == 'messages':
+            for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
+                field_info[notification_type_name] = notification_type_class.default_messages
+
+
         # Update type of fields returned...
+        model_field = None
+        if serializer and hasattr(serializer, 'Meta') and hasattr(serializer.Meta, 'model'):
+            try:
+                model_field = serializer.Meta.model._meta.get_field(field.field_name)
+            except Exception:
+                pass
         if field.field_name == 'type':
             field_info['type'] = 'choice'
-        elif field.field_name == 'url':
+        elif field.field_name in ('url', 'custom_virtualenv', 'token'):
             field_info['type'] = 'string'
         elif field.field_name in ('related', 'summary_fields'):
             field_info['type'] = 'object'
+        elif isinstance(field, PositiveIntegerField):
+            field_info['type'] = 'integer'
         elif field.field_name in ('created', 'modified'):
             field_info['type'] = 'datetime'
+        elif (
+            RelatedField in field.__class__.__bases__ or
+            isinstance(model_field, ForeignKey)
+        ):
+            field_info['type'] = 'id'
+        elif (
+            isinstance(field, JSONField) or
+            isinstance(model_field, JSONField) or
+            isinstance(field, DRFJSONField) or
+            isinstance(getattr(field, 'model_field', None), JSONField) or
+            field.field_name == 'credential_passwords'
+        ):
+            field_info['type'] = 'json'
+        elif (
+            isinstance(field, ManyRelatedField) and
+            field.field_name == 'credentials'
+            # launch-time credentials
+        ):
+            field_info['type'] = 'list_of_ids'
+        elif isinstance(model_field, BooleanField):
+            field_info['type'] = 'boolean'
 
         return field_info
 
@@ -156,10 +204,13 @@ class Metadata(metadata.SimpleMetadata):
             finally:
                 view.request = request
 
-            for field, meta in actions[method].items():
+            for field, meta in list(actions[method].items()):
                 if not isinstance(meta, dict):
                     continue
 
+                if field == "pod_spec_override":
+                    meta['default'] = PodManager().pod_definition
+
                 # Add type choices if available from the serializer.
                 if field == 'type' and hasattr(serializer, 'get_type_choices'):
                     meta['choices'] = serializer.get_type_choices()
@@ -212,6 +263,16 @@ class Metadata(metadata.SimpleMetadata):
         if getattr(view, 'related_search_fields', None):
             metadata['related_search_fields'] = view.related_search_fields
 
+        # include role names in metadata
+        roles = []
+        model = getattr(view, 'model', None)
+        if model:
+            for field in model._meta.get_fields():
+                if type(field) is ImplicitRoleField:
+                    roles.append(field.name)
+        if len(roles) > 0:
+            metadata['object_roles'] = roles
+
         from rest_framework import generics
         if isinstance(view, generics.ListAPIView) and hasattr(view, 'paginator'):
             metadata['max_page_size'] = view.paginator.max_page_size
@@ -231,28 +292,13 @@ class RoleMetadata(Metadata):
         return metadata
 
 
-# TODO: Tower 3.3 remove class and all uses in views.py when API v1 is removed
-class JobTypeMetadata(Metadata):
-        def get_field_info(self, field):
-            res = super(JobTypeMetadata, self).get_field_info(field)
-
-            if field.field_name == 'job_type':
-                index = 0
-                for choice in res['choices']:
-                    if choice[0] == 'scan':
-                        res['choices'].pop(index)
-                        break
-                    index += 1
-            return res
-
-
 class SublistAttachDetatchMetadata(Metadata):
 
     def determine_actions(self, request, view):
         actions = super(SublistAttachDetatchMetadata, self).determine_actions(request, view)
         method = 'POST'
         if method in actions:
-            for field in actions[method]:
+            for field in list(actions[method].keys()):
                 if field == 'id':
                     continue
                 actions[method].pop(field)

awx/api/metrics.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    MetricsView
+)
+
+
+urls = [
+    url(r'^$', MetricsView.as_view(), name='metrics_view'),
+]
+
+__all__ = ['urls']

awx/api/pagination.py

@@ -3,14 +3,28 @@
 
 # Django REST Framework
 from django.conf import settings
+from django.core.paginator import Paginator as DjangoPaginator
 from rest_framework import pagination
+from rest_framework.response import Response
 from rest_framework.utils.urls import replace_query_param
 
 
+class DisabledPaginator(DjangoPaginator):
+
+    @property
+    def num_pages(self):
+        return 1
+
+    @property
+    def count(self):
+        return 200
+
+
 class Pagination(pagination.PageNumberPagination):
 
     page_size_query_param = 'page_size'
     max_page_size = settings.MAX_PAGE_SIZE
+    count_disabled = False
 
     def get_next_link(self):
         if not self.page.has_next():
@@ -18,7 +32,7 @@ class Pagination(pagination.PageNumberPagination):
         url = self.request and self.request.get_full_path() or ''
         url = url.encode('utf-8')
         page_number = self.page.next_page_number()
-        return replace_query_param(url, self.page_query_param, page_number)
+        return replace_query_param(self.cap_page_size(url), self.page_query_param, page_number)
 
     def get_previous_link(self):
         if not self.page.has_previous():
@@ -26,4 +40,30 @@ class Pagination(pagination.PageNumberPagination):
         url = self.request and self.request.get_full_path() or ''
         url = url.encode('utf-8')
         page_number = self.page.previous_page_number()
-        return replace_query_param(url, self.page_query_param, page_number)
+        return replace_query_param(self.cap_page_size(url), self.page_query_param, page_number)
+
+    def cap_page_size(self, url):
+        if int(self.request.query_params.get(self.page_size_query_param, 0)) > self.max_page_size:
+            url = replace_query_param(url, self.page_size_query_param, self.max_page_size)
+        return url
+
+    def get_html_context(self):
+        context = super().get_html_context()
+        context['page_links'] = [pl._replace(url=self.cap_page_size(pl.url))
+                                 for pl in context['page_links']]
+
+        return context
+
+    def paginate_queryset(self, queryset, request, **kwargs):
+        self.count_disabled = 'count_disabled' in request.query_params
+        try:
+            if self.count_disabled:
+                self.django_paginator_class = DisabledPaginator
+            return super(Pagination, self).paginate_queryset(queryset, request, **kwargs)
+        finally:
+            self.django_paginator_class = DjangoPaginator
+
+    def get_paginated_response(self, data):
+        if self.count_disabled:
+            return Response({'results': data})
+        return super(Pagination, self).get_paginated_response(data)

awx/api/parsers.py

@@ -4,7 +4,7 @@ import json
 
 # Django
 from django.conf import settings
-from django.utils import six
+from django.utils.encoding import smart_str
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
@@ -25,7 +25,7 @@ class JSONParser(parsers.JSONParser):
         encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)
 
         try:
-            data = stream.read().decode(encoding)
+            data = smart_str(stream.read(), encoding=encoding)
             if not data:
                 return {}
             obj = json.loads(data, object_pairs_hook=OrderedDict)
@@ -33,4 +33,4 @@ class JSONParser(parsers.JSONParser):
                 raise ParseError(_('JSON parse error - not a JSON object'))
             return obj
         except ValueError as exc:
-            raise ParseError(_('JSON parse error - %s\nPossible cause: trailing comma.' % six.text_type(exc)))
+            raise ParseError(_('JSON parse error - %s\nPossible cause: trailing comma.' % str(exc)))

awx/api/permissions.py

@@ -9,15 +9,15 @@ from rest_framework.exceptions import MethodNotAllowed, PermissionDenied
 from rest_framework import permissions
 
 # AWX
-from awx.main.access import * # noqa
-from awx.main.models import * # noqa
+from awx.main.access import check_user_access
+from awx.main.models import Inventory, UnifiedJob
 from awx.main.utils import get_object_or_400
 
 logger = logging.getLogger('awx.api.permissions')
 
-__all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission',
+__all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission', 'VariableDataPermission',
            'TaskPermission', 'ProjectUpdatePermission', 'InventoryInventorySourcesUpdatePermission',
-           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission',]
+           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission', 'WorkflowApprovalPermission']
 
 
 class ModelAccessPermission(permissions.BasePermission):
@@ -74,12 +74,8 @@ class ModelAccessPermission(permissions.BasePermission):
             # FIXME: For some reason this needs to return True
             # because it is first called with obj=None?
             return True
-        if getattr(view, 'is_variable_data', False):
-            return check_user_access(request.user, view.model, 'change', obj,
-                                     dict(variables=request.data))
-        else:
-            return check_user_access(request.user, view.model, 'change', obj,
-                                     request.data)
+        return check_user_access(request.user, view.model, 'change', obj,
+                                 request.data)
 
     def check_patch_permissions(self, request, view, obj=None):
         return self.check_put_permissions(request, view, obj)
@@ -99,12 +95,11 @@ class ModelAccessPermission(permissions.BasePermission):
         '''
 
         # Don't allow anonymous users. 401, not 403, hence no raised exception.
-        if not request.user or request.user.is_anonymous():
+        if not request.user or request.user.is_anonymous:
             return False
 
         # Always allow superusers
-        if getattr(view, 'always_allow_superuser', True) and request.user.is_superuser \
-                and not hasattr(request.user, 'oauth_scopes'):
+        if getattr(view, 'always_allow_superuser', True) and request.user.is_superuser:
             return True
 
         # Check if view supports the request method before checking permission
@@ -164,6 +159,15 @@ class JobTemplateCallbackPermission(ModelAccessPermission):
             return True
 
 
+class VariableDataPermission(ModelAccessPermission):
+
+    def check_put_permissions(self, request, view, obj=None):
+        if not obj:
+            return True
+        return check_user_access(request.user, view.model, 'change', obj,
+                                 dict(variables=request.data))
+
+
 class TaskPermission(ModelAccessPermission):
     '''
     Permission checks used for API callbacks from running a task.
@@ -192,6 +196,17 @@ class TaskPermission(ModelAccessPermission):
             return False
 
 
+class WorkflowApprovalPermission(ModelAccessPermission):
+    '''
+    Permission check used by workflow `approval` and `deny` views to determine
+    who has access to approve and deny paused workflow nodes
+    '''
+
+    def check_post_permissions(self, request, view, obj=None):
+        approval = get_object_or_400(view.model, pk=view.kwargs['pk'])
+        return check_user_access(request.user, view.model, 'approve_or_deny', approval)
+
+
 class ProjectUpdatePermission(ModelAccessPermission):
     '''
     Permission check used by ProjectUpdateView to determine who can update projects
@@ -235,3 +250,7 @@ class InstanceGroupTowerPermission(ModelAccessPermission):
             return False
         return super(InstanceGroupTowerPermission, self).has_object_permission(request, view, obj)
 
+
+class WebhookKeyPermission(permissions.BasePermission):
+    def has_object_permission(self, request, view, obj):
+        return request.user.can_access(view.model, 'admin', obj, request.data)

awx/api/renderers.py

@@ -1,12 +1,13 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
+from django.utils.safestring import SafeText
+from prometheus_client.parser import text_string_to_metric_families
+
 # Django REST Framework
 from rest_framework import renderers
 from rest_framework.request import override_method
 
-import six
-
 
 class BrowsableAPIRenderer(renderers.BrowsableAPIRenderer):
     '''
@@ -20,6 +21,19 @@ class BrowsableAPIRenderer(renderers.BrowsableAPIRenderer):
             return renderers.JSONRenderer()
         return renderer
 
+    def get_content(self, renderer, data, accepted_media_type, renderer_context):
+        if isinstance(data, SafeText):
+            # Older versions of Django (pre-2.0) have a py3 bug which causes
+            # bytestrings marked as "safe" to not actually get _treated_ as
+            # safe; this causes certain embedded strings (like the stdout HTML
+            # view) to be improperly escaped
+            # see: https://github.com/ansible/awx/issues/3108
+            # https://code.djangoproject.com/ticket/28121
+            return data
+        return super(BrowsableAPIRenderer, self).get_content(renderer, data,
+                                                             accepted_media_type,
+                                                             renderer_context)
+
     def get_context(self, data, accepted_media_type, renderer_context):
         # Store the associated response status to know how to populate the raw
         # data form.
@@ -71,8 +85,8 @@ class PlainTextRenderer(renderers.BaseRenderer):
     format = 'txt'
 
     def render(self, data, media_type=None, renderer_context=None):
-        if not isinstance(data, six.string_types):
-            data = six.text_type(data)
+        if not isinstance(data, str):
+            data = str(data)
         return data.encode(self.charset)
 
 
@@ -90,3 +104,21 @@ class AnsiTextRenderer(PlainTextRenderer):
 class AnsiDownloadRenderer(PlainTextRenderer):
 
     format = "ansi_download"
+
+
+class PrometheusJSONRenderer(renderers.JSONRenderer):
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if isinstance(data, dict):
+            # HTTP errors are {'detail': ErrorDetail(string='...', code=...)}
+            return super(PrometheusJSONRenderer, self).render(
+                data, accepted_media_type, renderer_context
+            )
+        parsed_metrics = text_string_to_metric_families(data)
+        data = {}
+        for family in parsed_metrics:
+            for sample in family.samples:
+                data[sample[0]] = {"labels": sample[1], "value": sample[2]}
+        return super(PrometheusJSONRenderer, self).render(
+            data, accepted_media_type, renderer_context
+        )

awx/api/swagger.py

@@ -13,6 +13,17 @@ from rest_framework.views import APIView
 from rest_framework_swagger import renderers
 
 
+class SuperUserSchemaGenerator(SchemaGenerator):
+
+    def has_view_permissions(self, path, method, view):
+        #
+        # Generate the Swagger schema as if you were a superuser and
+        # permissions didn't matter; this short-circuits the schema path
+        # discovery to include _all_ potential paths in the API.
+        #
+        return True
+
+
 class AutoSchema(DRFAuthSchema):
 
     def get_link(self, path, method, base_url):
@@ -42,7 +53,6 @@ class AutoSchema(DRFAuthSchema):
         return link
 
     def get_description(self, path, method):
-        self.view._request = self.view.request
         setattr(self.view.request, 'swagger_method', method)
         description = super(AutoSchema, self).get_description(path, method)
         return description
@@ -59,7 +69,7 @@ class SwaggerSchemaView(APIView):
     ]
 
     def get(self, request):
-        generator = SchemaGenerator(
+        generator = SuperUserSchemaGenerator(
             title='Ansible Tower API',
             patterns=None,
             urlconf=None

awx/api/templates/api/_list_common.md

@@ -54,8 +54,6 @@ within all designated text fields of a model.
 
     ?search=findme
 
-_Added in AWX 1.4_
-
 (_Added in Ansible Tower 3.1.0_) Search across related fields:
 
     ?related__search=findme
@@ -84,7 +82,7 @@ To exclude results matching certain criteria, prefix the field parameter with
 
     ?not__field=value
 
-(_Added in AWX 1.4_) By default, all query string filters are AND'ed together, so
+By default, all query string filters are AND'ed together, so
 only the results matching *all* filters will be returned.  To combine results
 matching *any* one of multiple criteria, prefix each query string parameter
 with `or__`:

awx/api/templates/api/_schedule_detail.md

@@ -5,7 +5,7 @@ The following lists the expected format and details of our rrules:
 * INTERVAL is required
 * SECONDLY is not supported
 * TZID is not supported
-* RRULE must preceed the rule statements
+* RRULE must precede the rule statements
 * BYDAY is supported but not BYDAY with a numerical prefix
 * BYYEARDAY and BYWEEKNO are not supported
 * Only one rrule statement per schedule is supported

awx/api/templates/api/api_o_auth_authorization_root_view.md

@@ -29,17 +29,6 @@ to the redirect_uri specified in the application. The client application will th
 AWX will respond with the `access_token`, `token_type`, `refresh_token`, and `expires_in`. For more
 information on testing this flow, refer to [django-oauth-toolkit](http://django-oauth-toolkit.readthedocs.io/en/latest/tutorial/tutorial_01.html#test-your-authorization-server).
 
-## Create Token for an Application using Implicit grant type
-Suppose we have an application "admin's app" of grant type `implicit`.
-In API browser, first make sure the user is logged in via session auth, then visit authorization
-endpoint with given parameters:
-```text
-http://localhost:8013/api/o/authorize/?response_type=token&client_id=L0uQQWW8pKX51hoqIRQGsuqmIdPi2AcXZ9EJRGmj&scope=read
-```
-Here the value of `client_id` should be the same as that of `client_id` field of underlying application.
-On success, an authorization page should be displayed asking the logged in user to grant/deny the access token.
-Once the user clicks on 'grant', the API browser will try POSTing to the same endpoint with the same parameters 
-in POST body, on success a 302 redirect will be returned.  
 
 ## Create Token for an Application using Password grant type
 
@@ -56,6 +45,7 @@ For example:
 
 ```bash
 curl -X POST \
+  -H "Content-Type: application/x-www-form-urlencoded" \
   -d "grant_type=password&username=<username>&password=<password>&scope=read" \
   -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569e
 IaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
@@ -85,6 +75,7 @@ format:
 The `/api/o/token/` endpoint is used for refreshing access token:
 ```bash
 curl -X POST \
+  -H "Content-Type: application/x-www-form-urlencoded" \
   -d "grant_type=refresh_token&refresh_token=AL0NK9TTpv0qp54dGbC4VUZtsZ9r8z" \
   -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569eIaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
   http://localhost:8013/api/o/token/ -i
@@ -114,6 +105,7 @@ Revoking is done by POSTing to `/api/o/revoke_token/` with the token to revoke a
 
 ```bash
 curl -X POST -d "token=rQONsve372fQwuc2pn76k3IHDCYpi7" \
+  -H "Content-Type: application/x-www-form-urlencoded" \
   -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569eIaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
   http://localhost:8013/api/o/revoke_token/ -i
 ```

awx/api/templates/api/inventory_script_view.md

@@ -10,7 +10,7 @@ object containing groups, including the hosts, children and variables for each
 group.  The response data is equivalent to that returned by passing the
 `--list` argument to an inventory script.
 
-_(Added in AWX 1.3)_ Specify a query string of `?hostvars=1` to retrieve the JSON
+Specify a query string of `?hostvars=1` to retrieve the JSON
 object above including all host variables.  The `['_meta']['hostvars']` object
 in the response contains an entry for each host with its variables.  This
 response format can be used with Ansible 1.3 and later to avoid making a
@@ -18,11 +18,19 @@ separate API request for each host.  Refer to
 [Tuning the External Inventory Script](http://docs.ansible.com/developing_inventory.html#tuning-the-external-inventory-script)
 for more information on this feature.
 
-_(Added in AWX 1.4)_ By default, the inventory script will only return hosts that
+By default, the inventory script will only return hosts that
 are enabled in the inventory.  This feature allows disabled hosts to be skipped
 when running jobs without removing them from the inventory.  Specify a query
 string of `?all=1` to return all hosts, including disabled ones.
 
+Specify a query string of `?towervars=1` to add variables
+to the hostvars of each host that specifies its enabled state and database ID.
+
+Specify a query string of `?subset=slice2of5` to produce an inventory that
+has a restricted number of hosts according to the rules of job slicing.
+
+To apply multiple query strings, join them with the `&` character, like `?hostvars=1&all=1`.
+
 ## Host Response
 
 Make a GET request to this resource with a query string similar to

awx/api/templates/api/inventory_source_cancel.md

@@ -1,7 +1,7 @@
 # Cancel Inventory Update
 
 Make a GET request to this resource to determine if the inventory update can be
-cancelled.  The response will include the following field:
+canceled.  The response will include the following field:
 
 * `can_cancel`: Indicates whether this update can be canceled (boolean,
   read-only)

awx/api/templates/api/job_cancel.md

@@ -1,7 +1,7 @@
 {% ifmeth GET %}
-# Determine if a Job can be cancelled
+# Determine if a Job can be canceled
 
-Make a GET request to this resource to determine if the job can be cancelled.
+Make a GET request to this resource to determine if the job can be canceled.
 The response will include the following field:
 
 * `can_cancel`: Indicates whether this job can be canceled (boolean, read-only)

awx/api/templates/api/job_template_callback.md

@@ -8,15 +8,15 @@ job template.
 
 For example, using curl:
 
-    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY"}'  http://server/api/v1/job_templates/N/callback/
+    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY"}'  http://server/api/v2/job_templates/N/callback/
 
 Or using wget:
 
-    wget -O /dev/null --post-data='{"host_config_key": "HOST_CONFIG_KEY"}' --header=Content-Type:application/json http://server/api/v1/job_templates/N/callback/
+    wget -O /dev/null --post-data='{"host_config_key": "HOST_CONFIG_KEY"}' --header=Content-Type:application/json http://server/api/v2/job_templates/N/callback/
 
 You may also pass `extra_vars` to the callback:
 
-    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY", "extra_vars": {"key": "value"}}'  http://server/api/v1/job_templates/N/callback/
+    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY", "extra_vars": {"key": "value"}}'  http://server/api/v2/job_templates/N/callback/
     
 The response will return status 202 if the request is valid, 403 for an
 invalid host config key, or 400 if the host cannot be determined from the
@@ -30,7 +30,7 @@ A GET request may be used to verify that the correct host will be selected.
 This request must authenticate as a valid user with permission to edit the
 job template.  For example:
 
-    curl http://user:password@server/api/v1/job_templates/N/callback/
+    curl http://user:password@server/api/v2/job_templates/N/callback/
 
 The response will include the host config key as well as the host name(s)
 that would match the request:

awx/api/templates/api/job_template_survey_spec.md

@@ -48,7 +48,7 @@ Here is a more comprehensive example showing the various question types and thei
         	"min": 5,
         	"max": "",
         	"required": false,
-        	"default": "yes"
+        	"default": "Leeloo Minai Lekarariba-Laminai-Tchai Ekbat De Sebat"
             },
             {
         	"type": "text",
@@ -57,9 +57,9 @@ Here is a more comprehensive example showing the various question types and thei
         	"variable": "short_answer",
         	"choices": "",
         	"min": "",
-        	"max": 5,
+        	"max": 7,
         	"required": false,
-        	"default": "yes"
+        	"default": "leeloo"
             },
             {
         	"type": "text",
@@ -70,7 +70,7 @@ Here is a more comprehensive example showing the various question types and thei
         	"min": "",
         	"max": "",
         	"required": true,
-        	"default": "yes"
+        	"default": "NOT OPTIONAL"
             },
             {
         	"type": "multiplechoice",
@@ -81,7 +81,7 @@ Here is a more comprehensive example showing the various question types and thei
         	"min": "",
         	"max": "",
         	"required": false,
-        	"default": "yes"
+        	"default": "one"
             },
             {
         	"type": "multiselect",
@@ -92,7 +92,7 @@ Here is a more comprehensive example showing the various question types and thei
         	"min": "",
         	"max": "",
         	"required": false,
-        	"default": "yes"
+        	"default": "one\nthree"
             },
             {
                 "type": "integer",

awx/api/templates/api/project_update_cancel.md

@@ -1,7 +1,7 @@
 # Cancel Project Update
 
 Make a GET request to this resource to determine if the project update can be
-cancelled.  The response will include the following field:
+canceled.  The response will include the following field:
 
 * `can_cancel`: Indicates whether this update can be canceled (boolean,
   read-only)

awx/api/templates/api/retrieve_api_view.md

@@ -1,7 +1,3 @@
-{% if has_named_url %}
-### Note: starting from api v2, this resource object can be accessed via its named URL.
-{% endif %}
-
 # Retrieve {{ model_verbose_name|title|anora }}:
 
 Make GET request to this resource to retrieve a single {{ model_verbose_name }}

awx/api/templates/api/retrieve_destroy_api_view.md

@@ -1,7 +1,3 @@
-{% if has_named_url %}
-### Note: starting from api v2, this resource object can be accessed via its named URL.
-{% endif %}
-
 {% ifmeth GET %}
 # Retrieve {{ model_verbose_name|title|anora }}:
 

awx/api/templates/api/retrieve_update_api_view.md

@@ -1,7 +1,3 @@
-{% if has_named_url %}
-### Note: starting from api v2, this resource object can be accessed via its named URL.
-{% endif %}
-
 {% ifmeth GET %}
 # Retrieve {{ model_verbose_name|title|anora }}:
 

awx/api/templates/api/retrieve_update_destroy_api_view.md

@@ -1,7 +1,3 @@
-{% if has_named_url %}
-### Note: starting from api v2, this resource object can be accessed via its named URL.
-{% endif %}
-
 {% ifmeth GET %}
 # Retrieve {{ model_verbose_name|title|anora }}:
 

awx/api/templates/api/system_job_template_launch.md

@@ -3,7 +3,7 @@ Launch a Job Template:
 Make a POST request to this resource to launch the system job template.
 
 Variables specified inside of the parameter `extra_vars` are passed to the
-system job task as command line parameters. These tasks can be ran manually
+system job task as command line parameters. These tasks can be run manually
 on the host system via the `awx-manage` command.
 
 For example on `cleanup_jobs` and `cleanup_activitystream`:

awx/api/templates/api/team_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a Team:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected team.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/user_me_list.md

@@ -6,4 +6,4 @@ One result should be returned containing the following fields:
 
 {% include "api/_result_fields_common.md" %}
 
-Use the primary URL for the user (/api/v1/users/N/) to modify the user.
+Use the primary URL for the user (/api/v2/users/N/) to modify the user.

awx/api/templates/api/user_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a User:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected user.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/webhook_key_view.md

@@ -0,0 +1,12 @@
+Webhook Secret Key:
+
+Make a GET request to this resource to obtain the secret key for a job
+template or workflow job template configured to be triggered by
+webhook events.  The response will include the following fields:
+
+* `webhook_key`: Secret key that needs to be copied and added to the
+  webhook configuration of the service this template will be receiving
+  webhook events from (string, read-only)
+
+Make an empty POST request to this resource to generate a new
+replacement `webhook_key`.

awx/api/urls/__init__.py

@@ -4,4 +4,7 @@
 from __future__ import absolute_import, unicode_literals
 from .urls import urlpatterns
 
-__all__ = ['urlpatterns']
+__all__ = ['urlpatterns', 'app_name']
+
+
+app_name = 'api'

awx/api/urls/credential.py

@@ -12,6 +12,8 @@ from awx.api.views import (
     CredentialOwnerUsersList,
     CredentialOwnerTeamsList,
     CredentialCopy,
+    CredentialInputSourceSubList,
+    CredentialExternalTest,
 )
 
 
@@ -24,6 +26,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/owner_users/$', CredentialOwnerUsersList.as_view(), name='credential_owner_users_list'),
     url(r'^(?P<pk>[0-9]+)/owner_teams/$', CredentialOwnerTeamsList.as_view(), name='credential_owner_teams_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', CredentialCopy.as_view(), name='credential_copy'),
+    url(r'^(?P<pk>[0-9]+)/input_sources/$', CredentialInputSourceSubList.as_view(), name='credential_input_source_sublist'),
+    url(r'^(?P<pk>[0-9]+)/test/$', CredentialExternalTest.as_view(), name='credential_external_test'),
 ]
 
 __all__ = ['urls']

awx/api/urls/credential_input_source.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2019 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    CredentialInputSourceDetail,
+    CredentialInputSourceList,
+)
+
+
+urls = [
+    url(r'^$', CredentialInputSourceList.as_view(), name='credential_input_source_list'),
+    url(r'^(?P<pk>[0-9]+)/$', CredentialInputSourceDetail.as_view(), name='credential_input_source_detail'),
+]
+
+__all__ = ['urls']

awx/api/urls/credential_type.py

@@ -8,6 +8,7 @@ from awx.api.views import (
     CredentialTypeDetail,
     CredentialTypeCredentialList,
     CredentialTypeActivityStreamList,
+    CredentialTypeExternalTest,
 )
 
 
@@ -16,6 +17,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/$', CredentialTypeDetail.as_view(), name='credential_type_detail'),
     url(r'^(?P<pk>[0-9]+)/credentials/$', CredentialTypeCredentialList.as_view(), name='credential_type_credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', CredentialTypeActivityStreamList.as_view(), name='credential_type_activity_stream_list'),
+    url(r'^(?P<pk>[0-9]+)/test/$', CredentialTypeExternalTest.as_view(), name='credential_type_external_test'),
 ]
 
 __all__ = ['urls']

awx/api/urls/host.py

@@ -16,8 +16,6 @@ from awx.api.views import (
     HostSmartInventoriesList,
     HostAdHocCommandsList,
     HostAdHocCommandEventsList,
-    HostFactVersionsList,
-    HostFactCompareView,
     HostInsights,
 )
 
@@ -35,8 +33,6 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/smart_inventories/$', HostSmartInventoriesList.as_view(), name='host_smart_inventories_list'),
     url(r'^(?P<pk>[0-9]+)/ad_hoc_commands/$', HostAdHocCommandsList.as_view(), name='host_ad_hoc_commands_list'),
     url(r'^(?P<pk>[0-9]+)/ad_hoc_command_events/$', HostAdHocCommandEventsList.as_view(), name='host_ad_hoc_command_events_list'),
-    url(r'^(?P<pk>[0-9]+)/fact_versions/$', HostFactVersionsList.as_view(), name='host_fact_versions_list'),
-    url(r'^(?P<pk>[0-9]+)/fact_view/$', HostFactCompareView.as_view(), name='host_fact_compare_view'),
     url(r'^(?P<pk>[0-9]+)/insights/$', HostInsights.as_view(), name='host_insights'),
 ]
 

awx/api/urls/inventory_source.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     InventorySourceCredentialsList,
     InventorySourceGroupsList,
     InventorySourceHostsList,
-    InventorySourceNotificationTemplatesAnyList,
     InventorySourceNotificationTemplatesErrorList,
+    InventorySourceNotificationTemplatesStartedList,
     InventorySourceNotificationTemplatesSuccessList,
 )
 
@@ -29,8 +29,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', InventorySourceCredentialsList.as_view(), name='inventory_source_credentials_list'),
     url(r'^(?P<pk>[0-9]+)/groups/$', InventorySourceGroupsList.as_view(), name='inventory_source_groups_list'),
     url(r'^(?P<pk>[0-9]+)/hosts/$', InventorySourceHostsList.as_view(), name='inventory_source_hosts_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', InventorySourceNotificationTemplatesAnyList.as_view(),
-        name='inventory_source_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', InventorySourceNotificationTemplatesStartedList.as_view(),
+        name='inventory_source_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', InventorySourceNotificationTemplatesErrorList.as_view(),
         name='inventory_source_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', InventorySourceNotificationTemplatesSuccessList.as_view(),

awx/api/urls/job.py

@@ -6,7 +6,6 @@ from django.conf.urls import url
 from awx.api.views import (
     JobList,
     JobDetail,
-    JobStart,
     JobCancel,
     JobRelaunch,
     JobCreateSchedule,
@@ -23,7 +22,6 @@ from awx.api.views import (
 urls = [
     url(r'^$', JobList.as_view(), name='job_list'),
     url(r'^(?P<pk>[0-9]+)/$', JobDetail.as_view(), name='job_detail'),
-    url(r'^(?P<pk>[0-9]+)/start/$', JobStart.as_view(), name='job_start'),  # Todo: Remove In 3.3
     url(r'^(?P<pk>[0-9]+)/cancel/$', JobCancel.as_view(), name='job_cancel'),
     url(r'^(?P<pk>[0-9]+)/relaunch/$', JobRelaunch.as_view(), name='job_relaunch'),
     url(r'^(?P<pk>[0-9]+)/create_schedule/$', JobCreateSchedule.as_view(), name='job_create_schedule'),

awx/api/urls/job_template.py

@@ -1,19 +1,20 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     JobTemplateList,
     JobTemplateDetail,
     JobTemplateLaunch,
     JobTemplateJobsList,
+    JobTemplateSliceWorkflowJobsList,
     JobTemplateCallback,
     JobTemplateSchedulesList,
     JobTemplateSurveySpec,
     JobTemplateActivityStreamList,
-    JobTemplateNotificationTemplatesAnyList,
     JobTemplateNotificationTemplatesErrorList,
+    JobTemplateNotificationTemplatesStartedList,
     JobTemplateNotificationTemplatesSuccessList,
     JobTemplateInstanceGroupsList,
     JobTemplateAccessList,
@@ -28,12 +29,13 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/$', JobTemplateDetail.as_view(), name='job_template_detail'),
     url(r'^(?P<pk>[0-9]+)/launch/$', JobTemplateLaunch.as_view(), name='job_template_launch'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', JobTemplateJobsList.as_view(), name='job_template_jobs_list'),
+    url(r'^(?P<pk>[0-9]+)/slice_workflow_jobs/$', JobTemplateSliceWorkflowJobsList.as_view(), name='job_template_slice_workflow_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/callback/$', JobTemplateCallback.as_view(), name='job_template_callback'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', JobTemplateSchedulesList.as_view(), name='job_template_schedules_list'),
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', JobTemplateSurveySpec.as_view(), name='job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', JobTemplateActivityStreamList.as_view(), name='job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', JobTemplateNotificationTemplatesAnyList.as_view(),
-        name='job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', JobTemplateNotificationTemplatesStartedList.as_view(),
+        name='job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', JobTemplateNotificationTemplatesErrorList.as_view(),
         name='job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', JobTemplateNotificationTemplatesSuccessList.as_view(),
@@ -43,6 +45,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/oauth.py

@@ -1,18 +0,0 @@
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.
-
-from django.conf.urls import url
-
-from oauth2_provider.urls import base_urlpatterns
-
-from awx.api.views import (
-    ApiOAuthAuthorizationRootView,
-)
-
-
-urls = [
-    url(r'^$', ApiOAuthAuthorizationRootView.as_view(), name='oauth_authorization_root_view'),
-] + base_urlpatterns
-
-
-__all__ = ['urls']

awx/api/urls/oauth2.py

@@ -11,7 +11,6 @@ from awx.api.views import (
     OAuth2TokenList,
     OAuth2TokenDetail,
     OAuth2TokenActivityStreamList,
-    OAuth2PersonalTokenList
 )
 
 
@@ -42,8 +41,7 @@ urls = [
         r'^tokens/(?P<pk>[0-9]+)/activity_stream/$',
         OAuth2TokenActivityStreamList.as_view(),
         name='o_auth2_token_activity_stream_list'
-    ),
-    url(r'^personal_tokens/$', OAuth2PersonalTokenList.as_view(), name='o_auth2_personal_token_list'),    
+    ),   
 ]
 
 __all__ = ['urls']

awx/api/urls/oauth2_root.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from oauthlib import oauth2
+from oauth2_provider import views
+
+from awx.api.views import (
+    ApiOAuthAuthorizationRootView,
+)
+
+
+class TokenView(views.TokenView):
+
+    def create_token_response(self, request):
+        try:
+            return super(TokenView, self).create_token_response(request)
+        except oauth2.AccessDeniedError as e:
+            return request.build_absolute_uri(), {}, str(e), '403'
+
+
+urls = [
+    url(r'^$', ApiOAuthAuthorizationRootView.as_view(), name='oauth_authorization_root_view'),
+    url(r"^authorize/$", views.AuthorizationView.as_view(), name="authorize"),
+    url(r"^token/$", TokenView.as_view(), name="token"),
+    url(r"^revoke_token/$", views.RevokeTokenView.as_view(), name="revoke-token"),
+]
+
+
+__all__ = ['urls']

awx/api/urls/organization.py

@@ -15,9 +15,10 @@ from awx.api.views import (
     OrganizationCredentialList,
     OrganizationActivityStreamList,
     OrganizationNotificationTemplatesList,
-    OrganizationNotificationTemplatesAnyList,
     OrganizationNotificationTemplatesErrorList,
+    OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
+    OrganizationNotificationTemplatesApprovalList,
     OrganizationInstanceGroupsList,
     OrganizationObjectRolesList,
     OrganizationAccessList,
@@ -25,7 +26,7 @@ from awx.api.views import (
 )
 
 
-urls = [ 
+urls = [
     url(r'^$', OrganizationList.as_view(), name='organization_list'),
     url(r'^(?P<pk>[0-9]+)/$', OrganizationDetail.as_view(), name='organization_detail'),
     url(r'^(?P<pk>[0-9]+)/users/$', OrganizationUsersList.as_view(), name='organization_users_list'),
@@ -37,12 +38,14 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', OrganizationCredentialList.as_view(), name='organization_credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', OrganizationActivityStreamList.as_view(), name='organization_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates/$', OrganizationNotificationTemplatesList.as_view(), name='organization_notification_templates_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', OrganizationNotificationTemplatesAnyList.as_view(),
-        name='organization_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', OrganizationNotificationTemplatesStartedList.as_view(),
+        name='organization_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', OrganizationNotificationTemplatesErrorList.as_view(),
         name='organization_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', OrganizationNotificationTemplatesSuccessList.as_view(),
         name='organization_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', OrganizationNotificationTemplatesApprovalList.as_view(),
+        name='organization_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/instance_groups/$', OrganizationInstanceGroupsList.as_view(), name='organization_instance_groups_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', OrganizationObjectRolesList.as_view(), name='organization_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', OrganizationAccessList.as_view(), name='organization_access_list'),

awx/api/urls/project.py

@@ -14,8 +14,8 @@ from awx.api.views import (
     ProjectUpdatesList,
     ProjectActivityStreamList,
     ProjectSchedulesList,
-    ProjectNotificationTemplatesAnyList,
     ProjectNotificationTemplatesErrorList,
+    ProjectNotificationTemplatesStartedList,
     ProjectNotificationTemplatesSuccessList,
     ProjectObjectRolesList,
     ProjectAccessList,
@@ -34,10 +34,11 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/project_updates/$', ProjectUpdatesList.as_view(), name='project_updates_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', ProjectActivityStreamList.as_view(), name='project_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', ProjectSchedulesList.as_view(), name='project_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', ProjectNotificationTemplatesAnyList.as_view(), name='project_notification_templates_any_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', ProjectNotificationTemplatesErrorList.as_view(), name='project_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', ProjectNotificationTemplatesSuccessList.as_view(),
         name='project_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', ProjectNotificationTemplatesStartedList.as_view(),
+        name='project_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', ProjectObjectRolesList.as_view(), name='project_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', ProjectAccessList.as_view(), name='project_access_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', ProjectCopy.as_view(), name='project_copy'),

awx/api/urls/system_job_template.py

@@ -9,8 +9,8 @@ from awx.api.views import (
     SystemJobTemplateLaunch,
     SystemJobTemplateJobsList,
     SystemJobTemplateSchedulesList,
-    SystemJobTemplateNotificationTemplatesAnyList,
     SystemJobTemplateNotificationTemplatesErrorList,
+    SystemJobTemplateNotificationTemplatesStartedList,
     SystemJobTemplateNotificationTemplatesSuccessList,
 )
 
@@ -21,8 +21,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/launch/$', SystemJobTemplateLaunch.as_view(), name='system_job_template_launch'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', SystemJobTemplateJobsList.as_view(), name='system_job_template_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', SystemJobTemplateSchedulesList.as_view(), name='system_job_template_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', SystemJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='system_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', SystemJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='system_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', SystemJobTemplateNotificationTemplatesErrorList.as_view(),
         name='system_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', SystemJobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/urls.py

@@ -11,10 +11,10 @@ from awx.api.generics import (
 )
 from awx.api.views import (
     ApiRootView,
-    ApiV1RootView,
     ApiV2RootView,
-    ApiV1PingView,
-    ApiV1ConfigView,
+    ApiV2PingView,
+    ApiV2ConfigView,
+    ApiV2SubscriptionView,
     AuthView,
     UserMeList,
     DashboardView,
@@ -34,6 +34,8 @@ from awx.api.views import (
     OAuth2ApplicationDetail,
 )
 
+from awx.api.views.metrics import MetricsView
+
 from .organization import urls as organization_urls
 from .user import urls as user_urls
 from .project import urls as project_urls
@@ -47,6 +49,7 @@ from .inventory_update import urls as inventory_update_urls
 from .inventory_script import urls as inventory_script_urls
 from .credential_type import urls as credential_type_urls
 from .credential import urls as credential_urls
+from .credential_input_source import urls as credential_input_source_urls
 from .role import urls as role_urls
 from .job_template import urls as job_template_urls
 from .job import urls as job_urls
@@ -67,14 +70,32 @@ from .schedule import urls as schedule_urls
 from .activity_stream import urls as activity_stream_urls
 from .instance import urls as instance_urls
 from .instance_group import urls as instance_group_urls
-from .user_oauth import urls as user_oauth_urls
-from .oauth import urls as oauth_urls
+from .oauth2 import urls as oauth2_urls
+from .oauth2_root import urls as oauth2_root_urls
+from .workflow_approval_template import urls as workflow_approval_template_urls
+from .workflow_approval import urls as workflow_approval_urls
 
 
-v1_urls = [
-    url(r'^$', ApiV1RootView.as_view(), name='api_v1_root_view'),
-    url(r'^ping/$', ApiV1PingView.as_view(), name='api_v1_ping_view'),
-    url(r'^config/$', ApiV1ConfigView.as_view(), name='api_v1_config_view'),
+v2_urls = [
+    url(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
+    url(r'^credential_types/', include(credential_type_urls)),
+    url(r'^credential_input_sources/', include(credential_input_source_urls)),
+    url(r'^hosts/(?P<pk>[0-9]+)/ansible_facts/$', HostAnsibleFactsDetail.as_view(), name='host_ansible_facts_detail'),
+    url(r'^jobs/(?P<pk>[0-9]+)/extra_credentials/$', JobExtraCredentialsList.as_view(), name='job_extra_credentials_list'),
+    url(r'^jobs/(?P<pk>[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'),
+    url(r'^job_templates/(?P<pk>[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'),
+    url(r'^job_templates/(?P<pk>[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'),
+    url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'),
+    url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'),
+    url(r'^applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
+    url(r'^applications/(?P<pk>[0-9]+)/$', OAuth2ApplicationDetail.as_view(), name='o_auth2_application_detail'),
+    url(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
+    url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
+    url(r'^', include(oauth2_urls)),
+    url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
+    url(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'),
+    url(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'),
+    url(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'),
     url(r'^auth/$', AuthView.as_view()),
     url(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     url(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),
@@ -114,30 +135,15 @@ v1_urls = [
     url(r'^unified_job_templates/$', UnifiedJobTemplateList.as_view(), name='unified_job_template_list'),
     url(r'^unified_jobs/$', UnifiedJobList.as_view(), name='unified_job_list'),
     url(r'^activity_stream/', include(activity_stream_urls)),
+    url(r'^workflow_approval_templates/', include(workflow_approval_template_urls)),
+    url(r'^workflow_approvals/', include(workflow_approval_urls)),
 ]
 
-v2_urls = [
-    url(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
-    url(r'^credential_types/', include(credential_type_urls)),
-    url(r'^hosts/(?P<pk>[0-9]+)/ansible_facts/$', HostAnsibleFactsDetail.as_view(), name='host_ansible_facts_detail'),
-    url(r'^jobs/(?P<pk>[0-9]+)/extra_credentials/$', JobExtraCredentialsList.as_view(), name='job_extra_credentials_list'),
-    url(r'^jobs/(?P<pk>[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'),
-    url(r'^job_templates/(?P<pk>[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'),
-    url(r'^job_templates/(?P<pk>[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'),
-    url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'),
-    url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'),
-    url(r'^applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
-    url(r'^applications/(?P<pk>[0-9]+)/$', OAuth2ApplicationDetail.as_view(), name='o_auth2_application_detail'),
-    url(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
-    url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
-    url(r'^', include(user_oauth_urls)),
-]
 
 app_name = 'api'
 urlpatterns = [
     url(r'^$', ApiRootView.as_view(), name='api_root_view'),
     url(r'^(?P<version>(v2))/', include(v2_urls)),
-    url(r'^(?P<version>(v1|v2))/', include(v1_urls)),
     url(r'^login/$', LoggedLoginView.as_view(
         template_name='rest_framework/login.html',
         extra_context={'inside_login_context': True}
@@ -145,7 +151,7 @@ urlpatterns = [
     url(r'^logout/$', LoggedLogoutView.as_view(
         next_page='/api/', redirect_field_name='next'
     ), name='logout'),
-    url(r'^o/', include(oauth_urls)),
+    url(r'^o/', include(oauth2_root_urls)),
 ]
 if settings.SETTINGS_MODULE == 'awx.settings.development':
     from awx.api.swagger import SwaggerSchemaView

awx/api/urls/user.py

@@ -15,8 +15,8 @@ from awx.api.views import (
     UserActivityStreamList,
     UserAccessList,
     OAuth2ApplicationList,
-    OAuth2TokenList,
-    OAuth2PersonalTokenList,
+    OAuth2UserTokenList,
+    UserPersonalTokenList,
     UserAuthorizedTokenList,
 )
 
@@ -32,9 +32,9 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', UserActivityStreamList.as_view(), name='user_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', UserAccessList.as_view(), name='user_access_list'),
     url(r'^(?P<pk>[0-9]+)/applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
-    url(r'^(?P<pk>[0-9]+)/tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
+    url(r'^(?P<pk>[0-9]+)/tokens/$', OAuth2UserTokenList.as_view(), name='o_auth2_token_list'),
     url(r'^(?P<pk>[0-9]+)/authorized_tokens/$', UserAuthorizedTokenList.as_view(), name='user_authorized_token_list'),
-    url(r'^(?P<pk>[0-9]+)/personal_tokens/$', OAuth2PersonalTokenList.as_view(), name='o_auth2_personal_token_list'),
+    url(r'^(?P<pk>[0-9]+)/personal_tokens/$', UserPersonalTokenList.as_view(), name='user_personal_token_list'),
     
 ] 
 

awx/api/urls/webhooks.py

@@ -0,0 +1,14 @@
+from django.conf.urls import url
+
+from awx.api.views import (
+    WebhookKeyView,
+    GithubWebhookReceiver,
+    GitlabWebhookReceiver,
+)
+
+
+urlpatterns = [
+    url(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
+    url(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
+    url(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
+]

awx/api/urls/workflow_approval.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalList,
+    WorkflowApprovalDetail,
+    WorkflowApprovalApprove,
+    WorkflowApprovalDeny,
+)
+
+
+urls = [
+    url(r'^$', WorkflowApprovalList.as_view(), name='workflow_approval_list'),
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalDetail.as_view(), name='workflow_approval_detail'),
+    url(r'^(?P<pk>[0-9]+)/approve/$', WorkflowApprovalApprove.as_view(), name='workflow_approval_approve'),
+    url(r'^(?P<pk>[0-9]+)/deny/$', WorkflowApprovalDeny.as_view(), name='workflow_approval_deny'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_approval_template.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalTemplateDetail,
+    WorkflowApprovalTemplateJobsList,
+)
+
+
+urls = [
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalTemplateDetail.as_view(), name='workflow_approval_template_detail'),
+    url(r'^(?P<pk>[0-9]+)/approvals/$', WorkflowApprovalTemplateJobsList.as_view(), name='workflow_approval_template_jobs_list'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     WorkflowJobTemplateList,
@@ -13,9 +13,10 @@ from awx.api.views import (
     WorkflowJobTemplateSurveySpec,
     WorkflowJobTemplateWorkflowNodesList,
     WorkflowJobTemplateActivityStreamList,
-    WorkflowJobTemplateNotificationTemplatesAnyList,
     WorkflowJobTemplateNotificationTemplatesErrorList,
+    WorkflowJobTemplateNotificationTemplatesStartedList,
     WorkflowJobTemplateNotificationTemplatesSuccessList,
+    WorkflowJobTemplateNotificationTemplatesApprovalList,
     WorkflowJobTemplateAccessList,
     WorkflowJobTemplateObjectRolesList,
     WorkflowJobTemplateLabelList,
@@ -32,15 +33,18 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', WorkflowJobTemplateSurveySpec.as_view(), name='workflow_job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/workflow_nodes/$', WorkflowJobTemplateWorkflowNodesList.as_view(), name='workflow_job_template_workflow_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', WorkflowJobTemplateActivityStreamList.as_view(), name='workflow_job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', WorkflowJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='workflow_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', WorkflowJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='workflow_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', WorkflowJobTemplateNotificationTemplatesErrorList.as_view(),
         name='workflow_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', WorkflowJobTemplateNotificationTemplatesSuccessList.as_view(),
         name='workflow_job_template_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', WorkflowJobTemplateNotificationTemplatesApprovalList.as_view(),
+        name='workflow_job_template_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', WorkflowJobTemplateAccessList.as_view(), name='workflow_job_template_access_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', WorkflowJobTemplateObjectRolesList.as_view(), name='workflow_job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobTemplateLabelList.as_view(), name='workflow_job_template_label_list'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'workflow_job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/workflow_job_template_node.py

@@ -10,6 +10,7 @@ from awx.api.views import (
     WorkflowJobTemplateNodeFailureNodesList,
     WorkflowJobTemplateNodeAlwaysNodesList,
     WorkflowJobTemplateNodeCredentialsList,
+    WorkflowJobTemplateNodeCreateApproval,
 )
 
 
@@ -20,6 +21,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/failure_nodes/$', WorkflowJobTemplateNodeFailureNodesList.as_view(), name='workflow_job_template_node_failure_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/always_nodes/$', WorkflowJobTemplateNodeAlwaysNodesList.as_view(), name='workflow_job_template_node_always_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/credentials/$', WorkflowJobTemplateNodeCredentialsList.as_view(), name='workflow_job_template_node_credentials_list'),
+    url(r'^(?P<pk>[0-9]+)/create_approval_template/$', WorkflowJobTemplateNodeCreateApproval.as_view(), name='workflow_job_template_node_create_approval'),
 ]
 
 __all__ = ['urls']

awx/api/versioning.py

@@ -2,7 +2,7 @@
 # All Rights Reserved.
 
 from django.conf import settings
-from django.core.urlresolvers import NoReverseMatch
+from django.urls import NoReverseMatch
 
 from rest_framework.reverse import _reverse
 from rest_framework.versioning import URLPathVersioning as BaseVersioning
@@ -27,19 +27,6 @@ def drf_reverse(viewname, args=None, kwargs=None, request=None, format=None, **e
     return url
 
 
-def get_request_version(request):
-    """
-    The API version of a request as an integer i.e., 1 or 2
-    """
-    version = settings.REST_FRAMEWORK['DEFAULT_VERSION']
-    if request and hasattr(request, 'version'):
-        version = request.version
-        if version is None:
-            # For requests to /api/
-            return None
-    return int(version.lstrip('v'))
-
-
 def reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra):
     if request is None or getattr(request, 'version', None) is None:
         # We need the "current request" to determine the correct version to

awx/api/views/inventory.py

@@ -0,0 +1,211 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+# Python
+import logging
+
+# Django
+from django.conf import settings
+from django.db.models import Q
+from django.contrib.contenttypes.models import ContentType
+from django.utils.translation import ugettext_lazy as _
+
+# Django REST Framework
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
+from rest_framework import status
+
+# AWX
+from awx.main.models import (
+    ActivityStream,
+    Inventory,
+    JobTemplate,
+    Role,
+    User,
+    InstanceGroup,
+    InventoryUpdateEvent,
+    InventoryUpdate,
+    InventorySource,
+    CustomInventoryScript,
+)
+from awx.api.generics import (
+    ListCreateAPIView,
+    RetrieveUpdateDestroyAPIView,
+    SubListAPIView,
+    SubListAttachDetachAPIView,
+    ResourceAccessList,
+    CopyAPIView,
+)
+
+from awx.api.serializers import (
+    InventorySerializer,
+    ActivityStreamSerializer,
+    RoleSerializer,
+    InstanceGroupSerializer,
+    InventoryUpdateEventSerializer,
+    CustomInventoryScriptSerializer,
+    JobTemplateSerializer,
+)
+from awx.api.views.mixin import (
+    RelatedJobsPreventDeleteMixin,
+    ControlledByScmMixin,
+)
+
+logger = logging.getLogger('awx.api.views.organization')
+
+
+class InventoryUpdateEventsList(SubListAPIView):
+
+    model = InventoryUpdateEvent
+    serializer_class = InventoryUpdateEventSerializer
+    parent_model = InventoryUpdate
+    relationship = 'inventory_update_events'
+    name = _('Inventory Update Events List')
+    search_fields = ('stdout',)
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        response['X-UI-Max-Events'] = settings.MAX_UI_JOB_EVENTS
+        return super(InventoryUpdateEventsList, self).finalize_response(request, response, *args, **kwargs)
+
+
+class InventoryScriptList(ListCreateAPIView):
+
+    deprecated = True
+
+    model = CustomInventoryScript
+    serializer_class = CustomInventoryScriptSerializer
+
+
+class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
+
+    deprecated = True
+
+    model = CustomInventoryScript
+    serializer_class = CustomInventoryScriptSerializer
+
+    def destroy(self, request, *args, **kwargs):
+        instance = self.get_object()
+        can_delete = request.user.can_access(self.model, 'delete', instance)
+        if not can_delete:
+            raise PermissionDenied(_("Cannot delete inventory script."))
+        for inv_src in InventorySource.objects.filter(source_script=instance):
+            inv_src.source_script = None
+            inv_src.save()
+        return super(InventoryScriptDetail, self).destroy(request, *args, **kwargs)
+
+
+class InventoryScriptObjectRolesList(SubListAPIView):
+
+    deprecated = True
+
+    model = Role
+    serializer_class = RoleSerializer
+    parent_model = CustomInventoryScript
+    search_fields = ('role_field', 'content_type__model',)
+
+    def get_queryset(self):
+        po = self.get_parent_object()
+        content_type = ContentType.objects.get_for_model(self.parent_model)
+        return Role.objects.filter(content_type=content_type, object_id=po.pk)
+
+
+class InventoryScriptCopy(CopyAPIView):
+
+    deprecated = True
+
+    model = CustomInventoryScript
+    copy_return_serializer_class = CustomInventoryScriptSerializer
+
+
+class InventoryList(ListCreateAPIView):
+
+    model = Inventory
+    serializer_class = InventorySerializer
+
+
+class InventoryDetail(RelatedJobsPreventDeleteMixin, ControlledByScmMixin, RetrieveUpdateDestroyAPIView):
+
+    model = Inventory
+    serializer_class = InventorySerializer
+
+    def update(self, request, *args, **kwargs):
+        obj = self.get_object()
+        kind = self.request.data.get('kind') or kwargs.get('kind')
+
+        # Do not allow changes to an Inventory kind.
+        if kind is not None and obj.kind != kind:
+            return self.http_method_not_allowed(request, *args, **kwargs)
+        return super(InventoryDetail, self).update(request, *args, **kwargs)
+
+    def destroy(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if not request.user.can_access(self.model, 'delete', obj):
+            raise PermissionDenied()
+        self.check_related_active_jobs(obj)  # related jobs mixin
+        try:
+            obj.schedule_deletion(getattr(request.user, 'id', None))
+            return Response(status=status.HTTP_202_ACCEPTED)
+        except RuntimeError as e:
+            return Response(dict(error=_("{0}".format(e))), status=status.HTTP_400_BAD_REQUEST)
+
+
+class InventoryActivityStreamList(SubListAPIView):
+
+    model = ActivityStream
+    serializer_class = ActivityStreamSerializer
+    parent_model = Inventory
+    relationship = 'activitystream_set'
+    search_fields = ('changes',)
+
+    def get_queryset(self):
+        parent = self.get_parent_object()
+        self.check_parent_access(parent)
+        qs = self.request.user.get_queryset(self.model)
+        return qs.filter(Q(inventory=parent) | Q(host__in=parent.hosts.all()) | Q(group__in=parent.groups.all()))
+
+
+class InventoryInstanceGroupsList(SubListAttachDetachAPIView):
+
+    model = InstanceGroup
+    serializer_class = InstanceGroupSerializer
+    parent_model = Inventory
+    relationship = 'instance_groups'
+
+
+class InventoryAccessList(ResourceAccessList):
+
+    model = User # needs to be User for AccessLists's
+    parent_model = Inventory
+
+
+class InventoryObjectRolesList(SubListAPIView):
+
+    model = Role
+    serializer_class = RoleSerializer
+    parent_model = Inventory
+    search_fields = ('role_field', 'content_type__model',)
+
+    def get_queryset(self):
+        po = self.get_parent_object()
+        content_type = ContentType.objects.get_for_model(self.parent_model)
+        return Role.objects.filter(content_type=content_type, object_id=po.pk)
+
+
+class InventoryJobTemplateList(SubListAPIView):
+
+    model = JobTemplate
+    serializer_class = JobTemplateSerializer
+    parent_model = Inventory
+    relationship = 'jobtemplates'
+
+    def get_queryset(self):
+        parent = self.get_parent_object()
+        self.check_parent_access(parent)
+        qs = self.request.user.get_queryset(self.model)
+        return qs.filter(inventory=parent)
+
+
+class InventoryCopy(CopyAPIView):
+
+    model = Inventory
+    copy_return_serializer_class = InventorySerializer

awx/api/views/metrics.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+# Python
+import logging
+
+# Django
+from django.utils.translation import ugettext_lazy as _
+
+# Django REST Framework
+from rest_framework.response import Response
+from rest_framework.exceptions import PermissionDenied
+
+
+# AWX
+# from awx.main.analytics import collectors
+from awx.main.analytics.metrics import metrics
+from awx.api import renderers
+
+from awx.api.generics import (
+    APIView,
+)
+
+
+logger = logging.getLogger('awx.main.analytics')
+
+
+class MetricsView(APIView):
+
+    name = _('Metrics')
+    swagger_topic = 'Metrics'
+
+    renderer_classes = [renderers.PlainTextRenderer,
+                        renderers.PrometheusJSONRenderer,
+                        renderers.BrowsableAPIRenderer,]
+
+    def get(self, request):
+        ''' Show Metrics Details '''
+        if (request.user.is_superuser or request.user.is_system_auditor):
+            return Response(metrics().decode('UTF-8'))
+        raise PermissionDenied()

awx/api/views/mixin.py

@@ -0,0 +1,280 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+import dateutil
+import logging
+
+from django.db.models import (
+    Count,
+    F,
+)
+from django.db import transaction
+from django.shortcuts import get_object_or_404
+from django.utils.timezone import now
+from django.utils.translation import ugettext_lazy as _
+
+from rest_framework.permissions import SAFE_METHODS
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
+from rest_framework import status
+
+from awx.main.constants import ACTIVE_STATES
+from awx.main.utils import (
+    get_object_or_400,
+    parse_yaml_or_json,
+)
+from awx.main.models.ha import (
+    Instance,
+    InstanceGroup,
+)
+from awx.main.models.organization import Team
+from awx.main.models.projects import Project
+from awx.main.models.inventory import Inventory
+from awx.main.models.jobs import JobTemplate
+from awx.api.exceptions import ActiveJobConflict
+
+logger = logging.getLogger('awx.api.views.mixin')
+
+
+class UnifiedJobDeletionMixin(object):
+    '''
+    Special handling when deleting a running unified job object.
+    '''
+    def destroy(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if not request.user.can_access(self.model, 'delete', obj):
+            raise PermissionDenied()
+        try:
+            if obj.unified_job_node.workflow_job.status in ACTIVE_STATES:
+                raise PermissionDenied(detail=_('Cannot delete job resource when associated workflow job is running.'))
+        except self.model.unified_job_node.RelatedObjectDoesNotExist:
+            pass
+        # Still allow deletion of new status, because these can be manually created
+        if obj.status in ACTIVE_STATES and obj.status != 'new':
+            raise PermissionDenied(detail=_("Cannot delete running job resource."))
+        elif not obj.event_processing_finished:
+            # Prohibit deletion if job events are still coming in
+            if obj.finished and now() < obj.finished + dateutil.relativedelta.relativedelta(minutes=1):
+                # less than 1 minute has passed since job finished and events are not in
+                return Response({"error": _("Job has not finished processing events.")},
+                                status=status.HTTP_400_BAD_REQUEST)
+            else:
+                # if it has been > 1 minute, events are probably lost
+                logger.warning('Allowing deletion of {} through the API without all events '
+                               'processed.'.format(obj.log_format))
+        obj.delete()
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class InstanceGroupMembershipMixin(object):
+    '''
+    This mixin overloads attach/detach so that it calls InstanceGroup.save(),
+    triggering a background recalculation of policy-based instance group
+    membership.
+    '''
+    def attach(self, request, *args, **kwargs):
+        response = super(InstanceGroupMembershipMixin, self).attach(request, *args, **kwargs)
+        sub_id, res = self.attach_validate(request)
+        if status.is_success(response.status_code):
+            if self.parent_model is Instance:
+                ig_obj = get_object_or_400(self.model, pk=sub_id)
+                inst_name = ig_obj.hostname
+            else:
+                inst_name = get_object_or_400(self.model, pk=sub_id).hostname
+            with transaction.atomic():
+                ig_qs = InstanceGroup.objects.select_for_update()
+                if self.parent_model is Instance:
+                    ig_obj = get_object_or_400(ig_qs, pk=sub_id)
+                else:
+                    # similar to get_parent_object, but selected for update
+                    parent_filter = {
+                        self.lookup_field: self.kwargs.get(self.lookup_field, None),
+                    }
+                    ig_obj = get_object_or_404(ig_qs, **parent_filter)
+                if inst_name not in ig_obj.policy_instance_list:
+                    ig_obj.policy_instance_list.append(inst_name)
+                    ig_obj.save(update_fields=['policy_instance_list'])
+        return response
+
+    def is_valid_relation(self, parent, sub, created=False):
+        if sub.is_isolated():
+            return {'error': _('Isolated instances may not be added or removed from instances groups via the API.')}
+        if self.parent_model is InstanceGroup:
+            ig_obj = self.get_parent_object()
+            if ig_obj.controller_id is not None:
+                return {'error': _('Isolated instance group membership may not be managed via the API.')}
+        return None
+
+    def unattach_validate(self, request):
+        (sub_id, res) = super(InstanceGroupMembershipMixin, self).unattach_validate(request)
+        if res:
+            return (sub_id, res)
+        sub = get_object_or_400(self.model, pk=sub_id)
+        attach_errors = self.is_valid_relation(None, sub)
+        if attach_errors:
+            return (sub_id, Response(attach_errors, status=status.HTTP_400_BAD_REQUEST))
+        return (sub_id, res)
+
+    def unattach(self, request, *args, **kwargs):
+        response = super(InstanceGroupMembershipMixin, self).unattach(request, *args, **kwargs)
+        if status.is_success(response.status_code):
+            sub_id = request.data.get('id', None)
+            if self.parent_model is Instance:
+                inst_name = self.get_parent_object().hostname
+            else:
+                inst_name = get_object_or_400(self.model, pk=sub_id).hostname
+            with transaction.atomic():
+                ig_qs = InstanceGroup.objects.select_for_update()
+                if self.parent_model is Instance:
+                    ig_obj = get_object_or_400(ig_qs, pk=sub_id)
+                else:
+                    # similar to get_parent_object, but selected for update
+                    parent_filter = {
+                        self.lookup_field: self.kwargs.get(self.lookup_field, None),
+                    }
+                    ig_obj = get_object_or_404(ig_qs, **parent_filter)
+                if inst_name in ig_obj.policy_instance_list:
+                    ig_obj.policy_instance_list.pop(ig_obj.policy_instance_list.index(inst_name))
+                    ig_obj.save(update_fields=['policy_instance_list'])
+        return response
+
+
+class RelatedJobsPreventDeleteMixin(object):
+    def perform_destroy(self, obj):
+        self.check_related_active_jobs(obj)
+        return super(RelatedJobsPreventDeleteMixin, self).perform_destroy(obj)
+
+    def check_related_active_jobs(self, obj):
+        active_jobs = obj.get_active_jobs()
+        if len(active_jobs) > 0:
+            raise ActiveJobConflict(active_jobs)
+        time_cutoff = now() - dateutil.relativedelta.relativedelta(minutes=1)
+        recent_jobs = obj._get_related_jobs().filter(finished__gte = time_cutoff)
+        for unified_job in recent_jobs.get_real_instances():
+            if not unified_job.event_processing_finished:
+                raise PermissionDenied(_(
+                    'Related job {} is still processing events.'
+                ).format(unified_job.log_format))
+
+
+class OrganizationCountsMixin(object):
+
+    def get_serializer_context(self, *args, **kwargs):
+        full_context = super(OrganizationCountsMixin, self).get_serializer_context(*args, **kwargs)
+
+        if self.request is None:
+            return full_context
+
+        db_results = {}
+        org_qs = self.model.accessible_objects(self.request.user, 'read_role')
+        org_id_list = org_qs.values('id')
+        if len(org_id_list) == 0:
+            if self.request.method == 'POST':
+                full_context['related_field_counts'] = {}
+            return full_context
+
+        inv_qs = Inventory.accessible_objects(self.request.user, 'read_role')
+        project_qs = Project.accessible_objects(self.request.user, 'read_role')
+
+        # Produce counts of Foreign Key relationships
+        db_results['inventories'] = inv_qs\
+            .values('organization').annotate(Count('organization')).order_by('organization')
+
+        db_results['teams'] = Team.accessible_objects(
+            self.request.user, 'read_role').values('organization').annotate(
+            Count('organization')).order_by('organization')
+
+        JT_project_reference = 'project__organization'
+        JT_inventory_reference = 'inventory__organization'
+        db_results['job_templates_project'] = JobTemplate.accessible_objects(
+            self.request.user, 'read_role').exclude(
+            project__organization=F(JT_inventory_reference)).values(JT_project_reference).annotate(
+            Count(JT_project_reference)).order_by(JT_project_reference)
+
+        db_results['job_templates_inventory'] = JobTemplate.accessible_objects(
+            self.request.user, 'read_role').values(JT_inventory_reference).annotate(
+            Count(JT_inventory_reference)).order_by(JT_inventory_reference)
+
+        db_results['projects'] = project_qs\
+            .values('organization').annotate(Count('organization')).order_by('organization')
+
+        # Other members and admins of organization are always viewable
+        db_results['users'] = org_qs.annotate(
+            users=Count('member_role__members', distinct=True),
+            admins=Count('admin_role__members', distinct=True)
+        ).values('id', 'users', 'admins')
+
+        count_context = {}
+        for org in org_id_list:
+            org_id = org['id']
+            count_context[org_id] = {
+                'inventories': 0, 'teams': 0, 'users': 0, 'job_templates': 0,
+                'admins': 0, 'projects': 0}
+
+        for res, count_qs in db_results.items():
+            if res == 'job_templates_project':
+                org_reference = JT_project_reference
+            elif res == 'job_templates_inventory':
+                org_reference = JT_inventory_reference
+            elif res == 'users':
+                org_reference = 'id'
+            else:
+                org_reference = 'organization'
+            for entry in count_qs:
+                org_id = entry[org_reference]
+                if org_id in count_context:
+                    if res == 'users':
+                        count_context[org_id]['admins'] = entry['admins']
+                        count_context[org_id]['users'] = entry['users']
+                        continue
+                    count_context[org_id][res] = entry['%s__count' % org_reference]
+
+        # Combine the counts for job templates by project and inventory
+        for org in org_id_list:
+            org_id = org['id']
+            count_context[org_id]['job_templates'] = 0
+            for related_path in ['job_templates_project', 'job_templates_inventory']:
+                if related_path in count_context[org_id]:
+                    count_context[org_id]['job_templates'] += count_context[org_id].pop(related_path)
+
+        full_context['related_field_counts'] = count_context
+
+        return full_context
+
+
+class ControlledByScmMixin(object):
+    '''
+    Special method to reset SCM inventory commit hash
+    if anything that it manages changes.
+    '''
+
+    def _reset_inv_src_rev(self, obj):
+        if self.request.method in SAFE_METHODS or not obj:
+            return
+        project_following_sources = obj.inventory_sources.filter(
+            update_on_project_update=True, source='scm')
+        if project_following_sources:
+            # Allow inventory changes unrelated to variables
+            if self.model == Inventory and (
+                    not self.request or not self.request.data or
+                    parse_yaml_or_json(self.request.data.get('variables', '')) == parse_yaml_or_json(obj.variables)):
+                return
+            project_following_sources.update(scm_last_revision='')
+
+    def get_object(self):
+        obj = super(ControlledByScmMixin, self).get_object()
+        self._reset_inv_src_rev(obj)
+        return obj
+
+    def get_parent_object(self):
+        obj = super(ControlledByScmMixin, self).get_parent_object()
+        self._reset_inv_src_rev(obj)
+        return obj
+
+
+class NoTruncateMixin(object):
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.query_params.get('no_truncate'):
+            context.update(no_truncate=True)
+        return context

awx/api/views/organization.py

@@ -0,0 +1,227 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+# Python
+import logging
+
+# Django
+from django.db.models import Count
+from django.contrib.contenttypes.models import ContentType
+
+# AWX
+from awx.main.models import (
+    ActivityStream,
+    Inventory,
+    Project,
+    JobTemplate,
+    WorkflowJobTemplate,
+    Organization,
+    NotificationTemplate,
+    Role,
+    User,
+    Team,
+    InstanceGroup,
+)
+from awx.api.generics import (
+    ListCreateAPIView,
+    RetrieveUpdateDestroyAPIView,
+    SubListAPIView,
+    SubListCreateAttachDetachAPIView,
+    SubListAttachDetachAPIView,
+    ResourceAccessList,
+    BaseUsersList,
+)
+
+from awx.api.serializers import (
+    OrganizationSerializer,
+    InventorySerializer,
+    ProjectSerializer,
+    UserSerializer,
+    TeamSerializer,
+    ActivityStreamSerializer,
+    RoleSerializer,
+    NotificationTemplateSerializer,
+    WorkflowJobTemplateSerializer,
+    InstanceGroupSerializer,
+)
+from awx.api.views.mixin import (
+    RelatedJobsPreventDeleteMixin,
+    OrganizationCountsMixin,
+)
+
+logger = logging.getLogger('awx.api.views.organization')
+
+
+class OrganizationList(OrganizationCountsMixin, ListCreateAPIView):
+
+    model = Organization
+    serializer_class = OrganizationSerializer
+
+    def get_queryset(self):
+        qs = Organization.accessible_objects(self.request.user, 'read_role')
+        qs = qs.select_related('admin_role', 'auditor_role', 'member_role', 'read_role')
+        qs = qs.prefetch_related('created_by', 'modified_by')
+        return qs
+
+
+class OrganizationDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
+
+    model = Organization
+    serializer_class = OrganizationSerializer
+
+    def get_serializer_context(self, *args, **kwargs):
+        full_context = super(OrganizationDetail, self).get_serializer_context(*args, **kwargs)
+
+        if not hasattr(self, 'kwargs') or 'pk' not in self.kwargs:
+            return full_context
+        org_id = int(self.kwargs['pk'])
+
+        org_counts = {}
+        access_kwargs = {'accessor': self.request.user, 'role_field': 'read_role'}
+        direct_counts = Organization.objects.filter(id=org_id).annotate(
+            users=Count('member_role__members', distinct=True),
+            admins=Count('admin_role__members', distinct=True)
+        ).values('users', 'admins')
+
+        if not direct_counts:
+            return full_context
+
+        org_counts = direct_counts[0]
+        org_counts['inventories'] = Inventory.accessible_objects(**access_kwargs).filter(
+            organization__id=org_id).count()
+        org_counts['teams'] = Team.accessible_objects(**access_kwargs).filter(
+            organization__id=org_id).count()
+        org_counts['projects'] = Project.accessible_objects(**access_kwargs).filter(
+            organization__id=org_id).count()
+        org_counts['job_templates'] = JobTemplate.accessible_objects(**access_kwargs).filter(
+            project__organization__id=org_id).count()
+
+        full_context['related_field_counts'] = {}
+        full_context['related_field_counts'][org_id] = org_counts
+
+        return full_context
+
+
+class OrganizationInventoriesList(SubListAPIView):
+
+    model = Inventory
+    serializer_class = InventorySerializer
+    parent_model = Organization
+    relationship = 'inventories'
+
+
+class OrganizationUsersList(BaseUsersList):
+
+    model = User
+    serializer_class = UserSerializer
+    parent_model = Organization
+    relationship = 'member_role.members'
+    ordering = ('username',)
+
+
+class OrganizationAdminsList(BaseUsersList):
+
+    model = User
+    serializer_class = UserSerializer
+    parent_model = Organization
+    relationship = 'admin_role.members'
+    ordering = ('username',)
+
+
+class OrganizationProjectsList(SubListCreateAttachDetachAPIView):
+
+    model = Project
+    serializer_class = ProjectSerializer
+    parent_model = Organization
+    relationship = 'projects'
+    parent_key = 'organization'
+
+
+class OrganizationWorkflowJobTemplatesList(SubListCreateAttachDetachAPIView):
+
+    model = WorkflowJobTemplate
+    serializer_class = WorkflowJobTemplateSerializer
+    parent_model = Organization
+    relationship = 'workflows'
+    parent_key = 'organization'
+
+
+class OrganizationTeamsList(SubListCreateAttachDetachAPIView):
+
+    model = Team
+    serializer_class = TeamSerializer
+    parent_model = Organization
+    relationship = 'teams'
+    parent_key = 'organization'
+
+
+class OrganizationActivityStreamList(SubListAPIView):
+
+    model = ActivityStream
+    serializer_class = ActivityStreamSerializer
+    parent_model = Organization
+    relationship = 'activitystream_set'
+    search_fields = ('changes',)
+
+
+class OrganizationNotificationTemplatesList(SubListCreateAttachDetachAPIView):
+
+    model = NotificationTemplate
+    serializer_class = NotificationTemplateSerializer
+    parent_model = Organization
+    relationship = 'notification_templates'
+    parent_key = 'organization'
+
+
+class OrganizationNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView):
+
+    model = NotificationTemplate
+    serializer_class = NotificationTemplateSerializer
+    parent_model = Organization
+
+
+class OrganizationNotificationTemplatesStartedList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class OrganizationNotificationTemplatesErrorList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_error'
+
+
+class OrganizationNotificationTemplatesSuccessList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_success'
+
+
+class OrganizationNotificationTemplatesApprovalList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_approvals'
+
+
+class OrganizationInstanceGroupsList(SubListAttachDetachAPIView):
+
+    model = InstanceGroup
+    serializer_class = InstanceGroupSerializer
+    parent_model = Organization
+    relationship = 'instance_groups'
+
+
+class OrganizationAccessList(ResourceAccessList):
+
+    model = User # needs to be User for AccessLists's
+    parent_model = Organization
+
+
+class OrganizationObjectRolesList(SubListAPIView):
+
+    model = Role
+    serializer_class = RoleSerializer
+    parent_model = Organization
+    search_fields = ('role_field', 'content_type__model',)
+
+    def get_queryset(self):
+        po = self.get_parent_object()
+        content_type = ContentType.objects.get_for_model(self.parent_model)
+        return Role.objects.filter(content_type=content_type, object_id=po.pk)

awx/api/views/root.py

@@ -0,0 +1,318 @@
+# Copyright (c) 2018 Ansible, Inc.
+# All Rights Reserved.
+
+import logging
+import operator
+import json
+from collections import OrderedDict
+
+from django.conf import settings
+from django.utils.encoding import smart_text
+from django.utils.decorators import method_decorator
+from django.views.decorators.csrf import ensure_csrf_cookie
+from django.template.loader import render_to_string
+from django.utils.translation import ugettext_lazy as _
+
+from rest_framework.permissions import AllowAny, IsAuthenticated
+from rest_framework.response import Response
+from rest_framework import status
+
+import requests
+
+from awx.api.generics import APIView
+from awx.main.ha import is_ha_environment
+from awx.main.utils import (
+    get_awx_version,
+    get_ansible_version,
+    get_custom_venv_choices,
+    to_python_boolean,
+)
+from awx.api.versioning import reverse, drf_reverse
+from awx.conf.license import get_license
+from awx.main.constants import PRIVILEGE_ESCALATION_METHODS
+from awx.main.models import (
+    Project,
+    Organization,
+    Instance,
+    InstanceGroup,
+    JobTemplate,
+)
+
+logger = logging.getLogger('awx.api.views.root')
+
+
+class ApiRootView(APIView):
+
+    permission_classes = (AllowAny,)
+    name = _('REST API')
+    versioning_class = None
+    swagger_topic = 'Versioning'
+
+    @method_decorator(ensure_csrf_cookie)
+    def get(self, request, format=None):
+        ''' List supported API versions '''
+
+        v2 = reverse('api:api_v2_root_view', kwargs={'version': 'v2'})
+        data = OrderedDict()
+        data['description'] = _('AWX REST API')
+        data['current_version'] = v2
+        data['available_versions'] = dict(v2 = v2)
+        data['oauth2'] = drf_reverse('api:oauth_authorization_root_view')
+        data['custom_logo'] = settings.CUSTOM_LOGO
+        data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
+        data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
+        return Response(data)
+
+
+class ApiOAuthAuthorizationRootView(APIView):
+
+    permission_classes = (AllowAny,)
+    name = _("API OAuth 2 Authorization Root")
+    versioning_class = None
+    swagger_topic = 'Authentication'
+
+    def get(self, request, format=None):
+        data = OrderedDict()
+        data['authorize'] = drf_reverse('api:authorize')
+        data['token'] = drf_reverse('api:token')
+        data['revoke_token'] = drf_reverse('api:revoke-token')
+        return Response(data)
+
+
+class ApiVersionRootView(APIView):
+
+    permission_classes = (AllowAny,)
+    swagger_topic = 'Versioning'
+
+    def get(self, request, format=None):
+        ''' List top level resources '''
+        data = OrderedDict()
+        data['ping'] = reverse('api:api_v2_ping_view', request=request)
+        data['instances'] = reverse('api:instance_list', request=request)
+        data['instance_groups'] = reverse('api:instance_group_list', request=request)
+        data['config'] = reverse('api:api_v2_config_view', request=request)
+        data['settings'] = reverse('api:setting_category_list', request=request)
+        data['me'] = reverse('api:user_me_list', request=request)
+        data['dashboard'] = reverse('api:dashboard_view', request=request)
+        data['organizations'] = reverse('api:organization_list', request=request)
+        data['users'] = reverse('api:user_list', request=request)
+        data['projects'] = reverse('api:project_list', request=request)
+        data['project_updates'] = reverse('api:project_update_list', request=request)
+        data['teams'] = reverse('api:team_list', request=request)
+        data['credentials'] = reverse('api:credential_list', request=request)
+        data['credential_types'] = reverse('api:credential_type_list', request=request)
+        data['credential_input_sources'] = reverse('api:credential_input_source_list', request=request)
+        data['applications'] = reverse('api:o_auth2_application_list', request=request)
+        data['tokens'] = reverse('api:o_auth2_token_list', request=request)
+        data['metrics'] = reverse('api:metrics_view', request=request)
+        data['inventory'] = reverse('api:inventory_list', request=request)
+        data['inventory_scripts'] = reverse('api:inventory_script_list', request=request)
+        data['inventory_sources'] = reverse('api:inventory_source_list', request=request)
+        data['inventory_updates'] = reverse('api:inventory_update_list', request=request)
+        data['groups'] = reverse('api:group_list', request=request)
+        data['hosts'] = reverse('api:host_list', request=request)
+        data['job_templates'] = reverse('api:job_template_list', request=request)
+        data['jobs'] = reverse('api:job_list', request=request)
+        data['job_events'] = reverse('api:job_event_list', request=request)
+        data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request)
+        data['system_job_templates'] = reverse('api:system_job_template_list', request=request)
+        data['system_jobs'] = reverse('api:system_job_list', request=request)
+        data['schedules'] = reverse('api:schedule_list', request=request)
+        data['roles'] = reverse('api:role_list', request=request)
+        data['notification_templates'] = reverse('api:notification_template_list', request=request)
+        data['notifications'] = reverse('api:notification_list', request=request)
+        data['labels'] = reverse('api:label_list', request=request)
+        data['unified_job_templates'] = reverse('api:unified_job_template_list', request=request)
+        data['unified_jobs'] = reverse('api:unified_job_list', request=request)
+        data['activity_stream'] = reverse('api:activity_stream_list', request=request)
+        data['workflow_job_templates'] = reverse('api:workflow_job_template_list', request=request)
+        data['workflow_jobs'] = reverse('api:workflow_job_list', request=request)
+        data['workflow_approvals'] = reverse('api:workflow_approval_list', request=request)
+        data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
+        data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
+        return Response(data)
+
+
+class ApiV2RootView(ApiVersionRootView):
+    name = _('Version 2')
+
+
+class ApiV2PingView(APIView):
+    """A simple view that reports very basic information about this
+    instance, which is acceptable to be public information.
+    """
+    permission_classes = (AllowAny,)
+    authentication_classes = ()
+    name = _('Ping')
+    swagger_topic = 'System Configuration'
+
+    def get(self, request, format=None):
+        """Return some basic information about this instance
+
+        Everything returned here should be considered public / insecure, as
+        this requires no auth and is intended for use by the installer process.
+        """
+        response = {
+            'ha': is_ha_environment(),
+            'version': get_awx_version(),
+            'active_node': settings.CLUSTER_HOST_ID,
+            'install_uuid': settings.INSTALL_UUID,
+        }
+
+        response['instances'] = []
+        for instance in Instance.objects.all():
+            response['instances'].append(dict(node=instance.hostname, uuid=instance.uuid, heartbeat=instance.modified,
+                                              capacity=instance.capacity, version=instance.version))
+            sorted(response['instances'], key=operator.itemgetter('node'))
+        response['instance_groups'] = []
+        for instance_group in InstanceGroup.objects.prefetch_related('instances'):
+            response['instance_groups'].append(dict(name=instance_group.name,
+                                                    capacity=instance_group.capacity,
+                                                    instances=[x.hostname for x in instance_group.instances.all()]))
+        return Response(response)
+
+
+class ApiV2SubscriptionView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    name = _('Configuration')
+    swagger_topic = 'System Configuration'
+
+    def check_permissions(self, request):
+        super(ApiV2SubscriptionView, self).check_permissions(request)
+        if not request.user.is_superuser and request.method.lower() not in {'options', 'head'}:
+            self.permission_denied(request)  # Raises PermissionDenied exception.
+
+    def post(self, request):
+        from awx.main.utils.common import get_licenser
+        data = request.data.copy()
+        if data.get('rh_password') == '$encrypted$':
+            data['rh_password'] = settings.REDHAT_PASSWORD
+        try:
+            user, pw = data.get('rh_username'), data.get('rh_password')
+            validated = get_licenser().validate_rh(user, pw)
+            if user:
+                settings.REDHAT_USERNAME = data['rh_username']
+            if pw:
+                settings.REDHAT_PASSWORD = data['rh_password']
+        except Exception as exc:
+            msg = _("Invalid License")
+            if (
+                isinstance(exc, requests.exceptions.HTTPError) and
+                getattr(getattr(exc, 'response', None), 'status_code', None) == 401
+            ):
+                msg = _("The provided credentials are invalid (HTTP 401).")
+            if isinstance(exc, (ValueError, OSError)) and exc.args:
+                msg = exc.args[0]
+            logger.exception(smart_text(u"Invalid license submitted."),
+                             extra=dict(actor=request.user.username))
+            return Response({"error": msg}, status=status.HTTP_400_BAD_REQUEST)
+
+        return Response(validated)
+
+
+class ApiV2ConfigView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    name = _('Configuration')
+    swagger_topic = 'System Configuration'
+
+    def check_permissions(self, request):
+        super(ApiV2ConfigView, self).check_permissions(request)
+        if not request.user.is_superuser and request.method.lower() not in {'options', 'head', 'get'}:
+            self.permission_denied(request)  # Raises PermissionDenied exception.
+
+    def get(self, request, format=None):
+        '''Return various sitewide configuration settings'''
+
+        if request.user.is_superuser or request.user.is_system_auditor:
+            license_data = get_license(show_key=True)
+        else:
+            license_data = get_license(show_key=False)
+        if not license_data.get('valid_key', False):
+            license_data = {}
+        if license_data and 'features' in license_data and 'activity_streams' in license_data['features']:
+            # FIXME: Make the final setting value dependent on the feature?
+            license_data['features']['activity_streams'] &= settings.ACTIVITY_STREAM_ENABLED
+
+        pendo_state = settings.PENDO_TRACKING_STATE if settings.PENDO_TRACKING_STATE in ('off', 'anonymous', 'detailed') else 'off'
+
+        data = dict(
+            time_zone=settings.TIME_ZONE,
+            license_info=license_data,
+            version=get_awx_version(),
+            ansible_version=get_ansible_version(),
+            eula=render_to_string("eula.md") if license_data.get('license_type', 'UNLICENSED') != 'open' else '',
+            analytics_status=pendo_state,
+            become_methods=PRIVILEGE_ESCALATION_METHODS,
+        )
+
+        # If LDAP is enabled, user_ldap_fields will return a list of field
+        # names that are managed by LDAP and should be read-only for users with
+        # a non-empty ldap_dn attribute.
+        if getattr(settings, 'AUTH_LDAP_SERVER_URI', None):
+            user_ldap_fields = ['username', 'password']
+            user_ldap_fields.extend(getattr(settings, 'AUTH_LDAP_USER_ATTR_MAP', {}).keys())
+            user_ldap_fields.extend(getattr(settings, 'AUTH_LDAP_USER_FLAGS_BY_GROUP', {}).keys())
+            data['user_ldap_fields'] = user_ldap_fields
+
+        if request.user.is_superuser \
+                or request.user.is_system_auditor \
+                or Organization.accessible_objects(request.user, 'admin_role').exists() \
+                or Organization.accessible_objects(request.user, 'auditor_role').exists() \
+                or Organization.accessible_objects(request.user, 'project_admin_role').exists():
+            data.update(dict(
+                project_base_dir = settings.PROJECTS_ROOT,
+                project_local_paths = Project.get_local_path_choices(),
+                custom_virtualenvs = get_custom_venv_choices()
+            ))
+        elif JobTemplate.accessible_objects(request.user, 'admin_role').exists():
+            data['custom_virtualenvs'] = get_custom_venv_choices()
+
+        return Response(data)
+
+    def post(self, request):
+        if not isinstance(request.data, dict):
+            return Response({"error": _("Invalid license data")}, status=status.HTTP_400_BAD_REQUEST)
+        if "eula_accepted" not in request.data:
+            return Response({"error": _("Missing 'eula_accepted' property")}, status=status.HTTP_400_BAD_REQUEST)
+        try:
+            eula_accepted = to_python_boolean(request.data["eula_accepted"])
+        except ValueError:
+            return Response({"error": _("'eula_accepted' value is invalid")}, status=status.HTTP_400_BAD_REQUEST)
+
+        if not eula_accepted:
+            return Response({"error": _("'eula_accepted' must be True")}, status=status.HTTP_400_BAD_REQUEST)
+        request.data.pop("eula_accepted")
+        try:
+            data_actual = json.dumps(request.data)
+        except Exception:
+            logger.info(smart_text(u"Invalid JSON submitted for license."),
+                        extra=dict(actor=request.user.username))
+            return Response({"error": _("Invalid JSON")}, status=status.HTTP_400_BAD_REQUEST)
+        try:
+            from awx.main.utils.common import get_licenser
+            license_data = json.loads(data_actual)
+            license_data_validated = get_licenser(**license_data).validate()
+        except Exception:
+            logger.warning(smart_text(u"Invalid license submitted."),
+                           extra=dict(actor=request.user.username))
+            return Response({"error": _("Invalid License")}, status=status.HTTP_400_BAD_REQUEST)
+
+        # If the license is valid, write it to the database.
+        if license_data_validated['valid_key']:
+            settings.LICENSE = license_data
+            settings.TOWER_URL_BASE = "{}://{}".format(request.scheme, request.get_host())
+            return Response(license_data_validated)
+
+        logger.warning(smart_text(u"Invalid license submitted."),
+                       extra=dict(actor=request.user.username))
+        return Response({"error": _("Invalid license")}, status=status.HTTP_400_BAD_REQUEST)
+
+    def delete(self, request):
+        try:
+            settings.LICENSE = {}
+            return Response(status=status.HTTP_204_NO_CONTENT)
+        except Exception:
+            # FIX: Log
+            return Response({"error": _("Failed to remove license.")}, status=status.HTTP_400_BAD_REQUEST)

awx/api/views/webhooks.py

@@ -0,0 +1,246 @@
+from hashlib import sha1
+import hmac
+import logging
+import urllib.parse
+
+from django.utils.encoding import force_bytes
+from django.utils.translation import ugettext_lazy as _
+from django.views.decorators.csrf import csrf_exempt
+
+from rest_framework import status
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+
+from awx.api import serializers
+from awx.api.generics import APIView, GenericAPIView
+from awx.api.permissions import WebhookKeyPermission
+from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
+
+
+logger = logging.getLogger('awx.api.views.webhooks')
+
+
+class WebhookKeyView(GenericAPIView):
+    serializer_class = serializers.EmptySerializer
+    permission_classes = (WebhookKeyPermission,)
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        self.model = qs_models.get(self.kwargs['model_kwarg'])
+
+        return super().get_queryset()
+
+    def get(self, request, *args, **kwargs):
+        obj = self.get_object()
+
+        return Response({'webhook_key': obj.webhook_key})
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        obj.rotate_webhook_key()
+        obj.save(update_fields=['webhook_key'])
+
+        return Response({'webhook_key': obj.webhook_key}, status=status.HTTP_201_CREATED)
+
+
+class WebhookReceiverBase(APIView):
+    lookup_url_kwarg = None
+    lookup_field = 'pk'
+
+    permission_classes = (AllowAny,)
+    authentication_classes = ()
+
+    ref_keys = {}
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        model = qs_models.get(self.kwargs['model_kwarg'])
+        if model is None:
+            raise PermissionDenied
+
+        return model.objects.filter(webhook_service=self.service).exclude(webhook_key='')
+
+    def get_object(self):
+        queryset = self.get_queryset()
+        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
+        filter_kwargs = {self.lookup_field: self.kwargs[lookup_url_kwarg]}
+
+        obj = queryset.filter(**filter_kwargs).first()
+        if obj is None:
+            raise PermissionDenied
+
+        return obj
+
+    def get_event_type(self):
+        raise NotImplementedError
+
+    def get_event_guid(self):
+        raise NotImplementedError
+
+    def get_event_status_api(self):
+        raise NotImplementedError
+
+    def get_event_ref(self):
+        key = self.ref_keys.get(self.get_event_type(), '')
+        value = self.request.data
+        for element in key.split('.'):
+            try:
+                if element.isdigit():
+                    value = value[int(element)]
+                else:
+                    value = (value or {}).get(element)
+            except Exception:
+                value = None
+        if value == '0000000000000000000000000000000000000000':  # a deleted ref
+            value = None
+        return value
+
+    def get_signature(self):
+        raise NotImplementedError
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        logger.debug("header signature: %s", self.get_signature())
+        logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
+            raise PermissionDenied
+
+    @csrf_exempt
+    def post(self, request, *args, **kwargs):
+        # Ensure that the full contents of the request are captured for multiple uses.
+        request.body
+
+        logger.debug(
+            "headers: {}\n"
+            "data: {}\n".format(request.headers, request.data)
+        )
+        obj = self.get_object()
+        self.check_signature(obj)
+
+        event_type = self.get_event_type()
+        event_guid = self.get_event_guid()
+        event_ref = self.get_event_ref()
+        status_api = self.get_event_status_api()
+
+        kwargs = {
+            'unified_job_template_id': obj.id,
+            'webhook_service': obj.webhook_service,
+            'webhook_guid': event_guid,
+        }
+        if WorkflowJob.objects.filter(**kwargs).exists() or Job.objects.filter(**kwargs).exists():
+            # Short circuit if this webhook has already been received and acted upon.
+            logger.debug("Webhook previously received, returning without action.")
+            return Response({'message': _("Webhook previously received, aborting.")},
+                            status=status.HTTP_202_ACCEPTED)
+
+        kwargs = {
+            '_eager_fields': {
+                'launch_type': 'webhook',
+                'webhook_service': obj.webhook_service,
+                'webhook_credential': obj.webhook_credential,
+                'webhook_guid': event_guid,
+            },
+            'extra_vars': {
+                'tower_webhook_event_type': event_type,
+                'tower_webhook_event_guid': event_guid,
+                'tower_webhook_event_ref': event_ref,
+                'tower_webhook_status_api': status_api,
+                'tower_webhook_payload': request.data,
+            }
+        }
+
+        new_job = obj.create_unified_job(**kwargs)
+        new_job.signal_start()
+
+        return Response({'message': "Job queued."}, status=status.HTTP_202_ACCEPTED)
+
+
+class GithubWebhookReceiver(WebhookReceiverBase):
+    service = 'github'
+
+    ref_keys = {
+        'pull_request': 'pull_request.head.sha',
+        'pull_request_review': 'pull_request.head.sha',
+        'pull_request_review_comment': 'pull_request.head.sha',
+        'push': 'after',
+        'release': 'release.tag_name',
+        'commit_comment': 'comment.commit_id',
+        'create': 'ref',
+        'page_build': 'build.commit',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITHUB_EVENT')
+
+    def get_event_guid(self):
+        return self.request.META.get('HTTP_X_GITHUB_DELIVERY')
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'pull_request':
+            return
+        return self.request.data.get('pull_request', {}).get('statuses_url')
+
+    def get_signature(self):
+        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
+        if not header_sig:
+            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
+            raise PermissionDenied
+        hash_alg, signature = header_sig.split('=')
+        if hash_alg != 'sha1':
+            logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
+            raise PermissionDenied
+        return force_bytes(signature)
+
+
+class GitlabWebhookReceiver(WebhookReceiverBase):
+    service = 'gitlab'
+
+    ref_keys = {
+        'Push Hook': 'checkout_sha',
+        'Tag Push Hook': 'checkout_sha',
+        'Merge Request Hook': 'object_attributes.last_commit.id',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITLAB_EVENT')
+
+    def get_event_guid(self):
+        # GitLab does not provide a unique identifier on events, so construct one.
+        h = sha1()
+        h.update(force_bytes(self.request.body))
+        return h.hexdigest()
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'Merge Request Hook':
+            return
+        project = self.request.data.get('project', {})
+        repo_url = project.get('web_url')
+        if not repo_url:
+            return
+        parsed = urllib.parse.urlparse(repo_url)
+
+        return "{}://{}/api/v4/projects/{}/statuses/{}".format(
+            parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
+
+    def get_signature(self):
+        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        # GitLab only returns the secret token, not an hmac hash.  Use
+        # the hmac `compare_digest` helper function to prevent timing
+        # analysis by attackers.
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
+            raise PermissionDenied

awx/celery.py

@@ -1,25 +0,0 @@
-
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.
-
-from __future__ import absolute_import, unicode_literals
-
-import os
-from celery import Celery
-from django.conf import settings # noqa
-
-
-try:
-    import awx.devonly # noqa
-    MODE = 'development'
-except ImportError: # pragma: no cover
-    MODE = 'production'
-
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
-
-app = Celery('awx')
-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
-
-if __name__ == '__main__':
-    app.start()

awx/conf/apps.py

@@ -2,8 +2,6 @@
 from django.apps import AppConfig
 # from django.core import checks
 from django.utils.translation import ugettext_lazy as _
-from awx.main.utils.handlers import configure_external_logger
-from django.conf import settings
 
 
 class ConfConfig(AppConfig):
@@ -11,16 +9,7 @@ class ConfConfig(AppConfig):
     name = 'awx.conf'
     verbose_name = _('Configuration')
 
-    def configure_oauth2_provider(self, settings):
-        from oauth2_provider import settings as o_settings
-        o_settings.oauth2_settings = o_settings.OAuth2ProviderSettings(
-            settings.OAUTH2_PROVIDER, o_settings.DEFAULTS,
-            o_settings.IMPORT_STRINGS, o_settings.MANDATORY
-        )
-
     def ready(self):
         self.module.autodiscover()
         from .settings import SettingsWrapper
         SettingsWrapper.initialize()
-        configure_external_logger(settings)
-        self.configure_oauth2_provider(settings)

awx/conf/conf.py

@@ -78,9 +78,6 @@ register(
     # the other settings change, the cached value for this setting will be
     # cleared to require it to be recomputed.
     depends_on=['ANSIBLE_COW_SELECTION'],
-    # Optional; licensed feature required to be able to view or modify this
-    # setting.
-    feature_required='rebranding',
     # Optional; field is stored encrypted in the database and only $encrypted$
     # is returned via the API.
     encrypted=True,

awx/conf/fields.py

@@ -1,16 +1,19 @@
 # Python
+import os
+import re
 import logging
-import urlparse
+import urllib.parse as urlparse
 from collections import OrderedDict
 
 # Django
-from django.core.validators import URLValidator
+from django.core.validators import URLValidator, _lazy_re_compile
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
-from rest_framework.fields import *  # noqa
-
-import six
+from rest_framework.fields import (  # noqa
+    BooleanField, CharField, ChoiceField, DictField, EmailField,
+    IntegerField, ListField, NullBooleanField
+)
 
 logger = logging.getLogger('awx.conf.fields')
 
@@ -71,7 +74,7 @@ class StringListBooleanField(ListField):
                 return False
             elif value in NullBooleanField.NULL_VALUES:
                 return None
-            elif isinstance(value, basestring):
+            elif isinstance(value, str):
                 return self.child.to_representation(value)
         except TypeError:
             pass
@@ -88,22 +91,70 @@ class StringListBooleanField(ListField):
                 return False
             elif data in NullBooleanField.NULL_VALUES:
                 return None
-            elif isinstance(data, basestring):
+            elif isinstance(data, str):
                 return self.child.run_validation(data)
         except TypeError:
             pass
         self.fail('type_error', input_type=type(data))
 
 
+class StringListPathField(StringListField):
+
+    default_error_messages = {
+        'type_error': _('Expected list of strings but got {input_type} instead.'),
+        'path_error': _('{path} is not a valid path choice.'),
+    }
+
+    def to_internal_value(self, paths):
+        if isinstance(paths, (list, tuple)):
+            for p in paths:
+                if not isinstance(p, str):
+                    self.fail('type_error', input_type=type(p))
+                if not os.path.exists(p):
+                    self.fail('path_error', path=p)
+
+            return super(StringListPathField, self).to_internal_value(sorted({os.path.normpath(path) for path in paths}))
+        else:
+            self.fail('type_error', input_type=type(paths))
+
+
 class URLField(CharField):
+    # these lines set up a custom regex that allow numbers in the
+    # top-level domain
+    tld_re = (
+        r'\.'                                              # dot
+        r'(?!-)'                                           # can't start with a dash
+        r'(?:[a-z' + URLValidator.ul + r'0-9' + '-]{2,63}' # domain label, this line was changed from the original URLValidator
+        r'|xn--[a-z0-9]{1,59})'                            # or punycode label
+        r'(?<!-)'                                          # can't end with a dash
+        r'\.?'                                             # may have a trailing dot
+    )
+
+    host_re = '(' + URLValidator.hostname_re + URLValidator.domain_re + tld_re + '|localhost)'
+
+    regex = _lazy_re_compile(
+        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
+        r'(?:[^\s:@/]+(?::[^\s:@/]*)?@)?'  # user:pass authentication
+        r'(?:' + URLValidator.ipv4_re + '|' + URLValidator.ipv6_re + '|' + host_re + ')'
+        r'(?::\d{2,5})?'  # port
+        r'(?:[/?#][^\s]*)?'  # resource path
+        r'\Z', re.IGNORECASE)
 
     def __init__(self, **kwargs):
         schemes = kwargs.pop('schemes', None)
+        regex = kwargs.pop('regex', None)
         self.allow_plain_hostname = kwargs.pop('allow_plain_hostname', False)
+        self.allow_numbers_in_top_level_domain = kwargs.pop('allow_numbers_in_top_level_domain', True)
         super(URLField, self).__init__(**kwargs)
         validator_kwargs = dict(message=_('Enter a valid URL'))
         if schemes is not None:
             validator_kwargs['schemes'] = schemes
+        if regex is not None:
+            validator_kwargs['regex'] = regex
+        if self.allow_numbers_in_top_level_domain and regex is None:
+            # default behavior is to allow numbers in the top level domain
+            # if a custom regex isn't provided
+            validator_kwargs['regex'] = URLField.regex
         self.validators.append(URLValidator(**validator_kwargs))
 
     def to_representation(self, value):
@@ -139,7 +190,7 @@ class KeyValueField(DictField):
     def to_internal_value(self, data):
         ret = super(KeyValueField, self).to_internal_value(data)
         for value in data.values():
-            if not isinstance(value, six.string_types + six.integer_types + (float,)):
+            if not isinstance(value, (str, int, float)):
                 if isinstance(value, OrderedDict):
                     value = dict(value)
                 self.fail('invalid_child', input=value)

awx/conf/license.py

@@ -1,64 +1,18 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
-# Django
-from django.core.signals import setting_changed
-from django.dispatch import receiver
-from django.utils.translation import ugettext_lazy as _
 
-# Django REST Framework
-from rest_framework.exceptions import APIException
-
-# Tower
-from awx.main.utils.common import get_licenser
-from awx.main.utils import memoize, memoize_delete
-
-__all__ = ['LicenseForbids', 'get_license', 'get_licensed_features',
-           'feature_enabled', 'feature_exists']
-
-
-class LicenseForbids(APIException):
-    status_code = 402
-    default_detail = _('Your Tower license does not allow that.')
+__all__ = ['get_license']
 
 
 def _get_validated_license_data():
+    from awx.main.utils.common import get_licenser
     return get_licenser().validate()
 
 
-@receiver(setting_changed)
-def _on_setting_changed(sender, **kwargs):
-    # Clear cached result above when license changes.
-    if kwargs.get('setting', None) == 'LICENSE':
-        memoize_delete('feature_enabled')
-
-
 def get_license(show_key=False):
     """Return a dictionary representing the active license on this Tower instance."""
     license_data = _get_validated_license_data()
     if not show_key:
         license_data.pop('license_key', None)
     return license_data
-
-
-def get_licensed_features():
-    """Return a set of all features enabled by the active license."""
-    features = set()
-    for feature, enabled in _get_validated_license_data().get('features', {}).items():
-        if enabled:
-            features.add(feature)
-    return features
-
-
-@memoize(track_function=True)
-def feature_enabled(name):
-    """Return True if the requested feature is enabled, False otherwise."""
-    validated_license_data = _get_validated_license_data()
-    if validated_license_data.get('license_type', 'UNLICENSED') == 'open':
-        return True
-    return validated_license_data.get('features', {}).get(name, False)
-
-
-def feature_exists(name):
-    """Return True if the requested feature name exists, False otherwise."""
-    return bool(name in _get_validated_license_data().get('features', {}))

awx/conf/management/commands/migrate_to_database_settings.py

@@ -1,480 +0,0 @@
-# Copyright (c) 2016 Ansible, Inc.
-# All Rights Reserved.
-
-# Python
-import base64
-import collections
-import difflib
-import json
-import os
-import shutil
-
-# Django
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-from django.utils.text import slugify
-from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
-
-# Tower
-from awx import MODE
-from awx.conf import settings_registry
-from awx.conf.fields import empty, SkipField
-from awx.conf.models import Setting
-from awx.conf.utils import comment_assignments
-
-
-class Command(BaseCommand):
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            'category',
-            nargs='*',
-            type=str,
-        )
-        parser.add_argument(
-            '--dry-run',
-            action='store_true',
-            dest='dry_run',
-            default=False,
-            help=_('Only show which settings would be commented/migrated.'),
-        )
-        parser.add_argument(
-            '--skip-errors',
-            action='store_true',
-            dest='skip_errors',
-            default=False,
-            help=_('Skip over settings that would raise an error when commenting/migrating.'),
-        )
-        parser.add_argument(
-            '--no-comment',
-            action='store_true',
-            dest='no_comment',
-            default=False,
-            help=_('Skip commenting out settings in files.'),
-        )
-        parser.add_argument(
-            '--comment-only',
-            action='store_true',
-            dest='comment_only',
-            default=False,
-            help=_('Skip migrating and only comment out settings in files.'),
-        )
-        parser.add_argument(
-            '--backup-suffix',
-            dest='backup_suffix',
-            default=now().strftime('.%Y%m%d%H%M%S'),
-            help=_('Backup existing settings files with this suffix.'),
-        )
-
-    @transaction.atomic
-    def handle(self, *args, **options):
-        self.verbosity = int(options.get('verbosity', 1))
-        self.dry_run = bool(options.get('dry_run', False))
-        self.skip_errors = bool(options.get('skip_errors', False))
-        self.no_comment = bool(options.get('no_comment', False))
-        self.comment_only = bool(options.get('comment_only', False))
-        self.backup_suffix = options.get('backup_suffix', '')
-        self.categories = options.get('category', None) or ['all']
-        self.style.HEADING = self.style.MIGRATE_HEADING
-        self.style.LABEL = self.style.MIGRATE_LABEL
-        self.style.OK = self.style.SQL_FIELD
-        self.style.SKIP = self.style.WARNING
-        self.style.VALUE = self.style.SQL_KEYWORD
-
-        # Determine if any categories provided are invalid.
-        category_slugs = []
-        invalid_categories = []
-        for category in self.categories:
-            category_slug = slugify(category)
-            if category_slug in settings_registry.get_registered_categories():
-                if category_slug not in category_slugs:
-                    category_slugs.append(category_slug)
-            else:
-                if category not in invalid_categories:
-                    invalid_categories.append(category)
-        if len(invalid_categories) == 1:
-            raise CommandError('Invalid setting category: {}'.format(invalid_categories[0]))
-        elif len(invalid_categories) > 1:
-            raise CommandError('Invalid setting categories: {}'.format(', '.join(invalid_categories)))
-
-        # Build a list of all settings to be migrated.
-        registered_settings = []
-        for category_slug in category_slugs:
-            for registered_setting in settings_registry.get_registered_settings(category_slug=category_slug, read_only=False):
-                if registered_setting not in registered_settings:
-                    registered_settings.append(registered_setting)
-
-        self._migrate_settings(registered_settings)
-
-    def _get_settings_file_patterns(self):
-        if MODE == 'development':
-            return [
-                '/etc/tower/settings.py',
-                '/etc/tower/conf.d/*.py',
-                os.path.join(os.path.dirname(__file__), '..', '..', '..', 'settings', 'local_*.py')
-            ]
-        else:
-            return [
-                os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py'),
-                os.path.join(os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/'), '*.py'),
-            ]
-
-    def _get_license_file(self):
-        return os.environ.get('AWX_LICENSE_FILE', '/etc/tower/license')
-
-    def _comment_license_file(self, dry_run=True):
-        license_file = self._get_license_file()
-        diff_lines = []
-        if os.path.exists(license_file):
-            try:
-                raw_license_data = open(license_file).read()
-                json.loads(raw_license_data)
-            except Exception as e:
-                raise CommandError('Error reading license from {0}: {1!r}'.format(license_file, e))
-            if self.backup_suffix:
-                backup_license_file = '{}{}'.format(license_file, self.backup_suffix)
-            else:
-                backup_license_file = '{}.old'.format(license_file)
-            diff_lines = list(difflib.unified_diff(
-                raw_license_data.splitlines(),
-                [],
-                fromfile=backup_license_file,
-                tofile=license_file,
-                lineterm='',
-            ))
-            if not dry_run:
-                if self.backup_suffix:
-                    shutil.copy2(license_file, backup_license_file)
-                os.remove(license_file)
-        return diff_lines
-
-    def _get_local_settings_file(self):
-        if MODE == 'development':
-            static_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'ui', 'static')
-        else:
-            static_root = settings.STATIC_ROOT
-        return os.path.join(static_root, 'local_settings.json')
-
-    def _comment_local_settings_file(self, dry_run=True):
-        local_settings_file = self._get_local_settings_file()
-        diff_lines = []
-        if os.path.exists(local_settings_file):
-            try:
-                raw_local_settings_data = open(local_settings_file).read()
-                json.loads(raw_local_settings_data)
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading local settings from {0}: {1!r}'.format(local_settings_file, e))
-                return diff_lines
-            if self.backup_suffix:
-                backup_local_settings_file = '{}{}'.format(local_settings_file, self.backup_suffix)
-            else:
-                backup_local_settings_file = '{}.old'.format(local_settings_file)
-            diff_lines = list(difflib.unified_diff(
-                raw_local_settings_data.splitlines(),
-                [],
-                fromfile=backup_local_settings_file,
-                tofile=local_settings_file,
-                lineterm='',
-            ))
-            if not dry_run:
-                if self.backup_suffix:
-                    shutil.copy2(local_settings_file, backup_local_settings_file)
-                os.remove(local_settings_file)
-        return diff_lines
-
-    def _get_custom_logo_file(self):
-        if MODE == 'development':
-            static_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'ui', 'static')
-        else:
-            static_root = settings.STATIC_ROOT
-        return os.path.join(static_root, 'assets', 'custom_console_logo.png')
-
-    def _comment_custom_logo_file(self, dry_run=True):
-        custom_logo_file = self._get_custom_logo_file()
-        diff_lines = []
-        if os.path.exists(custom_logo_file):
-            try:
-                raw_custom_logo_data = open(custom_logo_file).read()
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading custom logo from {0}: {1!r}'.format(custom_logo_file, e))
-                return diff_lines
-            if self.backup_suffix:
-                backup_custom_logo_file = '{}{}'.format(custom_logo_file, self.backup_suffix)
-            else:
-                backup_custom_logo_file = '{}.old'.format(custom_logo_file)
-            diff_lines = list(difflib.unified_diff(
-                ['<PNG Image ({} bytes)>'.format(len(raw_custom_logo_data))],
-                [],
-                fromfile=backup_custom_logo_file,
-                tofile=custom_logo_file,
-                lineterm='',
-            ))
-            if not dry_run:
-                if self.backup_suffix:
-                    shutil.copy2(custom_logo_file, backup_custom_logo_file)
-                os.remove(custom_logo_file)
-        return diff_lines
-
-    def _check_if_needs_comment(self, patterns, setting):
-        files_to_comment = []
-        # If any diffs are returned, this setting needs to be commented.
-        diffs = comment_assignments(patterns, setting, dry_run=True)
-        if setting == 'LICENSE':
-            diffs.extend(self._comment_license_file(dry_run=True))
-        elif setting == 'CUSTOM_LOGIN_INFO':
-            diffs.extend(self._comment_local_settings_file(dry_run=True))
-        elif setting == 'CUSTOM_LOGO':
-            diffs.extend(self._comment_custom_logo_file(dry_run=True))
-        for diff in diffs:
-            for line in diff.splitlines():
-                if line.startswith('+++ '):
-                    files_to_comment.append(line[4:])
-        return files_to_comment
-
-    def _check_if_needs_migration(self, setting):
-        # Check whether the current value differs from the default.
-        default_value = settings.DEFAULTS_SNAPSHOT.get(setting, empty)
-        if default_value is empty and setting != 'LICENSE':
-            field = settings_registry.get_setting_field(setting, read_only=True)
-            try:
-                default_value = field.get_default()
-            except SkipField:
-                pass
-        current_value = getattr(settings, setting, empty)
-        if setting == 'CUSTOM_LOGIN_INFO' and current_value in {empty, ''}:
-            local_settings_file = self._get_local_settings_file()
-            try:
-                if os.path.exists(local_settings_file):
-                    local_settings = json.load(open(local_settings_file))
-                    current_value = local_settings.get('custom_login_info', '')
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading custom login info from {0}: {1!r}'.format(local_settings_file, e))
-        if setting == 'CUSTOM_LOGO' and current_value in {empty, ''}:
-            custom_logo_file = self._get_custom_logo_file()
-            try:
-                if os.path.exists(custom_logo_file):
-                    custom_logo_data = open(custom_logo_file).read()
-                    if custom_logo_data:
-                        current_value = 'data:image/png;base64,{}'.format(base64.b64encode(custom_logo_data))
-                    else:
-                        current_value = ''
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading custom logo from {0}: {1!r}'.format(custom_logo_file, e))
-        if current_value != default_value:
-            if current_value is empty:
-                current_value = None
-            return current_value
-        return empty
-
-    def _display_tbd(self, setting, files_to_comment, migrate_value, comment_error=None, migrate_error=None):
-        if self.verbosity >= 1:
-            if files_to_comment:
-                if migrate_value is not empty:
-                    action = 'Migrate + Comment'
-                else:
-                    action = 'Comment'
-                if comment_error or migrate_error:
-                    action = self.style.ERROR('{} (skipped)'.format(action))
-                else:
-                    action = self.style.OK(action)
-                self.stdout.write('  {}: {}'.format(
-                    self.style.LABEL(setting),
-                    action,
-                ))
-                if self.verbosity >= 2:
-                    if migrate_error:
-                        self.stdout.write('    - Migrate value: {}'.format(
-                            self.style.ERROR(migrate_error),
-                        ))
-                    elif migrate_value is not empty:
-                        self.stdout.write('    - Migrate value: {}'.format(
-                            self.style.VALUE(repr(migrate_value)),
-                        ))
-                    if comment_error:
-                        self.stdout.write('    - Comment: {}'.format(
-                            self.style.ERROR(comment_error),
-                        ))
-                    elif files_to_comment:
-                        for file_to_comment in files_to_comment:
-                            self.stdout.write('    - Comment in: {}'.format(
-                                self.style.VALUE(file_to_comment),
-                            ))
-            else:
-                if self.verbosity >= 2:
-                    self.stdout.write('  {}: {}'.format(
-                        self.style.LABEL(setting),
-                        self.style.SKIP('No Migration'),
-                    ))
-
-    def _display_migrate(self, setting, action, display_value):
-        if self.verbosity >= 1:
-            if action == 'No Change':
-                action = self.style.SKIP(action)
-            else:
-                action = self.style.OK(action)
-            self.stdout.write('  {}: {}'.format(
-                self.style.LABEL(setting),
-                action,
-            ))
-            if self.verbosity >= 2:
-                for line in display_value.splitlines():
-                    self.stdout.write('    {}'.format(
-                        self.style.VALUE(line),
-                    ))
-
-    def _display_diff_summary(self, filename, added, removed):
-        self.stdout.write('  {} {}{} {}{}'.format(
-            self.style.LABEL(filename),
-            self.style.ERROR('-'),
-            self.style.ERROR(int(removed)),
-            self.style.OK('+'),
-            self.style.OK(str(added)),
-        ))
-
-    def _display_comment(self, diffs):
-        for diff in diffs:
-            if self.verbosity >= 2:
-                for line in diff.splitlines():
-                    display_line = line
-                    if line.startswith('--- ') or line.startswith('+++ '):
-                        display_line = self.style.LABEL(line)
-                    elif line.startswith('-'):
-                        display_line = self.style.ERROR(line)
-                    elif line.startswith('+'):
-                        display_line = self.style.OK(line)
-                    elif line.startswith('@@'):
-                        display_line = self.style.VALUE(line)
-                    if line.startswith('--- ') or line.startswith('+++ '):
-                        self.stdout.write('  ' + display_line)
-                    else:
-                        self.stdout.write('    ' + display_line)
-            elif self.verbosity >= 1:
-                filename, lines_added, lines_removed = None, 0, 0
-                for line in diff.splitlines():
-                    if line.startswith('+++ '):
-                        if filename:
-                            self._display_diff_summary(filename, lines_added, lines_removed)
-                        filename, lines_added, lines_removed = line[4:], 0, 0
-                    elif line.startswith('+'):
-                        lines_added += 1
-                    elif line.startswith('-'):
-                        lines_removed += 1
-                if filename:
-                    self._display_diff_summary(filename, lines_added, lines_removed)
-
-    def _discover_settings(self, registered_settings):
-        if self.verbosity >= 1:
-            self.stdout.write(self.style.HEADING('Discovering settings to be migrated and commented:'))
-
-        # Determine which settings need to be commented/migrated.
-        to_migrate = collections.OrderedDict()
-        to_comment = collections.OrderedDict()
-        patterns = self._get_settings_file_patterns()
-
-        for name in registered_settings:
-            comment_error, migrate_error = None, None
-            files_to_comment = []
-            try:
-                files_to_comment = self._check_if_needs_comment(patterns, name)
-            except Exception as e:
-                comment_error = 'Error commenting {0}: {1!r}'.format(name, e)
-                if not self.skip_errors:
-                    raise CommandError(comment_error)
-            if files_to_comment:
-                to_comment[name] = files_to_comment
-            migrate_value = empty
-            if files_to_comment:
-                migrate_value = self._check_if_needs_migration(name)
-                if migrate_value is not empty:
-                    field = settings_registry.get_setting_field(name)
-                    assert not field.read_only
-                    try:
-                        data = field.to_representation(migrate_value)
-                        setting_value = field.run_validation(data)
-                        db_value = field.to_representation(setting_value)
-                        to_migrate[name] = db_value
-                    except Exception as e:
-                        to_comment.pop(name)
-                        migrate_error = 'Unable to assign value {0!r} to setting "{1}: {2!s}".'.format(migrate_value, name, e)
-                        if not self.skip_errors:
-                            raise CommandError(migrate_error)
-            self._display_tbd(name, files_to_comment, migrate_value, comment_error, migrate_error)
-        if self.verbosity == 1 and not to_migrate and not to_comment:
-            self.stdout.write('  No settings found to migrate or comment!')
-        return (to_migrate, to_comment)
-
-    def _migrate(self, to_migrate):
-        if self.verbosity >= 1:
-            if self.dry_run:
-                self.stdout.write(self.style.HEADING('Migrating settings to database (dry-run):'))
-            else:
-                self.stdout.write(self.style.HEADING('Migrating settings to database:'))
-            if not to_migrate:
-                self.stdout.write('  No settings to migrate!')
-
-        # Now migrate those settings to the database.
-        for name, db_value in to_migrate.items():
-            display_value = json.dumps(db_value, indent=4)
-            setting = Setting.objects.filter(key=name, user__isnull=True).order_by('pk').first()
-            action = 'No Change'
-            if not setting:
-                action = 'Migrated'
-                if not self.dry_run:
-                    Setting.objects.create(key=name, user=None, value=db_value)
-            elif setting.value != db_value or type(setting.value) != type(db_value):
-                action = 'Updated'
-                if not self.dry_run:
-                    setting.value = db_value
-                    setting.save(update_fields=['value'])
-            self._display_migrate(name, action, display_value)
-
-    def _comment(self, to_comment):
-        if self.verbosity >= 1:
-            if bool(self.dry_run or self.no_comment):
-                self.stdout.write(self.style.HEADING('Commenting settings in files (dry-run):'))
-            else:
-                self.stdout.write(self.style.HEADING('Commenting settings in files:'))
-            if not to_comment:
-                self.stdout.write('  No settings to comment!')
-
-        # Now comment settings in settings files.
-        if to_comment:
-            to_comment_patterns = []
-            license_file_to_comment = None
-            local_settings_file_to_comment = None
-            custom_logo_file_to_comment = None
-            for files_to_comment in to_comment.values():
-                for file_to_comment in files_to_comment:
-                    if file_to_comment == self._get_license_file():
-                        license_file_to_comment = file_to_comment
-                    elif file_to_comment == self._get_local_settings_file():
-                        local_settings_file_to_comment = file_to_comment
-                    elif file_to_comment == self._get_custom_logo_file():
-                        custom_logo_file_to_comment = file_to_comment
-                    elif file_to_comment not in to_comment_patterns:
-                        to_comment_patterns.append(file_to_comment)
-            # Run once in dry-run mode to catch any errors from updating the files.
-            diffs = comment_assignments(to_comment_patterns, to_comment.keys(), dry_run=True, backup_suffix=self.backup_suffix)
-            # Then, if really updating, run again.
-            if not self.dry_run and not self.no_comment:
-                diffs = comment_assignments(to_comment_patterns, to_comment.keys(), dry_run=False, backup_suffix=self.backup_suffix)
-                if license_file_to_comment:
-                    diffs.extend(self._comment_license_file(dry_run=False))
-                if local_settings_file_to_comment:
-                    diffs.extend(self._comment_local_settings_file(dry_run=False))
-                if custom_logo_file_to_comment:
-                    diffs.extend(self._comment_custom_logo_file(dry_run=False))
-            self._display_comment(diffs)
-
-    def _migrate_settings(self, registered_settings):
-        to_migrate, to_comment = self._discover_settings(registered_settings)
-
-        if not bool(self.comment_only):
-            self._migrate(to_migrate)
-        self._comment(to_comment)

awx/conf/migrations/0001_initial.py

@@ -21,7 +21,8 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('key', models.CharField(max_length=255)),
                 ('value', jsonfield.fields.JSONField(null=True)),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False,
+                                           to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True)),
             ],
             options={
                 'abstract': False,

awx/conf/migrations/0004_v320_reencrypt.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from django.db import migrations
-from awx.conf.migrations import _reencrypt
 
 
 class Migration(migrations.Migration):
@@ -12,5 +11,8 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(_reencrypt.replace_aesecb_fernet),
+        # This list is intentionally empty.
+        # Tower 3.2 included several data migrations that are no longer
+        # necessary (this list is now empty because Tower 3.2 is past EOL and
+        # cannot be directly upgraded to modern versions of Tower)
     ]

awx/conf/migrations/0005_v330_rename_two_session_settings.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from django.db import migrations
+from awx.conf.migrations import _rename_setting
+    
+    
+def copy_session_settings(apps, schema_editor):
+    _rename_setting.rename_setting(apps, schema_editor, old_key='AUTH_TOKEN_PER_USER', new_key='SESSIONS_PER_USER')
+    _rename_setting.rename_setting(apps, schema_editor, old_key='AUTH_TOKEN_EXPIRATION', new_key='SESSION_COOKIE_AGE')
+
+
+def reverse_copy_session_settings(apps, schema_editor):
+    _rename_setting.rename_setting(apps, schema_editor, old_key='SESSION_COOKIE_AGE', new_key='AUTH_TOKEN_EXPIRATION')
+    _rename_setting.rename_setting(apps, schema_editor, old_key='SESSIONS_PER_USER', new_key='AUTH_TOKEN_PER_USER')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('conf', '0004_v320_reencrypt'),
+    ]
+
+    operations = [
+        migrations.RunPython(copy_session_settings, reverse_copy_session_settings),
+    ]
+    

awx/conf/migrations/0006_v331_ldap_group_type.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+# AWX
+from awx.conf.migrations._ldap_group_type import fill_ldap_group_type_params
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('conf', '0005_v330_rename_two_session_settings'),
+    ]
+
+    operations = [
+        migrations.RunPython(fill_ldap_group_type_params),
+    ]

awx/conf/migrations/_ldap_group_type.py

@@ -0,0 +1,30 @@
+
+import inspect
+
+from django.conf import settings
+from django.utils.timezone import now
+
+
+def fill_ldap_group_type_params(apps, schema_editor):
+    group_type = settings.AUTH_LDAP_GROUP_TYPE
+    Setting = apps.get_model('conf', 'Setting')
+
+    group_type_params = {'name_attr': 'cn', 'member_attr': 'member'}
+    qs = Setting.objects.filter(key='AUTH_LDAP_GROUP_TYPE_PARAMS')
+    entry = None
+    if qs.exists():
+        entry = qs[0]
+        group_type_params = entry.value
+    else:
+        entry = Setting(key='AUTH_LDAP_GROUP_TYPE_PARAMS',
+                        value=group_type_params,
+                        created=now(),
+                        modified=now())
+
+    init_attrs = set(inspect.getargspec(group_type.__init__).args[1:])
+    for k in list(group_type_params.keys()):
+        if k not in init_attrs:
+            del group_type_params[k]
+
+    entry.value = group_type_params
+    entry.save()

awx/conf/migrations/_reencrypt.py

@@ -1,31 +1,13 @@
 import base64
 import hashlib
 
-import six
-from django.utils.encoding import smart_str
-
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.ciphers import Cipher
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
 from cryptography.hazmat.primitives.ciphers.modes import ECB
 
-from awx.conf import settings_registry
 
-
-__all__ = ['replace_aesecb_fernet', 'get_encryption_key', 'encrypt_field',
-           'decrypt_value', 'decrypt_value', 'should_decrypt_field']
-
-
-def replace_aesecb_fernet(apps, schema_editor):
-    from awx.main.utils.encryption import encrypt_field
-    Setting = apps.get_model('conf', 'Setting')
-
-    for setting in Setting.objects.filter().order_by('pk'):
-        if settings_registry.is_setting_encrypted(setting.key):
-            if should_decrypt_field(setting.value):
-                setting.value = decrypt_field(setting, 'value')
-                setting.value = encrypt_field(setting, 'value')
-            setting.save()
+__all__ = ['get_encryption_key', 'decrypt_field']
 
 
 def get_encryption_key(field_name, pk=None):
@@ -77,38 +59,3 @@ def decrypt_field(instance, field_name, subfield=None):
     key = get_encryption_key(field_name, getattr(instance, 'pk', None))
 
     return decrypt_value(key, value)
-
-
-def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False):
-    '''
-    Return content of the given instance and field name encrypted.
-    '''
-    value = getattr(instance, field_name)
-    if isinstance(value, dict) and subfield is not None:
-        value = value[subfield]
-    if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'):
-        return value
-    if skip_utf8:
-        utf8 = False
-    else:
-        utf8 = type(value) == six.text_type
-    value = smart_str(value)
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
-    encryptor = Cipher(AES(key), ECB(), default_backend()).encryptor()
-    block_size = 16
-    while len(value) % block_size != 0:
-        value += '\x00'
-    encrypted = encryptor.update(value) + encryptor.finalize()
-    b64data = base64.b64encode(encrypted)
-    tokens = ['$encrypted', 'AES', b64data]
-    if utf8:
-        # If the value to encrypt is utf-8, we need to add a marker so we
-        # know to decode the data when it's decrypted later
-        tokens.insert(1, 'UTF8')
-    return '$'.join(tokens)
-
-
-def should_decrypt_field(value):
-    if hasattr(value, 'startswith'):
-        return value.startswith('$encrypted$') and '$AESCBC$' not in value
-    return False

awx/conf/migrations/_rename_setting.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import logging
+from django.utils.timezone import now
+from django.conf import settings
+
+logger = logging.getLogger('awx.conf.settings')
+
+__all__ = ['rename_setting']    
+    
+    
+def rename_setting(apps, schema_editor, old_key, new_key):
+    
+    old_setting = None
+    Setting = apps.get_model('conf', 'Setting')
+    if Setting.objects.filter(key=new_key).exists() or hasattr(settings, new_key):
+        logger.info('Setting ' + new_key + ' unexpectedly exists before this migration, it will be replaced by the value of the ' + old_key + ' setting.')
+        Setting.objects.filter(key=new_key).delete()
+    # Look for db setting, which wouldn't be picked up by SettingsWrapper because the register method is gone
+    if Setting.objects.filter(key=old_key).exists():
+        old_setting = Setting.objects.filter(key=old_key).last().value
+        Setting.objects.filter(key=old_key).delete()
+    # Look for "on-disk" setting (/etc/tower/conf.d)
+    if hasattr(settings, old_key):
+        old_setting = getattr(settings, old_key)
+    if old_setting is not None:
+        Setting.objects.create(key=new_key, 
+                               value=old_setting, 
+                               created=now(),
+                               modified=now()
+                               )
+        

awx/conf/models.py

@@ -33,7 +33,7 @@ class Setting(CreatedModifiedModel):
         on_delete=models.CASCADE,
     ))
 
-    def __unicode__(self):
+    def __str__(self):
         try:
             json_value = json.dumps(self.value)
         except ValueError:
@@ -78,6 +78,14 @@ class Setting(CreatedModifiedModel):
     def get_cache_id_key(self, key):
         return '{}_ID'.format(key)
 
+    def display_value(self):
+        if self.key == 'LICENSE' and 'license_key' in self.value:
+            # don't log the license key in activity stream
+            value = self.value.copy()
+            value['license_key'] = '********'
+            return value
+        return self.value
+
 
 import awx.conf.signals  # noqa
 

awx/conf/registry.py

@@ -68,7 +68,7 @@ class SettingsRegistry(object):
     def get_dependent_settings(self, setting):
         return self._dependent_settings.get(setting, set())
 
-    def get_registered_categories(self, features_enabled=None):
+    def get_registered_categories(self):
         categories = {
             'all': _('All'),
             'changed': _('Changed'),
@@ -77,10 +77,6 @@ class SettingsRegistry(object):
             category_slug = kwargs.get('category_slug', None)
             if category_slug is None or category_slug in categories:
                 continue
-            if features_enabled is not None:
-                feature_required = kwargs.get('feature_required', None)
-                if feature_required and feature_required not in features_enabled:
-                    continue
             if category_slug == 'user':
                 categories['user'] = _('User')
                 categories['user-defaults'] = _('User-Defaults')
@@ -88,7 +84,7 @@ class SettingsRegistry(object):
                 categories[category_slug] = kwargs.get('category', None) or category_slug
         return categories
 
-    def get_registered_settings(self, category_slug=None, read_only=None, features_enabled=None, slugs_to_ignore=set()):
+    def get_registered_settings(self, category_slug=None, read_only=None, slugs_to_ignore=set()):
         setting_names = []
         if category_slug == 'user-defaults':
             category_slug = 'user'
@@ -100,14 +96,10 @@ class SettingsRegistry(object):
             if kwargs.get('category_slug', None) in slugs_to_ignore:
                 continue
             if (read_only in {True, False} and kwargs.get('read_only', False) != read_only and
-                    setting not in ('AWX_ISOLATED_PRIVATE_KEY', 'AWX_ISOLATED_PUBLIC_KEY')):
+                    setting not in ('INSTALL_UUID', 'AWX_ISOLATED_PRIVATE_KEY', 'AWX_ISOLATED_PUBLIC_KEY')):
                 # Note: Doesn't catch fields that set read_only via __init__;
                 # read-only field kwargs should always include read_only=True.
                 continue
-            if features_enabled is not None:
-                feature_required = kwargs.get('feature_required', None)
-                if feature_required and feature_required not in features_enabled:
-                    continue
             setting_names.append(setting)
         return setting_names
 
@@ -135,7 +127,6 @@ class SettingsRegistry(object):
         category = field_kwargs.pop('category', None)
         depends_on = frozenset(field_kwargs.pop('depends_on', None) or [])
         placeholder = field_kwargs.pop('placeholder', empty)
-        feature_required = field_kwargs.pop('feature_required', empty)
         encrypted = bool(field_kwargs.pop('encrypted', False))
         defined_in_file = bool(field_kwargs.pop('defined_in_file', False))
         if getattr(field_kwargs.get('child', None), 'source', None) is not None:
@@ -146,8 +137,6 @@ class SettingsRegistry(object):
         field_instance.depends_on = depends_on
         if placeholder is not empty:
             field_instance.placeholder = placeholder
-        if feature_required is not empty:
-            field_instance.feature_required = feature_required
         field_instance.defined_in_file = defined_in_file
         if field_instance.defined_in_file:
             field_instance.help_text = (

awx/conf/serializers.py

@@ -1,8 +1,6 @@
 # Django REST Framework
 from rest_framework import serializers
 
-import six
-
 # Tower
 from awx.api.fields import VerbatimField
 from awx.api.serializers import BaseSerializer
@@ -47,12 +45,12 @@ class SettingFieldMixin(object):
     """Mixin to use a registered setting field class for API display/validation."""
 
     def to_representation(self, obj):
-        if getattr(self, 'encrypted', False) and isinstance(obj, six.string_types) and obj:
+        if getattr(self, 'encrypted', False) and isinstance(obj, str) and obj:
             return '$encrypted$'
         return obj
 
     def to_internal_value(self, value):
-        if getattr(self, 'encrypted', False) and isinstance(value, six.string_types) and value.startswith('$encrypted$'):
+        if getattr(self, 'encrypted', False) and isinstance(value, str) and value.startswith('$encrypted$'):
             raise serializers.SkipField()
         obj = super(SettingFieldMixin, self).to_internal_value(value)
         return super(SettingFieldMixin, self).to_representation(obj)
@@ -90,7 +88,7 @@ class SettingSingletonSerializer(serializers.Serializer):
                 continue
             extra_kwargs = {}
             # Make LICENSE and AWX_ISOLATED_KEY_GENERATION read-only here;
-            # LICENSE is only updated via /api/v1/config/
+            # LICENSE is only updated via /api/v2/config/
             # AWX_ISOLATED_KEY_GENERATION is only set/unset via the setup playbook
             if key in ('LICENSE', 'AWX_ISOLATED_KEY_GENERATION'):
                 extra_kwargs['read_only'] = True

awx/conf/settings.py

@@ -2,18 +2,21 @@
 from collections import namedtuple
 import contextlib
 import logging
+import re
 import sys
 import threading
 import time
-
-import six
+import traceback
+import urllib.parse
+from io import StringIO
 
 # Django
 from django.conf import LazySettings
 from django.conf import settings, UserSettingsHolder
 from django.core.cache import cache as django_cache
 from django.core.exceptions import ImproperlyConfigured
-from django.db import ProgrammingError, OperationalError
+from django.db import transaction, connection
+from django.db.utils import Error as DBError
 from django.utils.functional import cached_property
 
 # Django REST Framework
@@ -21,7 +24,6 @@ from rest_framework.fields import empty, SkipField
 
 # Tower
 from awx.main.utils import encrypt_field, decrypt_field
-from awx.main.utils.db import get_tower_migration_version
 from awx.conf import settings_registry
 from awx.conf.models import Setting
 from awx.conf.migrations._reencrypt import decrypt_field as old_decrypt_field
@@ -58,17 +60,74 @@ SETTING_CACHE_DEFAULTS = True
 __all__ = ['SettingsWrapper', 'get_settings_to_cache', 'SETTING_CACHE_NOTSET']
 
 
+def normalize_broker_url(value):
+    parts = value.rsplit('@', 1)
+    match = re.search('(amqp://[^:]+:)(.*)', parts[0])
+    if match:
+        prefix, password = match.group(1), match.group(2)
+        parts[0] = prefix + urllib.parse.quote(password)
+    return '@'.join(parts)
+
+
 @contextlib.contextmanager
-def _log_database_error():
+def _ctit_db_wrapper(trans_safe=False):
+    '''
+    Wrapper to avoid undesired actions by Django ORM when managing settings
+    if only getting a setting, can use trans_safe=True, which will avoid
+    throwing errors if the prior context was a broken transaction.
+    Any database errors will be logged, but exception will be suppressed.
+    '''
+    rollback_set = None
+    is_atomic = None
     try:
+        if trans_safe:
+            is_atomic = connection.in_atomic_block
+            if is_atomic:
+                rollback_set = transaction.get_rollback()
+                if rollback_set:
+                    logger.debug('Obtaining database settings in spite of broken transaction.')
+                    transaction.set_rollback(False)
         yield
-    except (ProgrammingError, OperationalError) as e:
-        if get_tower_migration_version() < '310':
-            logger.info('Using default settings until version 3.1 migration.')
+    except DBError:
+        # We want the _full_ traceback with the context
+        # First we get the current call stack, which constitutes the "top",
+        # it has the context up to the point where the context manager is used
+        top_stack = StringIO()
+        traceback.print_stack(file=top_stack)
+        top_lines = top_stack.getvalue().strip('\n').split('\n')
+        top_stack.close()
+        # Get "bottom" stack from the local error that happened
+        # inside of the "with" block this wraps
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        bottom_stack = StringIO()
+        traceback.print_tb(exc_traceback, file=bottom_stack)
+        bottom_lines = bottom_stack.getvalue().strip('\n').split('\n')
+        # Glue together top and bottom where overlap is found
+        bottom_cutoff = 0
+        for i, line in enumerate(bottom_lines):
+            if line in top_lines:
+                # start of overlapping section, take overlap from bottom
+                top_lines = top_lines[:top_lines.index(line)]
+                bottom_cutoff = i
+                break
+        bottom_lines = bottom_lines[bottom_cutoff:]
+        tb_lines = top_lines + bottom_lines
+
+        tb_string = '\n'.join(
+            ['Traceback (most recent call last):'] +
+            tb_lines +
+            ['{}: {}'.format(exc_type.__name__, str(exc_value))]
+        )
+        bottom_stack.close()
+        # Log the combined stack
+        if trans_safe:
+            if 'check_migrations' not in sys.argv:
+                logger.debug('Database settings are not available, using defaults, error:\n{}'.format(tb_string))
         else:
-            logger.warning('Database settings are not available, using defaults (%s)', e, exc_info=True)
+            logger.debug('Error modifying something related to database settings.\n{}'.format(tb_string))
     finally:
-        pass
+        if trans_safe and is_atomic and rollback_set:
+            transaction.set_rollback(rollback_set)
 
 
 def filter_sensitive(registry, key, value):
@@ -104,15 +163,6 @@ class EncryptedCacheProxy(object):
     def get(self, key, **kwargs):
         value = self.cache.get(key, **kwargs)
         value = self._handle_encryption(self.decrypter, key, value)
-
-        # python-memcached auto-encodes unicode on cache set in python2
-        # https://github.com/linsomniac/python-memcached/issues/79
-        # https://github.com/linsomniac/python-memcached/blob/288c159720eebcdf667727a859ef341f1e908308/memcache.py#L961
-        if six.PY2 and isinstance(value, six.binary_type):
-            try:
-                six.text_type(value)
-            except UnicodeDecodeError:
-                value = value.decode('utf-8')
         logger.debug('cache get(%r, %r) -> %r', key, empty, filter_sensitive(self.registry, key, value))
         return value
 
@@ -245,7 +295,7 @@ class SettingsWrapper(UserSettingsHolder):
             self.__dict__['_awx_conf_preload_expires'] = time.time() + SETTING_CACHE_TIMEOUT
             # Check for any settings that have been defined in Python files and
             # make those read-only to avoid overriding in the database.
-            if not self._awx_conf_init_readonly and 'migrate_to_database_settings' not in sys.argv:
+            if not self._awx_conf_init_readonly:
                 defaults_snapshot = self._get_default('DEFAULTS_SNAPSHOT')
                 for key in get_writeable_settings(self.registry):
                     init_default = defaults_snapshot.get(key, None)
@@ -327,8 +377,9 @@ class SettingsWrapper(UserSettingsHolder):
             setting = None
             setting_id = None
             if not field.read_only or name in (
-                # these two values are read-only - however - we *do* want
+                # these values are read-only - however - we *do* want
                 # to fetch their value from the database
+                'INSTALL_UUID',
                 'AWX_ISOLATED_PRIVATE_KEY',
                 'AWX_ISOLATED_PUBLIC_KEY',
             ):
@@ -388,11 +439,20 @@ class SettingsWrapper(UserSettingsHolder):
     def __getattr__(self, name):
         value = empty
         if name in self.all_supported_settings:
-            with _log_database_error():
+            with _ctit_db_wrapper(trans_safe=True):
                 value = self._get_local(name)
         if value is not empty:
             return value
-        return self._get_default(name)
+        value = self._get_default(name)
+        # sometimes users specify RabbitMQ passwords that contain
+        # unescaped : and @ characters that confused urlparse, e.g.,
+        # amqp://guest:a@ns:ibl3#@localhost:5672//
+        #
+        # detect these scenarios, and automatically escape the user's
+        # password so it just works
+        if name == 'BROKER_URL':
+            value = normalize_broker_url(value)
+        return value
 
     def _set_local(self, name, value):
         field = self.registry.get_setting_field(name)
@@ -420,7 +480,7 @@ class SettingsWrapper(UserSettingsHolder):
 
     def __setattr__(self, name, value):
         if name in self.all_supported_settings:
-            with _log_database_error():
+            with _ctit_db_wrapper():
                 self._set_local(name, value)
         else:
             setattr(self.default_settings, name, value)
@@ -436,14 +496,14 @@ class SettingsWrapper(UserSettingsHolder):
 
     def __delattr__(self, name):
         if name in self.all_supported_settings:
-            with _log_database_error():
+            with _ctit_db_wrapper():
                 self._del_local(name)
         else:
             delattr(self.default_settings, name)
 
     def __dir__(self):
         keys = []
-        with _log_database_error():
+        with _ctit_db_wrapper(trans_safe=True):
             for setting in Setting.objects.filter(
                     key__in=self.all_supported_settings, user__isnull=True):
                 # Skip returning settings that have been overridden but are
@@ -460,7 +520,7 @@ class SettingsWrapper(UserSettingsHolder):
     def is_overridden(self, setting):
         set_locally = False
         if setting in self.all_supported_settings:
-            with _log_database_error():
+            with _ctit_db_wrapper(trans_safe=True):
                 set_locally = Setting.objects.filter(key=setting, user__isnull=True).exists()
         set_on_default = getattr(self.default_settings, 'is_overridden', lambda s: False)(setting)
         return (set_locally or set_on_default)

awx/conf/signals.py

@@ -9,15 +9,11 @@ from django.core.cache import cache
 from django.dispatch import receiver
 
 # Tower
-import awx.main.signals
 from awx.conf import settings_registry
 from awx.conf.models import Setting
-from awx.conf.serializers import SettingSerializer
 
 logger = logging.getLogger('awx.conf.signals')
 
-awx.main.signals.model_serializer_mapping[Setting] = SettingSerializer
-
 __all__ = []
 
 

awx/conf/tests/functional/conftest.py

@@ -1,7 +1,8 @@
+import urllib.parse
+
 import pytest
 
-from django.core.urlresolvers import resolve
-from django.utils.six.moves.urllib.parse import urlparse
+from django.urls import resolve
 from django.contrib.auth.models import User
 
 from rest_framework.test import (
@@ -33,7 +34,7 @@ def admin():
 @pytest.fixture
 def api_request(admin):
     def rf(verb, url, data=None, user=admin):
-        view, view_args, view_kwargs = resolve(urlparse(url)[2])
+        view, view_args, view_kwargs = resolve(urllib.parse.urlparse(url)[2])
         request = getattr(APIRequestFactory(), verb)(url, data=data, format='json')
         if user:
             force_authenticate(request, user=user)

awx/conf/tests/functional/test_api.py

@@ -1,5 +1,5 @@
 import pytest
-import mock
+from unittest import mock
 
 from rest_framework import serializers
 
@@ -8,6 +8,7 @@ from awx.main.utils.encryption import decrypt_field
 from awx.conf import fields
 from awx.conf.registry import settings_registry
 from awx.conf.models import Setting
+from awx.sso import fields as sso_fields
 
 
 @pytest.fixture
@@ -65,41 +66,6 @@ def test_non_admin_user_does_not_see_categories(api_request, dummy_setting, norm
         assert not response.data['results']
 
 
-@pytest.mark.django_db
-@mock.patch(
-    'awx.conf.views.VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE',
-    {
-        1: set([]),
-        2: set(['foobar']),
-    }
-)
-def test_version_specific_category_slug_to_exclude_does_not_show_up(api_request, dummy_setting):
-    with dummy_setting(
-        'FOO_BAR',
-        field_class=fields.IntegerField,
-        category='FooBar',
-        category_slug='foobar'
-    ):
-        response = api_request(
-            'get',
-            reverse('api:setting_category_list',
-                    kwargs={'version': 'v2'})
-        )
-        for item in response.data['results']:
-            assert item['slug'] != 'foobar'
-        response = api_request(
-            'get',
-            reverse('api:setting_category_list',
-                    kwargs={'version': 'v1'})
-        )
-        contains = False
-        for item in response.data['results']:
-            if item['slug'] != 'foobar':
-                contains = True
-                break
-        assert contains
-
-
 @pytest.mark.django_db
 def test_setting_singleton_detail_retrieve(api_request, dummy_setting):
     with dummy_setting(
@@ -172,7 +138,7 @@ def test_setting_signleton_retrieve_hierachy(api_request, dummy_setting):
 
 
 @pytest.mark.django_db
-def test_setting_signleton_retrieve_readonly(api_request, dummy_setting):
+def test_setting_singleton_retrieve_readonly(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.IntegerField,
@@ -218,6 +184,30 @@ def test_setting_singleton_update(api_request, dummy_setting):
         assert response.data['FOO_BAR'] == 4
 
 
+@pytest.mark.django_db
+def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, dummy_setting):
+    # Some HybridDictField subclasses have a child of _Forbidden,
+    # indicating that only the defined fields can be filled in.  Make
+    # sure that the _Forbidden validator doesn't get used for the
+    # fields.  See also https://github.com/ansible/awx/issues/4099.
+    with dummy_setting(
+        'FOO_BAR',
+        field_class=sso_fields.SAMLOrgAttrField,
+        category='FooBar',
+        category_slug='foobar',
+    ), mock.patch('awx.conf.views.handle_setting_changes'):
+        api_request(
+            'patch',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}),
+            data={'FOO_BAR': {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}}
+        )
+        response = api_request(
+            'get',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'})
+        )
+        assert response.data['FOO_BAR'] == {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}
+
+
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting(
@@ -241,7 +231,7 @@ def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy
 
 
 @pytest.mark.django_db
-def test_setting_singleton_update_dont_change_encripted_mark(api_request, dummy_setting):
+def test_setting_singleton_update_dont_change_encrypted_mark(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.CharField,
@@ -338,13 +328,14 @@ def test_setting_singleton_delete_no_read_only_fields(api_request, dummy_setting
 
 @pytest.mark.django_db
 def test_setting_logging_test(api_request):
-    with mock.patch('awx.conf.views.BaseHTTPSHandler.perform_test') as mock_func:
+    with mock.patch('awx.conf.views.AWXProxyHandler.perform_test') as mock_func:
         api_request(
             'post',
             reverse('api:setting_logging_test'),
             data={'LOG_AGGREGATOR_HOST': 'http://foobar', 'LOG_AGGREGATOR_TYPE': 'logstash'}
         )
-        test_arguments = mock_func.call_args[0][0]
-        assert test_arguments.LOG_AGGREGATOR_HOST == 'http://foobar'
-        assert test_arguments.LOG_AGGREGATOR_TYPE == 'logstash'
-        assert test_arguments.LOG_AGGREGATOR_LEVEL == 'DEBUG'
+        call = mock_func.call_args_list[0]
+        args, kwargs = call
+        given_settings = kwargs['custom_settings']
+        assert given_settings.LOG_AGGREGATOR_HOST == 'http://foobar'
+        assert given_settings.LOG_AGGREGATOR_TYPE == 'logstash'