Merge pull request #5656 from ryanpetrello/pygments-minus-minus

remove some unnecessary callback receiver debugging code

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.dput.cf

@@ -1,6 +0,0 @@
-[mini_dinstall]
-fqdn = localhost
-method = local
-incoming = FIXME/deb-repo/mini-dinstall/incoming
-run_dinstall = 0
-post_upload_command = mini-dinstall -b -v

.editorconfig

@@ -1,20 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = true
-
-[Makefile]
-indent_style = tab
-
-[**.py]
-indent_style = space
-indent_size = 4
-
-[**.{js,less,html}]
-indent_style = space
-indent_size = 4
-
-[**.{json}]
-indent_style = space
-indent_size = 2

.env

@@ -1,2 +1,3 @@
 PYTHONUNBUFFERED=true
+SELENIUM_DOCKER_TAG=latest
 

.github/BOTMETA.yml

@@ -1,3 +1,4 @@
+---
 files:
     awx/ui/:
         labels: component:ui

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,12 +7,6 @@ about: Create a report to help us improve
 ##### ISSUE TYPE
  - Bug Report
 
-##### COMPONENT NAME
-<!-- Pick the area of AWX for this issue, you can have multiple, delete the rest: -->
- - API
- - UI
- - Installer
-
 ##### SUMMARY
 <!-- Briefly describe the problem. -->
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -7,16 +7,5 @@ about: Suggest an idea for this project
 ##### ISSUE TYPE
  - Feature Idea
 
-##### COMPONENT NAME
-<!-- Pick the area of AWX for this issue, you can have multiple, delete the rest: -->
- - API
- - UI
- - Installer
-
 ##### SUMMARY
 <!-- Briefly describe the problem or desired enhancement. -->
-
-##### ADDITIONAL INFORMATION
-
-<!-- Include any links to sosreport, database dumps, screenshots or other
-information. -->

.gitignore

@@ -8,6 +8,7 @@ reference-schema.json
 .tags1
 
 # Tower
+awx-dev
 awx/settings/local_*.py*
 awx/*.sqlite3
 awx/*.sqlite3_*
@@ -27,8 +28,12 @@ awx/ui/build_test
 awx/ui/client/languages
 awx/ui/templates/ui/index.html
 awx/ui/templates/ui/installing.html
+awx/ui_next/node_modules/
+awx/ui_next/coverage/
+awx/ui_next/build/locales/_build
 /tower-license
 /tower-license/**
+tools/prometheus/data
 
 # Tower setup playbook testing
 setup/test/roles/postgresql
@@ -120,6 +125,7 @@ local/
 requirements/vendor
 .i18n_built
 .idea/*
+*credentials*.y*ml*
 
 # AWX python libs populated by requirements.txt
 awx/lib/.deps_built
@@ -127,4 +133,13 @@ awx/lib/site-packages
 venv/*
 use_dev_supervisor.txt
 
+
+# Ansible module tests
+/awx_collection_test_venv/
+/awx_collection/*.tar.gz
+/awx_collection/galaxy.yml
+/sanity/
+
 .idea/*
+*.unison.tmp
+*.#

.mini-dinstall.cf

@@ -1,16 +0,0 @@
-[DEFAULT]
-archivedir = FIXME/deb-repo
-mail_to =
-verify_sigs = false
-architectures = all, amd64
-archive_style = flat
-generate_release = true
-mail_on_success = false
-release_codename = ansible-tower
-release_description = Ansible Tower
-release_label = ansible-tower
-release_origin = ansible-tower
-
-[trusty]
-
-[precise]

.pylintrc

@@ -1,6 +0,0 @@
-[MASTER]
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=site-packages,ui,migrations,data
-

.yamllint

@@ -0,0 +1,12 @@
+---
+ignore: |
+  .tox
+  awx/main/tests/data/inventory/plugins/**
+  # vault files
+  awx/main/tests/data/ansible_utils/playbooks/valid/vault.yml
+  awx/ui/test/e2e/tests/smoke-vars.yml
+
+extends: default
+
+rules:
+  line-length: disable

CHANGELOG.md

@@ -0,0 +1,128 @@
+# Changelog
+
+This is a list of high-level changes for each release of AWX. A full list of commits can be found at `https://github.com/ansible/awx/releases/tag/<version>`.
+
+## 9.1.1 (Jan 14, 2020)
+
+- Fixed a bug that caused database migrations on Kubernetes installs to hang https://github.com/ansible/awx/pull/5579
+- Upgraded Python-level app dependencies in AWX virtual environment https://github.com/ansible/awx/pull/5407
+- Running jobs no longer block associated inventory updates https://github.com/ansible/awx/pull/5519
+- Fixed invalid_response SAML error https://github.com/ansible/awx/pull/5577
+- Optimized the callback receiver to drastically improve the write speed of stdout for parallel jobs (https://github.com/ansible/awx/pull/5618)
+
+## 9.1.0 (Dec 17, 2019)
+- Added a command to generate a new SECRET_KEY and rekey the secrets in the database
+- Removed project update locking when jobs using it are running
+- Fixed slow queries for /api/v2/instances and /api/v2/instance_groups when smart inventories are used
+- Fixed a partial password disclosure when special characters existed in the RabbitMQ password (CVE-2019-19342)
+- Fixed hang in error handling for source control checkouts
+- Fixed an error on subsequent job runs that override the branch of a project on an instance that did not have a prior project checkout
+- Fixed an issue where jobs launched in isolated or container groups would incorrectly timeout
+- Fixed an incorrect link to instance groups documentation in the user interface
+- Fixed editing of inventory on Workflow templates
+- Fixed multiple issues with OAuth2 token cleanup system jobs
+- Fixed a bug that broke email notifications for workflow approval/deny https://github.com/ansible/awx/issues/5401
+- Updated SAML implementation to automatically login if authorization already exists
+- Updated AngularJS to 1.7.9 for CVE-2019-10768
+
+## 9.0.1 (Nov 4, 2019)
+
+- Fixed a bug in the installer that broke certain types of k8s installs https://github.com/ansible/awx/issues/5205
+
+## 9.0.0 (Oct 31, 2019)
+
+- Updated AWX images to use centos:8 as the parent image.
+- Updated to ansible-runner 1.4.4 to address various bugs.
+- Added oc and kubectl to the AWX images to support new container-based execution introduced in 8.0.0.
+- Added some optimizations to speed up the deletion of large Inventory Groups.
+- Fixed a bug that broke webhook launches for Job Templates that define a survey (https://github.com/ansible/awx/issues/5062).
+- Fixed a bug in the CLI which incorrectly parsed launch time arguments for `awx job_templates launch` and `awx workflow_job_templates launch` (https://github.com/ansible/awx/issues/5093).
+- Fixed a bug that caused inventory updates using "sourced from a project" to stop working (https://github.com/ansible/awx/issues/4750).
+- Fixed a bug that caused Slack notifications to sometimes show the wrong bot avatar (https://github.com/ansible/awx/pull/5125).
+- Fixed a bug that prevented the use of digits in Tower's URL settings (https://github.com/ansible/awx/issues/5081).
+
+## 8.0.0 (Oct 21, 2019)
+
+- The Ansible Tower Ansible modules have been migrated to a new official Ansible AWX collection: https://galaxy.ansible.com/awx/AWX
+  Please note that this functionality is only supported in Ansible 2.9+
+- AWX now supports the ability to launch jobs from external webhooks (GitHub and GitLab integration are supported).
+- AWX now supports Container Groups, a new feature that allows you to schedule and run playbooks on single-use kubernetes pods on-demand.
+- AWX now supports sending notifications when Workflow steps are approved, denied, or time out.
+- AWX now records the user who approved or denied Workflow steps.
+- AWX now supports fetching Ansible Collections from private galaxy servers.
+- AWX now checks the user's ansible.cfg for paths where role/collections may live when running project updates.
+- AWX now uses PostgreSQL 10 by default.
+- AWX now warns more loudly about underlying AMQP connectivity issues (https://github.com/ansible/awx/pull/4857).
+- Added a few optimizations to drastically improve dashboard performance for larger AWX installs (installs with several hundred thousand jobs or more).
+- Updated to the latest version of Ansible's VMWare inventory script (which adds support for vmware_guest_facts).
+- Deprecated /api/v2/inventory_scripts/ (this endpoint - and the Custom Inventory Script feature - will be removed in a future release of AWX).
+- Fixed a bug which prevented Organization Admins from removing users from their own Organization (https://github.com/ansible/awx/issues/2979)
+- Fixed a bug which sometimes caused cluster nodes to fail to re-join with a cryptic error, "No instance found with the current cluster host id" (https://github.com/ansible/awx/issues/4294)
+- Fixed a bug that prevented the use of launch-time passphrases when using credential plugins (https://github.com/ansible/awx/pull/4807)
+- Fixed a bug that caused notifications assigned at the Organization level not to take effect for Workflows in that Organization (https://github.com/ansible/awx/issues/4712)
+- Fixed a bug which caused a notable amount of CPU overhead on RabbitMQ health checks (https://github.com/ansible/awx/pull/5009)
+- Fixed a bug which sometimes caused the <return> key to stop functioning in <textarea> elements (https://github.com/ansible/awx/issues/4192)
+- Fixed a bug which caused request contention when the same OAuth2.0 token was used in multiple simultaneous requests (https://github.com/ansible/awx/issues/4694)
+- Fixed a bug related to parsing multiple choice survey options (https://github.com/ansible/awx/issues/4452).
+- Fixed a bug that caused single-sign-on icons on the login page to fail to render in certain Windows browsers (https://github.com/ansible/awx/issues/3924)
+- Fixed a number of bugs that caused certain OAuth2 settings to not be properly respected, such as REFRESH_TOKEN_EXPIRE_SECONDS.
+- Fixed a number of bugs in the AWX CLI, including a bug which sometimes caused long lines of stdout output to be unexpectedly truncated.
+- Fixed a number of bugs on the job details UI which sometimes caused auto-scrolling stdout to become stuck.
+- Fixed a bug which caused LDAP authentication to fail if the TLD of the server URL contained digits (https://github.com/ansible/awx/issues/3646)
+- Fixed a bug which broke HashiCorp Vault integration on older versions of HashiCorp Vault.
+
+## 7.0.0 (Sept 4, 2019)
+
+- AWX now detects and installs Ansible Collections defined in your project (note - this feature only works in Ansible 2.9+) (https://github.com/ansible/awx/issues/2534)
+- AWX now includes an official command line client.  Keep an eye out for a follow-up email on this mailing list for information on how to install it and try it out.
+- Added the ability to provide a specific SCM branch on jobs (https://github.com/ansible/awx/issues/282)
+- Added support for Workflow Approval Nodes, a new feature which allows you to add "pause and wait for approval" steps into your workflows (https://github.com/ansible/awx/issues/1206)
+- Added the ability to specify a specific HTTP method for webhook notifications (POST vs PUT) (https://github.com/ansible/awx/pull/4124)
+- Added the ability to specify a username and password for HTTP Basic Authorization for webhook notifications (https://github.com/ansible/awx/pull/4124)
+- Added support for customizing the text content of notifications (https://github.com/ansible/awx/issues/79)
+- Added the ability to enable and disable hosts in dynamic inventory (https://github.com/ansible/awx/pull/4420)
+- Added the description (if any) to the Job Template list (https://github.com/ansible/awx/issues/4359)
+- Added new metrics for instance hostnames and pending jobs to the /api/v2/metrics/ endpoint (https://github.com/ansible/awx/pull/4375)
+- Changed AWX's on/off toggle buttons to a non-text based style to simplify internationalization (https://github.com/ansible/awx/pull/4425)
+- Events emitted by ansible for adhoc commands are now sent to the external log aggregrator (https://github.com/ansible/awx/issues/4545)
+- Fixed a bug which allowed a user to make an organization credential in another organization without permissions to that organization (https://github.com/ansible/awx/pull/4483)
+- Fixed a bug that caused `extra_vars` on workflows to break when edited (https://github.com/ansible/awx/issues/4293)
+- Fixed a slow SQL query that caused performance issues when large numbers of groups exist (https://github.com/ansible/awx/issues/4461)
+- Fixed a few minor bugs in survey field validation (https://github.com/ansible/awx/pull/4509) (https://github.com/ansible/awx/pull/4479)
+- Fixed a bug that sometimes resulted in orphaned `ansible_runner_pi` directories in `/tmp` after playbook execution (https://github.com/ansible/awx/pull/4409)
+- Fixed a bug that caused the `is_system_auditor` flag in LDAP configuration to not work (https://github.com/ansible/awx/pull/4396)
+- Fixed a bug which caused schedules to disappear from the UI when toggled off (https://github.com/ansible/awx/pull/4378)
+- Fixed a bug that sometimes caused stdout content to contain extraneous blank lines in newer versions of Ansible (https://github.com/ansible/awx/pull/4391)
+- Updated to the latest Django security release, 2.2.4 (https://github.com/ansible/awx/pull/4410) (https://www.djangoproject.com/weblog/2019/aug/01/security-releases/)
+- Updated the default version of git to a version that includes support for x509 certificates (https://github.com/ansible/awx/issues/4362)
+- Removed the deprecated `credential` field from `/api/v2/workflow_job_templates/N/` (as part of the `/api/v1/` removal in prior AWX versions - https://github.com/ansible/awx/pull/4490).
+
+## 6.1.0 (Jul 18, 2019)
+
+- Updated AWX to use Django 2.2.2.
+- Updated the provided openstacksdk version to support new functionality (such as Nova scheduler_hints)
+- Added the ability to specify a custom cacert for the HashiCorp Vault credential plugin
+- Fixed a number of bugs related to path lookups for the HashiCorp Vault credential plugin
+- Fixed a bug which prevented signed SSH certificates from working, including the HashiCorp Vault Signed SSH backend
+- Fixed a bug which prevented custom logos from displaying on the login page (as a result of a new Content Security Policy in 6.0.0)
+- Fixed a bug which broke websocket connectivity in Apple Safari (as a result of a new Content Security Policy in 6.0.0)
+- Fixed a bug on the job output page that occasionally caused the "up" and "down" buttons to not load additional output
+- Fixed a bug on the job output page that caused quoted task names to display incorrectly
+
+## 6.0.0 (Jul 1, 2019)
+
+- Removed support for "Any" notification templates and their API endpoints e.g., /api/v2/job_templates/N/notification_templates/any/ (https://github.com/ansible/awx/issues/4022)
+- Fixed a bug which prevented credentials from properly being applied to inventory sources (https://github.com/ansible/awx/issues/4059)
+- Fixed a bug which can cause the task dispatcher to hang indefinitely when external logging support (e.g., Splunk, Logstash) is enabled (https://github.com/ansible/awx/issues/4181)
+- Fixed a bug which causes slow stdout display when running jobs against smart inventories. (https://github.com/ansible/awx/issues/3106)
+- Fixed a bug that caused SSL verification flags to fail to be respected for LDAP authentication in certain environments. (https://github.com/ansible/awx/pull/4190)
+- Added a simple Content Security Policy (https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) to restrict access to third-party resources in the browser. (https://github.com/ansible/awx/pull/4167)
+- Updated ovirt4 library dependencies to work with newer versions of oVirt (https://github.com/ansible/awx/issues/4138)
+
+## 5.0.0 (Jun 21, 2019)
+
+- Bump Django Rest Framework from 3.7.7 to 3.9.4
+- Bump setuptools / pip dependencies
+- Fixed bug where Recent Notification list would not appear
+- Added notifications on job start
+- Default to Ansible 2.8

CONTRIBUTING.md

@@ -83,12 +83,10 @@ If you're not using Docker for Mac, or Docker for Windows, you may need, or choo
 (host)$ pip install docker-compose
 ```
 
-#### Node and npm
+#### Frontend Development
 
-The AWX UI requires the following:
+See [the ui development documentation](awx/ui/README.md).
 
-- Node 8.x LTS
-- NPM 6.x LTS
 
 ### Build the environment
 
@@ -136,6 +134,8 @@ Run the following to build the AWX UI:
 ```bash
 (host) $ make ui-devel
 ```
+See [the ui development documentation](awx/ui/README.md) for more information on using the frontend development, build, and test tooling.
+
 ### Running the environment
 
 #### Start the containers
@@ -156,8 +156,8 @@ If you start a second terminal session, you can take a look at the running conta
 
 $ docker ps
 CONTAINER ID        IMAGE                                              COMMAND                  CREATED             STATUS              PORTS                                                                                                                                      NAMES
-aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:6899-6999->6899-6999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
-e4c0afeb548c        postgres:9.6                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
+aa4a75d6d77b        gcr.io/ansible-tower-engineering/awx_devel:devel   "/tini -- /bin/sh ..."   23 seconds ago      Up 15 seconds       0.0.0.0:5555->5555/tcp, 0.0.0.0:7899-7999->7899-7999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 22/tcp, 0.0.0.0:8080->8080/tcp   tools_awx_1
+e4c0afeb548c        postgres:10                                       "docker-entrypoint..."   26 seconds ago      Up 23 seconds       5432/tcp                                                                                                                                   tools_postgres_1
 0089699d5afd        tools_logstash                                     "/docker-entrypoin..."   26 seconds ago      Up 25 seconds                                                                                                                                                  tools_logstash_1
 4d4ff0ced266        memcached:alpine                                   "docker-entrypoint..."   26 seconds ago      Up 25 seconds       0.0.0.0:11211->11211/tcp                                                                                                                   tools_memcached_1
 92842acd64cd        rabbitmq:3-management                              "docker-entrypoint..."   26 seconds ago      Up 24 seconds       4369/tcp, 5671-5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp                                                                    tools_rabbitmq_1

INSTALL.md

@@ -4,41 +4,45 @@ This document provides a guide for installing AWX.
 
 ## Table of contents
 
-- [Getting started](#getting-started)
-  - [Clone the repo](#clone-the-repo)
-  - [AWX branding](#awx-branding)
-  - [Prerequisites](#prerequisites)
-  - [System Requirements](#system-requirements)
-  - [AWX Tunables](#awx-tunables)
-  - [Choose a deployment platform](#choose-a-deployment-platform)
-  - [Official vs Building Images](#official-vs-building-images)
-- [OpenShift](#openshift)
-  - [Prerequisites](#prerequisites-1)
-    - [Deploying to Minishift](#deploying-to-minishift)
-  - [Pre-build steps](#pre-build-steps)
-  - [PostgreSQL](#postgresql)
-  - [Start the build](#start-the-build)
-  - [Post build](#post-build)
-  - [Accessing AWX](#accessing-awx)
-- [Kubernetes](#kubernetes)
-  - [Prerequisites](#prerequisites-2)
-  - [Pre-build steps](#pre-build-steps-1)
-  - [Configuring Helm](#configuring-helm)
-  - [Start the build](#start-the-build-1)
-  - [Accessing AWX](#accessing-awx-1)
-  - [SSL Termination](#ssl-termination)
-- [Docker or Docker Compose](#docker-or-docker-compose)
-  - [Prerequisites](#prerequisites-3)
-  - [Pre-build steps](#pre-build-steps-2)
-    - [Deploying to a remote host](#deploying-to-a-remote-host)
-    - [Inventory variables](#inventory-variables)
+- [Installing AWX](#installing-awx)
+  * [Getting started](#getting-started)
+    + [Clone the repo](#clone-the-repo)
+    + [AWX branding](#awx-branding)
+    + [Prerequisites](#prerequisites)
+    + [System Requirements](#system-requirements)
+    + [AWX Tunables](#awx-tunables)
+    + [Choose a deployment platform](#choose-a-deployment-platform)
+    + [Official vs Building Images](#official-vs-building-images)
+  * [Upgrading from previous versions](#upgrading-from-previous-versions)
+  * [OpenShift](#openshift)
+    + [Prerequisites](#prerequisites-1)
+    + [Pre-install steps](#pre-install-steps)
+      - [Deploying to Minishift](#deploying-to-minishift)
+      - [PostgreSQL](#postgresql)
+    + [Run the installer](#run-the-installer)
+    + [Post-install](#post-install)
+    + [Accessing AWX](#accessing-awx)
+  * [Kubernetes](#kubernetes)
+    + [Prerequisites](#prerequisites-2)
+    + [Pre-install steps](#pre-install-steps-1)
+    + [Configuring Helm](#configuring-helm)
+    + [Run the installer](#run-the-installer-1)
+    + [Post-install](#post-install-1)
+    + [Accessing AWX](#accessing-awx-1)
+    + [SSL Termination](#ssl-termination)
+  * [Docker-Compose](#docker-compose)
+    + [Prerequisites](#prerequisites-3)
+    + [Pre-install steps](#pre-install-steps-2)
+      - [Deploying to a remote host](#deploying-to-a-remote-host)
+      - [Inventory variables](#inventory-variables)
       - [Docker registry](#docker-registry)
-      - [PostgreSQL](#postgresql-1)
       - [Proxy settings](#proxy-settings)
-  - [Start the build](#start-the-build-2)
-  - [Post build](#post-build-1)
-  - [Accessing AWX](#accessing-awx-2)
+      - [PostgreSQL](#postgresql-1)
+    + [Run the installer](#run-the-installer-2)
+    + [Post-install](#post-install-2)
+    + [Accessing AWX](#accessing-awx-2)
 
+    
 ## Getting started
 
 ### Clone the repo
@@ -57,23 +61,26 @@ To install the assets, clone the `awx-logos` repo so that it is next to your `aw
 
 Before you can run a deployment, you'll need the following installed in your local environment:
 
-- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) Requires Version 2.4+
+- [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) Requires Version 2.8+
 - [Docker](https://docs.docker.com/engine/installation/)
-- [docker-py](https://github.com/docker/docker-py) Python module
+    + A recent version
+- [docker](https://pypi.org/project/docker/) Python module
+    + This is incompatible with `docker-py`. If you have previously installed `docker-py`, please uninstall it.
+    + We use this module instead of `docker-py` because it is what the `docker-compose` Python module requires.
 - [GNU Make](https://www.gnu.org/software/make/)
 - [Git](https://git-scm.com/) Requires Version 1.8.4+
-- [Node 8.x LTS version](https://nodejs.org/en/download/)
+- [Node 10.x LTS version](https://nodejs.org/en/download/)
 - [NPM 6.x LTS](https://docs.npmjs.com/)
 
 ### System Requirements
 
 The system that runs the AWX service will need to satisfy the following requirements
 
-- At leasts 4GB of memory
+- At least 4GB of memory
 - At least 2 cpu cores
 - At least 20GB of space
 - Running Docker, Openshift, or Kubernetes
-- If you choose to use an external PostgreSQL database, please note that the minimum version is 9.4.
+- If you choose to use an external PostgreSQL database, please note that the minimum version is 9.6+.
 
 ### AWX Tunables
 
@@ -81,14 +88,14 @@ The system that runs the AWX service will need to satisfy the following requirem
 
 ### Choose a deployment platform
 
-We currently support running AWX as a containerized application using Docker images deployed to either an OpenShift cluster, docker-compose or a standalone Docker daemon. The remainder of this document will walk you through the process of building the images, and deploying them to either platform.
+We currently support running AWX as a containerized application using Docker images deployed to either an OpenShift cluster, a Kubernetes cluster, or docker-compose. The remainder of this document will walk you through the process of building the images, and deploying them to either platform.
 
 The [installer](./installer) directory contains an [inventory](./installer/inventory) file, and a playbook, [install.yml](./installer/install.yml). You'll begin by setting variables in the inventory file according to the platform you wish to use, and then you'll start the image build and deployment process by running the playbook.
 
 In the sections below, you'll find deployment details and instructions for each platform:
 - [OpenShift](#openshift)
 - [Kubernetes](#kubernetes)
-- [Docker or Docker Compose](#docker-or-docker-compose).
+- [Docker Compose](#docker-compose).
 
 ### Official vs Building Images
 
@@ -111,12 +118,34 @@ If these variables are present then all deployments will use these hosted images
 
 > Multiple versions are provided. `latest` always pulls the most recent. You may also select version numbers at different granularities: 1, 1.0, 1.0.1, 1.0.0.123
 
+
+## Upgrading from previous versions
+
+Upgrading AWX involves rerunning the install playbook. Download a newer release from [https://github.com/ansible/awx/releases](https://github.com/ansible/awx/releases) and re-populate the inventory file with your customized variables.
+
+For convenience, you can create a file called `vars.yml`:
+
+```
+admin_password: 'adminpass'
+pg_password: 'pgpass'
+rabbitmq_password: 'rabbitpass'
+secret_key: 'mysupersecret'
+```
+
+And pass it to the installer:
+
+```
+$ ansible-playbook -i inventory install.yml -e @vars.yml
+```
+
 ## OpenShift
 
 ### Prerequisites
 
 To complete a deployment to OpenShift, you will obviously need access to an OpenShift cluster. For demo and testing purposes, you can use [Minishift](https://github.com/minishift/minishift) to create a single node cluster running inside a virtual machine.
 
+When using OpenShift for deploying AWX make sure you have correct privileges to add the security context 'privileged', otherwise the installation will fail. The privileged context is needed because of the use of [the bubblewrap tool](https://github.com/containers/bubblewrap) to add an additional layer of security when using containers.
+
 You will also need to have the `oc` command in your PATH. The `install.yml` playbook will call out to `oc` when logging into, and creating objects on the cluster.
 
 The default resource requests per-deployment requires:
@@ -128,9 +157,9 @@ This can be tuned by overriding the variables found in [/installer/roles/kuberne
 
 For more detail on how resource requests are formed see: [https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources](https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources)
 
-### Pre-build steps
+### Pre-install steps
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
+Before starting the install, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
 
 *openshift_host*
 
@@ -190,22 +219,22 @@ $ eval $(minishift docker-env)
 
 By default, AWX will deploy a PostgreSQL pod inside of your cluster. You will need to create a [Persistent Volume Claim](https://docs.openshift.org/latest/dev_guide/persistent_volumes.html) which is named `postgresql` by default, and can be overridden by setting the `openshift_pg_pvc_name` variable. For testing and demo purposes, you may set `openshift_pg_emptydir=yes`.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information. When setting `pg_hostname` the installer will assume you have configured the database in that location and will not launch the postgresql pod.
 
-### Start the build
+### Run the installer
 
-To start the build, you will pass two *extra* variables on the command line. The first is *openshift_password*, which is the password for the *openshift_user*, and the second is *docker_registry_password*, which is the password associated with *docker_registry_username*.
+To start the install, you will pass two *extra* variables on the command line. The first is *openshift_password*, which is the password for the *openshift_user*, and the second is *docker_registry_password*, which is the password associated with *docker_registry_username*.
 
 If you're using the OpenShift internal registry, then you'll pass an access token for the *docker_registry_password* value, rather than a password. The `oc whoami -t` command will generate the required token, as long as you're logged into the cluster via `oc cluster login`.
 
-To start the build and deployment, run the following (docker_registry_password is optional if using official images):
+Run the following command (docker_registry_password is optional if using official images):
 
 ```bash
-# Start the build and deployment
+# Start the install
 $ ansible-playbook -i inventory install.yml -e openshift_password=developer  -e docker_registry_password=$(oc whoami -t)
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, check the status of the deployment by running `oc get pods`:
 
@@ -322,9 +351,9 @@ This can be tuned by overriding the variables found in [/installer/roles/kuberne
 
 For more detail on how resource requests are formed see: [https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)
 
-### Pre-build steps
+### Pre-install steps
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section uncommenting when necessary. Make sure the openshift and standalone docker sections are commented out:
+Before starting the install process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section uncommenting when necessary. Make sure the openshift and standalone docker sections are commented out:
 
 *kubernetes_context*
 
@@ -344,7 +373,7 @@ If you want the AWX installer to manage creating the database pod (rather than i
 
 Newer Kubernetes clusters with RBAC enabled will need to make sure a service account is created, make sure to follow the instructions here [https://docs.helm.sh/using_helm/#role-based-access-control](https://docs.helm.sh/using_helm/#role-based-access-control)
 
-### Start the build
+### Run the installer
 
 After making changes to the `inventory` file use `ansible-playbook` to begin the install
 
@@ -352,7 +381,7 @@ After making changes to the `inventory` file use `ansible-playbook` to begin the
 $ ansible-playbook -i inventory install.yml
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, check the status of the deployment by running `kubectl get pods --namespace awx` (replace awx with the namespace you used):
 
@@ -391,16 +420,16 @@ If your provider is able to allocate an IP Address from the Ingress controller t
 Unlike Openshift's `Route` the Kubernetes `Ingress` doesn't yet handle SSL termination. As such the default configuration will only expose AWX through HTTP on port 80. You are responsible for configuring SSL support until support is added (either to Kubernetes or AWX itself).
 
 
-## Docker or Docker-Compose
+## Docker-Compose
 
 ### Prerequisites
 
 - [Docker](https://docs.docker.com/engine/installation/) on the host where AWX will be deployed. After installing Docker, the Docker service must be started (depending on your OS, you may have to add the local user that uses Docker to the ``docker`` group, refer to the documentation for details)
-- [docker-py](https://github.com/docker/docker-py) Python module.
+- [docker-compose](https://pypi.org/project/docker-compose/) Python module.
+    + This also installs the `docker` Python module, which is incompatible with `docker-py`. If you have previously installed `docker-py`, please uninstall it.
+- [Docker Compose](https://docs.docker.com/compose/install/).
 
-If you're installing using Docker Compose, you'll need [Docker Compose](https://docs.docker.com/compose/install/).
-
-### Pre-build steps
+### Pre-install steps
 
 #### Deploying to a remote host
 
@@ -431,7 +460,7 @@ If you choose to use the official images then the remote host will be the one to
 
 #### Inventory variables
 
-Before starting the build process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
+Before starting the install process, review the [inventory](./installer/inventory) file, and uncomment and provide values for the following variables found in the `[all:vars]` section:
 
 *postgres_data_dir*
 
@@ -441,13 +470,21 @@ Before starting the build process, review the [inventory](./installer/inventory)
 
 > Provide a port number that can be mapped from the Docker daemon host to the web server running inside the AWX container. Defaults to *80*.
 
-*use_docker_compose*
+*host_port_ssl*
 
-> Switch to ``true`` to use Docker Compose instead of the standalone Docker install.
+> Provide a port number that can be mapped from the Docker daemon host to the web server running inside the AWX container for SSL support. Defaults to *443*, only works if you also set `ssl_certificate` (see below).
+
+*ssl_certificate*
+
+> Optionally, provide the path to a file that contains a certificate and its private key.
 
 *docker_compose_dir*
 
-When using docker-compose, the `docker-compose.yml` file will be created there (default `/var/lib/awx`).
+> When using docker-compose, the `docker-compose.yml` file will be created there (default `/tmp/awxcompose`).
+
+*custom_venv_dir*
+
+> Adds the custom venv environments from the local host to be passed into the containers at install.
 
 *ca_trust_dir*
 
@@ -496,11 +533,11 @@ If you wish to tag and push built images to a Docker registry, set the following
 
 AWX requires access to a PostgreSQL database, and by default, one will be created and deployed in a container, and data will be persisted to a host volume. In this scenario, you must set the value of `postgres_data_dir` to a path that can be mounted to the container. When the container is stopped, the database files will still exist in the specified path.
 
-If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_database`, and `pg_port` with the connection information.
+If you wish to use an external database, in the inventory file, set the value of `pg_hostname`, and update `pg_username`, `pg_password`, `pg_admin_password`, `pg_database`, and `pg_port` with the connection information.
 
-### Start the build
+### Run the installer
 
-If you are not pushing images to a Docker registry, start the build by running the following:
+If you are not pushing images to a Docker registry, start the install by running the following:
 
 ```bash
 # Set the working directory to installer
@@ -520,14 +557,14 @@ $ cd installer
 $ ansible-playbook -i inventory -e docker_registry_password=password install.yml
 ```
 
-### Post build
+### Post-install
 
 After the playbook run completes, Docker will report up to 5 running containers. If you chose to use an existing PostgresSQL database, then it will report 4. You can view the running containers using the `docker ps` command, as follows:
 
 ```bash
 CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                NAMES
 e240ed8209cd        awx_task:1.0.0.8    "/tini -- /bin/sh ..."   2 minutes ago       Up About a minute   8052/tcp                             awx_task
-1cfd02601690        awx_web:1.0.0.8     "/tini -- /bin/sh ..."   2 minutes ago       Up About a minute   0.0.0.0:80->8052/tcp                 awx_web
+1cfd02601690        awx_web:1.0.0.8     "/tini -- /bin/sh ..."   2 minutes ago       Up About a minute   0.0.0.0:443->8052/tcp                 awx_web
 55a552142bcd        memcached:alpine    "docker-entrypoint..."   2 minutes ago       Up 2 minutes        11211/tcp                            memcached
 84011c072aad        rabbitmq:3          "docker-entrypoint..."   2 minutes ago       Up 2 minutes        4369/tcp, 5671-5672/tcp, 25672/tcp   rabbitmq
 97e196120ab3        postgres:9.6        "docker-entrypoint..."   2 minutes ago       Up 2 minutes        5432/tcp                             postgres
@@ -597,14 +634,3 @@ Added instance awx to tower
 The AWX web server is accessible on the deployment host, using the *host_port* value set in the *inventory* file. The default URL is [http://localhost](http://localhost).
 
 You will prompted with a login dialog. The default administrator username is `admin`, and the password is `password`.
-
-### Maintenance using docker-compose
-
-After the installation, maintenance operations with docker-compose can be done by using the  `docker-compose.yml` file created at the location pointed by `docker_compose_dir`.
-
-Among the possible operations, you may:
-
-- Stop AWX : `docker-compose stop`
-- Upgrade AWX : `docker-compose pull && docker-compose up --force-recreate`
-
-See the [docker-compose documentation](https://docs.docker.com/compose/) for details.

Makefile

@@ -18,6 +18,7 @@ COMPOSE_TAG ?= $(GIT_BRANCH)
 COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
+COLLECTION_VENV ?= /awx_devel/awx_collection_test_venv
 SCL_PREFIX ?=
 CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
@@ -25,6 +26,9 @@ DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering
 # Python packages to install only from source (not from binary wheels)
 # Comma separated list
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio
+# These should be upgraded in the AWX and Ansible venv before attempting
+# to install the actual requirements
+VENV_BOOTSTRAP ?= pip==19.3.1 setuptools==41.6.0
 
 # Determine appropriate shasum command
 UNAME_S := $(shell uname -s)
@@ -59,20 +63,23 @@ UI_RELEASE_FLAG_FILE = awx/ui/.release_built
 I18N_FLAG_FILE = .i18n_built
 
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
-	develop refresh adduser migrate dbchange dbshell runserver \
-	receiver test test_unit test_ansible test_coverage coverage_html \
+	develop refresh adduser migrate dbchange runserver \
+	receiver test test_unit test_coverage coverage_html \
 	dev_build release_build release_clean sdist \
 	ui-docker-machine ui-docker ui-release ui-devel \
 	ui-test ui-deps ui-test-ci VERSION
 
 # remove ui build artifacts
-clean-ui:
+clean-ui: clean-languages
 	rm -rf awx/ui/static/
 	rm -rf awx/ui/node_modules/
 	rm -rf awx/ui/test/unit/reports/
 	rm -rf awx/ui/test/spec/reports/
 	rm -rf awx/ui/test/e2e/reports/
 	rm -rf awx/ui/client/languages/
+	rm -rf awx/ui_next/node_modules/
+	rm -rf awx/ui_next/coverage/
+	rm -rf awx/ui_next/build/locales/_build/
 	rm -f $(UI_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_DEPS_FLAG_FILE)
 	rm -f $(UI_RELEASE_FLAG_FILE)
@@ -91,21 +98,31 @@ clean-schema:
 	rm -rf schema.json
 	rm -rf reference-schema.json
 
+clean-languages:
+	rm -f $(I18N_FLAG_FILE)
+	find . -type f -regex ".*\.mo$$" -delete
+
 # Remove temporary build files, compiled Python files.
-clean: clean-ui clean-dist
+clean: clean-ui clean-api clean-awxkit clean-dist
 	rm -rf awx/public
 	rm -rf awx/lib/site-packages
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3
-	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
 	mkdir tmp
+
+clean-api:
 	rm -rf build $(NAME)-$(VERSION) *.egg-info
 	find . -type f -regex ".*\.py[co]$$" -delete
 	find . -type d -name "__pycache__" -delete
+	rm -f awx/awx_test.sqlite3*
+	rm -rf requirements/vendor
+	rm -rf awx/projects
+
+clean-awxkit:
+	rm -rf awxkit/*.egg-info awxkit/.tox
 
 # convenience target to assert environment variables are defined
 guard-%:
@@ -116,16 +133,16 @@ guard-%:
 
 virtualenv: virtualenv_ansible virtualenv_awx
 
+# virtualenv_* targets do not use --system-site-packages to prevent bugs installing packages
+# but Ansible venvs are expected to have this, so that must be done after venv creation
 virtualenv_ansible:
 	if [ "$(VENV_BASE)" ]; then \
 		if [ ! -d "$(VENV_BASE)" ]; then \
 			mkdir $(VENV_BASE); \
 		fi; \
 		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
-			virtualenv -p python --system-site-packages $(VENV_BASE)/ansible && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed six packaging appdirs && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==36.0.1 && \
-			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==9.0.1; \
+			virtualenv -p python $(VENV_BASE)/ansible && \
+			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP); \
 		fi; \
 	fi
 
@@ -134,62 +151,69 @@ virtualenv_ansible_py3:
 		if [ ! -d "$(VENV_BASE)" ]; then \
 			mkdir $(VENV_BASE); \
 		fi; \
-		if [ ! -d "$(VENV_BASE)/ansible3" ]; then \
-			python3 -m venv --system-site-packages $(VENV_BASE)/ansible3; \
+		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
+			virtualenv -p $(PYTHON) $(VENV_BASE)/ansible; \
+			$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP); \
 		fi; \
 	fi
 
+# flit is needed for offline install of certain packages, specifically ptyprocess
+# it is needed for setup, but not always recognized as a setup dependency
+# similar to pip, setuptools, and wheel, these are all needed here as a bootstrapping issues
 virtualenv_awx:
 	if [ "$(VENV_BASE)" ]; then \
 		if [ ! -d "$(VENV_BASE)" ]; then \
 			mkdir $(VENV_BASE); \
 		fi; \
 		if [ ! -d "$(VENV_BASE)/awx" ]; then \
-			$(PYTHON) -m venv $(VENV_BASE)/awx; \
+			virtualenv -p $(PYTHON) $(VENV_BASE)/awx; \
+			$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP) && \
+			$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) flit; \
 		fi; \
 	fi
 
+# --ignore-install flag is not used because *.txt files should specify exact versions
 requirements_ansible: virtualenv_ansible
 	if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
-	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed -r /dev/stdin ; \
+	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \
 	else \
-	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
+	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \
 	fi
 	$(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt
+	# Same effect as using --system-site-packages flag on venv creation
+	rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt
 
 requirements_ansible_py3: virtualenv_ansible_py3
-	cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible3/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin
-	$(VENV_BASE)/ansible3/bin/pip3 install ansible  # can't inherit from system ansible, it's py2
-	$(VENV_BASE)/ansible3/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt
+	if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
+	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) -r /dev/stdin ; \
+	else \
+	    cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \
+	fi
+	$(VENV_BASE)/ansible/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt
+	# Same effect as using --system-site-packages flag on venv creation
+	rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt
 
 requirements_ansible_dev:
 	if [ "$(VENV_BASE)" ]; then \
 		$(VENV_BASE)/ansible/bin/pip install pytest mock; \
 	fi
 
-requirements_isolated:
-	if [ ! -d "$(VENV_BASE)/awx" ]; then \
-		$(PYTHON) -m venv $(VENV_BASE)/awx; \
-	fi;
-	echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg
-	$(VENV_BASE)/awx/bin/pip install -r requirements/requirements_isolated.txt
-
 # Install third-party requirements needed for AWX's environment.
+# this does not use system site packages intentionally
 requirements_awx: virtualenv_awx
 	if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
-	    cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed -r /dev/stdin ; \
+	    cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \
 	else \
-	    cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
+	    cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \
 	fi
-	echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg
-	#$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
+	$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
 
 requirements_awx_dev:
 	$(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt
 
 requirements: requirements_ansible requirements_awx
 
-requirements_dev: requirements requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev
+requirements_dev: requirements_awx requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev
 
 requirements_test: requirements
 
@@ -221,7 +245,7 @@ init:
 	if [ "$(AWX_GROUP_QUEUES)" == "tower,thepentagon" ]; then \
 		$(MANAGEMENT_COMMAND) provision_instance --hostname=isolated; \
 		$(MANAGEMENT_COMMAND) register_queue --queuename='thepentagon' --hostnames=isolated --controller=tower; \
-		$(MANAGEMENT_COMMAND) generate_isolated_key > /awx_devel/awx/main/expect/authorized_keys; \
+		$(MANAGEMENT_COMMAND) generate_isolated_key > /awx_devel/awx/main/isolated/authorized_keys; \
 	fi;
 
 # Refresh development environment after pulling new code.
@@ -242,10 +266,6 @@ migrate:
 dbchange:
 	$(MANAGEMENT_COMMAND) makemigrations
 
-# access database shell, asks for password
-dbshell:
-	sudo -u postgres psql -d awx-dev
-
 server_noattach:
 	tmux new-session -d -s awx 'exec make uwsgi'
 	tmux rename-window 'AWX'
@@ -272,15 +292,7 @@ supervisor:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	supervisord --pidfile=/tmp/supervisor_pid
-
-# Alternate approach to tmux to run all development tasks specified in
-# Procfile.
-honcho:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	honcho start -f tools/docker-compose/Procfile
+	supervisord --pidfile=/tmp/supervisor_pid -n
 
 collectstatic:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -292,7 +304,7 @@ uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1-once="exec:awx-manage run_dispatcher --reload"
+    uwsgi -b 32768 --socket 127.0.0.1:8050 --module=awx.wsgi:application --home=/venv/awx --chdir=/awx_devel/ --vacuum --processes=5 --harakiri=120 --master --no-orphans --py-autoreload 1 --max-requests=1000 --stats /tmp/stats.socket --lazy-apps --logformat "%(addr) %(method) %(uri) - %(proto) %(status)" --hook-accepting1="exec:supervisorctl restart tower-processes:awx-dispatcher tower-processes:awx-receiver"
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -356,7 +368,8 @@ pylint: reports
 	@(set -o pipefail && $@ | reports/$@.report)
 
 genschema: reports
-	$(MAKE) swagger PYTEST_ARGS="--genschema"
+	$(MAKE) swagger PYTEST_ARGS="--genschema --create-db "
+	mv swagger.json schema.json
 
 swagger: reports
 	@if [ "$(VENV_BASE)" ]; then \
@@ -368,7 +381,7 @@ check: flake8 pep8 # pyflakes pylint
 
 awx-link:
 	cp -R /tmp/awx.egg-info /awx_devel/ || true
-	sed -i "s/placeholder/$(shell git describe --long | sed 's/\./\\./g')/" /awx_devel/awx.egg-info/PKG-INFO
+	sed -i "s/placeholder/$(shell cat VERSION)/" /awx_devel/awx.egg-info/PKG-INFO
 	cp -f /tmp/awx.egg-link /venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.egg-link
 
 TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests
@@ -379,9 +392,43 @@ test:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
+	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
 	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
-test_combined: test_ansible test
+prepare_collection_venv:
+	rm -rf $(COLLECTION_VENV)
+	mkdir $(COLLECTION_VENV)
+	$(VENV_BASE)/awx/bin/pip install --target=$(COLLECTION_VENV) git+https://github.com/ansible/tower-cli.git
+
+COLLECTION_TEST_DIRS ?= awx_collection/test/awx
+COLLECTION_PACKAGE ?= awx
+COLLECTION_NAMESPACE ?= awx
+
+test_collection:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	PYTHONPATH=$(COLLECTION_VENV):/awx_devel/awx_collection:$PYTHONPATH:/usr/lib/python3.6/site-packages py.test $(COLLECTION_TEST_DIRS)
+
+flake8_collection:
+	flake8 awx_collection/  # Different settings, in main exclude list
+
+test_collection_all: prepare_collection_venv test_collection flake8_collection
+
+test_collection_sanity:
+	rm -rf sanity
+	mkdir -p sanity/ansible_collections/awx
+	cp -Ra awx_collection sanity/ansible_collections/awx/awx  # symlinks do not work
+	cd sanity/ansible_collections/awx/awx && git init && git add . # requires both this file structure and a git repo, so there you go
+	cd sanity/ansible_collections/awx/awx && ansible-test sanity
+
+build_collection:
+	ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION)
+	ansible-galaxy collection build awx_collection --force --output-path=awx_collection
+
+install_collection: build_collection
+	rm -rf ~/.ansible/collections/ansible_collections/awx/awx
+	ansible-galaxy collection install awx_collection/awx-awx-$(VERSION).tar.gz
 
 test_unit:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -389,12 +436,6 @@ test_unit:
 	fi; \
 	py.test awx/main/tests/unit awx/conf/tests/unit awx/sso/tests/unit
 
-test_ansible:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/ansible/bin/activate; \
-	fi; \
-	py.test awx/lib/tests -c awx/lib/tests/pytest.ini
-
 # Run all API unit tests with coverage enabled.
 test_coverage:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -483,7 +524,7 @@ $(UI_RELEASE_FLAG_FILE): $(I18N_FLAG_FILE) $(UI_RELEASE_DEPS_FLAG_FILE)
 	touch $(UI_RELEASE_FLAG_FILE)
 
 $(UI_RELEASE_DEPS_FLAG_FILE):
-	PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 $(NPM_BIN) --unsafe-perm --prefix awx/ui install --no-save awx/ui
+	PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 $(NPM_BIN) --unsafe-perm --prefix awx/ui ci --no-save awx/ui
 	touch $(UI_RELEASE_DEPS_FLAG_FILE)
 
 # END UI RELEASE TASKS
@@ -498,7 +539,7 @@ $(UI_DEPS_FLAG_FILE):
 		rm -rf awx/ui/node_modules; \
 		rm -f ${UI_RELEASE_DEPS_FLAG_FILE}; \
 	fi; \
-	$(NPM_BIN) --unsafe-perm --prefix awx/ui install --no-save awx/ui
+	$(NPM_BIN) --unsafe-perm --prefix awx/ui ci --no-save awx/ui
 	touch $(UI_DEPS_FLAG_FILE)
 
 ui-docker-machine: $(UI_DEPS_FLAG_FILE)
@@ -515,6 +556,10 @@ ui-devel: $(UI_DEPS_FLAG_FILE)
 ui-test: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) --prefix awx/ui run test
 
+ui-lint: $(UI_DEPS_FLAG_FILE)
+	$(NPM_BIN) run --prefix awx/ui jshint
+	$(NPM_BIN) run --prefix awx/ui lint
+
 # A standard go-to target for API developers to use building the frontend
 ui: clean-ui ui-devel
 
@@ -526,9 +571,36 @@ jshint: $(UI_DEPS_FLAG_FILE)
 	$(NPM_BIN) run --prefix awx/ui jshint
 	$(NPM_BIN) run --prefix awx/ui lint
 
+ui-zuul-lint-and-test: $(UI_DEPS_FLAG_FILE)
+	$(NPM_BIN) run --prefix awx/ui jshint
+	$(NPM_BIN) run --prefix awx/ui lint
+	$(NPM_BIN) --prefix awx/ui run test:ci
+	$(NPM_BIN) --prefix awx/ui run unit
+
 # END UI TASKS
 # --------------------------------------
 
+# UI NEXT TASKS
+# --------------------------------------
+
+ui-next-lint:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+
+ui-next-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+ui-next-zuul-lint-and-test:
+	$(NPM_BIN) --prefix awx/ui_next install
+	$(NPM_BIN) run --prefix awx/ui_next lint
+	$(NPM_BIN) run --prefix awx/ui_next prettier-check
+	$(NPM_BIN) run --prefix awx/ui_next test
+
+# END UI NEXT TASKS
+# --------------------------------------
+
 # Build a pip-installable package into dist/ with a timestamped version number.
 dev_build:
 	$(PYTHON) setup.py dev_build
@@ -560,45 +632,46 @@ setup-bundle-build:
 	mkdir -p $@
 
 docker-auth:
-	if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
-		docker login -u oauth2accesstoken -p "$(IMAGE_REPOSITORY_AUTH)" $(IMAGE_REPOSITORY_BASE); \
+	@if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
+		echo "$(IMAGE_REPOSITORY_AUTH)" | docker login -u oauth2accesstoken --password-stdin $(IMAGE_REPOSITORY_BASE); \
 	fi;
 
+# This directory is bind-mounted inside of the development container and
+# needs to be pre-created for permissions to be set correctly. Otherwise,
+# Docker will create this directory as root.
+awx/projects:
+	@mkdir -p $@
+
 # Docker isolated rampart
-docker-isolated:
-	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml create
-	docker start tools_awx_1
-	docker start tools_isolated_1
-	echo "__version__ = '`git describe --long | cut -d - -f 1-1`'" | docker exec -i tools_isolated_1 /bin/bash -c "cat > /venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.py"
+docker-compose-isolated: awx/projects
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml up
 
 # Docker Compose Development environment
-docker-compose: docker-auth
-	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml up --no-recreate awx
+docker-compose: docker-auth awx/projects
+	CURRENT_UID=$(shell id -u) OS="$(shell docker info | grep 'Operating System')" TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml up --no-recreate awx
 
-docker-compose-cluster: docker-auth
+docker-compose-cluster: docker-auth awx/projects
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml up
 
-docker-compose-test: docker-auth
-	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /bin/bash
+docker-compose-credential-plugins: docker-auth awx/projects
+	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
+	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx
 
-docker-compose-runtest:
+docker-compose-test: docker-auth awx/projects
+	cd tools && CURRENT_UID=$(shell id -u) OS="$(shell docker info | grep 'Operating System')" TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /bin/bash
+
+docker-compose-runtest: awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh
 
-docker-compose-build-swagger:
+docker-compose-build-swagger: awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh swagger
 
-docker-compose-genschema:
-	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh genschema
-	mv swagger.json schema.json
-
-docker-compose-detect-schema-change:
-	$(MAKE) docker-compose-genschema
+detect-schema-change: genschema
 	curl https://s3.amazonaws.com/awx-public-ci-files/schema.json -o reference-schema.json
 	# Ignore differences in whitespace with -b
 	diff -u -b reference-schema.json schema.json
 
-docker-compose-clean:
+docker-compose-clean: awx/projects
 	cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm -w /awx_devel --service-ports awx make clean
 	cd tools && TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose rm -sf
 
@@ -606,12 +679,13 @@ docker-compose-build: awx-devel-build
 
 # Base development image build
 awx-devel-build:
-	docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile .
+	docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile \
+		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 	docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 
 # For use when developing on "isolated" AWX deployments
-awx-isolated-build:
+docker-compose-isolated-build: awx-devel-build
 	docker build -t ansible/awx_isolated -f tools/docker-isolated/Dockerfile .
 	docker tag ansible/awx_isolated $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG)
 	#docker push $(DEV_DOCKER_TAG_BASE)/awx_isolated:$(COMPOSE_TAG)
@@ -625,12 +699,15 @@ docker-clean:
 docker-refresh: docker-clean docker-compose
 
 # Docker Development Environment with Elastic Stack Connected
-docker-compose-elk: docker-auth
+docker-compose-elk: docker-auth awx/projects
 	CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
-docker-compose-cluster-elk: docker-auth
+docker-compose-cluster-elk: docker-auth awx/projects
 	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose-cluster.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
+prometheus:
+	docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml
+
 minishift-dev:
 	ansible-playbook -i localhost, -e devtree_directory=$(CURDIR) tools/clusterdevel/start_minishift_dev.yml
 
@@ -643,7 +720,7 @@ clean-elk:
 	docker rm tools_kibana_1
 
 psql-container:
-	docker run -it --net tools_default --rm postgres:9.6 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
+	docker run -it --net tools_default --rm postgres:10 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
 
 VERSION:
 	@echo "awx: $(VERSION)"

VERSION

@@ -1 +1 @@
-3.0.0
+9.1.1

awx/__init__.py

@@ -24,43 +24,32 @@ except ImportError: # pragma: no cover
 import hashlib
 
 try:
-    import django
-    from django.utils.encoding import force_bytes
-    from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-    from django.db.backends.base import schema
+    import django  # noqa: F401
     HAS_DJANGO = True
 except ImportError:
     HAS_DJANGO = False
+else:
+    from django.db.backends.base import schema
+    from django.db.backends.utils import names_digest
 
 
 if HAS_DJANGO is True:
-    # This line exists to make sure we don't regress on FIPS support if we
-    # upgrade Django; if you're upgrading Django and see this error,
-    # update the version check below, and confirm that FIPS still works.
-    if django.__version__ != '1.11.16':
-        raise RuntimeError("Django version other than 1.11.16 detected {}. \
-                Subclassing BaseDatabaseSchemaEditor is known to work for Django 1.11.16 \
-                and may not work in newer Django versions.".format(django.__version__))
 
-
-    class FipsBaseDatabaseSchemaEditor(BaseDatabaseSchemaEditor):
-
-        @classmethod
-        def _digest(cls, *args):
+    # See upgrade blocker note in requirements/README.md
+    try:
+        names_digest('foo', 'bar', 'baz', length=8)
+    except ValueError:
+        def names_digest(*args, length):
             """
-            Generates a 32-bit digest of a set of arguments that can be used to
-            shorten identifying names.
+            Generate a 32-bit digest of a set of arguments that can be used to shorten
+            identifying names.  Support for use in FIPS environments.
             """
-            try:
-                h = hashlib.md5()
-            except ValueError:
-                h = hashlib.md5(usedforsecurity=False)
+            h = hashlib.md5(usedforsecurity=False)
             for arg in args:
-                h.update(force_bytes(arg))
-            return h.hexdigest()[:8]
+                h.update(arg.encode())
+            return h.hexdigest()[:length]
 
-
-    schema.BaseDatabaseSchemaEditor = FipsBaseDatabaseSchemaEditor
+        schema.names_digest = names_digest
 
 
 def find_commands(management_dir):
@@ -80,6 +69,23 @@ def find_commands(management_dir):
     return commands
 
 
+def oauth2_getattribute(self, attr):
+    # Custom method to override
+    # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__
+    from django.conf import settings
+    val = None
+    if 'migrate' not in sys.argv:
+        # certain Django OAuth Toolkit migrations actually reference
+        # setting lookups for references to model classes (e.g.,
+        # oauth2_settings.REFRESH_TOKEN_MODEL)
+        # If we're doing an OAuth2 setting lookup *while running* a migration,
+        # don't do our usual "Configure Tower in Tower" database setting lookup
+        val = settings.OAUTH2_PROVIDER.get(attr)
+    if val is None:
+        val = object.__getattribute__(self, attr)
+    return val
+
+
 def prepare_env():
     # Update the default settings environment variable based on current mode.
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
@@ -91,16 +97,12 @@ def prepare_env():
     # Monkeypatch Django find_commands to also work with .pyc files.
     import django.core.management
     django.core.management.find_commands = find_commands
-    # Fixup sys.modules reference to django.utils.six to allow jsonfield to
-    # work when using Django 1.4.
-    import django.utils
-    try:
-        import django.utils.six
-    except ImportError: # pragma: no cover
-        import six
-        sys.modules['django.utils.six'] = sys.modules['six']
-        django.utils.six = sys.modules['django.utils.six']
-        from django.utils import six # noqa
+
+    # Monkeypatch Oauth2 toolkit settings class to check for settings
+    # in django.conf settings each time, not just once during import
+    import oauth2_provider.settings
+    oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ = oauth2_getattribute
+
     # Use the AWX_TEST_DATABASE_* environment variables to specify the test
     # database settings to use when management command is run as an external
     # program via unit tests.

awx/api/conf.py

@@ -38,12 +38,15 @@ register(
     'OAUTH2_PROVIDER',
     field_class=OAuth2ProviderField,
     default={'ACCESS_TOKEN_EXPIRE_SECONDS': oauth2_settings.ACCESS_TOKEN_EXPIRE_SECONDS,
-             'AUTHORIZATION_CODE_EXPIRE_SECONDS': 600},
+             'AUTHORIZATION_CODE_EXPIRE_SECONDS': oauth2_settings.AUTHORIZATION_CODE_EXPIRE_SECONDS,
+             'REFRESH_TOKEN_EXPIRE_SECONDS': oauth2_settings.REFRESH_TOKEN_EXPIRE_SECONDS},
     label=_('OAuth 2 Timeout Settings'),
     help_text=_('Dictionary for customizing OAuth 2 timeouts, available items are '
                 '`ACCESS_TOKEN_EXPIRE_SECONDS`, the duration of access tokens in the number '
-                'of seconds, and `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
-                'authorization codes in the number of seconds.'),
+                'of seconds, `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
+                'authorization codes in the number of seconds, and `REFRESH_TOKEN_EXPIRE_SECONDS`, '
+                'the duration of refresh tokens, after expired access tokens, '
+                'in the number of seconds.'),
     category=_('Authentication'),
     category_slug='authentication',
 )
@@ -59,3 +62,15 @@ register(
     category=_('Authentication'),
     category_slug='authentication',
 )
+register(
+    'LOGIN_REDIRECT_OVERRIDE',
+    field_class=fields.CharField,
+    allow_blank=True,
+    required=False,
+    default='',
+    label=_('Login redirect override URL'),
+    help_text=_('URL to which unauthorized users will be redirected to log in. '
+                'If blank, users will be sent to the Tower login page.'),
+    category=_('Authentication'),
+    category_slug='authentication',
+)

awx/api/fields.py

@@ -80,7 +80,7 @@ class OAuth2ProviderField(fields.DictField):
     default_error_messages = {
         'invalid_key_names': _('Invalid key names: {invalid_key_names}'),
     }
-    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS'}
+    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS', 'REFRESH_TOKEN_EXPIRE_SECONDS'}
     child = fields.IntegerField(min_value=1)
 
     def to_internal_value(self, data):
@@ -101,6 +101,10 @@ class DeprecatedCredentialField(serializers.IntegerField):
         super(DeprecatedCredentialField, self).__init__(**kwargs)
 
     def to_internal_value(self, pk):
+        try:
+            pk = int(pk)
+        except ValueError:
+            self.fail('invalid')
         try:
             Credential.objects.get(pk=pk)
         except ObjectDoesNotExist:

awx/api/filters.py

@@ -24,20 +24,6 @@ from rest_framework.filters import BaseFilterBackend
 # AWX
 from awx.main.utils import get_type_for_model, to_python_boolean
 from awx.main.utils.db import get_all_field_names
-from awx.main.models.credential import CredentialType
-
-
-class V1CredentialFilterBackend(BaseFilterBackend):
-    '''
-    For /api/v1/ requests, filter out v2 (custom) credentials
-    '''
-
-    def filter_queryset(self, request, queryset, view):
-        # TODO: remove in 3.3
-        from awx.api.versioning import get_request_version
-        if get_request_version(request) == 1:
-            queryset = queryset.filter(credential_type__managed_by_tower=True)
-        return queryset
 
 
 class TypeFilterBackend(BaseFilterBackend):
@@ -140,7 +126,7 @@ class FieldLookupBackend(BaseFilterBackend):
     '''
 
     RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
-                      'search', 'type', 'host_filter')
+                      'search', 'type', 'host_filter', 'count_disabled', 'no_truncate')
 
     SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                          'startswith', 'istartswith', 'endswith', 'iendswith',
@@ -223,7 +209,7 @@ class FieldLookupBackend(BaseFilterBackend):
                 raise ValueError('%s is not searchable' % new_lookup[:-8])
             new_lookups = []
             for rm_field in related_model._meta.fields:
-                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description'):
+                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'):
                     new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
             return value, new_lookups
         else:
@@ -292,39 +278,6 @@ class FieldLookupBackend(BaseFilterBackend):
                     key = key[5:]
                     q_not = True
 
-                # Make legacy v1 Job/Template fields work for backwards compatability
-                # TODO: remove after API v1 deprecation period
-                if queryset.model._meta.object_name in ('JobTemplate', 'Job') and key in (
-                        'credential', 'vault_credential', 'cloud_credential', 'network_credential'
-                ) or queryset.model._meta.object_name in ('InventorySource', 'InventoryUpdate') and key == 'credential':
-                    key = 'credentials'
-
-                # Make legacy v1 Credential fields work for backwards compatability
-                # TODO: remove after API v1 deprecation period
-                #
-                # convert v1 `Credential.kind` queries to `Credential.credential_type__pk`
-                if queryset.model._meta.object_name == 'Credential' and key == 'kind':
-                    key = key.replace('kind', 'credential_type')
-
-                    if 'ssh' in values:
-                        # In 3.2, SSH and Vault became separate credential types, but in the v1 API,
-                        # they're both still "kind=ssh"
-                        # under the hood, convert `/api/v1/credentials/?kind=ssh` to
-                        # `/api/v1/credentials/?or__credential_type=<ssh_pk>&or__credential_type=<vault_pk>`
-                        values = set(values)
-                        values.add('vault')
-                        values = list(values)
-                        q_or = True
-
-                    for i, kind in enumerate(values):
-                        if kind == 'vault':
-                            type_ = CredentialType.objects.get(kind=kind)
-                        else:
-                            type_ = CredentialType.from_v1_kind(kind)
-                        if type_ is None:
-                            raise ParseError(_('cannot filter on kind %s') % kind)
-                        values[i] = type_.pk
-
                 # Convert value(s) to python and add to the appropriate list.
                 for value in values:
                     if q_int:
@@ -402,6 +355,8 @@ class OrderByBackend(BaseFilterBackend):
                         order_by = value.split(',')
                     else:
                         order_by = (value,)
+            if order_by is None:
+                order_by = self.get_default_ordering(view)
             if order_by:
                 order_by = self._validate_ordering_fields(queryset.model, order_by)
 
@@ -428,6 +383,12 @@ class OrderByBackend(BaseFilterBackend):
             # Return a 400 for invalid field names.
             raise ParseError(*e.args)
 
+    def get_default_ordering(self, view):
+        ordering = getattr(view, 'ordering', None)
+        if isinstance(ordering, str):
+            return (ordering,)
+        return ordering
+
     def _validate_ordering_fields(self, model, order_by):
         for field_name in order_by:
             # strip off the negation prefix `-` if it exists

awx/api/generics.py

@@ -5,7 +5,6 @@
 import inspect
 import logging
 import time
-import six
 import urllib.parse
 
 # Django
@@ -32,17 +31,23 @@ from rest_framework.permissions import AllowAny
 from rest_framework.renderers import StaticHTMLRenderer, JSONRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
-# cryptography
-from cryptography.fernet import InvalidToken
-
 # AWX
 from awx.api.filters import FieldLookupBackend
-from awx.main.models import *  # noqa
+from awx.main.models import (
+    UnifiedJob, UnifiedJobTemplate, User, Role, Credential,
+    WorkflowJobTemplateNode, WorkflowApprovalTemplate
+)
 from awx.main.access import access_registry
-from awx.main.utils import * # noqa
+from awx.main.utils import (
+    camelcase_to_underscore,
+    get_search_fields,
+    getattrd,
+    get_object_or_400,
+    decrypt_field
+)
 from awx.main.utils.db import get_all_field_names
 from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer, UserSerializer
-from awx.api.versioning import URLPathVersioning, get_request_version
+from awx.api.versioning import URLPathVersioning
 from awx.api.metadata import SublistAttachDetatchMetadata, Metadata
 
 __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView',
@@ -87,7 +92,7 @@ class LoggedLoginView(auth_views.LoginView):
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
         if request.user.is_authenticated:
-            logger.info(smart_text(u"User {} logged in.".format(self.request.user.username)))
+            logger.info(smart_text(u"User {} logged in from {}".format(self.request.user.username,request.META.get('REMOTE_ADDR', None))))
             ret.set_cookie('userLoggedIn', 'true')
             current_user = UserSerializer(self.request.user)
             current_user = smart_text(JSONRenderer().render(current_user.data))
@@ -96,6 +101,8 @@ class LoggedLoginView(auth_views.LoginView):
 
             return ret
         else:
+            if 'username' in self.request.POST:
+                logger.warn(smart_text(u"Login failed for user {} from {}".format(self.request.POST.get('username'),request.META.get('REMOTE_ADDR', None))))
             ret.status_code = 401
             return ret
 
@@ -113,39 +120,12 @@ class LoggedLogoutView(auth_views.LogoutView):
         return ret
 
 
-def get_view_name(cls, suffix=None):
-    '''
-    Wrapper around REST framework get_view_name() to support get_name() method
-    and view_name property on a view class.
-    '''
-    name = ''
-    if hasattr(cls, 'get_name') and callable(cls.get_name):
-        name = cls().get_name()
-    elif hasattr(cls, 'view_name'):
-        if callable(cls.view_name):
-            name = cls.view_name()
-        else:
-            name = cls.view_name
-    if name:
-        return ('%s %s' % (name, suffix)) if suffix else name
-    return views.get_view_name(cls, suffix=None)
+def get_view_description(view, html=False):
+    '''Wrapper around REST framework get_view_description() to continue
+    to support our historical div.
 
-
-def get_view_description(cls, request, html=False):
     '''
-    Wrapper around REST framework get_view_description() to support
-    get_description() method and view_description property on a view class.
-    '''
-    if hasattr(cls, 'get_description') and callable(cls.get_description):
-        desc = cls().get_description(request, html=html)
-        cls = type(cls.__name__, (object,), {'__doc__': desc})
-    elif hasattr(cls, 'view_description'):
-        if callable(cls.view_description):
-            view_desc = cls.view_description()
-        else:
-            view_desc = cls.view_description
-        cls = type(cls.__name__, (object,), {'__doc__': view_desc})
-    desc = views.get_view_description(cls, html=html)
+    desc = views.get_view_description(view, html=html)
     if html:
         desc = '<div class="description">%s</div>' % desc
     return mark_safe(desc)
@@ -225,6 +205,9 @@ class APIView(views.APIView):
             response['X-API-Query-Count'] = len(q_times)
             response['X-API-Query-Time'] = '%0.3fs' % sum(q_times)
 
+        if getattr(self, 'deprecated', False):
+            response['Warning'] = '299 awx "This resource has been deprecated and will be removed in a future release."'  # noqa
+
         return response
 
     def get_authenticate_header(self, request):
@@ -258,14 +241,6 @@ class APIView(views.APIView):
         # `curl https://user:pass@tower.example.org/api/v2/job_templates/N/launch/`
         return 'Bearer realm=api authorization_url=/api/o/authorize/'
 
-    def get_view_description(self, html=False):
-        """
-        Return some descriptive text for the view, as used in OPTIONS responses
-        and in the browsable API.
-        """
-        func = self.settings.VIEW_DESCRIPTION_FUNCTION
-        return func(self.__class__, getattr(self, '_request', None), html)
-
     def get_description_context(self):
         return {
             'view': self,
@@ -274,20 +249,14 @@ class APIView(views.APIView):
             'swagger_method': getattr(self.request, 'swagger_method', None),
         }
 
-    def get_description(self, request, html=False):
-        self.request = request
+    @property
+    def description(self):
         template_list = []
         for klass in inspect.getmro(type(self)):
             template_basename = camelcase_to_underscore(klass.__name__)
             template_list.append('api/%s.md' % template_basename)
         context = self.get_description_context()
 
-        # "v2" -> 2
-        default_version = int(settings.REST_FRAMEWORK['DEFAULT_VERSION'].lstrip('v'))
-        request_version = get_request_version(self.request)
-        if request_version is not None and request_version < default_version:
-            context['deprecated'] = True
-
         description = render_to_string(template_list, context)
         if context.get('deprecated') and context.get('swagger_method') is None:
             # render deprecation messages at the very top
@@ -383,12 +352,14 @@ class GenericAPIView(generics.GenericAPIView, APIView):
                     'model_verbose_name_plural': smart_text(self.model._meta.verbose_name_plural),
                 })
             serializer = self.get_serializer()
+            metadata = self.metadata_class()
+            metadata.request = self.request
             for method, key in [
                 ('GET', 'serializer_fields'),
                 ('POST', 'serializer_create_fields'),
                 ('PUT', 'serializer_update_fields')
             ]:
-                d[key] = self.metadata_class().get_serializer_info(serializer, method=method)
+                d[key] = metadata.get_serializer_info(serializer, method=method)
         d['settings'] = settings
         return d
 
@@ -434,21 +405,21 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
                 continue
             if getattr(field, 'related_model', None):
                 fields.add('{}__search'.format(field.name))
-        for rel in self.model._meta.related_objects:
-            name = rel.related_name
-            if isinstance(rel, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
+        for related in self.model._meta.related_objects:
+            name = related.related_name
+            if isinstance(related, OneToOneRel) and self.model._meta.verbose_name.startswith('unified'):
                 # Add underscores for polymorphic subclasses for user utility
-                name = rel.related_model._meta.verbose_name.replace(" ", "_")
+                name = related.related_model._meta.verbose_name.replace(" ", "_")
             if skip_related_name(name) or name.endswith('+'):
                 continue
             fields.add('{}__search'.format(name))
-        m2m_rel = []
-        m2m_rel += self.model._meta.local_many_to_many
+        m2m_related = []
+        m2m_related += self.model._meta.local_many_to_many
         if issubclass(self.model, UnifiedJobTemplate) and self.model != UnifiedJobTemplate:
-            m2m_rel += UnifiedJobTemplate._meta.local_many_to_many
+            m2m_related += UnifiedJobTemplate._meta.local_many_to_many
         if issubclass(self.model, UnifiedJob) and self.model != UnifiedJob:
-            m2m_rel += UnifiedJob._meta.local_many_to_many
-        for relationship in m2m_rel:
+            m2m_related += UnifiedJob._meta.local_many_to_many
+        for relationship in m2m_related:
             if skip_related_name(relationship.name):
                 continue
             if relationship.related_model._meta.app_label != 'main':
@@ -521,9 +492,12 @@ class SubListAPIView(ParentMixin, ListAPIView):
         parent = self.get_parent_object()
         self.check_parent_access(parent)
         qs = self.request.user.get_queryset(self.model).distinct()
-        sublist_qs = getattrd(parent, self.relationship).distinct()
+        sublist_qs = self.get_sublist_queryset(parent)
         return qs & sublist_qs
 
+    def get_sublist_queryset(self, parent):
+        return getattrd(parent, self.relationship).distinct()
+
 
 class DestroyAPIView(generics.DestroyAPIView):
 
@@ -600,7 +574,7 @@ class SubListCreateAPIView(SubListAPIView, ListCreateAPIView):
                             status=status.HTTP_400_BAD_REQUEST)
 
         # Verify we have permission to add the object as given.
-        if not request.user.can_access(self.model, 'add', serializer.initial_data):
+        if not request.user.can_access(self.model, 'add', serializer.validated_data):
             raise PermissionDenied()
 
         # save the object through the serializer, reload and returned the saved
@@ -809,6 +783,7 @@ class RetrieveUpdateDestroyAPIView(RetrieveUpdateAPIView, DestroyAPIView):
 class ResourceAccessList(ParentMixin, ListAPIView):
 
     serializer_class = ResourceAccessListElementSerializer
+    ordering = ('username',)
 
     def get_queryset(self):
         obj = self.get_parent_object()
@@ -835,10 +810,6 @@ class CopyAPIView(GenericAPIView):
     new_in_330 = True
     new_in_api_v2 = True
 
-    def v1_not_allowed(self):
-        return Response({'detail': 'Action only possible starting with v2 API.'},
-                        status=status.HTTP_404_NOT_FOUND)
-
     def _get_copy_return_serializer(self, *args, **kwargs):
         if not self.copy_return_serializer_class:
             return self.get_serializer(*args, **kwargs)
@@ -852,17 +823,20 @@ class CopyAPIView(GenericAPIView):
     def _decrypt_model_field_if_needed(obj, field_name, field_val):
         if field_name in getattr(type(obj), 'REENCRYPTION_BLACKLIST_AT_COPY', []):
             return field_val
-        if isinstance(field_val, dict):
+        if isinstance(obj, Credential) and field_name == 'inputs':
+            for secret in obj.credential_type.secret_fields:
+                if secret in field_val:
+                    field_val[secret] = decrypt_field(obj, secret)
+        elif isinstance(field_val, dict):
             for sub_field in field_val:
-                if isinstance(sub_field, six.string_types) \
-                        and isinstance(field_val[sub_field], six.string_types):
-                    try:
-                        field_val[sub_field] = decrypt_field(obj, field_name, sub_field)
-                    except InvalidToken:
-                        # Catching the corner case with v1 credential fields
-                        field_val[sub_field] = decrypt_field(obj, sub_field)
-        elif isinstance(field_val, six.string_types):
-            field_val = decrypt_field(obj, field_name)
+                if isinstance(sub_field, str) \
+                        and isinstance(field_val[sub_field], str):
+                    field_val[sub_field] = decrypt_field(obj, field_name, sub_field)
+        elif isinstance(field_val, str):
+            try:
+                field_val = decrypt_field(obj, field_name)
+            except AttributeError:
+                return field_val
         return field_val
 
     def _build_create_dict(self, obj):
@@ -915,8 +889,23 @@ class CopyAPIView(GenericAPIView):
                 create_kwargs[field.name] = CopyAPIView._decrypt_model_field_if_needed(
                     obj, field.name, field_val
                 )
+
+        # WorkflowJobTemplateNodes that represent an approval are *special*;
+        # when we copy them, we actually want to *copy* the UJT they point at
+        # rather than share the template reference between nodes in disparate
+        # workflows
+        if (
+            isinstance(obj, WorkflowJobTemplateNode) and
+            isinstance(getattr(obj, 'unified_job_template'), WorkflowApprovalTemplate)
+        ):
+            new_approval_template, sub_objs = CopyAPIView.copy_model_obj(
+                None, None, WorkflowApprovalTemplate,
+                obj.unified_job_template, creater
+            )
+            create_kwargs['unified_job_template'] = new_approval_template
+
         new_obj = model.objects.create(**create_kwargs)
-        logger.debug(six.text_type('Deep copy: Created new object {}({})').format(
+        logger.debug('Deep copy: Created new object {}({})'.format(
             new_obj, model
         ))
         # Need to save separatedly because Djang-crum get_current_user would
@@ -942,21 +931,20 @@ class CopyAPIView(GenericAPIView):
         return ret
 
     def get(self, request, *args, **kwargs):
-        if get_request_version(request) < 2:
-            return self.v1_not_allowed()
         obj = self.get_object()
         if not request.user.can_access(obj.__class__, 'read', obj):
             raise PermissionDenied()
         create_kwargs = self._build_create_dict(obj)
         for key in create_kwargs:
             create_kwargs[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key]
-        can_copy = request.user.can_access(self.model, 'add', create_kwargs) and \
-            request.user.can_access(self.model, 'copy_related', obj)
+        try:
+            can_copy = request.user.can_access(self.model, 'add', create_kwargs) and \
+                request.user.can_access(self.model, 'copy_related', obj)
+        except PermissionDenied:
+            return Response({'can_copy': False})
         return Response({'can_copy': can_copy})
 
     def post(self, request, *args, **kwargs):
-        if get_request_version(request) < 2:
-            return self.v1_not_allowed()
         obj = self.get_object()
         create_kwargs = self._build_create_dict(obj)
         create_kwargs_check = {}
@@ -973,7 +961,7 @@ class CopyAPIView(GenericAPIView):
             None, None, self.model, obj, request.user, create_kwargs=create_kwargs,
             copy_name=serializer.validated_data.get('name', '')
         )
-        if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role:
+        if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role.members.all():
             new_obj.admin_role.members.add(request.user)
         if sub_objs:
             permission_check_func = None

awx/api/metadata.py

@@ -5,6 +5,8 @@ from collections import OrderedDict
 
 # Django
 from django.core.exceptions import PermissionDenied
+from django.db.models.fields import PositiveIntegerField, BooleanField
+from django.db.models.fields.related import ForeignKey
 from django.http import Http404
 from django.utils.encoding import force_text, smart_text
 from django.utils.translation import ugettext_lazy as _
@@ -14,10 +16,13 @@ from rest_framework import exceptions
 from rest_framework import metadata
 from rest_framework import serializers
 from rest_framework.relations import RelatedField, ManyRelatedField
+from rest_framework.fields import JSONField as DRFJSONField
 from rest_framework.request import clone_request
 
 # AWX
+from awx.main.fields import JSONField, ImplicitRoleField
 from awx.main.models import InventorySource, NotificationTemplate
+from awx.main.scheduler.kubernetes import PodManager
 
 
 class Metadata(metadata.SimpleMetadata):
@@ -68,6 +73,8 @@ class Metadata(metadata.SimpleMetadata):
             else:
                 for model_field in serializer.Meta.model._meta.fields:
                     if field.field_name == model_field.name:
+                        if getattr(model_field, '__accepts_json__', None):
+                            field_info['type'] = 'json'
                         field_info['filterable'] = True
                         break
                 else:
@@ -114,15 +121,55 @@ class Metadata(metadata.SimpleMetadata):
             for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
                 field_info[notification_type_name] = notification_type_class.init_parameters
 
+        # Special handling of notification messages where the required properties
+        # are conditional on the type selected.
+        try:
+            view_model = field.context['view'].model
+        except (AttributeError, KeyError):
+            view_model = None
+        if view_model == NotificationTemplate and field.field_name == 'messages':
+            for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES:
+                field_info[notification_type_name] = notification_type_class.default_messages
+
+
         # Update type of fields returned...
+        model_field = None
+        if serializer and hasattr(serializer, 'Meta') and hasattr(serializer.Meta, 'model'):
+            try:
+                model_field = serializer.Meta.model._meta.get_field(field.field_name)
+            except Exception:
+                pass
         if field.field_name == 'type':
             field_info['type'] = 'choice'
-        elif field.field_name == 'url':
+        elif field.field_name in ('url', 'custom_virtualenv', 'token'):
             field_info['type'] = 'string'
         elif field.field_name in ('related', 'summary_fields'):
             field_info['type'] = 'object'
+        elif isinstance(field, PositiveIntegerField):
+            field_info['type'] = 'integer'
         elif field.field_name in ('created', 'modified'):
             field_info['type'] = 'datetime'
+        elif (
+            RelatedField in field.__class__.__bases__ or
+            isinstance(model_field, ForeignKey)
+        ):
+            field_info['type'] = 'id'
+        elif (
+            isinstance(field, JSONField) or
+            isinstance(model_field, JSONField) or
+            isinstance(field, DRFJSONField) or
+            isinstance(getattr(field, 'model_field', None), JSONField) or
+            field.field_name == 'credential_passwords'
+        ):
+            field_info['type'] = 'json'
+        elif (
+            isinstance(field, ManyRelatedField) and
+            field.field_name == 'credentials'
+            # launch-time credentials
+        ):
+            field_info['type'] = 'list_of_ids'
+        elif isinstance(model_field, BooleanField):
+            field_info['type'] = 'boolean'
 
         return field_info
 
@@ -161,6 +208,9 @@ class Metadata(metadata.SimpleMetadata):
                 if not isinstance(meta, dict):
                     continue
 
+                if field == "pod_spec_override":
+                    meta['default'] = PodManager().pod_definition
+
                 # Add type choices if available from the serializer.
                 if field == 'type' and hasattr(serializer, 'get_type_choices'):
                     meta['choices'] = serializer.get_type_choices()
@@ -213,6 +263,16 @@ class Metadata(metadata.SimpleMetadata):
         if getattr(view, 'related_search_fields', None):
             metadata['related_search_fields'] = view.related_search_fields
 
+        # include role names in metadata
+        roles = []
+        model = getattr(view, 'model', None)
+        if model:
+            for field in model._meta.get_fields():
+                if type(field) is ImplicitRoleField:
+                    roles.append(field.name)
+        if len(roles) > 0:
+            metadata['object_roles'] = roles
+
         from rest_framework import generics
         if isinstance(view, generics.ListAPIView) and hasattr(view, 'paginator'):
             metadata['max_page_size'] = view.paginator.max_page_size
@@ -232,28 +292,13 @@ class RoleMetadata(Metadata):
         return metadata
 
 
-# TODO: Tower 3.3 remove class and all uses in views.py when API v1 is removed
-class JobTypeMetadata(Metadata):
-        def get_field_info(self, field):
-            res = super(JobTypeMetadata, self).get_field_info(field)
-
-            if field.field_name == 'job_type':
-                index = 0
-                for choice in res['choices']:
-                    if choice[0] == 'scan':
-                        res['choices'].pop(index)
-                        break
-                    index += 1
-            return res
-
-
 class SublistAttachDetatchMetadata(Metadata):
 
     def determine_actions(self, request, view):
         actions = super(SublistAttachDetatchMetadata, self).determine_actions(request, view)
         method = 'POST'
         if method in actions:
-            for field in actions[method]:
+            for field in list(actions[method].keys()):
                 if field == 'id':
                     continue
                 actions[method].pop(field)

awx/api/metrics.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    MetricsView
+)
+
+
+urls = [
+    url(r'^$', MetricsView.as_view(), name='metrics_view'),
+]
+
+__all__ = ['urls']

awx/api/pagination.py

@@ -3,14 +3,28 @@
 
 # Django REST Framework
 from django.conf import settings
+from django.core.paginator import Paginator as DjangoPaginator
 from rest_framework import pagination
+from rest_framework.response import Response
 from rest_framework.utils.urls import replace_query_param
 
 
+class DisabledPaginator(DjangoPaginator):
+
+    @property
+    def num_pages(self):
+        return 1
+
+    @property
+    def count(self):
+        return 200
+
+
 class Pagination(pagination.PageNumberPagination):
 
     page_size_query_param = 'page_size'
     max_page_size = settings.MAX_PAGE_SIZE
+    count_disabled = False
 
     def get_next_link(self):
         if not self.page.has_next():
@@ -18,7 +32,7 @@ class Pagination(pagination.PageNumberPagination):
         url = self.request and self.request.get_full_path() or ''
         url = url.encode('utf-8')
         page_number = self.page.next_page_number()
-        return replace_query_param(url, self.page_query_param, page_number)
+        return replace_query_param(self.cap_page_size(url), self.page_query_param, page_number)
 
     def get_previous_link(self):
         if not self.page.has_previous():
@@ -26,4 +40,30 @@ class Pagination(pagination.PageNumberPagination):
         url = self.request and self.request.get_full_path() or ''
         url = url.encode('utf-8')
         page_number = self.page.previous_page_number()
-        return replace_query_param(url, self.page_query_param, page_number)
+        return replace_query_param(self.cap_page_size(url), self.page_query_param, page_number)
+
+    def cap_page_size(self, url):
+        if int(self.request.query_params.get(self.page_size_query_param, 0)) > self.max_page_size:
+            url = replace_query_param(url, self.page_size_query_param, self.max_page_size)
+        return url
+
+    def get_html_context(self):
+        context = super().get_html_context()
+        context['page_links'] = [pl._replace(url=self.cap_page_size(pl.url))
+                                 for pl in context['page_links']]
+
+        return context
+
+    def paginate_queryset(self, queryset, request, **kwargs):
+        self.count_disabled = 'count_disabled' in request.query_params
+        try:
+            if self.count_disabled:
+                self.django_paginator_class = DisabledPaginator
+            return super(Pagination, self).paginate_queryset(queryset, request, **kwargs)
+        finally:
+            self.django_paginator_class = DjangoPaginator
+
+    def get_paginated_response(self, data):
+        if self.count_disabled:
+            return Response({'results': data})
+        return super(Pagination, self).get_paginated_response(data)

awx/api/parsers.py

@@ -4,7 +4,6 @@ import json
 
 # Django
 from django.conf import settings
-from django.utils import six
 from django.utils.encoding import smart_str
 from django.utils.translation import ugettext_lazy as _
 
@@ -34,4 +33,4 @@ class JSONParser(parsers.JSONParser):
                 raise ParseError(_('JSON parse error - not a JSON object'))
             return obj
         except ValueError as exc:
-            raise ParseError(_('JSON parse error - %s\nPossible cause: trailing comma.' % six.text_type(exc)))
+            raise ParseError(_('JSON parse error - %s\nPossible cause: trailing comma.' % str(exc)))

awx/api/permissions.py

@@ -9,15 +9,15 @@ from rest_framework.exceptions import MethodNotAllowed, PermissionDenied
 from rest_framework import permissions
 
 # AWX
-from awx.main.access import * # noqa
-from awx.main.models import * # noqa
+from awx.main.access import check_user_access
+from awx.main.models import Inventory, UnifiedJob
 from awx.main.utils import get_object_or_400
 
 logger = logging.getLogger('awx.api.permissions')
 
-__all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission',
+__all__ = ['ModelAccessPermission', 'JobTemplateCallbackPermission', 'VariableDataPermission',
            'TaskPermission', 'ProjectUpdatePermission', 'InventoryInventorySourcesUpdatePermission',
-           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission',]
+           'UserPermission', 'IsSuperUser', 'InstanceGroupTowerPermission', 'WorkflowApprovalPermission']
 
 
 class ModelAccessPermission(permissions.BasePermission):
@@ -74,12 +74,8 @@ class ModelAccessPermission(permissions.BasePermission):
             # FIXME: For some reason this needs to return True
             # because it is first called with obj=None?
             return True
-        if getattr(view, 'is_variable_data', False):
-            return check_user_access(request.user, view.model, 'change', obj,
-                                     dict(variables=request.data))
-        else:
-            return check_user_access(request.user, view.model, 'change', obj,
-                                     request.data)
+        return check_user_access(request.user, view.model, 'change', obj,
+                                 request.data)
 
     def check_patch_permissions(self, request, view, obj=None):
         return self.check_put_permissions(request, view, obj)
@@ -99,12 +95,11 @@ class ModelAccessPermission(permissions.BasePermission):
         '''
 
         # Don't allow anonymous users. 401, not 403, hence no raised exception.
-        if not request.user or request.user.is_anonymous():
+        if not request.user or request.user.is_anonymous:
             return False
 
         # Always allow superusers
-        if getattr(view, 'always_allow_superuser', True) and request.user.is_superuser \
-                and not hasattr(request.user, 'oauth_scopes'):
+        if getattr(view, 'always_allow_superuser', True) and request.user.is_superuser:
             return True
 
         # Check if view supports the request method before checking permission
@@ -164,6 +159,15 @@ class JobTemplateCallbackPermission(ModelAccessPermission):
             return True
 
 
+class VariableDataPermission(ModelAccessPermission):
+
+    def check_put_permissions(self, request, view, obj=None):
+        if not obj:
+            return True
+        return check_user_access(request.user, view.model, 'change', obj,
+                                 dict(variables=request.data))
+
+
 class TaskPermission(ModelAccessPermission):
     '''
     Permission checks used for API callbacks from running a task.
@@ -192,6 +196,17 @@ class TaskPermission(ModelAccessPermission):
             return False
 
 
+class WorkflowApprovalPermission(ModelAccessPermission):
+    '''
+    Permission check used by workflow `approval` and `deny` views to determine
+    who has access to approve and deny paused workflow nodes
+    '''
+
+    def check_post_permissions(self, request, view, obj=None):
+        approval = get_object_or_400(view.model, pk=view.kwargs['pk'])
+        return check_user_access(request.user, view.model, 'approve_or_deny', approval)
+
+
 class ProjectUpdatePermission(ModelAccessPermission):
     '''
     Permission check used by ProjectUpdateView to determine who can update projects
@@ -235,3 +250,7 @@ class InstanceGroupTowerPermission(ModelAccessPermission):
             return False
         return super(InstanceGroupTowerPermission, self).has_object_permission(request, view, obj)
 
+
+class WebhookKeyPermission(permissions.BasePermission):
+    def has_object_permission(self, request, view, obj):
+        return request.user.can_access(view.model, 'admin', obj, request.data)

awx/api/renderers.py

@@ -1,12 +1,13 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
+from django.utils.safestring import SafeText
+from prometheus_client.parser import text_string_to_metric_families
+
 # Django REST Framework
 from rest_framework import renderers
 from rest_framework.request import override_method
 
-import six
-
 
 class BrowsableAPIRenderer(renderers.BrowsableAPIRenderer):
     '''
@@ -20,6 +21,19 @@ class BrowsableAPIRenderer(renderers.BrowsableAPIRenderer):
             return renderers.JSONRenderer()
         return renderer
 
+    def get_content(self, renderer, data, accepted_media_type, renderer_context):
+        if isinstance(data, SafeText):
+            # Older versions of Django (pre-2.0) have a py3 bug which causes
+            # bytestrings marked as "safe" to not actually get _treated_ as
+            # safe; this causes certain embedded strings (like the stdout HTML
+            # view) to be improperly escaped
+            # see: https://github.com/ansible/awx/issues/3108
+            # https://code.djangoproject.com/ticket/28121
+            return data
+        return super(BrowsableAPIRenderer, self).get_content(renderer, data,
+                                                             accepted_media_type,
+                                                             renderer_context)
+
     def get_context(self, data, accepted_media_type, renderer_context):
         # Store the associated response status to know how to populate the raw
         # data form.
@@ -71,8 +85,8 @@ class PlainTextRenderer(renderers.BaseRenderer):
     format = 'txt'
 
     def render(self, data, media_type=None, renderer_context=None):
-        if not isinstance(data, six.string_types):
-            data = six.text_type(data)
+        if not isinstance(data, str):
+            data = str(data)
         return data.encode(self.charset)
 
 
@@ -90,3 +104,21 @@ class AnsiTextRenderer(PlainTextRenderer):
 class AnsiDownloadRenderer(PlainTextRenderer):
 
     format = "ansi_download"
+
+
+class PrometheusJSONRenderer(renderers.JSONRenderer):
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if isinstance(data, dict):
+            # HTTP errors are {'detail': ErrorDetail(string='...', code=...)}
+            return super(PrometheusJSONRenderer, self).render(
+                data, accepted_media_type, renderer_context
+            )
+        parsed_metrics = text_string_to_metric_families(data)
+        data = {}
+        for family in parsed_metrics:
+            for sample in family.samples:
+                data[sample[0]] = {"labels": sample[1], "value": sample[2]}
+        return super(PrometheusJSONRenderer, self).render(
+            data, accepted_media_type, renderer_context
+        )

awx/api/swagger.py

@@ -13,6 +13,17 @@ from rest_framework.views import APIView
 from rest_framework_swagger import renderers
 
 
+class SuperUserSchemaGenerator(SchemaGenerator):
+
+    def has_view_permissions(self, path, method, view):
+        #
+        # Generate the Swagger schema as if you were a superuser and
+        # permissions didn't matter; this short-circuits the schema path
+        # discovery to include _all_ potential paths in the API.
+        #
+        return True
+
+
 class AutoSchema(DRFAuthSchema):
 
     def get_link(self, path, method, base_url):
@@ -42,7 +53,6 @@ class AutoSchema(DRFAuthSchema):
         return link
 
     def get_description(self, path, method):
-        self.view._request = self.view.request
         setattr(self.view.request, 'swagger_method', method)
         description = super(AutoSchema, self).get_description(path, method)
         return description
@@ -59,7 +69,7 @@ class SwaggerSchemaView(APIView):
     ]
 
     def get(self, request):
-        generator = SchemaGenerator(
+        generator = SuperUserSchemaGenerator(
             title='Ansible Tower API',
             patterns=None,
             urlconf=None

awx/api/templates/api/_schedule_detail.md

@@ -5,7 +5,7 @@ The following lists the expected format and details of our rrules:
 * INTERVAL is required
 * SECONDLY is not supported
 * TZID is not supported
-* RRULE must preceed the rule statements
+* RRULE must precede the rule statements
 * BYDAY is supported but not BYDAY with a numerical prefix
 * BYYEARDAY and BYWEEKNO are not supported
 * Only one rrule statement per schedule is supported

awx/api/templates/api/api_o_auth_authorization_root_view.md

@@ -29,17 +29,6 @@ to the redirect_uri specified in the application. The client application will th
 AWX will respond with the `access_token`, `token_type`, `refresh_token`, and `expires_in`. For more
 information on testing this flow, refer to [django-oauth-toolkit](http://django-oauth-toolkit.readthedocs.io/en/latest/tutorial/tutorial_01.html#test-your-authorization-server).
 
-## Create Token for an Application using Implicit grant type
-Suppose we have an application "admin's app" of grant type `implicit`.
-In API browser, first make sure the user is logged in via session auth, then visit authorization
-endpoint with given parameters:
-```text
-http://localhost:8013/api/o/authorize/?response_type=token&client_id=L0uQQWW8pKX51hoqIRQGsuqmIdPi2AcXZ9EJRGmj&scope=read
-```
-Here the value of `client_id` should be the same as that of `client_id` field of underlying application.
-On success, an authorization page should be displayed asking the logged in user to grant/deny the access token.
-Once the user clicks on 'grant', the API browser will try POSTing to the same endpoint with the same parameters 
-in POST body, on success a 302 redirect will be returned.  
 
 ## Create Token for an Application using Password grant type
 
@@ -56,6 +45,7 @@ For example:
 
 ```bash
 curl -X POST \
+  -H "Content-Type: application/x-www-form-urlencoded" \
   -d "grant_type=password&username=<username>&password=<password>&scope=read" \
   -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569e
 IaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
@@ -85,6 +75,7 @@ format:
 The `/api/o/token/` endpoint is used for refreshing access token:
 ```bash
 curl -X POST \
+  -H "Content-Type: application/x-www-form-urlencoded" \
   -d "grant_type=refresh_token&refresh_token=AL0NK9TTpv0qp54dGbC4VUZtsZ9r8z" \
   -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569eIaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
   http://localhost:8013/api/o/token/ -i
@@ -114,6 +105,7 @@ Revoking is done by POSTing to `/api/o/revoke_token/` with the token to revoke a
 
 ```bash
 curl -X POST -d "token=rQONsve372fQwuc2pn76k3IHDCYpi7" \
+  -H "Content-Type: application/x-www-form-urlencoded" \
   -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569eIaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
   http://localhost:8013/api/o/revoke_token/ -i
 ```

awx/api/templates/api/inventory_source_cancel.md

@@ -1,7 +1,7 @@
 # Cancel Inventory Update
 
 Make a GET request to this resource to determine if the inventory update can be
-cancelled.  The response will include the following field:
+canceled.  The response will include the following field:
 
 * `can_cancel`: Indicates whether this update can be canceled (boolean,
   read-only)

awx/api/templates/api/job_cancel.md

@@ -1,7 +1,7 @@
 {% ifmeth GET %}
-# Determine if a Job can be cancelled
+# Determine if a Job can be canceled
 
-Make a GET request to this resource to determine if the job can be cancelled.
+Make a GET request to this resource to determine if the job can be canceled.
 The response will include the following field:
 
 * `can_cancel`: Indicates whether this job can be canceled (boolean, read-only)

awx/api/templates/api/job_template_callback.md

@@ -8,15 +8,15 @@ job template.
 
 For example, using curl:
 
-    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY"}'  http://server/api/v1/job_templates/N/callback/
+    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY"}'  http://server/api/v2/job_templates/N/callback/
 
 Or using wget:
 
-    wget -O /dev/null --post-data='{"host_config_key": "HOST_CONFIG_KEY"}' --header=Content-Type:application/json http://server/api/v1/job_templates/N/callback/
+    wget -O /dev/null --post-data='{"host_config_key": "HOST_CONFIG_KEY"}' --header=Content-Type:application/json http://server/api/v2/job_templates/N/callback/
 
 You may also pass `extra_vars` to the callback:
 
-    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY", "extra_vars": {"key": "value"}}'  http://server/api/v1/job_templates/N/callback/
+    curl -H "Content-Type: application/json" -d '{"host_config_key": "HOST_CONFIG_KEY", "extra_vars": {"key": "value"}}'  http://server/api/v2/job_templates/N/callback/
     
 The response will return status 202 if the request is valid, 403 for an
 invalid host config key, or 400 if the host cannot be determined from the
@@ -30,7 +30,7 @@ A GET request may be used to verify that the correct host will be selected.
 This request must authenticate as a valid user with permission to edit the
 job template.  For example:
 
-    curl http://user:password@server/api/v1/job_templates/N/callback/
+    curl http://user:password@server/api/v2/job_templates/N/callback/
 
 The response will include the host config key as well as the host name(s)
 that would match the request:

awx/api/templates/api/project_update_cancel.md

@@ -1,7 +1,7 @@
 # Cancel Project Update
 
 Make a GET request to this resource to determine if the project update can be
-cancelled.  The response will include the following field:
+canceled.  The response will include the following field:
 
 * `can_cancel`: Indicates whether this update can be canceled (boolean,
   read-only)

awx/api/templates/api/system_job_template_launch.md

@@ -3,7 +3,7 @@ Launch a Job Template:
 Make a POST request to this resource to launch the system job template.
 
 Variables specified inside of the parameter `extra_vars` are passed to the
-system job task as command line parameters. These tasks can be ran manually
+system job task as command line parameters. These tasks can be run manually
 on the host system via the `awx-manage` command.
 
 For example on `cleanup_jobs` and `cleanup_activitystream`:

awx/api/templates/api/team_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a Team:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected team.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/user_me_list.md

@@ -6,4 +6,4 @@ One result should be returned containing the following fields:
 
 {% include "api/_result_fields_common.md" %}
 
-Use the primary URL for the user (/api/v1/users/N/) to modify the user.
+Use the primary URL for the user (/api/v2/users/N/) to modify the user.

awx/api/templates/api/user_roles_list.md

@@ -1,6 +1,6 @@
+{% ifmeth GET %}
 # List Roles for a User:
 
-{% ifmeth GET %}
 Make a GET request to this resource to retrieve a list of roles associated with the selected user.
 
 {% include "api/_list_common.md" %}

awx/api/templates/api/webhook_key_view.md

@@ -0,0 +1,12 @@
+Webhook Secret Key:
+
+Make a GET request to this resource to obtain the secret key for a job
+template or workflow job template configured to be triggered by
+webhook events.  The response will include the following fields:
+
+* `webhook_key`: Secret key that needs to be copied and added to the
+  webhook configuration of the service this template will be receiving
+  webhook events from (string, read-only)
+
+Make an empty POST request to this resource to generate a new
+replacement `webhook_key`.

awx/api/urls/__init__.py

@@ -4,4 +4,7 @@
 from __future__ import absolute_import, unicode_literals
 from .urls import urlpatterns
 
-__all__ = ['urlpatterns']
+__all__ = ['urlpatterns', 'app_name']
+
+
+app_name = 'api'

awx/api/urls/credential.py

@@ -12,6 +12,8 @@ from awx.api.views import (
     CredentialOwnerUsersList,
     CredentialOwnerTeamsList,
     CredentialCopy,
+    CredentialInputSourceSubList,
+    CredentialExternalTest,
 )
 
 
@@ -24,6 +26,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/owner_users/$', CredentialOwnerUsersList.as_view(), name='credential_owner_users_list'),
     url(r'^(?P<pk>[0-9]+)/owner_teams/$', CredentialOwnerTeamsList.as_view(), name='credential_owner_teams_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', CredentialCopy.as_view(), name='credential_copy'),
+    url(r'^(?P<pk>[0-9]+)/input_sources/$', CredentialInputSourceSubList.as_view(), name='credential_input_source_sublist'),
+    url(r'^(?P<pk>[0-9]+)/test/$', CredentialExternalTest.as_view(), name='credential_external_test'),
 ]
 
 __all__ = ['urls']

awx/api/urls/credential_input_source.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2019 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    CredentialInputSourceDetail,
+    CredentialInputSourceList,
+)
+
+
+urls = [
+    url(r'^$', CredentialInputSourceList.as_view(), name='credential_input_source_list'),
+    url(r'^(?P<pk>[0-9]+)/$', CredentialInputSourceDetail.as_view(), name='credential_input_source_detail'),
+]
+
+__all__ = ['urls']

awx/api/urls/credential_type.py

@@ -8,6 +8,7 @@ from awx.api.views import (
     CredentialTypeDetail,
     CredentialTypeCredentialList,
     CredentialTypeActivityStreamList,
+    CredentialTypeExternalTest,
 )
 
 
@@ -16,6 +17,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/$', CredentialTypeDetail.as_view(), name='credential_type_detail'),
     url(r'^(?P<pk>[0-9]+)/credentials/$', CredentialTypeCredentialList.as_view(), name='credential_type_credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', CredentialTypeActivityStreamList.as_view(), name='credential_type_activity_stream_list'),
+    url(r'^(?P<pk>[0-9]+)/test/$', CredentialTypeExternalTest.as_view(), name='credential_type_external_test'),
 ]
 
 __all__ = ['urls']

awx/api/urls/host.py

@@ -16,8 +16,6 @@ from awx.api.views import (
     HostSmartInventoriesList,
     HostAdHocCommandsList,
     HostAdHocCommandEventsList,
-    HostFactVersionsList,
-    HostFactCompareView,
     HostInsights,
 )
 
@@ -35,8 +33,6 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/smart_inventories/$', HostSmartInventoriesList.as_view(), name='host_smart_inventories_list'),
     url(r'^(?P<pk>[0-9]+)/ad_hoc_commands/$', HostAdHocCommandsList.as_view(), name='host_ad_hoc_commands_list'),
     url(r'^(?P<pk>[0-9]+)/ad_hoc_command_events/$', HostAdHocCommandEventsList.as_view(), name='host_ad_hoc_command_events_list'),
-    url(r'^(?P<pk>[0-9]+)/fact_versions/$', HostFactVersionsList.as_view(), name='host_fact_versions_list'),
-    url(r'^(?P<pk>[0-9]+)/fact_view/$', HostFactCompareView.as_view(), name='host_fact_compare_view'),
     url(r'^(?P<pk>[0-9]+)/insights/$', HostInsights.as_view(), name='host_insights'),
 ]
 

awx/api/urls/inventory_source.py

@@ -13,8 +13,8 @@ from awx.api.views import (
     InventorySourceCredentialsList,
     InventorySourceGroupsList,
     InventorySourceHostsList,
-    InventorySourceNotificationTemplatesAnyList,
     InventorySourceNotificationTemplatesErrorList,
+    InventorySourceNotificationTemplatesStartedList,
     InventorySourceNotificationTemplatesSuccessList,
 )
 
@@ -29,8 +29,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', InventorySourceCredentialsList.as_view(), name='inventory_source_credentials_list'),
     url(r'^(?P<pk>[0-9]+)/groups/$', InventorySourceGroupsList.as_view(), name='inventory_source_groups_list'),
     url(r'^(?P<pk>[0-9]+)/hosts/$', InventorySourceHostsList.as_view(), name='inventory_source_hosts_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', InventorySourceNotificationTemplatesAnyList.as_view(),
-        name='inventory_source_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', InventorySourceNotificationTemplatesStartedList.as_view(),
+        name='inventory_source_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', InventorySourceNotificationTemplatesErrorList.as_view(),
         name='inventory_source_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', InventorySourceNotificationTemplatesSuccessList.as_view(),

awx/api/urls/job.py

@@ -6,7 +6,6 @@ from django.conf.urls import url
 from awx.api.views import (
     JobList,
     JobDetail,
-    JobStart,
     JobCancel,
     JobRelaunch,
     JobCreateSchedule,
@@ -23,7 +22,6 @@ from awx.api.views import (
 urls = [
     url(r'^$', JobList.as_view(), name='job_list'),
     url(r'^(?P<pk>[0-9]+)/$', JobDetail.as_view(), name='job_detail'),
-    url(r'^(?P<pk>[0-9]+)/start/$', JobStart.as_view(), name='job_start'),  # Todo: Remove In 3.3
     url(r'^(?P<pk>[0-9]+)/cancel/$', JobCancel.as_view(), name='job_cancel'),
     url(r'^(?P<pk>[0-9]+)/relaunch/$', JobRelaunch.as_view(), name='job_relaunch'),
     url(r'^(?P<pk>[0-9]+)/create_schedule/$', JobCreateSchedule.as_view(), name='job_create_schedule'),

awx/api/urls/job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     JobTemplateList,
@@ -13,8 +13,8 @@ from awx.api.views import (
     JobTemplateSchedulesList,
     JobTemplateSurveySpec,
     JobTemplateActivityStreamList,
-    JobTemplateNotificationTemplatesAnyList,
     JobTemplateNotificationTemplatesErrorList,
+    JobTemplateNotificationTemplatesStartedList,
     JobTemplateNotificationTemplatesSuccessList,
     JobTemplateInstanceGroupsList,
     JobTemplateAccessList,
@@ -34,8 +34,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/schedules/$', JobTemplateSchedulesList.as_view(), name='job_template_schedules_list'),
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', JobTemplateSurveySpec.as_view(), name='job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', JobTemplateActivityStreamList.as_view(), name='job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', JobTemplateNotificationTemplatesAnyList.as_view(),
-        name='job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', JobTemplateNotificationTemplatesStartedList.as_view(),
+        name='job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', JobTemplateNotificationTemplatesErrorList.as_view(),
         name='job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', JobTemplateNotificationTemplatesSuccessList.as_view(),
@@ -45,6 +45,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/organization.py

@@ -15,9 +15,10 @@ from awx.api.views import (
     OrganizationCredentialList,
     OrganizationActivityStreamList,
     OrganizationNotificationTemplatesList,
-    OrganizationNotificationTemplatesAnyList,
     OrganizationNotificationTemplatesErrorList,
+    OrganizationNotificationTemplatesStartedList,
     OrganizationNotificationTemplatesSuccessList,
+    OrganizationNotificationTemplatesApprovalList,
     OrganizationInstanceGroupsList,
     OrganizationObjectRolesList,
     OrganizationAccessList,
@@ -25,7 +26,7 @@ from awx.api.views import (
 )
 
 
-urls = [ 
+urls = [
     url(r'^$', OrganizationList.as_view(), name='organization_list'),
     url(r'^(?P<pk>[0-9]+)/$', OrganizationDetail.as_view(), name='organization_detail'),
     url(r'^(?P<pk>[0-9]+)/users/$', OrganizationUsersList.as_view(), name='organization_users_list'),
@@ -37,12 +38,14 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/credentials/$', OrganizationCredentialList.as_view(), name='organization_credential_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', OrganizationActivityStreamList.as_view(), name='organization_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates/$', OrganizationNotificationTemplatesList.as_view(), name='organization_notification_templates_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', OrganizationNotificationTemplatesAnyList.as_view(),
-        name='organization_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', OrganizationNotificationTemplatesStartedList.as_view(),
+        name='organization_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', OrganizationNotificationTemplatesErrorList.as_view(),
         name='organization_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', OrganizationNotificationTemplatesSuccessList.as_view(),
         name='organization_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', OrganizationNotificationTemplatesApprovalList.as_view(),
+        name='organization_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/instance_groups/$', OrganizationInstanceGroupsList.as_view(), name='organization_instance_groups_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', OrganizationObjectRolesList.as_view(), name='organization_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', OrganizationAccessList.as_view(), name='organization_access_list'),

awx/api/urls/project.py

@@ -14,8 +14,8 @@ from awx.api.views import (
     ProjectUpdatesList,
     ProjectActivityStreamList,
     ProjectSchedulesList,
-    ProjectNotificationTemplatesAnyList,
     ProjectNotificationTemplatesErrorList,
+    ProjectNotificationTemplatesStartedList,
     ProjectNotificationTemplatesSuccessList,
     ProjectObjectRolesList,
     ProjectAccessList,
@@ -34,10 +34,11 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/project_updates/$', ProjectUpdatesList.as_view(), name='project_updates_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', ProjectActivityStreamList.as_view(), name='project_activity_stream_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', ProjectSchedulesList.as_view(), name='project_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', ProjectNotificationTemplatesAnyList.as_view(), name='project_notification_templates_any_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', ProjectNotificationTemplatesErrorList.as_view(), name='project_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', ProjectNotificationTemplatesSuccessList.as_view(),
         name='project_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', ProjectNotificationTemplatesStartedList.as_view(),
+        name='project_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', ProjectObjectRolesList.as_view(), name='project_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', ProjectAccessList.as_view(), name='project_access_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', ProjectCopy.as_view(), name='project_copy'),

awx/api/urls/system_job_template.py

@@ -9,8 +9,8 @@ from awx.api.views import (
     SystemJobTemplateLaunch,
     SystemJobTemplateJobsList,
     SystemJobTemplateSchedulesList,
-    SystemJobTemplateNotificationTemplatesAnyList,
     SystemJobTemplateNotificationTemplatesErrorList,
+    SystemJobTemplateNotificationTemplatesStartedList,
     SystemJobTemplateNotificationTemplatesSuccessList,
 )
 
@@ -21,8 +21,8 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/launch/$', SystemJobTemplateLaunch.as_view(), name='system_job_template_launch'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', SystemJobTemplateJobsList.as_view(), name='system_job_template_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', SystemJobTemplateSchedulesList.as_view(), name='system_job_template_schedules_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', SystemJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='system_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', SystemJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='system_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', SystemJobTemplateNotificationTemplatesErrorList.as_view(),
         name='system_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', SystemJobTemplateNotificationTemplatesSuccessList.as_view(),

awx/api/urls/urls.py

@@ -11,10 +11,10 @@ from awx.api.generics import (
 )
 from awx.api.views import (
     ApiRootView,
-    ApiV1RootView,
     ApiV2RootView,
-    ApiV1PingView,
-    ApiV1ConfigView,
+    ApiV2PingView,
+    ApiV2ConfigView,
+    ApiV2SubscriptionView,
     AuthView,
     UserMeList,
     DashboardView,
@@ -34,6 +34,8 @@ from awx.api.views import (
     OAuth2ApplicationDetail,
 )
 
+from awx.api.views.metrics import MetricsView
+
 from .organization import urls as organization_urls
 from .user import urls as user_urls
 from .project import urls as project_urls
@@ -47,6 +49,7 @@ from .inventory_update import urls as inventory_update_urls
 from .inventory_script import urls as inventory_script_urls
 from .credential_type import urls as credential_type_urls
 from .credential import urls as credential_urls
+from .credential_input_source import urls as credential_input_source_urls
 from .role import urls as role_urls
 from .job_template import urls as job_template_urls
 from .job import urls as job_urls
@@ -69,12 +72,30 @@ from .instance import urls as instance_urls
 from .instance_group import urls as instance_group_urls
 from .oauth2 import urls as oauth2_urls
 from .oauth2_root import urls as oauth2_root_urls
+from .workflow_approval_template import urls as workflow_approval_template_urls
+from .workflow_approval import urls as workflow_approval_urls
 
 
-v1_urls = [
-    url(r'^$', ApiV1RootView.as_view(), name='api_v1_root_view'),
-    url(r'^ping/$', ApiV1PingView.as_view(), name='api_v1_ping_view'),
-    url(r'^config/$', ApiV1ConfigView.as_view(), name='api_v1_config_view'),
+v2_urls = [
+    url(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
+    url(r'^credential_types/', include(credential_type_urls)),
+    url(r'^credential_input_sources/', include(credential_input_source_urls)),
+    url(r'^hosts/(?P<pk>[0-9]+)/ansible_facts/$', HostAnsibleFactsDetail.as_view(), name='host_ansible_facts_detail'),
+    url(r'^jobs/(?P<pk>[0-9]+)/extra_credentials/$', JobExtraCredentialsList.as_view(), name='job_extra_credentials_list'),
+    url(r'^jobs/(?P<pk>[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'),
+    url(r'^job_templates/(?P<pk>[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'),
+    url(r'^job_templates/(?P<pk>[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'),
+    url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'),
+    url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'),
+    url(r'^applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
+    url(r'^applications/(?P<pk>[0-9]+)/$', OAuth2ApplicationDetail.as_view(), name='o_auth2_application_detail'),
+    url(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
+    url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
+    url(r'^', include(oauth2_urls)),
+    url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
+    url(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'),
+    url(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'),
+    url(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'),
     url(r'^auth/$', AuthView.as_view()),
     url(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     url(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),
@@ -114,30 +135,15 @@ v1_urls = [
     url(r'^unified_job_templates/$', UnifiedJobTemplateList.as_view(), name='unified_job_template_list'),
     url(r'^unified_jobs/$', UnifiedJobList.as_view(), name='unified_job_list'),
     url(r'^activity_stream/', include(activity_stream_urls)),
+    url(r'^workflow_approval_templates/', include(workflow_approval_template_urls)),
+    url(r'^workflow_approvals/', include(workflow_approval_urls)),
 ]
 
-v2_urls = [
-    url(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
-    url(r'^credential_types/', include(credential_type_urls)),
-    url(r'^hosts/(?P<pk>[0-9]+)/ansible_facts/$', HostAnsibleFactsDetail.as_view(), name='host_ansible_facts_detail'),
-    url(r'^jobs/(?P<pk>[0-9]+)/extra_credentials/$', JobExtraCredentialsList.as_view(), name='job_extra_credentials_list'),
-    url(r'^jobs/(?P<pk>[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'),
-    url(r'^job_templates/(?P<pk>[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'),
-    url(r'^job_templates/(?P<pk>[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'),
-    url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'),
-    url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'),
-    url(r'^applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
-    url(r'^applications/(?P<pk>[0-9]+)/$', OAuth2ApplicationDetail.as_view(), name='o_auth2_application_detail'),
-    url(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
-    url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
-    url(r'^', include(oauth2_urls)),
-]
 
 app_name = 'api'
 urlpatterns = [
     url(r'^$', ApiRootView.as_view(), name='api_root_view'),
     url(r'^(?P<version>(v2))/', include(v2_urls)),
-    url(r'^(?P<version>(v1|v2))/', include(v1_urls)),
     url(r'^login/$', LoggedLoginView.as_view(
         template_name='rest_framework/login.html',
         extra_context={'inside_login_context': True}

awx/api/urls/webhooks.py

@@ -0,0 +1,14 @@
+from django.conf.urls import url
+
+from awx.api.views import (
+    WebhookKeyView,
+    GithubWebhookReceiver,
+    GitlabWebhookReceiver,
+)
+
+
+urlpatterns = [
+    url(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
+    url(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
+    url(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
+]

awx/api/urls/workflow_approval.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalList,
+    WorkflowApprovalDetail,
+    WorkflowApprovalApprove,
+    WorkflowApprovalDeny,
+)
+
+
+urls = [
+    url(r'^$', WorkflowApprovalList.as_view(), name='workflow_approval_list'),
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalDetail.as_view(), name='workflow_approval_detail'),
+    url(r'^(?P<pk>[0-9]+)/approve/$', WorkflowApprovalApprove.as_view(), name='workflow_approval_approve'),
+    url(r'^(?P<pk>[0-9]+)/deny/$', WorkflowApprovalDeny.as_view(), name='workflow_approval_deny'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_approval_template.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.conf.urls import url
+
+from awx.api.views import (
+    WorkflowApprovalTemplateDetail,
+    WorkflowApprovalTemplateJobsList,
+)
+
+
+urls = [
+    url(r'^(?P<pk>[0-9]+)/$', WorkflowApprovalTemplateDetail.as_view(), name='workflow_approval_template_detail'),
+    url(r'^(?P<pk>[0-9]+)/approvals/$', WorkflowApprovalTemplateJobsList.as_view(), name='workflow_approval_template_jobs_list'),
+]
+
+__all__ = ['urls']

awx/api/urls/workflow_job_template.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2017 Ansible, Inc.
 # All Rights Reserved.
 
-from django.conf.urls import url
+from django.conf.urls import include, url
 
 from awx.api.views import (
     WorkflowJobTemplateList,
@@ -13,9 +13,10 @@ from awx.api.views import (
     WorkflowJobTemplateSurveySpec,
     WorkflowJobTemplateWorkflowNodesList,
     WorkflowJobTemplateActivityStreamList,
-    WorkflowJobTemplateNotificationTemplatesAnyList,
     WorkflowJobTemplateNotificationTemplatesErrorList,
+    WorkflowJobTemplateNotificationTemplatesStartedList,
     WorkflowJobTemplateNotificationTemplatesSuccessList,
+    WorkflowJobTemplateNotificationTemplatesApprovalList,
     WorkflowJobTemplateAccessList,
     WorkflowJobTemplateObjectRolesList,
     WorkflowJobTemplateLabelList,
@@ -32,15 +33,18 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', WorkflowJobTemplateSurveySpec.as_view(), name='workflow_job_template_survey_spec'),
     url(r'^(?P<pk>[0-9]+)/workflow_nodes/$', WorkflowJobTemplateWorkflowNodesList.as_view(), name='workflow_job_template_workflow_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$', WorkflowJobTemplateActivityStreamList.as_view(), name='workflow_job_template_activity_stream_list'),
-    url(r'^(?P<pk>[0-9]+)/notification_templates_any/$', WorkflowJobTemplateNotificationTemplatesAnyList.as_view(),
-        name='workflow_job_template_notification_templates_any_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_started/$', WorkflowJobTemplateNotificationTemplatesStartedList.as_view(),
+        name='workflow_job_template_notification_templates_started_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_error/$', WorkflowJobTemplateNotificationTemplatesErrorList.as_view(),
         name='workflow_job_template_notification_templates_error_list'),
     url(r'^(?P<pk>[0-9]+)/notification_templates_success/$', WorkflowJobTemplateNotificationTemplatesSuccessList.as_view(),
         name='workflow_job_template_notification_templates_success_list'),
+    url(r'^(?P<pk>[0-9]+)/notification_templates_approvals/$', WorkflowJobTemplateNotificationTemplatesApprovalList.as_view(),
+        name='workflow_job_template_notification_templates_approvals_list'),
     url(r'^(?P<pk>[0-9]+)/access_list/$', WorkflowJobTemplateAccessList.as_view(), name='workflow_job_template_access_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', WorkflowJobTemplateObjectRolesList.as_view(), name='workflow_job_template_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobTemplateLabelList.as_view(), name='workflow_job_template_label_list'),
+    url(r'^(?P<pk>[0-9]+)/', include('awx.api.urls.webhooks'), {'model_kwarg': 'workflow_job_templates'}),
 ]
 
 __all__ = ['urls']

awx/api/urls/workflow_job_template_node.py

@@ -10,6 +10,7 @@ from awx.api.views import (
     WorkflowJobTemplateNodeFailureNodesList,
     WorkflowJobTemplateNodeAlwaysNodesList,
     WorkflowJobTemplateNodeCredentialsList,
+    WorkflowJobTemplateNodeCreateApproval,
 )
 
 
@@ -20,6 +21,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/failure_nodes/$', WorkflowJobTemplateNodeFailureNodesList.as_view(), name='workflow_job_template_node_failure_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/always_nodes/$', WorkflowJobTemplateNodeAlwaysNodesList.as_view(), name='workflow_job_template_node_always_nodes_list'),
     url(r'^(?P<pk>[0-9]+)/credentials/$', WorkflowJobTemplateNodeCredentialsList.as_view(), name='workflow_job_template_node_credentials_list'),
+    url(r'^(?P<pk>[0-9]+)/create_approval_template/$', WorkflowJobTemplateNodeCreateApproval.as_view(), name='workflow_job_template_node_create_approval'),
 ]
 
 __all__ = ['urls']

awx/api/versioning.py

@@ -2,7 +2,7 @@
 # All Rights Reserved.
 
 from django.conf import settings
-from django.core.urlresolvers import NoReverseMatch
+from django.urls import NoReverseMatch
 
 from rest_framework.reverse import _reverse
 from rest_framework.versioning import URLPathVersioning as BaseVersioning
@@ -27,19 +27,6 @@ def drf_reverse(viewname, args=None, kwargs=None, request=None, format=None, **e
     return url
 
 
-def get_request_version(request):
-    """
-    The API version of a request as an integer i.e., 1 or 2
-    """
-    version = settings.REST_FRAMEWORK['DEFAULT_VERSION']
-    if request and hasattr(request, 'version'):
-        version = request.version
-        if version is None:
-            # For requests to /api/
-            return None
-    return int(version.lstrip('v'))
-
-
 def reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra):
     if request is None or getattr(request, 'version', None) is None:
         # We need the "current request" to determine the correct version to

awx/api/views/inventory.py

@@ -44,11 +44,9 @@ from awx.api.serializers import (
     InstanceGroupSerializer,
     InventoryUpdateEventSerializer,
     CustomInventoryScriptSerializer,
-    InventoryDetailSerializer,
     JobTemplateSerializer,
 )
 from awx.api.views.mixin import (
-    ActivityStreamEnforcementMixin,
     RelatedJobsPreventDeleteMixin,
     ControlledByScmMixin,
 )
@@ -62,7 +60,7 @@ class InventoryUpdateEventsList(SubListAPIView):
     serializer_class = InventoryUpdateEventSerializer
     parent_model = InventoryUpdate
     relationship = 'inventory_update_events'
-    view_name = _('Inventory Update Events List')
+    name = _('Inventory Update Events List')
     search_fields = ('stdout',)
 
     def finalize_response(self, request, response, *args, **kwargs):
@@ -72,12 +70,16 @@ class InventoryUpdateEventsList(SubListAPIView):
 
 class InventoryScriptList(ListCreateAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     serializer_class = CustomInventoryScriptSerializer
 
 
 class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     serializer_class = CustomInventoryScriptSerializer
 
@@ -94,6 +96,8 @@ class InventoryScriptDetail(RetrieveUpdateDestroyAPIView):
 
 class InventoryScriptObjectRolesList(SubListAPIView):
 
+    deprecated = True
+
     model = Role
     serializer_class = RoleSerializer
     parent_model = CustomInventoryScript
@@ -107,6 +111,8 @@ class InventoryScriptObjectRolesList(SubListAPIView):
 
 class InventoryScriptCopy(CopyAPIView):
 
+    deprecated = True
+
     model = CustomInventoryScript
     copy_return_serializer_class = CustomInventoryScriptSerializer
 
@@ -116,17 +122,11 @@ class InventoryList(ListCreateAPIView):
     model = Inventory
     serializer_class = InventorySerializer
 
-    def get_queryset(self):
-        qs = Inventory.accessible_objects(self.request.user, 'read_role')
-        qs = qs.select_related('admin_role', 'read_role', 'update_role', 'use_role', 'adhoc_role')
-        qs = qs.prefetch_related('created_by', 'modified_by', 'organization')
-        return qs
-
 
 class InventoryDetail(RelatedJobsPreventDeleteMixin, ControlledByScmMixin, RetrieveUpdateDestroyAPIView):
 
     model = Inventory
-    serializer_class = InventoryDetailSerializer
+    serializer_class = InventorySerializer
 
     def update(self, request, *args, **kwargs):
         obj = self.get_object()
@@ -149,7 +149,7 @@ class InventoryDetail(RelatedJobsPreventDeleteMixin, ControlledByScmMixin, Retri
             return Response(dict(error=_("{0}".format(e))), status=status.HTTP_400_BAD_REQUEST)
 
 
-class InventoryActivityStreamList(ActivityStreamEnforcementMixin, SubListAPIView):
+class InventoryActivityStreamList(SubListAPIView):
 
     model = ActivityStream
     serializer_class = ActivityStreamSerializer

awx/api/views/metrics.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+# Python
+import logging
+
+# Django
+from django.utils.translation import ugettext_lazy as _
+
+# Django REST Framework
+from rest_framework.response import Response
+from rest_framework.exceptions import PermissionDenied
+
+
+# AWX
+# from awx.main.analytics import collectors
+from awx.main.analytics.metrics import metrics
+from awx.api import renderers
+
+from awx.api.generics import (
+    APIView,
+)
+
+
+logger = logging.getLogger('awx.main.analytics')
+
+
+class MetricsView(APIView):
+
+    name = _('Metrics')
+    swagger_topic = 'Metrics'
+
+    renderer_classes = [renderers.PlainTextRenderer,
+                        renderers.PrometheusJSONRenderer,
+                        renderers.BrowsableAPIRenderer,]
+
+    def get(self, request):
+        ''' Show Metrics Details '''
+        if (request.user.is_superuser or request.user.is_system_auditor):
+            return Response(metrics().decode('UTF-8'))
+        raise PermissionDenied()

awx/api/views/mixin.py

@@ -31,48 +31,11 @@ from awx.main.models.organization import Team
 from awx.main.models.projects import Project
 from awx.main.models.inventory import Inventory
 from awx.main.models.jobs import JobTemplate
-from awx.conf.license import (
-    feature_enabled,
-    LicenseForbids,
-)
 from awx.api.exceptions import ActiveJobConflict
 
 logger = logging.getLogger('awx.api.views.mixin')
 
 
-class ActivityStreamEnforcementMixin(object):
-    '''
-    Mixin to check that license supports activity streams.
-    '''
-    def check_permissions(self, request):
-        ret = super(ActivityStreamEnforcementMixin, self).check_permissions(request)
-        if not feature_enabled('activity_streams'):
-            raise LicenseForbids(_('Your license does not allow use of the activity stream.'))
-        return ret
-
-
-class SystemTrackingEnforcementMixin(object):
-    '''
-    Mixin to check that license supports system tracking.
-    '''
-    def check_permissions(self, request):
-        ret = super(SystemTrackingEnforcementMixin, self).check_permissions(request)
-        if not feature_enabled('system_tracking'):
-            raise LicenseForbids(_('Your license does not permit use of system tracking.'))
-        return ret
-
-
-class WorkflowsEnforcementMixin(object):
-    '''
-    Mixin to check that license supports workflows.
-    '''
-    def check_permissions(self, request):
-        ret = super(WorkflowsEnforcementMixin, self).check_permissions(request)
-        if not feature_enabled('workflows') and request.method not in ('GET', 'OPTIONS', 'DELETE'):
-            raise LicenseForbids(_('Your license does not allow use of workflows.'))
-        return ret
-
-
 class UnifiedJobDeletionMixin(object):
     '''
     Special handling when deleting a running unified job object.
@@ -307,3 +270,11 @@ class ControlledByScmMixin(object):
         obj = super(ControlledByScmMixin, self).get_parent_object()
         self._reset_inv_src_rev(obj)
         return obj
+
+
+class NoTruncateMixin(object):
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.query_params.get('no_truncate'):
+            context.update(no_truncate=True)
+        return context

awx/api/views/organization.py

@@ -7,13 +7,8 @@ import logging
 # Django
 from django.db.models import Count
 from django.contrib.contenttypes.models import ContentType
-from django.utils.translation import ugettext_lazy as _
 
 # AWX
-from awx.conf.license import (
-    feature_enabled,
-    LicenseForbids,
-)
 from awx.main.models import (
     ActivityStream,
     Inventory,
@@ -50,7 +45,6 @@ from awx.api.serializers import (
     InstanceGroupSerializer,
 )
 from awx.api.views.mixin import (
-    ActivityStreamEnforcementMixin,
     RelatedJobsPreventDeleteMixin,
     OrganizationCountsMixin,
 )
@@ -69,24 +63,6 @@ class OrganizationList(OrganizationCountsMixin, ListCreateAPIView):
         qs = qs.prefetch_related('created_by', 'modified_by')
         return qs
 
-    def create(self, request, *args, **kwargs):
-        """Create a new organzation.
-
-        If there is already an organization and the license of this
-        instance does not permit multiple organizations, then raise
-        LicenseForbids.
-        """
-        # Sanity check: If the multiple organizations feature is disallowed
-        # by the license, then we are only willing to create this organization
-        # if no organizations exist in the system.
-        if (not feature_enabled('multiple_organizations') and
-                self.model.objects.exists()):
-            raise LicenseForbids(_('Your license only permits a single '
-                                   'organization to exist.'))
-
-        # Okay, create the organization as usual.
-        return super(OrganizationList, self).create(request, *args, **kwargs)
-
 
 class OrganizationDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
 
@@ -140,6 +116,7 @@ class OrganizationUsersList(BaseUsersList):
     serializer_class = UserSerializer
     parent_model = Organization
     relationship = 'member_role.members'
+    ordering = ('username',)
 
 
 class OrganizationAdminsList(BaseUsersList):
@@ -148,6 +125,7 @@ class OrganizationAdminsList(BaseUsersList):
     serializer_class = UserSerializer
     parent_model = Organization
     relationship = 'admin_role.members'
+    ordering = ('username',)
 
 
 class OrganizationProjectsList(SubListCreateAttachDetachAPIView):
@@ -177,7 +155,7 @@ class OrganizationTeamsList(SubListCreateAttachDetachAPIView):
     parent_key = 'organization'
 
 
-class OrganizationActivityStreamList(ActivityStreamEnforcementMixin, SubListAPIView):
+class OrganizationActivityStreamList(SubListAPIView):
 
     model = ActivityStream
     serializer_class = ActivityStreamSerializer
@@ -200,25 +178,28 @@ class OrganizationNotificationTemplatesAnyList(SubListCreateAttachDetachAPIView)
     model = NotificationTemplate
     serializer_class = NotificationTemplateSerializer
     parent_model = Organization
-    relationship = 'notification_templates_any'
 
 
-class OrganizationNotificationTemplatesErrorList(SubListCreateAttachDetachAPIView):
+class OrganizationNotificationTemplatesStartedList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_started'
+
+
+class OrganizationNotificationTemplatesErrorList(OrganizationNotificationTemplatesAnyList):
 
-    model = NotificationTemplate
-    serializer_class = NotificationTemplateSerializer
-    parent_model = Organization
     relationship = 'notification_templates_error'
 
 
-class OrganizationNotificationTemplatesSuccessList(SubListCreateAttachDetachAPIView):
+class OrganizationNotificationTemplatesSuccessList(OrganizationNotificationTemplatesAnyList):
 
-    model = NotificationTemplate
-    serializer_class = NotificationTemplateSerializer
-    parent_model = Organization
     relationship = 'notification_templates_success'
 
 
+class OrganizationNotificationTemplatesApprovalList(OrganizationNotificationTemplatesAnyList):
+
+    relationship = 'notification_templates_approvals'
+
+
 class OrganizationInstanceGroupsList(SubListAttachDetachAPIView):
 
     model = InstanceGroup
@@ -244,4 +225,3 @@ class OrganizationObjectRolesList(SubListAPIView):
         po = self.get_parent_object()
         content_type = ContentType.objects.get_for_model(self.parent_model)
         return Role.objects.filter(content_type=content_type, object_id=po.pk)
-

awx/api/views/root.py

@@ -17,6 +17,8 @@ from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 from rest_framework import status
 
+import requests
+
 from awx.api.generics import APIView
 from awx.main.ha import is_ha_environment
 from awx.main.utils import (
@@ -25,8 +27,9 @@ from awx.main.utils import (
     get_custom_venv_choices,
     to_python_boolean,
 )
-from awx.api.versioning import reverse, get_request_version, drf_reverse
-from awx.conf.license import get_license, feature_enabled
+from awx.api.versioning import reverse, drf_reverse
+from awx.conf.license import get_license
+from awx.main.constants import PRIVILEGE_ESCALATION_METHODS
 from awx.main.models import (
     Project,
     Organization,
@@ -41,7 +44,7 @@ logger = logging.getLogger('awx.api.views.root')
 class ApiRootView(APIView):
 
     permission_classes = (AllowAny,)
-    view_name = _('REST API')
+    name = _('REST API')
     versioning_class = None
     swagger_topic = 'Versioning'
 
@@ -49,23 +52,22 @@ class ApiRootView(APIView):
     def get(self, request, format=None):
         ''' List supported API versions '''
 
-        v1 = reverse('api:api_v1_root_view', kwargs={'version': 'v1'})
         v2 = reverse('api:api_v2_root_view', kwargs={'version': 'v2'})
         data = OrderedDict()
         data['description'] = _('AWX REST API')
         data['current_version'] = v2
-        data['available_versions'] = dict(v1 = v1, v2 = v2)
+        data['available_versions'] = dict(v2 = v2)
         data['oauth2'] = drf_reverse('api:oauth_authorization_root_view')
-        if feature_enabled('rebranding'):
-            data['custom_logo'] = settings.CUSTOM_LOGO
-            data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
+        data['custom_logo'] = settings.CUSTOM_LOGO
+        data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
+        data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
         return Response(data)
 
 
 class ApiOAuthAuthorizationRootView(APIView):
 
     permission_classes = (AllowAny,)
-    view_name = _("API OAuth 2 Authorization Root")
+    name = _("API OAuth 2 Authorization Root")
     versioning_class = None
     swagger_topic = 'Authentication'
 
@@ -85,10 +87,10 @@ class ApiVersionRootView(APIView):
     def get(self, request, format=None):
         ''' List top level resources '''
         data = OrderedDict()
-        data['ping'] = reverse('api:api_v1_ping_view', request=request)
+        data['ping'] = reverse('api:api_v2_ping_view', request=request)
         data['instances'] = reverse('api:instance_list', request=request)
         data['instance_groups'] = reverse('api:instance_group_list', request=request)
-        data['config'] = reverse('api:api_v1_config_view', request=request)
+        data['config'] = reverse('api:api_v2_config_view', request=request)
         data['settings'] = reverse('api:setting_category_list', request=request)
         data['me'] = reverse('api:user_me_list', request=request)
         data['dashboard'] = reverse('api:dashboard_view', request=request)
@@ -98,10 +100,11 @@ class ApiVersionRootView(APIView):
         data['project_updates'] = reverse('api:project_update_list', request=request)
         data['teams'] = reverse('api:team_list', request=request)
         data['credentials'] = reverse('api:credential_list', request=request)
-        if get_request_version(request) > 1:
-            data['credential_types'] = reverse('api:credential_type_list', request=request)
-            data['applications'] = reverse('api:o_auth2_application_list', request=request)
-            data['tokens'] = reverse('api:o_auth2_token_list', request=request)
+        data['credential_types'] = reverse('api:credential_type_list', request=request)
+        data['credential_input_sources'] = reverse('api:credential_input_source_list', request=request)
+        data['applications'] = reverse('api:o_auth2_application_list', request=request)
+        data['tokens'] = reverse('api:o_auth2_token_list', request=request)
+        data['metrics'] = reverse('api:metrics_view', request=request)
         data['inventory'] = reverse('api:inventory_list', request=request)
         data['inventory_scripts'] = reverse('api:inventory_script_list', request=request)
         data['inventory_sources'] = reverse('api:inventory_source_list', request=request)
@@ -124,26 +127,23 @@ class ApiVersionRootView(APIView):
         data['activity_stream'] = reverse('api:activity_stream_list', request=request)
         data['workflow_job_templates'] = reverse('api:workflow_job_template_list', request=request)
         data['workflow_jobs'] = reverse('api:workflow_job_list', request=request)
+        data['workflow_approvals'] = reverse('api:workflow_approval_list', request=request)
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
         return Response(data)
 
 
-class ApiV1RootView(ApiVersionRootView):
-    view_name = _('Version 1')
-
-
 class ApiV2RootView(ApiVersionRootView):
-    view_name = _('Version 2')
+    name = _('Version 2')
 
 
-class ApiV1PingView(APIView):
+class ApiV2PingView(APIView):
     """A simple view that reports very basic information about this
     instance, which is acceptable to be public information.
     """
     permission_classes = (AllowAny,)
     authentication_classes = ()
-    view_name = _('Ping')
+    name = _('Ping')
     swagger_topic = 'System Configuration'
 
     def get(self, request, format=None):
@@ -156,29 +156,69 @@ class ApiV1PingView(APIView):
             'ha': is_ha_environment(),
             'version': get_awx_version(),
             'active_node': settings.CLUSTER_HOST_ID,
+            'install_uuid': settings.INSTALL_UUID,
         }
 
         response['instances'] = []
         for instance in Instance.objects.all():
-            response['instances'].append(dict(node=instance.hostname, heartbeat=instance.modified,
+            response['instances'].append(dict(node=instance.hostname, uuid=instance.uuid, heartbeat=instance.modified,
                                               capacity=instance.capacity, version=instance.version))
             sorted(response['instances'], key=operator.itemgetter('node'))
         response['instance_groups'] = []
-        for instance_group in InstanceGroup.objects.all():
+        for instance_group in InstanceGroup.objects.prefetch_related('instances'):
             response['instance_groups'].append(dict(name=instance_group.name,
                                                     capacity=instance_group.capacity,
                                                     instances=[x.hostname for x in instance_group.instances.all()]))
         return Response(response)
 
 
-class ApiV1ConfigView(APIView):
+class ApiV2SubscriptionView(APIView):
 
     permission_classes = (IsAuthenticated,)
-    view_name = _('Configuration')
+    name = _('Configuration')
     swagger_topic = 'System Configuration'
 
     def check_permissions(self, request):
-        super(ApiV1ConfigView, self).check_permissions(request)
+        super(ApiV2SubscriptionView, self).check_permissions(request)
+        if not request.user.is_superuser and request.method.lower() not in {'options', 'head'}:
+            self.permission_denied(request)  # Raises PermissionDenied exception.
+
+    def post(self, request):
+        from awx.main.utils.common import get_licenser
+        data = request.data.copy()
+        if data.get('rh_password') == '$encrypted$':
+            data['rh_password'] = settings.REDHAT_PASSWORD
+        try:
+            user, pw = data.get('rh_username'), data.get('rh_password')
+            validated = get_licenser().validate_rh(user, pw)
+            if user:
+                settings.REDHAT_USERNAME = data['rh_username']
+            if pw:
+                settings.REDHAT_PASSWORD = data['rh_password']
+        except Exception as exc:
+            msg = _("Invalid License")
+            if (
+                isinstance(exc, requests.exceptions.HTTPError) and
+                getattr(getattr(exc, 'response', None), 'status_code', None) == 401
+            ):
+                msg = _("The provided credentials are invalid (HTTP 401).")
+            if isinstance(exc, (ValueError, OSError)) and exc.args:
+                msg = exc.args[0]
+            logger.exception(smart_text(u"Invalid license submitted."),
+                             extra=dict(actor=request.user.username))
+            return Response({"error": msg}, status=status.HTTP_400_BAD_REQUEST)
+
+        return Response(validated)
+
+
+class ApiV2ConfigView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    name = _('Configuration')
+    swagger_topic = 'System Configuration'
+
+    def check_permissions(self, request):
+        super(ApiV2ConfigView, self).check_permissions(request)
         if not request.user.is_superuser and request.method.lower() not in {'options', 'head', 'get'}:
             self.permission_denied(request)  # Raises PermissionDenied exception.
 
@@ -203,13 +243,14 @@ class ApiV1ConfigView(APIView):
             version=get_awx_version(),
             ansible_version=get_ansible_version(),
             eula=render_to_string("eula.md") if license_data.get('license_type', 'UNLICENSED') != 'open' else '',
-            analytics_status=pendo_state
+            analytics_status=pendo_state,
+            become_methods=PRIVILEGE_ESCALATION_METHODS,
         )
 
         # If LDAP is enabled, user_ldap_fields will return a list of field
         # names that are managed by LDAP and should be read-only for users with
         # a non-empty ldap_dn attribute.
-        if getattr(settings, 'AUTH_LDAP_SERVER_URI', None) and feature_enabled('ldap'):
+        if getattr(settings, 'AUTH_LDAP_SERVER_URI', None):
             user_ldap_fields = ['username', 'password']
             user_ldap_fields.extend(getattr(settings, 'AUTH_LDAP_USER_ATTR_MAP', {}).keys())
             user_ldap_fields.extend(getattr(settings, 'AUTH_LDAP_USER_FLAGS_BY_GROUP', {}).keys())
@@ -218,7 +259,8 @@ class ApiV1ConfigView(APIView):
         if request.user.is_superuser \
                 or request.user.is_system_auditor \
                 or Organization.accessible_objects(request.user, 'admin_role').exists() \
-                or Organization.accessible_objects(request.user, 'auditor_role').exists():
+                or Organization.accessible_objects(request.user, 'auditor_role').exists() \
+                or Organization.accessible_objects(request.user, 'project_admin_role').exists():
             data.update(dict(
                 project_base_dir = settings.PROJECTS_ROOT,
                 project_local_paths = Project.get_local_path_choices(),
@@ -274,6 +316,3 @@ class ApiV1ConfigView(APIView):
         except Exception:
             # FIX: Log
             return Response({"error": _("Failed to remove license.")}, status=status.HTTP_400_BAD_REQUEST)
-
-
-

awx/api/views/webhooks.py

@@ -0,0 +1,246 @@
+from hashlib import sha1
+import hmac
+import logging
+import urllib.parse
+
+from django.utils.encoding import force_bytes
+from django.utils.translation import ugettext_lazy as _
+from django.views.decorators.csrf import csrf_exempt
+
+from rest_framework import status
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
+
+from awx.api import serializers
+from awx.api.generics import APIView, GenericAPIView
+from awx.api.permissions import WebhookKeyPermission
+from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
+
+
+logger = logging.getLogger('awx.api.views.webhooks')
+
+
+class WebhookKeyView(GenericAPIView):
+    serializer_class = serializers.EmptySerializer
+    permission_classes = (WebhookKeyPermission,)
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        self.model = qs_models.get(self.kwargs['model_kwarg'])
+
+        return super().get_queryset()
+
+    def get(self, request, *args, **kwargs):
+        obj = self.get_object()
+
+        return Response({'webhook_key': obj.webhook_key})
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+        obj.rotate_webhook_key()
+        obj.save(update_fields=['webhook_key'])
+
+        return Response({'webhook_key': obj.webhook_key}, status=status.HTTP_201_CREATED)
+
+
+class WebhookReceiverBase(APIView):
+    lookup_url_kwarg = None
+    lookup_field = 'pk'
+
+    permission_classes = (AllowAny,)
+    authentication_classes = ()
+
+    ref_keys = {}
+
+    def get_queryset(self):
+        qs_models = {
+            'job_templates': JobTemplate,
+            'workflow_job_templates': WorkflowJobTemplate,
+        }
+        model = qs_models.get(self.kwargs['model_kwarg'])
+        if model is None:
+            raise PermissionDenied
+
+        return model.objects.filter(webhook_service=self.service).exclude(webhook_key='')
+
+    def get_object(self):
+        queryset = self.get_queryset()
+        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
+        filter_kwargs = {self.lookup_field: self.kwargs[lookup_url_kwarg]}
+
+        obj = queryset.filter(**filter_kwargs).first()
+        if obj is None:
+            raise PermissionDenied
+
+        return obj
+
+    def get_event_type(self):
+        raise NotImplementedError
+
+    def get_event_guid(self):
+        raise NotImplementedError
+
+    def get_event_status_api(self):
+        raise NotImplementedError
+
+    def get_event_ref(self):
+        key = self.ref_keys.get(self.get_event_type(), '')
+        value = self.request.data
+        for element in key.split('.'):
+            try:
+                if element.isdigit():
+                    value = value[int(element)]
+                else:
+                    value = (value or {}).get(element)
+            except Exception:
+                value = None
+        if value == '0000000000000000000000000000000000000000':  # a deleted ref
+            value = None
+        return value
+
+    def get_signature(self):
+        raise NotImplementedError
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        logger.debug("header signature: %s", self.get_signature())
+        logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
+            raise PermissionDenied
+
+    @csrf_exempt
+    def post(self, request, *args, **kwargs):
+        # Ensure that the full contents of the request are captured for multiple uses.
+        request.body
+
+        logger.debug(
+            "headers: {}\n"
+            "data: {}\n".format(request.headers, request.data)
+        )
+        obj = self.get_object()
+        self.check_signature(obj)
+
+        event_type = self.get_event_type()
+        event_guid = self.get_event_guid()
+        event_ref = self.get_event_ref()
+        status_api = self.get_event_status_api()
+
+        kwargs = {
+            'unified_job_template_id': obj.id,
+            'webhook_service': obj.webhook_service,
+            'webhook_guid': event_guid,
+        }
+        if WorkflowJob.objects.filter(**kwargs).exists() or Job.objects.filter(**kwargs).exists():
+            # Short circuit if this webhook has already been received and acted upon.
+            logger.debug("Webhook previously received, returning without action.")
+            return Response({'message': _("Webhook previously received, aborting.")},
+                            status=status.HTTP_202_ACCEPTED)
+
+        kwargs = {
+            '_eager_fields': {
+                'launch_type': 'webhook',
+                'webhook_service': obj.webhook_service,
+                'webhook_credential': obj.webhook_credential,
+                'webhook_guid': event_guid,
+            },
+            'extra_vars': {
+                'tower_webhook_event_type': event_type,
+                'tower_webhook_event_guid': event_guid,
+                'tower_webhook_event_ref': event_ref,
+                'tower_webhook_status_api': status_api,
+                'tower_webhook_payload': request.data,
+            }
+        }
+
+        new_job = obj.create_unified_job(**kwargs)
+        new_job.signal_start()
+
+        return Response({'message': "Job queued."}, status=status.HTTP_202_ACCEPTED)
+
+
+class GithubWebhookReceiver(WebhookReceiverBase):
+    service = 'github'
+
+    ref_keys = {
+        'pull_request': 'pull_request.head.sha',
+        'pull_request_review': 'pull_request.head.sha',
+        'pull_request_review_comment': 'pull_request.head.sha',
+        'push': 'after',
+        'release': 'release.tag_name',
+        'commit_comment': 'comment.commit_id',
+        'create': 'ref',
+        'page_build': 'build.commit',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITHUB_EVENT')
+
+    def get_event_guid(self):
+        return self.request.META.get('HTTP_X_GITHUB_DELIVERY')
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'pull_request':
+            return
+        return self.request.data.get('pull_request', {}).get('statuses_url')
+
+    def get_signature(self):
+        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
+        if not header_sig:
+            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
+            raise PermissionDenied
+        hash_alg, signature = header_sig.split('=')
+        if hash_alg != 'sha1':
+            logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
+            raise PermissionDenied
+        return force_bytes(signature)
+
+
+class GitlabWebhookReceiver(WebhookReceiverBase):
+    service = 'gitlab'
+
+    ref_keys = {
+        'Push Hook': 'checkout_sha',
+        'Tag Push Hook': 'checkout_sha',
+        'Merge Request Hook': 'object_attributes.last_commit.id',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_GITLAB_EVENT')
+
+    def get_event_guid(self):
+        # GitLab does not provide a unique identifier on events, so construct one.
+        h = sha1()
+        h.update(force_bytes(self.request.body))
+        return h.hexdigest()
+
+    def get_event_status_api(self):
+        if self.get_event_type() != 'Merge Request Hook':
+            return
+        project = self.request.data.get('project', {})
+        repo_url = project.get('web_url')
+        if not repo_url:
+            return
+        parsed = urllib.parse.urlparse(repo_url)
+
+        return "{}://{}/api/v4/projects/{}/statuses/{}".format(
+            parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
+
+    def get_signature(self):
+        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
+    def check_signature(self, obj):
+        if not obj.webhook_key:
+            raise PermissionDenied
+
+        # GitLab only returns the secret token, not an hmac hash.  Use
+        # the hmac `compare_digest` helper function to prevent timing
+        # analysis by attackers.
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
+            raise PermissionDenied

awx/conf/conf.py

@@ -78,9 +78,6 @@ register(
     # the other settings change, the cached value for this setting will be
     # cleared to require it to be recomputed.
     depends_on=['ANSIBLE_COW_SELECTION'],
-    # Optional; licensed feature required to be able to view or modify this
-    # setting.
-    feature_required='rebranding',
     # Optional; field is stored encrypted in the database and only $encrypted$
     # is returned via the API.
     encrypted=True,

awx/conf/fields.py

@@ -1,16 +1,19 @@
 # Python
+import os
+import re
 import logging
 import urllib.parse as urlparse
 from collections import OrderedDict
 
 # Django
-from django.core.validators import URLValidator
+from django.core.validators import URLValidator, _lazy_re_compile
 from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
-from rest_framework.fields import *  # noqa
-
-import six
+from rest_framework.fields import (  # noqa
+    BooleanField, CharField, ChoiceField, DictField, EmailField,
+    IntegerField, ListField, NullBooleanField
+)
 
 logger = logging.getLogger('awx.conf.fields')
 
@@ -95,15 +98,63 @@ class StringListBooleanField(ListField):
         self.fail('type_error', input_type=type(data))
 
 
+class StringListPathField(StringListField):
+
+    default_error_messages = {
+        'type_error': _('Expected list of strings but got {input_type} instead.'),
+        'path_error': _('{path} is not a valid path choice.'),
+    }
+
+    def to_internal_value(self, paths):
+        if isinstance(paths, (list, tuple)):
+            for p in paths:
+                if not isinstance(p, str):
+                    self.fail('type_error', input_type=type(p))
+                if not os.path.exists(p):
+                    self.fail('path_error', path=p)
+
+            return super(StringListPathField, self).to_internal_value(sorted({os.path.normpath(path) for path in paths}))
+        else:
+            self.fail('type_error', input_type=type(paths))
+
+
 class URLField(CharField):
+    # these lines set up a custom regex that allow numbers in the
+    # top-level domain
+    tld_re = (
+        r'\.'                                              # dot
+        r'(?!-)'                                           # can't start with a dash
+        r'(?:[a-z' + URLValidator.ul + r'0-9' + '-]{2,63}' # domain label, this line was changed from the original URLValidator
+        r'|xn--[a-z0-9]{1,59})'                            # or punycode label
+        r'(?<!-)'                                          # can't end with a dash
+        r'\.?'                                             # may have a trailing dot
+    )
+
+    host_re = '(' + URLValidator.hostname_re + URLValidator.domain_re + tld_re + '|localhost)'
+
+    regex = _lazy_re_compile(
+        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
+        r'(?:[^\s:@/]+(?::[^\s:@/]*)?@)?'  # user:pass authentication
+        r'(?:' + URLValidator.ipv4_re + '|' + URLValidator.ipv6_re + '|' + host_re + ')'
+        r'(?::\d{2,5})?'  # port
+        r'(?:[/?#][^\s]*)?'  # resource path
+        r'\Z', re.IGNORECASE)
 
     def __init__(self, **kwargs):
         schemes = kwargs.pop('schemes', None)
+        regex = kwargs.pop('regex', None)
         self.allow_plain_hostname = kwargs.pop('allow_plain_hostname', False)
+        self.allow_numbers_in_top_level_domain = kwargs.pop('allow_numbers_in_top_level_domain', True)
         super(URLField, self).__init__(**kwargs)
         validator_kwargs = dict(message=_('Enter a valid URL'))
         if schemes is not None:
             validator_kwargs['schemes'] = schemes
+        if regex is not None:
+            validator_kwargs['regex'] = regex
+        if self.allow_numbers_in_top_level_domain and regex is None:
+            # default behavior is to allow numbers in the top level domain
+            # if a custom regex isn't provided
+            validator_kwargs['regex'] = URLField.regex
         self.validators.append(URLValidator(**validator_kwargs))
 
     def to_representation(self, value):
@@ -139,7 +190,7 @@ class KeyValueField(DictField):
     def to_internal_value(self, data):
         ret = super(KeyValueField, self).to_internal_value(data)
         for value in data.values():
-            if not isinstance(value, six.string_types + six.integer_types + (float,)):
+            if not isinstance(value, (str, int, float)):
                 if isinstance(value, OrderedDict):
                     value = dict(value)
                 self.fail('invalid_child', input=value)

awx/conf/license.py

@@ -1,64 +1,18 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
-# Django
-from django.core.signals import setting_changed
-from django.dispatch import receiver
-from django.utils.translation import ugettext_lazy as _
 
-# Django REST Framework
-from rest_framework.exceptions import APIException
-
-# Tower
-from awx.main.utils.common import get_licenser
-from awx.main.utils import memoize, memoize_delete
-
-__all__ = ['LicenseForbids', 'get_license', 'get_licensed_features',
-           'feature_enabled', 'feature_exists']
-
-
-class LicenseForbids(APIException):
-    status_code = 402
-    default_detail = _('Your Tower license does not allow that.')
+__all__ = ['get_license']
 
 
 def _get_validated_license_data():
+    from awx.main.utils.common import get_licenser
     return get_licenser().validate()
 
 
-@receiver(setting_changed)
-def _on_setting_changed(sender, **kwargs):
-    # Clear cached result above when license changes.
-    if kwargs.get('setting', None) == 'LICENSE':
-        memoize_delete('feature_enabled')
-
-
 def get_license(show_key=False):
     """Return a dictionary representing the active license on this Tower instance."""
     license_data = _get_validated_license_data()
     if not show_key:
         license_data.pop('license_key', None)
     return license_data
-
-
-def get_licensed_features():
-    """Return a set of all features enabled by the active license."""
-    features = set()
-    for feature, enabled in _get_validated_license_data().get('features', {}).items():
-        if enabled:
-            features.add(feature)
-    return features
-
-
-@memoize(track_function=True)
-def feature_enabled(name):
-    """Return True if the requested feature is enabled, False otherwise."""
-    validated_license_data = _get_validated_license_data()
-    if validated_license_data.get('license_type', 'UNLICENSED') == 'open':
-        return True
-    return validated_license_data.get('features', {}).get(name, False)
-
-
-def feature_exists(name):
-    """Return True if the requested feature name exists, False otherwise."""
-    return bool(name in _get_validated_license_data().get('features', {}))

awx/conf/management/commands/migrate_to_database_settings.py

@@ -1,480 +0,0 @@
-# Copyright (c) 2016 Ansible, Inc.
-# All Rights Reserved.
-
-# Python
-import base64
-import collections
-import difflib
-import json
-import os
-import shutil
-
-# Django
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-from django.utils.text import slugify
-from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
-
-# Tower
-from awx import MODE
-from awx.conf import settings_registry
-from awx.conf.fields import empty, SkipField
-from awx.conf.models import Setting
-from awx.conf.utils import comment_assignments
-
-
-class Command(BaseCommand):
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            'category',
-            nargs='*',
-            type=str,
-        )
-        parser.add_argument(
-            '--dry-run',
-            action='store_true',
-            dest='dry_run',
-            default=False,
-            help=_('Only show which settings would be commented/migrated.'),
-        )
-        parser.add_argument(
-            '--skip-errors',
-            action='store_true',
-            dest='skip_errors',
-            default=False,
-            help=_('Skip over settings that would raise an error when commenting/migrating.'),
-        )
-        parser.add_argument(
-            '--no-comment',
-            action='store_true',
-            dest='no_comment',
-            default=False,
-            help=_('Skip commenting out settings in files.'),
-        )
-        parser.add_argument(
-            '--comment-only',
-            action='store_true',
-            dest='comment_only',
-            default=False,
-            help=_('Skip migrating and only comment out settings in files.'),
-        )
-        parser.add_argument(
-            '--backup-suffix',
-            dest='backup_suffix',
-            default=now().strftime('.%Y%m%d%H%M%S'),
-            help=_('Backup existing settings files with this suffix.'),
-        )
-
-    @transaction.atomic
-    def handle(self, *args, **options):
-        self.verbosity = int(options.get('verbosity', 1))
-        self.dry_run = bool(options.get('dry_run', False))
-        self.skip_errors = bool(options.get('skip_errors', False))
-        self.no_comment = bool(options.get('no_comment', False))
-        self.comment_only = bool(options.get('comment_only', False))
-        self.backup_suffix = options.get('backup_suffix', '')
-        self.categories = options.get('category', None) or ['all']
-        self.style.HEADING = self.style.MIGRATE_HEADING
-        self.style.LABEL = self.style.MIGRATE_LABEL
-        self.style.OK = self.style.SQL_FIELD
-        self.style.SKIP = self.style.WARNING
-        self.style.VALUE = self.style.SQL_KEYWORD
-
-        # Determine if any categories provided are invalid.
-        category_slugs = []
-        invalid_categories = []
-        for category in self.categories:
-            category_slug = slugify(category)
-            if category_slug in settings_registry.get_registered_categories():
-                if category_slug not in category_slugs:
-                    category_slugs.append(category_slug)
-            else:
-                if category not in invalid_categories:
-                    invalid_categories.append(category)
-        if len(invalid_categories) == 1:
-            raise CommandError('Invalid setting category: {}'.format(invalid_categories[0]))
-        elif len(invalid_categories) > 1:
-            raise CommandError('Invalid setting categories: {}'.format(', '.join(invalid_categories)))
-
-        # Build a list of all settings to be migrated.
-        registered_settings = []
-        for category_slug in category_slugs:
-            for registered_setting in settings_registry.get_registered_settings(category_slug=category_slug, read_only=False):
-                if registered_setting not in registered_settings:
-                    registered_settings.append(registered_setting)
-
-        self._migrate_settings(registered_settings)
-
-    def _get_settings_file_patterns(self):
-        if MODE == 'development':
-            return [
-                '/etc/tower/settings.py',
-                '/etc/tower/conf.d/*.py',
-                os.path.join(os.path.dirname(__file__), '..', '..', '..', 'settings', 'local_*.py')
-            ]
-        else:
-            return [
-                os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py'),
-                os.path.join(os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/'), '*.py'),
-            ]
-
-    def _get_license_file(self):
-        return os.environ.get('AWX_LICENSE_FILE', '/etc/tower/license')
-
-    def _comment_license_file(self, dry_run=True):
-        license_file = self._get_license_file()
-        diff_lines = []
-        if os.path.exists(license_file):
-            try:
-                raw_license_data = open(license_file).read()
-                json.loads(raw_license_data)
-            except Exception as e:
-                raise CommandError('Error reading license from {0}: {1!r}'.format(license_file, e))
-            if self.backup_suffix:
-                backup_license_file = '{}{}'.format(license_file, self.backup_suffix)
-            else:
-                backup_license_file = '{}.old'.format(license_file)
-            diff_lines = list(difflib.unified_diff(
-                raw_license_data.splitlines(),
-                [],
-                fromfile=backup_license_file,
-                tofile=license_file,
-                lineterm='',
-            ))
-            if not dry_run:
-                if self.backup_suffix:
-                    shutil.copy2(license_file, backup_license_file)
-                os.remove(license_file)
-        return diff_lines
-
-    def _get_local_settings_file(self):
-        if MODE == 'development':
-            static_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'ui', 'static')
-        else:
-            static_root = settings.STATIC_ROOT
-        return os.path.join(static_root, 'local_settings.json')
-
-    def _comment_local_settings_file(self, dry_run=True):
-        local_settings_file = self._get_local_settings_file()
-        diff_lines = []
-        if os.path.exists(local_settings_file):
-            try:
-                raw_local_settings_data = open(local_settings_file).read()
-                json.loads(raw_local_settings_data)
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading local settings from {0}: {1!r}'.format(local_settings_file, e))
-                return diff_lines
-            if self.backup_suffix:
-                backup_local_settings_file = '{}{}'.format(local_settings_file, self.backup_suffix)
-            else:
-                backup_local_settings_file = '{}.old'.format(local_settings_file)
-            diff_lines = list(difflib.unified_diff(
-                raw_local_settings_data.splitlines(),
-                [],
-                fromfile=backup_local_settings_file,
-                tofile=local_settings_file,
-                lineterm='',
-            ))
-            if not dry_run:
-                if self.backup_suffix:
-                    shutil.copy2(local_settings_file, backup_local_settings_file)
-                os.remove(local_settings_file)
-        return diff_lines
-
-    def _get_custom_logo_file(self):
-        if MODE == 'development':
-            static_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'ui', 'static')
-        else:
-            static_root = settings.STATIC_ROOT
-        return os.path.join(static_root, 'assets', 'custom_console_logo.png')
-
-    def _comment_custom_logo_file(self, dry_run=True):
-        custom_logo_file = self._get_custom_logo_file()
-        diff_lines = []
-        if os.path.exists(custom_logo_file):
-            try:
-                raw_custom_logo_data = open(custom_logo_file).read()
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading custom logo from {0}: {1!r}'.format(custom_logo_file, e))
-                return diff_lines
-            if self.backup_suffix:
-                backup_custom_logo_file = '{}{}'.format(custom_logo_file, self.backup_suffix)
-            else:
-                backup_custom_logo_file = '{}.old'.format(custom_logo_file)
-            diff_lines = list(difflib.unified_diff(
-                ['<PNG Image ({} bytes)>'.format(len(raw_custom_logo_data))],
-                [],
-                fromfile=backup_custom_logo_file,
-                tofile=custom_logo_file,
-                lineterm='',
-            ))
-            if not dry_run:
-                if self.backup_suffix:
-                    shutil.copy2(custom_logo_file, backup_custom_logo_file)
-                os.remove(custom_logo_file)
-        return diff_lines
-
-    def _check_if_needs_comment(self, patterns, setting):
-        files_to_comment = []
-        # If any diffs are returned, this setting needs to be commented.
-        diffs = comment_assignments(patterns, setting, dry_run=True)
-        if setting == 'LICENSE':
-            diffs.extend(self._comment_license_file(dry_run=True))
-        elif setting == 'CUSTOM_LOGIN_INFO':
-            diffs.extend(self._comment_local_settings_file(dry_run=True))
-        elif setting == 'CUSTOM_LOGO':
-            diffs.extend(self._comment_custom_logo_file(dry_run=True))
-        for diff in diffs:
-            for line in diff.splitlines():
-                if line.startswith('+++ '):
-                    files_to_comment.append(line[4:])
-        return files_to_comment
-
-    def _check_if_needs_migration(self, setting):
-        # Check whether the current value differs from the default.
-        default_value = settings.DEFAULTS_SNAPSHOT.get(setting, empty)
-        if default_value is empty and setting != 'LICENSE':
-            field = settings_registry.get_setting_field(setting, read_only=True)
-            try:
-                default_value = field.get_default()
-            except SkipField:
-                pass
-        current_value = getattr(settings, setting, empty)
-        if setting == 'CUSTOM_LOGIN_INFO' and current_value in {empty, ''}:
-            local_settings_file = self._get_local_settings_file()
-            try:
-                if os.path.exists(local_settings_file):
-                    local_settings = json.load(open(local_settings_file))
-                    current_value = local_settings.get('custom_login_info', '')
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading custom login info from {0}: {1!r}'.format(local_settings_file, e))
-        if setting == 'CUSTOM_LOGO' and current_value in {empty, ''}:
-            custom_logo_file = self._get_custom_logo_file()
-            try:
-                if os.path.exists(custom_logo_file):
-                    custom_logo_data = open(custom_logo_file).read()
-                    if custom_logo_data:
-                        current_value = 'data:image/png;base64,{}'.format(base64.b64encode(custom_logo_data))
-                    else:
-                        current_value = ''
-            except Exception as e:
-                if not self.skip_errors:
-                    raise CommandError('Error reading custom logo from {0}: {1!r}'.format(custom_logo_file, e))
-        if current_value != default_value:
-            if current_value is empty:
-                current_value = None
-            return current_value
-        return empty
-
-    def _display_tbd(self, setting, files_to_comment, migrate_value, comment_error=None, migrate_error=None):
-        if self.verbosity >= 1:
-            if files_to_comment:
-                if migrate_value is not empty:
-                    action = 'Migrate + Comment'
-                else:
-                    action = 'Comment'
-                if comment_error or migrate_error:
-                    action = self.style.ERROR('{} (skipped)'.format(action))
-                else:
-                    action = self.style.OK(action)
-                self.stdout.write('  {}: {}'.format(
-                    self.style.LABEL(setting),
-                    action,
-                ))
-                if self.verbosity >= 2:
-                    if migrate_error:
-                        self.stdout.write('    - Migrate value: {}'.format(
-                            self.style.ERROR(migrate_error),
-                        ))
-                    elif migrate_value is not empty:
-                        self.stdout.write('    - Migrate value: {}'.format(
-                            self.style.VALUE(repr(migrate_value)),
-                        ))
-                    if comment_error:
-                        self.stdout.write('    - Comment: {}'.format(
-                            self.style.ERROR(comment_error),
-                        ))
-                    elif files_to_comment:
-                        for file_to_comment in files_to_comment:
-                            self.stdout.write('    - Comment in: {}'.format(
-                                self.style.VALUE(file_to_comment),
-                            ))
-            else:
-                if self.verbosity >= 2:
-                    self.stdout.write('  {}: {}'.format(
-                        self.style.LABEL(setting),
-                        self.style.SKIP('No Migration'),
-                    ))
-
-    def _display_migrate(self, setting, action, display_value):
-        if self.verbosity >= 1:
-            if action == 'No Change':
-                action = self.style.SKIP(action)
-            else:
-                action = self.style.OK(action)
-            self.stdout.write('  {}: {}'.format(
-                self.style.LABEL(setting),
-                action,
-            ))
-            if self.verbosity >= 2:
-                for line in display_value.splitlines():
-                    self.stdout.write('    {}'.format(
-                        self.style.VALUE(line),
-                    ))
-
-    def _display_diff_summary(self, filename, added, removed):
-        self.stdout.write('  {} {}{} {}{}'.format(
-            self.style.LABEL(filename),
-            self.style.ERROR('-'),
-            self.style.ERROR(int(removed)),
-            self.style.OK('+'),
-            self.style.OK(str(added)),
-        ))
-
-    def _display_comment(self, diffs):
-        for diff in diffs:
-            if self.verbosity >= 2:
-                for line in diff.splitlines():
-                    display_line = line
-                    if line.startswith('--- ') or line.startswith('+++ '):
-                        display_line = self.style.LABEL(line)
-                    elif line.startswith('-'):
-                        display_line = self.style.ERROR(line)
-                    elif line.startswith('+'):
-                        display_line = self.style.OK(line)
-                    elif line.startswith('@@'):
-                        display_line = self.style.VALUE(line)
-                    if line.startswith('--- ') or line.startswith('+++ '):
-                        self.stdout.write('  ' + display_line)
-                    else:
-                        self.stdout.write('    ' + display_line)
-            elif self.verbosity >= 1:
-                filename, lines_added, lines_removed = None, 0, 0
-                for line in diff.splitlines():
-                    if line.startswith('+++ '):
-                        if filename:
-                            self._display_diff_summary(filename, lines_added, lines_removed)
-                        filename, lines_added, lines_removed = line[4:], 0, 0
-                    elif line.startswith('+'):
-                        lines_added += 1
-                    elif line.startswith('-'):
-                        lines_removed += 1
-                if filename:
-                    self._display_diff_summary(filename, lines_added, lines_removed)
-
-    def _discover_settings(self, registered_settings):
-        if self.verbosity >= 1:
-            self.stdout.write(self.style.HEADING('Discovering settings to be migrated and commented:'))
-
-        # Determine which settings need to be commented/migrated.
-        to_migrate = collections.OrderedDict()
-        to_comment = collections.OrderedDict()
-        patterns = self._get_settings_file_patterns()
-
-        for name in registered_settings:
-            comment_error, migrate_error = None, None
-            files_to_comment = []
-            try:
-                files_to_comment = self._check_if_needs_comment(patterns, name)
-            except Exception as e:
-                comment_error = 'Error commenting {0}: {1!r}'.format(name, e)
-                if not self.skip_errors:
-                    raise CommandError(comment_error)
-            if files_to_comment:
-                to_comment[name] = files_to_comment
-            migrate_value = empty
-            if files_to_comment:
-                migrate_value = self._check_if_needs_migration(name)
-                if migrate_value is not empty:
-                    field = settings_registry.get_setting_field(name)
-                    assert not field.read_only
-                    try:
-                        data = field.to_representation(migrate_value)
-                        setting_value = field.run_validation(data)
-                        db_value = field.to_representation(setting_value)
-                        to_migrate[name] = db_value
-                    except Exception as e:
-                        to_comment.pop(name)
-                        migrate_error = 'Unable to assign value {0!r} to setting "{1}: {2!s}".'.format(migrate_value, name, e)
-                        if not self.skip_errors:
-                            raise CommandError(migrate_error)
-            self._display_tbd(name, files_to_comment, migrate_value, comment_error, migrate_error)
-        if self.verbosity == 1 and not to_migrate and not to_comment:
-            self.stdout.write('  No settings found to migrate or comment!')
-        return (to_migrate, to_comment)
-
-    def _migrate(self, to_migrate):
-        if self.verbosity >= 1:
-            if self.dry_run:
-                self.stdout.write(self.style.HEADING('Migrating settings to database (dry-run):'))
-            else:
-                self.stdout.write(self.style.HEADING('Migrating settings to database:'))
-            if not to_migrate:
-                self.stdout.write('  No settings to migrate!')
-
-        # Now migrate those settings to the database.
-        for name, db_value in to_migrate.items():
-            display_value = json.dumps(db_value, indent=4)
-            setting = Setting.objects.filter(key=name, user__isnull=True).order_by('pk').first()
-            action = 'No Change'
-            if not setting:
-                action = 'Migrated'
-                if not self.dry_run:
-                    Setting.objects.create(key=name, user=None, value=db_value)
-            elif setting.value != db_value or type(setting.value) != type(db_value):
-                action = 'Updated'
-                if not self.dry_run:
-                    setting.value = db_value
-                    setting.save(update_fields=['value'])
-            self._display_migrate(name, action, display_value)
-
-    def _comment(self, to_comment):
-        if self.verbosity >= 1:
-            if bool(self.dry_run or self.no_comment):
-                self.stdout.write(self.style.HEADING('Commenting settings in files (dry-run):'))
-            else:
-                self.stdout.write(self.style.HEADING('Commenting settings in files:'))
-            if not to_comment:
-                self.stdout.write('  No settings to comment!')
-
-        # Now comment settings in settings files.
-        if to_comment:
-            to_comment_patterns = []
-            license_file_to_comment = None
-            local_settings_file_to_comment = None
-            custom_logo_file_to_comment = None
-            for files_to_comment in to_comment.values():
-                for file_to_comment in files_to_comment:
-                    if file_to_comment == self._get_license_file():
-                        license_file_to_comment = file_to_comment
-                    elif file_to_comment == self._get_local_settings_file():
-                        local_settings_file_to_comment = file_to_comment
-                    elif file_to_comment == self._get_custom_logo_file():
-                        custom_logo_file_to_comment = file_to_comment
-                    elif file_to_comment not in to_comment_patterns:
-                        to_comment_patterns.append(file_to_comment)
-            # Run once in dry-run mode to catch any errors from updating the files.
-            diffs = comment_assignments(to_comment_patterns, list(to_comment.keys()), dry_run=True, backup_suffix=self.backup_suffix)
-            # Then, if really updating, run again.
-            if not self.dry_run and not self.no_comment:
-                diffs = comment_assignments(to_comment_patterns, list(to_comment.keys()), dry_run=False, backup_suffix=self.backup_suffix)
-                if license_file_to_comment:
-                    diffs.extend(self._comment_license_file(dry_run=False))
-                if local_settings_file_to_comment:
-                    diffs.extend(self._comment_local_settings_file(dry_run=False))
-                if custom_logo_file_to_comment:
-                    diffs.extend(self._comment_custom_logo_file(dry_run=False))
-            self._display_comment(diffs)
-
-    def _migrate_settings(self, registered_settings):
-        to_migrate, to_comment = self._discover_settings(registered_settings)
-
-        if not bool(self.comment_only):
-            self._migrate(to_migrate)
-        self._comment(to_comment)

awx/conf/migrations/0001_initial.py

@@ -21,7 +21,8 @@ class Migration(migrations.Migration):
                 ('modified', models.DateTimeField(default=None, editable=False)),
                 ('key', models.CharField(max_length=255)),
                 ('value', jsonfield.fields.JSONField(null=True)),
-                ('user', models.ForeignKey(related_name='settings', default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)),
+                ('user', models.ForeignKey(related_name='settings', default=None, editable=False,
+                                           to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True)),
             ],
             options={
                 'abstract': False,

awx/conf/migrations/0004_v320_reencrypt.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from django.db import migrations
-from awx.conf.migrations import _reencrypt
 
 
 class Migration(migrations.Migration):
@@ -12,5 +11,8 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(_reencrypt.replace_aesecb_fernet),
+        # This list is intentionally empty.
+        # Tower 3.2 included several data migrations that are no longer
+        # necessary (this list is now empty because Tower 3.2 is past EOL and
+        # cannot be directly upgraded to modern versions of Tower)
     ]

awx/conf/migrations/_ldap_group_type.py

@@ -22,7 +22,7 @@ def fill_ldap_group_type_params(apps, schema_editor):
                         modified=now())
 
     init_attrs = set(inspect.getargspec(group_type.__init__).args[1:])
-    for k in group_type_params.keys():
+    for k in list(group_type_params.keys()):
         if k not in init_attrs:
             del group_type_params[k]
 

awx/conf/migrations/_reencrypt.py

@@ -1,31 +1,13 @@
 import base64
 import hashlib
 
-import six
-from django.utils.encoding import smart_str
-
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.ciphers import Cipher
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
 from cryptography.hazmat.primitives.ciphers.modes import ECB
 
-from awx.conf import settings_registry
 
-
-__all__ = ['replace_aesecb_fernet', 'get_encryption_key', 'encrypt_field',
-           'decrypt_value', 'decrypt_value', 'should_decrypt_field']
-
-
-def replace_aesecb_fernet(apps, schema_editor):
-    from awx.main.utils.encryption import encrypt_field
-    Setting = apps.get_model('conf', 'Setting')
-
-    for setting in Setting.objects.filter().order_by('pk'):
-        if settings_registry.is_setting_encrypted(setting.key):
-            if should_decrypt_field(setting.value):
-                setting.value = decrypt_field(setting, 'value')
-                setting.value = encrypt_field(setting, 'value')
-            setting.save()
+__all__ = ['get_encryption_key', 'decrypt_field']
 
 
 def get_encryption_key(field_name, pk=None):
@@ -77,38 +59,3 @@ def decrypt_field(instance, field_name, subfield=None):
     key = get_encryption_key(field_name, getattr(instance, 'pk', None))
 
     return decrypt_value(key, value)
-
-
-def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False):
-    '''
-    Return content of the given instance and field name encrypted.
-    '''
-    value = getattr(instance, field_name)
-    if isinstance(value, dict) and subfield is not None:
-        value = value[subfield]
-    if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'):
-        return value
-    if skip_utf8:
-        utf8 = False
-    else:
-        utf8 = type(value) == six.text_type
-    value = smart_str(value)
-    key = get_encryption_key(field_name, getattr(instance, 'pk', None))
-    encryptor = Cipher(AES(key), ECB(), default_backend()).encryptor()
-    block_size = 16
-    while len(value) % block_size != 0:
-        value += '\x00'
-    encrypted = encryptor.update(value) + encryptor.finalize()
-    b64data = base64.b64encode(encrypted)
-    tokens = ['$encrypted', 'AES', b64data]
-    if utf8:
-        # If the value to encrypt is utf-8, we need to add a marker so we
-        # know to decode the data when it's decrypted later
-        tokens.insert(1, 'UTF8')
-    return '$'.join(tokens)
-
-
-def should_decrypt_field(value):
-    if hasattr(value, 'startswith'):
-        return value.startswith('$encrypted$') and '$AESCBC$' not in value
-    return False

awx/conf/registry.py

@@ -68,7 +68,7 @@ class SettingsRegistry(object):
     def get_dependent_settings(self, setting):
         return self._dependent_settings.get(setting, set())
 
-    def get_registered_categories(self, features_enabled=None):
+    def get_registered_categories(self):
         categories = {
             'all': _('All'),
             'changed': _('Changed'),
@@ -77,10 +77,6 @@ class SettingsRegistry(object):
             category_slug = kwargs.get('category_slug', None)
             if category_slug is None or category_slug in categories:
                 continue
-            if features_enabled is not None:
-                feature_required = kwargs.get('feature_required', None)
-                if feature_required and feature_required not in features_enabled:
-                    continue
             if category_slug == 'user':
                 categories['user'] = _('User')
                 categories['user-defaults'] = _('User-Defaults')
@@ -88,7 +84,7 @@ class SettingsRegistry(object):
                 categories[category_slug] = kwargs.get('category', None) or category_slug
         return categories
 
-    def get_registered_settings(self, category_slug=None, read_only=None, features_enabled=None, slugs_to_ignore=set()):
+    def get_registered_settings(self, category_slug=None, read_only=None, slugs_to_ignore=set()):
         setting_names = []
         if category_slug == 'user-defaults':
             category_slug = 'user'
@@ -100,14 +96,10 @@ class SettingsRegistry(object):
             if kwargs.get('category_slug', None) in slugs_to_ignore:
                 continue
             if (read_only in {True, False} and kwargs.get('read_only', False) != read_only and
-                    setting not in ('AWX_ISOLATED_PRIVATE_KEY', 'AWX_ISOLATED_PUBLIC_KEY')):
+                    setting not in ('INSTALL_UUID', 'AWX_ISOLATED_PRIVATE_KEY', 'AWX_ISOLATED_PUBLIC_KEY')):
                 # Note: Doesn't catch fields that set read_only via __init__;
                 # read-only field kwargs should always include read_only=True.
                 continue
-            if features_enabled is not None:
-                feature_required = kwargs.get('feature_required', None)
-                if feature_required and feature_required not in features_enabled:
-                    continue
             setting_names.append(setting)
         return setting_names
 
@@ -135,7 +127,6 @@ class SettingsRegistry(object):
         category = field_kwargs.pop('category', None)
         depends_on = frozenset(field_kwargs.pop('depends_on', None) or [])
         placeholder = field_kwargs.pop('placeholder', empty)
-        feature_required = field_kwargs.pop('feature_required', empty)
         encrypted = bool(field_kwargs.pop('encrypted', False))
         defined_in_file = bool(field_kwargs.pop('defined_in_file', False))
         if getattr(field_kwargs.get('child', None), 'source', None) is not None:
@@ -146,8 +137,6 @@ class SettingsRegistry(object):
         field_instance.depends_on = depends_on
         if placeholder is not empty:
             field_instance.placeholder = placeholder
-        if feature_required is not empty:
-            field_instance.feature_required = feature_required
         field_instance.defined_in_file = defined_in_file
         if field_instance.defined_in_file:
             field_instance.help_text = (

awx/conf/serializers.py

@@ -1,8 +1,6 @@
 # Django REST Framework
 from rest_framework import serializers
 
-import six
-
 # Tower
 from awx.api.fields import VerbatimField
 from awx.api.serializers import BaseSerializer
@@ -47,12 +45,12 @@ class SettingFieldMixin(object):
     """Mixin to use a registered setting field class for API display/validation."""
 
     def to_representation(self, obj):
-        if getattr(self, 'encrypted', False) and isinstance(obj, six.string_types) and obj:
+        if getattr(self, 'encrypted', False) and isinstance(obj, str) and obj:
             return '$encrypted$'
         return obj
 
     def to_internal_value(self, value):
-        if getattr(self, 'encrypted', False) and isinstance(value, six.string_types) and value.startswith('$encrypted$'):
+        if getattr(self, 'encrypted', False) and isinstance(value, str) and value.startswith('$encrypted$'):
             raise serializers.SkipField()
         obj = super(SettingFieldMixin, self).to_internal_value(value)
         return super(SettingFieldMixin, self).to_representation(obj)
@@ -90,7 +88,7 @@ class SettingSingletonSerializer(serializers.Serializer):
                 continue
             extra_kwargs = {}
             # Make LICENSE and AWX_ISOLATED_KEY_GENERATION read-only here;
-            # LICENSE is only updated via /api/v1/config/
+            # LICENSE is only updated via /api/v2/config/
             # AWX_ISOLATED_KEY_GENERATION is only set/unset via the setup playbook
             if key in ('LICENSE', 'AWX_ISOLATED_KEY_GENERATION'):
                 extra_kwargs['read_only'] = True

awx/conf/settings.py

@@ -24,11 +24,12 @@ from rest_framework.fields import empty, SkipField
 
 # Tower
 from awx.main.utils import encrypt_field, decrypt_field
-from awx.main.utils.db import get_tower_migration_version
 from awx.conf import settings_registry
 from awx.conf.models import Setting
 from awx.conf.migrations._reencrypt import decrypt_field as old_decrypt_field
 
+import cachetools
+
 # FIXME: Gracefully handle when settings are accessed before the database is
 # ready (or during migrations).
 
@@ -90,45 +91,42 @@ def _ctit_db_wrapper(trans_safe=False):
                     transaction.set_rollback(False)
         yield
     except DBError:
-        if 'migrate' in sys.argv and get_tower_migration_version() < '310':
-            logger.info('Using default settings until version 3.1 migration.')
-        else:
-            # We want the _full_ traceback with the context
-            # First we get the current call stack, which constitutes the "top",
-            # it has the context up to the point where the context manager is used
-            top_stack = StringIO()
-            traceback.print_stack(file=top_stack)
-            top_lines = top_stack.getvalue().strip('\n').split('\n')
-            top_stack.close()
-            # Get "bottom" stack from the local error that happened
-            # inside of the "with" block this wraps
-            exc_type, exc_value, exc_traceback = sys.exc_info()
-            bottom_stack = StringIO()
-            traceback.print_tb(exc_traceback, file=bottom_stack)
-            bottom_lines = bottom_stack.getvalue().strip('\n').split('\n')
-            # Glue together top and bottom where overlap is found
-            bottom_cutoff = 0
-            for i, line in enumerate(bottom_lines):
-                if line in top_lines:
-                    # start of overlapping section, take overlap from bottom
-                    top_lines = top_lines[:top_lines.index(line)]
-                    bottom_cutoff = i
-                    break
-            bottom_lines = bottom_lines[bottom_cutoff:]
-            tb_lines = top_lines + bottom_lines
+        # We want the _full_ traceback with the context
+        # First we get the current call stack, which constitutes the "top",
+        # it has the context up to the point where the context manager is used
+        top_stack = StringIO()
+        traceback.print_stack(file=top_stack)
+        top_lines = top_stack.getvalue().strip('\n').split('\n')
+        top_stack.close()
+        # Get "bottom" stack from the local error that happened
+        # inside of the "with" block this wraps
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        bottom_stack = StringIO()
+        traceback.print_tb(exc_traceback, file=bottom_stack)
+        bottom_lines = bottom_stack.getvalue().strip('\n').split('\n')
+        # Glue together top and bottom where overlap is found
+        bottom_cutoff = 0
+        for i, line in enumerate(bottom_lines):
+            if line in top_lines:
+                # start of overlapping section, take overlap from bottom
+                top_lines = top_lines[:top_lines.index(line)]
+                bottom_cutoff = i
+                break
+        bottom_lines = bottom_lines[bottom_cutoff:]
+        tb_lines = top_lines + bottom_lines
 
-            tb_string = '\n'.join(
-                ['Traceback (most recent call last):'] +
-                tb_lines +
-                ['{}: {}'.format(exc_type.__name__, str(exc_value))]
-            )
-            bottom_stack.close()
-            # Log the combined stack
-            if trans_safe:
-                if 'check_migrations' not in sys.argv:
-                    logger.warning('Database settings are not available, using defaults, error:\n{}'.format(tb_string))
-            else:
-                logger.error('Error modifying something related to database settings.\n{}'.format(tb_string))
+        tb_string = '\n'.join(
+            ['Traceback (most recent call last):'] +
+            tb_lines +
+            ['{}: {}'.format(exc_type.__name__, str(exc_value))]
+        )
+        bottom_stack.close()
+        # Log the combined stack
+        if trans_safe:
+            if 'check_migrations' not in sys.argv:
+                logger.debug('Database settings are not available, using defaults, error:\n{}'.format(tb_string))
+        else:
+            logger.debug('Error modifying something related to database settings.\n{}'.format(tb_string))
     finally:
         if trans_safe and is_atomic and rollback_set:
             transaction.set_rollback(rollback_set)
@@ -140,6 +138,14 @@ def filter_sensitive(registry, key, value):
     return value
 
 
+# settings.__getattr__ is called *constantly*, and the LOG_AGGREGATOR_ ones are
+# so ubiquitous when external logging is enabled that they should kept in memory
+# with a short TTL to avoid even having to contact memcached
+# the primary use case for this optimization is the callback receiver
+# when external logging is enabled
+LOGGING_SETTINGS_CACHE = cachetools.TTLCache(maxsize=50, ttl=1)
+
+
 class EncryptedCacheProxy(object):
 
     def __init__(self, cache, registry, encrypter=None, decrypter=None):
@@ -299,7 +305,7 @@ class SettingsWrapper(UserSettingsHolder):
             self.__dict__['_awx_conf_preload_expires'] = time.time() + SETTING_CACHE_TIMEOUT
             # Check for any settings that have been defined in Python files and
             # make those read-only to avoid overriding in the database.
-            if not self._awx_conf_init_readonly and 'migrate_to_database_settings' not in sys.argv:
+            if not self._awx_conf_init_readonly:
                 defaults_snapshot = self._get_default('DEFAULTS_SNAPSHOT')
                 for key in get_writeable_settings(self.registry):
                     init_default = defaults_snapshot.get(key, None)
@@ -381,8 +387,9 @@ class SettingsWrapper(UserSettingsHolder):
             setting = None
             setting_id = None
             if not field.read_only or name in (
-                # these two values are read-only - however - we *do* want
+                # these values are read-only - however - we *do* want
                 # to fetch their value from the database
+                'INSTALL_UUID',
                 'AWX_ISOLATED_PRIVATE_KEY',
                 'AWX_ISOLATED_PUBLIC_KEY',
             ):
@@ -440,11 +447,17 @@ class SettingsWrapper(UserSettingsHolder):
         return self._get_default('SETTINGS_MODULE')
 
     def __getattr__(self, name):
+        if name.startswith('LOG_AGGREGATOR_'):
+            cached = LOGGING_SETTINGS_CACHE.get(name)
+            if cached:
+                return cached
         value = empty
         if name in self.all_supported_settings:
             with _ctit_db_wrapper(trans_safe=True):
                 value = self._get_local(name)
         if value is not empty:
+            if name.startswith('LOG_AGGREGATOR_'):
+                LOGGING_SETTINGS_CACHE[name] = value
             return value
         value = self._get_default(name)
         # sometimes users specify RabbitMQ passwords that contain

awx/conf/tests/functional/conftest.py

@@ -1,7 +1,8 @@
+import urllib.parse
+
 import pytest
 
-from django.core.urlresolvers import resolve
-from django.utils.six.moves.urllib.parse import urlparse
+from django.urls import resolve
 from django.contrib.auth.models import User
 
 from rest_framework.test import (
@@ -33,7 +34,7 @@ def admin():
 @pytest.fixture
 def api_request(admin):
     def rf(verb, url, data=None, user=admin):
-        view, view_args, view_kwargs = resolve(urlparse(url)[2])
+        view, view_args, view_kwargs = resolve(urllib.parse.urlparse(url)[2])
         request = getattr(APIRequestFactory(), verb)(url, data=data, format='json')
         if user:
             force_authenticate(request, user=user)

awx/conf/tests/functional/test_api.py

@@ -8,6 +8,7 @@ from awx.main.utils.encryption import decrypt_field
 from awx.conf import fields
 from awx.conf.registry import settings_registry
 from awx.conf.models import Setting
+from awx.sso import fields as sso_fields
 
 
 @pytest.fixture
@@ -65,41 +66,6 @@ def test_non_admin_user_does_not_see_categories(api_request, dummy_setting, norm
         assert not response.data['results']
 
 
-@pytest.mark.django_db
-@mock.patch(
-    'awx.conf.views.VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE',
-    {
-        1: set([]),
-        2: set(['foobar']),
-    }
-)
-def test_version_specific_category_slug_to_exclude_does_not_show_up(api_request, dummy_setting):
-    with dummy_setting(
-        'FOO_BAR',
-        field_class=fields.IntegerField,
-        category='FooBar',
-        category_slug='foobar'
-    ):
-        response = api_request(
-            'get',
-            reverse('api:setting_category_list',
-                    kwargs={'version': 'v2'})
-        )
-        for item in response.data['results']:
-            assert item['slug'] != 'foobar'
-        response = api_request(
-            'get',
-            reverse('api:setting_category_list',
-                    kwargs={'version': 'v1'})
-        )
-        contains = False
-        for item in response.data['results']:
-            if item['slug'] != 'foobar':
-                contains = True
-                break
-        assert contains
-
-
 @pytest.mark.django_db
 def test_setting_singleton_detail_retrieve(api_request, dummy_setting):
     with dummy_setting(
@@ -172,7 +138,7 @@ def test_setting_signleton_retrieve_hierachy(api_request, dummy_setting):
 
 
 @pytest.mark.django_db
-def test_setting_signleton_retrieve_readonly(api_request, dummy_setting):
+def test_setting_singleton_retrieve_readonly(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.IntegerField,
@@ -218,6 +184,30 @@ def test_setting_singleton_update(api_request, dummy_setting):
         assert response.data['FOO_BAR'] == 4
 
 
+@pytest.mark.django_db
+def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, dummy_setting):
+    # Some HybridDictField subclasses have a child of _Forbidden,
+    # indicating that only the defined fields can be filled in.  Make
+    # sure that the _Forbidden validator doesn't get used for the
+    # fields.  See also https://github.com/ansible/awx/issues/4099.
+    with dummy_setting(
+        'FOO_BAR',
+        field_class=sso_fields.SAMLOrgAttrField,
+        category='FooBar',
+        category_slug='foobar',
+    ), mock.patch('awx.conf.views.handle_setting_changes'):
+        api_request(
+            'patch',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}),
+            data={'FOO_BAR': {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}}
+        )
+        response = api_request(
+            'get',
+            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'})
+        )
+        assert response.data['FOO_BAR'] == {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}
+
+
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting(
@@ -241,7 +231,7 @@ def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy
 
 
 @pytest.mark.django_db
-def test_setting_singleton_update_dont_change_encripted_mark(api_request, dummy_setting):
+def test_setting_singleton_update_dont_change_encrypted_mark(api_request, dummy_setting):
     with dummy_setting(
         'FOO_BAR',
         field_class=fields.CharField,

awx/conf/tests/unit/test_fields.py

@@ -1,7 +1,7 @@
 import pytest
 
 from rest_framework.fields import ValidationError
-from awx.conf.fields import StringListBooleanField, ListTuplesField
+from awx.conf.fields import StringListBooleanField, StringListPathField, ListTuplesField, URLField
 
 
 class TestStringListBooleanField():
@@ -62,7 +62,7 @@ class TestListTuplesField():
     FIELD_VALUES = [
         ([('a', 'b'), ('abc', '123')], [("a", "b"), ("abc", "123")]),
     ]
-    
+
     FIELD_VALUES_INVALID = [
         ("abc", type("abc")),
         ([('a', 'b', 'c'), ('abc', '123', '456')], type(('a',))),
@@ -84,3 +84,71 @@ class TestListTuplesField():
         assert e.value.detail[0] == "Expected a list of tuples of max length 2 " \
             "but got {} instead.".format(t)
 
+
+class TestStringListPathField():
+
+    FIELD_VALUES = [
+        ((".", "..", "/"), [".", "..", "/"]),
+        (("/home",), ["/home"]),
+        (("///home///",), ["/home"]),
+        (("/home/././././",), ["/home"]),
+        (("/home", "/home", "/home/"), ["/home"]),
+        (["/home/", "/home/", "/opt/", "/opt/", "/var/"], ["/home", "/opt", "/var"])
+    ]
+
+    FIELD_VALUES_INVALID_TYPE = [
+        1.245,
+        {"a": "b"},
+        ("/home"),
+    ]
+
+    FIELD_VALUES_INVALID_PATH = [
+        "",
+        "~/",
+        "home",
+        "/invalid_path",
+        "/home/invalid_path",
+    ]
+
+    @pytest.mark.parametrize("value_in, value_known", FIELD_VALUES)
+    def test_to_internal_value_valid(self, value_in, value_known):
+        field = StringListPathField()
+        v = field.to_internal_value(value_in)
+        assert v == value_known
+
+    @pytest.mark.parametrize("value", FIELD_VALUES_INVALID_TYPE)
+    def test_to_internal_value_invalid_type(self, value):
+        field = StringListPathField()
+        with pytest.raises(ValidationError) as e:
+            field.to_internal_value(value)
+        assert e.value.detail[0] == "Expected list of strings but got {} instead.".format(type(value))
+
+    @pytest.mark.parametrize("value", FIELD_VALUES_INVALID_PATH)
+    def test_to_internal_value_invalid_path(self, value):
+        field = StringListPathField()
+        with pytest.raises(ValidationError) as e:
+            field.to_internal_value([value])
+        assert e.value.detail[0] == "{} is not a valid path choice.".format(value)
+
+
+class TestURLField():
+    regex = "^https://www.example.org$"
+
+    @pytest.mark.parametrize("url,schemes,regex, allow_numbers_in_top_level_domain, expect_no_error",[
+        ("ldap://www.example.org42", "ldap", None, True, True),
+        ("https://www.example.org42", "https", None, False, False),
+        ("https://www.example.org", None,  regex, None, True),
+        ("https://www.example3.org", None, regex, None, False),
+        ("ftp://www.example.org", "https", None, None, False)
+    ])
+    def test_urls(self, url, schemes, regex, allow_numbers_in_top_level_domain, expect_no_error):
+        kwargs = {}
+        kwargs.setdefault("allow_numbers_in_top_level_domain", allow_numbers_in_top_level_domain)
+        kwargs.setdefault("schemes", schemes)
+        kwargs.setdefault("regex", regex)
+        field = URLField(**kwargs)
+        if expect_no_error:
+            field.run_validators(url)
+        else:
+            with pytest.raises(ValidationError):
+                field.run_validators(url)

awx/conf/tests/unit/test_registry.py

@@ -119,20 +119,6 @@ def test_get_registered_read_only_settings(reg):
     ]
 
 
-def test_get_registered_settings_with_required_features(reg):
-    reg.register(
-        'AWX_SOME_SETTING_ENABLED',
-        field_class=fields.BooleanField,
-        category=_('System'),
-        category_slug='system',
-        feature_required='superpowers',
-    )
-    assert reg.get_registered_settings(features_enabled=[]) == []
-    assert reg.get_registered_settings(features_enabled=['superpowers']) == [
-        'AWX_SOME_SETTING_ENABLED'
-    ]
-
-
 def test_get_dependent_settings(reg):
     reg.register(
         'AWX_SOME_SETTING_ENABLED',
@@ -173,45 +159,6 @@ def test_get_registered_categories(reg):
     }
 
 
-def test_get_registered_categories_with_required_features(reg):
-    reg.register(
-        'AWX_SOME_SETTING_ENABLED',
-        field_class=fields.BooleanField,
-        category=_('System'),
-        category_slug='system',
-        feature_required='superpowers'
-    )
-    reg.register(
-        'AWX_SOME_OTHER_SETTING_ENABLED',
-        field_class=fields.BooleanField,
-        category=_('OtherSystem'),
-        category_slug='other-system',
-        feature_required='sortapowers'
-    )
-    assert reg.get_registered_categories(features_enabled=[]) == {
-        'all': _('All'),
-        'changed': _('Changed'),
-    }
-    assert reg.get_registered_categories(features_enabled=['superpowers']) == {
-        'all': _('All'),
-        'changed': _('Changed'),
-        'system': _('System'),
-    }
-    assert reg.get_registered_categories(features_enabled=['sortapowers']) == {
-        'all': _('All'),
-        'changed': _('Changed'),
-        'other-system': _('OtherSystem'),
-    }
-    assert reg.get_registered_categories(
-        features_enabled=['superpowers', 'sortapowers']
-    ) == {
-        'all': _('All'),
-        'changed': _('Changed'),
-        'system': _('System'),
-        'other-system': _('OtherSystem'),
-    }
-
-
 def test_is_setting_encrypted(reg):
     reg.register(
         'AWX_SOME_SETTING_ENABLED',
@@ -237,7 +184,6 @@ def test_simple_field(reg):
         category=_('System'),
         category_slug='system',
         placeholder='Example Value',
-        feature_required='superpowers'
     )
 
     field = reg.get_setting_field('AWX_SOME_SETTING')
@@ -246,7 +192,6 @@ def test_simple_field(reg):
     assert field.category_slug == 'system'
     assert field.default is empty
     assert field.placeholder == 'Example Value'
-    assert field.feature_required == 'superpowers'
 
 
 def test_field_with_custom_attribute(reg):

awx/conf/tests/unit/test_settings.py

@@ -13,7 +13,6 @@ from django.core.cache.backends.locmem import LocMemCache
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import ugettext_lazy as _
 import pytest
-import six
 
 from awx.conf import models, fields
 from awx.conf.settings import SettingsWrapper, EncryptedCacheProxy, SETTING_CACHE_NOTSET
@@ -70,7 +69,7 @@ def test_cached_settings_unicode_is_auto_decoded(settings):
 
     value = 'Iñtërnâtiônàlizætiøn'  # this simulates what python-memcached does on cache.set()
     settings.cache.set('DEBUG', value)
-    assert settings.cache.get('DEBUG') == six.u('Iñtërnâtiônàlizætiøn')
+    assert settings.cache.get('DEBUG') == 'Iñtërnâtiônàlizætiøn'
 
 
 def test_read_only_setting(settings):

awx/conf/utils.py

@@ -1,110 +1,9 @@
 #!/usr/bin/env python
 
-# Python
-import difflib
-import glob
-import os
-import shutil
-
-import six
-
 # AWX
 from awx.conf.registry import settings_registry
 
-__all__ = ['comment_assignments', 'conf_to_dict']
-
-
-def comment_assignments(patterns, assignment_names, dry_run=True, backup_suffix='.old'):
-    if isinstance(patterns, six.string_types):
-        patterns = [patterns]
-    diffs = []
-    for pattern in patterns:
-        for filename in sorted(glob.glob(pattern)):
-            filename = os.path.abspath(os.path.normpath(filename))
-            if backup_suffix:
-                backup_filename = '{}{}'.format(filename, backup_suffix)
-            else:
-                backup_filename = None
-            diff = comment_assignments_in_file(filename, assignment_names, dry_run, backup_filename)
-            if diff:
-                diffs.append(diff)
-    return diffs
-
-
-def comment_assignments_in_file(filename, assignment_names, dry_run=True, backup_filename=None):
-    from redbaron import RedBaron, indent
-
-    if isinstance(assignment_names, six.string_types):
-        assignment_names = [assignment_names]
-    else:
-        assignment_names = assignment_names[:]
-    current_file_data = open(filename).read()
-
-    for assignment_name in assignment_names[:]:
-        if assignment_name in current_file_data:
-            continue
-        if assignment_name in assignment_names:
-            assignment_names.remove(assignment_name)
-    if not assignment_names:
-        return ''
-
-    replace_lines = {}
-    rb = RedBaron(current_file_data)
-    for assignment_node in rb.find_all('assignment'):
-        for assignment_name in assignment_names:
-
-            # Only target direct assignments to a variable.
-            name_node = assignment_node.find('name', value=assignment_name)
-            if not name_node:
-                continue
-            if assignment_node.target.type != 'name':
-                continue
-
-            # Build a new node that comments out the existing assignment node.
-            indentation = '{}# '.format(assignment_node.indentation or '')
-            new_node_content = indent(assignment_node.dumps(), indentation)
-            new_node_lines = new_node_content.splitlines()
-            # Add a pass statement in case the assignment block is the only
-            # child in a parent code block to prevent a syntax error.
-            if assignment_node.indentation:
-                new_node_lines[0] = new_node_lines[0].replace(indentation, '{}pass  # '.format(assignment_node.indentation or ''), 1)
-            new_node_lines[0] = '{0}This setting is now configured via the Tower API.\n{1}'.format(indentation, new_node_lines[0])
-
-            # Store new node lines in dictionary to be replaced in file.
-            start_lineno = assignment_node.absolute_bounding_box.top_left.line
-            end_lineno = assignment_node.absolute_bounding_box.bottom_right.line
-            for n, new_node_line in enumerate(new_node_lines):
-                new_lineno = start_lineno + n
-                assert new_lineno <= end_lineno
-                replace_lines[new_lineno] = new_node_line
-
-    if not replace_lines:
-        return ''
-
-    # Iterate through all lines in current file and replace as needed.
-    current_file_lines = current_file_data.splitlines()
-    new_file_lines = []
-    for n, line in enumerate(current_file_lines):
-        new_file_lines.append(replace_lines.get(n + 1, line))
-    new_file_data = '\n'.join(new_file_lines)
-    new_file_lines = new_file_data.splitlines()
-
-    # If changed, syntax check and write the new file; return a diff of changes.
-    diff_lines = []
-    if new_file_data != current_file_data:
-        compile(new_file_data, filename, 'exec')
-        if backup_filename:
-            from_file = backup_filename
-        else:
-            from_file = '{}.old'.format(filename)
-        to_file = filename
-        diff_lines = list(difflib.unified_diff(current_file_lines, new_file_lines, fromfile=from_file, tofile=to_file, lineterm=''))
-        if not dry_run:
-            if backup_filename:
-                shutil.copy2(filename, backup_filename)
-            with open(filename, 'w') as fileobj:
-                fileobj.write(new_file_data)
-    return '\n'.join(diff_lines)
+__all__ = ['conf_to_dict']
 
 
 def conf_to_dict(obj):
@@ -112,10 +11,3 @@ def conf_to_dict(obj):
         'category': settings_registry.get_setting_category(obj.key),
         'name': obj.key,
     }
-
-
-if __name__ == '__main__':
-    pattern = os.path.join(os.path.dirname(__file__), '..', 'settings', 'local_*.py')
-    diffs = comment_assignments(pattern, ['AUTH_LDAP_ORGANIZATION_MAP'])
-    for diff in diffs:
-        print(diff)

awx/conf/views.py

@@ -17,13 +17,17 @@ from rest_framework import serializers
 from rest_framework import status
 
 # Tower
-from awx.api.generics import *  # noqa
+from awx.api.generics import (
+    APIView,
+    GenericAPIView,
+    ListAPIView,
+    RetrieveUpdateDestroyAPIView,
+)
 from awx.api.permissions import IsSuperUser
-from awx.api.versioning import reverse, get_request_version
-from awx.main.utils import *  # noqa
+from awx.api.versioning import reverse
+from awx.main.utils import camelcase_to_underscore
 from awx.main.utils.handlers import AWXProxyHandler, LoggingConnectivityException
 from awx.main.tasks import handle_setting_changes
-from awx.conf.license import get_licensed_features
 from awx.conf.models import Setting
 from awx.conf.serializers import SettingCategorySerializer, SettingSingletonSerializer
 from awx.conf import settings_registry
@@ -31,24 +35,17 @@ from awx.conf import settings_registry
 
 SettingCategory = collections.namedtuple('SettingCategory', ('url', 'slug', 'name'))
 
-VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE = {
-    1: set([
-        'named-url',
-    ]),
-    2: set([]),
-}
-
 
 class SettingCategoryList(ListAPIView):
 
     model = Setting  # Not exactly, but needed for the view.
     serializer_class = SettingCategorySerializer
     filter_backends = []
-    view_name = _('Setting Categories')
+    name = _('Setting Categories')
 
     def get_queryset(self):
         setting_categories = []
-        categories = settings_registry.get_registered_categories(features_enabled=get_licensed_features())
+        categories = settings_registry.get_registered_categories()
         if self.request.user.is_superuser or self.request.user.is_system_auditor:
             pass  # categories = categories
         elif 'user' in categories:
@@ -56,8 +53,6 @@ class SettingCategoryList(ListAPIView):
         else:
             categories = {}
         for category_slug in sorted(categories.keys()):
-            if category_slug in VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE[get_request_version(self.request)]:
-                continue
             url = reverse('api:setting_singleton_detail', kwargs={'category_slug': category_slug}, request=self.request)
             setting_categories.append(SettingCategory(url, category_slug, categories[category_slug]))
         return setting_categories
@@ -68,13 +63,11 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
     model = Setting  # Not exactly, but needed for the view.
     serializer_class = SettingSingletonSerializer
     filter_backends = []
-    view_name = _('Setting Detail')
+    name = _('Setting Detail')
 
     def get_queryset(self):
         self.category_slug = self.kwargs.get('category_slug', 'all')
-        all_category_slugs = list(settings_registry.get_registered_categories(features_enabled=get_licensed_features()).keys())
-        for slug_to_delete in VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE[get_request_version(self.request)]:
-            all_category_slugs.remove(slug_to_delete)
+        all_category_slugs = list(settings_registry.get_registered_categories().keys())
         if self.request.user.is_superuser or getattr(self.request.user, 'is_system_auditor', False):
             category_slugs = all_category_slugs
         else:
@@ -85,8 +78,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
             raise PermissionDenied()
 
         registered_settings = settings_registry.get_registered_settings(
-            category_slug=self.category_slug, read_only=False, features_enabled=get_licensed_features(),
-            slugs_to_ignore=VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE[get_request_version(self.request)]
+            category_slug=self.category_slug, read_only=False,
         )
         if self.category_slug == 'user':
             return Setting.objects.filter(key__in=registered_settings, user=self.request.user)
@@ -96,8 +88,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
     def get_object(self):
         settings_qs = self.get_queryset()
         registered_settings = settings_registry.get_registered_settings(
-            category_slug=self.category_slug, features_enabled=get_licensed_features(),
-            slugs_to_ignore=VERSION_SPECIFIC_CATEGORIES_TO_EXCLUDE[get_request_version(self.request)]
+            category_slug=self.category_slug,
         )
         all_settings = {}
         for setting in settings_qs:
@@ -135,7 +126,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
                 setting.value = value
                 setting.save(update_fields=['value'])
                 settings_change_list.append(key)
-        if settings_change_list and 'migrate_to_database_settings' not in sys.argv:
+        if settings_change_list:
             handle_setting_changes.delay(settings_change_list)
 
     def destroy(self, request, *args, **kwargs):
@@ -150,7 +141,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
                 continue
             setting.delete()
             settings_change_list.append(setting.key)
-        if settings_change_list and 'migrate_to_database_settings' not in sys.argv:
+        if settings_change_list:
             handle_setting_changes.delay(settings_change_list)
 
         # When TOWER_URL_BASE is deleted from the API, reset it to the hostname
@@ -163,7 +154,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
 
 class SettingLoggingTest(GenericAPIView):
 
-    view_name = _('Logging Connectivity Test')
+    name = _('Logging Connectivity Test')
     model = Setting
     serializer_class = SettingSingletonSerializer
     permission_classes = (IsSuperUser,)

awx/lib/awx_display_callback/__init__.py

@@ -1,25 +0,0 @@
-# Copyright (c) 2016 Ansible by Red Hat, Inc.
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# AWX Display Callback
-from . import cleanup  # noqa (registers control persistent cleanup)
-from . import display  # noqa (wraps ansible.display.Display methods)
-from .module import AWXDefaultCallbackModule, AWXMinimalCallbackModule
-
-__all__ = ['AWXDefaultCallbackModule', 'AWXMinimalCallbackModule']

awx/lib/awx_display_callback/cleanup.py

@@ -1,85 +0,0 @@
-# Copyright (c) 2016 Ansible by Red Hat, Inc.
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import atexit
-import glob
-import os
-import pwd
-
-# PSUtil
-try:
-    import psutil
-except ImportError:
-    raise ImportError('psutil is missing; {}bin/pip install psutil'.format(
-        os.environ['VIRTUAL_ENV']
-    ))
-
-__all__ = []
-
-main_pid = os.getpid()
-
-
-@atexit.register
-def terminate_ssh_control_masters():
-    # Only run this cleanup from the main process.
-    if os.getpid() != main_pid:
-        return
-    # Determine if control persist is being used and if any open sockets
-    # exist after running the playbook.
-    cp_path = os.environ.get('ANSIBLE_SSH_CONTROL_PATH', '')
-    if not cp_path:
-        return
-    cp_dir = os.path.dirname(cp_path)
-    if not os.path.exists(cp_dir):
-        return
-    cp_pattern = os.path.join(cp_dir, 'ansible-ssh-*')
-    cp_files = glob.glob(cp_pattern)
-    if not cp_files:
-        return
-
-    # Attempt to find any running control master processes.
-    username = pwd.getpwuid(os.getuid())[0]
-    ssh_cm_procs = []
-    for proc in psutil.process_iter():
-        try:
-            pname = proc.name()
-            pcmdline = proc.cmdline()
-            pusername = proc.username()
-        except psutil.NoSuchProcess:
-            continue
-        if pusername != username:
-            continue
-        if pname != 'ssh':
-            continue
-        for cp_file in cp_files:
-            if pcmdline and cp_file in pcmdline[0]:
-                ssh_cm_procs.append(proc)
-                break
-
-    # Terminate then kill control master processes.  Workaround older
-    # version of psutil that may not have wait_procs implemented.
-    for proc in ssh_cm_procs:
-        try:
-            proc.terminate()
-        except psutil.NoSuchProcess:
-            continue
-    procs_gone, procs_alive = psutil.wait_procs(ssh_cm_procs, timeout=5)
-    for proc in procs_alive:
-        proc.kill()

awx/lib/awx_display_callback/display.py

@@ -1,98 +0,0 @@
-# Copyright (c) 2016 Ansible by Red Hat, Inc.
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import functools
-import sys
-import uuid
-
-# Ansible
-from ansible.utils.display import Display
-
-# Tower Display Callback
-from .events import event_context
-
-__all__ = []
-
-
-def with_context(**context):
-    global event_context
-
-    def wrap(f):
-        @functools.wraps(f)
-        def wrapper(*args, **kwargs):
-            with event_context.set_local(**context):
-                return f(*args, **kwargs)
-        return wrapper
-    return wrap
-
-
-for attr in dir(Display):
-    if attr.startswith('_') or 'cow' in attr or 'prompt' in attr:
-        continue
-    if attr in ('display', 'v', 'vv', 'vvv', 'vvvv', 'vvvvv', 'vvvvvv', 'verbose'):
-        continue
-    if not callable(getattr(Display, attr)):
-        continue
-    setattr(Display, attr, with_context(**{attr: True})(getattr(Display, attr)))
-
-
-def with_verbosity(f):
-    global event_context
-
-    @functools.wraps(f)
-    def wrapper(*args, **kwargs):
-        host = args[2] if len(args) >= 3 else kwargs.get('host', None)
-        caplevel = args[3] if len(args) >= 4 else kwargs.get('caplevel', 2)
-        context = dict(verbose=True, verbosity=(caplevel + 1))
-        if host is not None:
-            context['remote_addr'] = host
-        with event_context.set_local(**context):
-            return f(*args, **kwargs)
-    return wrapper
-
-
-Display.verbose = with_verbosity(Display.verbose)
-
-
-def display_with_context(f):
-
-    @functools.wraps(f)
-    def wrapper(*args, **kwargs):
-        log_only = args[5] if len(args) >= 6 else kwargs.get('log_only', False)
-        stderr = args[3] if len(args) >= 4 else kwargs.get('stderr', False)
-        event_uuid = event_context.get().get('uuid', None)
-        with event_context.display_lock:
-            # If writing only to a log file or there is already an event UUID
-            # set (from a callback module method), skip dumping the event data.
-            if log_only or event_uuid:
-                return f(*args, **kwargs)
-            try:
-                fileobj = sys.stderr if stderr else sys.stdout
-                event_context.add_local(uuid=str(uuid.uuid4()))
-                event_context.dump_begin(fileobj)
-                return f(*args, **kwargs)
-            finally:
-                event_context.dump_end(fileobj)
-                event_context.remove_local(uuid=None)
-
-    return wrapper
-
-
-Display.display = display_with_context(Display.display)

awx/lib/awx_display_callback/events.py

@@ -1,186 +0,0 @@
-# Copyright (c) 2016 Ansible by Red Hat, Inc.
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import base64
-import contextlib
-import datetime
-import json
-import multiprocessing
-import os
-import stat
-import threading
-import uuid
-
-try:
-    import memcache
-except ImportError:
-    raise ImportError('python-memcached is missing; {}bin/pip install python-memcached'.format(
-        os.environ['VIRTUAL_ENV']
-    ))
-
-__all__ = ['event_context']
-
-
-class IsolatedFileWrite:
-    '''
-    Stand-in class that will write partial event data to a file as a
-    replacement for memcache when a job is running on an isolated host.
-    '''
-
-    def __init__(self):
-        self.private_data_dir = os.getenv('AWX_ISOLATED_DATA_DIR')
-
-    def set(self, key, value):
-        # Strip off the leading memcache key identifying characters :1:ev-
-        event_uuid = key[len(':1:ev-'):]
-        # Write data in a staging area and then atomic move to pickup directory
-        filename = '{}-partial.json'.format(event_uuid)
-        dropoff_location = os.path.join(self.private_data_dir, 'artifacts', 'job_events', filename)
-        write_location = '.'.join([dropoff_location, 'tmp'])
-        with os.fdopen(os.open(write_location, os.O_WRONLY | os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR), 'w') as f:
-            f.write(value)
-        os.rename(write_location, dropoff_location)
-
-
-class EventContext(object):
-    '''
-    Store global and local (per thread/process) data associated with callback
-    events and other display output methods.
-    '''
-
-    def __init__(self):
-        self.display_lock = multiprocessing.RLock()
-        cache_actual = os.getenv('CACHE', '127.0.0.1:11211')
-        if os.getenv('AWX_ISOLATED_DATA_DIR', False):
-            self.cache = IsolatedFileWrite()
-        else:
-            self.cache = memcache.Client([cache_actual], debug=0)
-
-    def add_local(self, **kwargs):
-        if not hasattr(self, '_local'):
-            self._local = threading.local()
-            self._local._ctx = {}
-        self._local._ctx.update(kwargs)
-
-    def remove_local(self, **kwargs):
-        if hasattr(self, '_local'):
-            for key in kwargs.keys():
-                self._local._ctx.pop(key, None)
-
-    @contextlib.contextmanager
-    def set_local(self, **kwargs):
-        try:
-            self.add_local(**kwargs)
-            yield
-        finally:
-            self.remove_local(**kwargs)
-
-    def get_local(self):
-        return getattr(getattr(self, '_local', None), '_ctx', {})
-
-    def add_global(self, **kwargs):
-        if not hasattr(self, '_global_ctx'):
-            self._global_ctx = {}
-        self._global_ctx.update(kwargs)
-
-    def remove_global(self, **kwargs):
-        if hasattr(self, '_global_ctx'):
-            for key in kwargs.keys():
-                self._global_ctx.pop(key, None)
-
-    @contextlib.contextmanager
-    def set_global(self, **kwargs):
-        try:
-            self.add_global(**kwargs)
-            yield
-        finally:
-            self.remove_global(**kwargs)
-
-    def get_global(self):
-        return getattr(self, '_global_ctx', {})
-
-    def get(self):
-        ctx = {}
-        ctx.update(self.get_global())
-        ctx.update(self.get_local())
-        return ctx
-
-    def get_begin_dict(self):
-        event_data = self.get()
-        if os.getenv('JOB_ID', ''):
-            event_data['job_id'] = int(os.getenv('JOB_ID', '0'))
-        if os.getenv('AD_HOC_COMMAND_ID', ''):
-            event_data['ad_hoc_command_id'] = int(os.getenv('AD_HOC_COMMAND_ID', '0'))
-        if os.getenv('PROJECT_UPDATE_ID', ''):
-            event_data['project_update_id'] = int(os.getenv('PROJECT_UPDATE_ID', '0'))
-        event_data.setdefault('pid', os.getpid())
-        event_data.setdefault('uuid', str(uuid.uuid4()))
-        event_data.setdefault('created', datetime.datetime.utcnow().isoformat())
-        if not event_data.get('parent_uuid', None) and event_data.get('job_id', None):
-            for key in ('task_uuid', 'play_uuid', 'playbook_uuid'):
-                parent_uuid = event_data.get(key, None)
-                if parent_uuid and parent_uuid != event_data.get('uuid', None):
-                    event_data['parent_uuid'] = parent_uuid
-                    break
-
-        event = event_data.pop('event', None)
-        if not event:
-            event = 'verbose'
-            for key in ('debug', 'verbose', 'deprecated', 'warning', 'system_warning', 'error'):
-                if event_data.get(key, False):
-                    event = key
-                    break
-        max_res = int(os.getenv("MAX_EVENT_RES", 700000))
-        if event not in ('playbook_on_stats',) and "res" in event_data and len(str(event_data['res'])) > max_res:
-            event_data['res'] = {}
-        event_dict = dict(event=event, event_data=event_data)
-        for key in list(event_data.keys()):
-            if key in ('job_id', 'ad_hoc_command_id', 'project_update_id', 'uuid', 'parent_uuid', 'created',):
-                event_dict[key] = event_data.pop(key)
-            elif key in ('verbosity', 'pid'):
-                event_dict[key] = event_data[key]
-        return event_dict
-
-    def get_end_dict(self):
-        return {}
-
-    def dump(self, fileobj, data, max_width=78, flush=False):
-        b64data = base64.b64encode(json.dumps(data).encode('utf-8')).decode()
-        with self.display_lock:
-            # pattern corresponding to OutputEventFilter expectation
-            fileobj.write(u'\x1b[K')
-            for offset in range(0, len(b64data), max_width):
-                chunk = b64data[offset:offset + max_width]
-                escaped_chunk = u'{}\x1b[{}D'.format(chunk, len(chunk))
-                fileobj.write(escaped_chunk)
-            fileobj.write(u'\x1b[K')
-            if flush:
-                fileobj.flush()
-
-    def dump_begin(self, fileobj):
-        begin_dict = self.get_begin_dict()
-        self.cache.set(":1:ev-{}".format(begin_dict['uuid']), json.dumps(begin_dict))
-        self.dump(fileobj, {'uuid': begin_dict['uuid']})
-
-    def dump_end(self, fileobj):
-        self.dump(fileobj, self.get_end_dict(), flush=True)
-
-
-event_context = EventContext()

awx/lib/awx_display_callback/minimal.py

@@ -1,29 +0,0 @@
-# Copyright (c) 2016 Ansible by Red Hat, Inc.
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import os
-
-# Ansible
-import ansible
-
-# Because of the way Ansible loads plugins, it's not possible to import
-# ansible.plugins.callback.minimal when being loaded as the minimal plugin. Ugh.
-with open(os.path.join(os.path.dirname(ansible.__file__), 'plugins', 'callback', 'minimal.py')) as in_file:
-    exec(in_file.read())

awx/lib/awx_display_callback/module.py

@@ -1,501 +0,0 @@
-# Copyright (c) 2016 Ansible by Red Hat, Inc.
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import codecs
-import contextlib
-import json
-import os
-import stat
-import sys
-import uuid
-from copy import copy
-
-# Ansible
-from ansible import constants as C
-from ansible.plugins.callback import CallbackBase
-from ansible.plugins.callback.default import CallbackModule as DefaultCallbackModule
-
-# AWX Display Callback
-from .events import event_context
-from .minimal import CallbackModule as MinimalCallbackModule
-
-CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result"  # noqa
-
-
-class BaseCallbackModule(CallbackBase):
-    '''
-    Callback module for logging ansible/ansible-playbook events.
-    '''
-
-    CALLBACK_VERSION = 2.0
-    CALLBACK_TYPE = 'stdout'
-
-    # These events should never have an associated play.
-    EVENTS_WITHOUT_PLAY = [
-        'playbook_on_start',
-        'playbook_on_stats',
-    ]
-
-    # These events should never have an associated task.
-    EVENTS_WITHOUT_TASK = EVENTS_WITHOUT_PLAY + [
-        'playbook_on_setup',
-        'playbook_on_notify',
-        'playbook_on_import_for_host',
-        'playbook_on_not_import_for_host',
-        'playbook_on_no_hosts_matched',
-        'playbook_on_no_hosts_remaining',
-    ]
-
-    def __init__(self):
-        super(BaseCallbackModule, self).__init__()
-        self.task_uuids = set()
-
-    @contextlib.contextmanager
-    def capture_event_data(self, event, **event_data):
-        event_data.setdefault('uuid', str(uuid.uuid4()))
-
-        if event not in self.EVENTS_WITHOUT_TASK:
-            task = event_data.pop('task', None)
-        else:
-            task = None
-
-        if event_data.get('res'):
-            if event_data['res'].get('_ansible_no_log', False):
-                event_data['res'] = {'censored': CENSORED}
-            if event_data['res'].get('results', []):
-                event_data['res']['results'] = copy(event_data['res']['results'])
-            for i, item in enumerate(event_data['res'].get('results', [])):
-                if isinstance(item, dict) and item.get('_ansible_no_log', False):
-                    event_data['res']['results'][i] = {'censored': CENSORED}
-
-        with event_context.display_lock:
-            try:
-                event_context.add_local(event=event, **event_data)
-                if task:
-                    self.set_task(task, local=True)
-                event_context.dump_begin(sys.stdout)
-                yield
-            finally:
-                event_context.dump_end(sys.stdout)
-                if task:
-                    self.clear_task(local=True)
-                event_context.remove_local(event=None, **event_data)
-
-    def set_playbook(self, playbook):
-        # NOTE: Ansible doesn't generate a UUID for playbook_on_start so do it for them.
-        self.playbook_uuid = str(uuid.uuid4())
-        file_name = getattr(playbook, '_file_name', '???')
-        event_context.add_global(playbook=file_name, playbook_uuid=self.playbook_uuid)
-        self.clear_play()
-
-    def set_play(self, play):
-        if hasattr(play, 'hosts'):
-            if isinstance(play.hosts, list):
-                pattern = ','.join(play.hosts)
-            else:
-                pattern = play.hosts
-        else:
-            pattern = ''
-        name = play.get_name().strip() or pattern
-        event_context.add_global(play=name, play_uuid=str(play._uuid), play_pattern=pattern)
-        self.clear_task()
-
-    def clear_play(self):
-        event_context.remove_global(play=None, play_uuid=None, play_pattern=None)
-        self.clear_task()
-
-    def set_task(self, task, local=False):
-        # FIXME: Task is "global" unless using free strategy!
-        task_ctx = dict(
-            task=(task.name or task.action),
-            task_uuid=str(task._uuid),
-            task_action=task.action,
-            task_args='',
-        )
-        try:
-            task_ctx['task_path'] = task.get_path()
-        except AttributeError:
-            pass
-
-        if C.DISPLAY_ARGS_TO_STDOUT:
-            if task.no_log:
-                task_ctx['task_args'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result"
-            else:
-                task_args = ', '.join(('%s=%s' % a for a in task.args.items()))
-                task_ctx['task_args'] = task_args
-        if getattr(task, '_role', None):
-            task_role = task._role._role_name
-        else:
-            task_role = getattr(task, 'role_name', '')
-        if task_role:
-            task_ctx['role'] = task_role
-        if local:
-            event_context.add_local(**task_ctx)
-        else:
-            event_context.add_global(**task_ctx)
-
-    def clear_task(self, local=False):
-        task_ctx = dict(task=None, task_path=None, task_uuid=None, task_action=None, task_args=None, role=None)
-        if local:
-            event_context.remove_local(**task_ctx)
-        else:
-            event_context.remove_global(**task_ctx)
-
-    def v2_playbook_on_start(self, playbook):
-        self.set_playbook(playbook)
-        event_data = dict(
-            uuid=self.playbook_uuid,
-        )
-        with self.capture_event_data('playbook_on_start', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_start(playbook)
-
-    def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None,
-                                   encrypt=None, confirm=False, salt_size=None,
-                                   salt=None, default=None):
-        event_data = dict(
-            varname=varname,
-            private=private,
-            prompt=prompt,
-            encrypt=encrypt,
-            confirm=confirm,
-            salt_size=salt_size,
-            salt=salt,
-            default=default,
-        )
-        with self.capture_event_data('playbook_on_vars_prompt', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_vars_prompt(
-                varname, private, prompt, encrypt, confirm, salt_size, salt,
-                default,
-            )
-
-    def v2_playbook_on_include(self, included_file):
-        event_data = dict(
-            included_file=included_file._filename if included_file is not None else None,
-        )
-        with self.capture_event_data('playbook_on_include', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_include(included_file)
-
-    def v2_playbook_on_play_start(self, play):
-        self.set_play(play)
-        if hasattr(play, 'hosts'):
-            if isinstance(play.hosts, list):
-                pattern = ','.join(play.hosts)
-            else:
-                pattern = play.hosts
-        else:
-            pattern = ''
-        name = play.get_name().strip() or pattern
-        event_data = dict(
-            name=name,
-            pattern=pattern,
-            uuid=str(play._uuid),
-        )
-        with self.capture_event_data('playbook_on_play_start', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_play_start(play)
-
-    def v2_playbook_on_import_for_host(self, result, imported_file):
-        # NOTE: Not used by Ansible 2.x.
-        with self.capture_event_data('playbook_on_import_for_host'):
-            super(BaseCallbackModule, self).v2_playbook_on_import_for_host(result, imported_file)
-
-    def v2_playbook_on_not_import_for_host(self, result, missing_file):
-        # NOTE: Not used by Ansible 2.x.
-        with self.capture_event_data('playbook_on_not_import_for_host'):
-            super(BaseCallbackModule, self).v2_playbook_on_not_import_for_host(result, missing_file)
-
-    def v2_playbook_on_setup(self):
-        # NOTE: Not used by Ansible 2.x.
-        with self.capture_event_data('playbook_on_setup'):
-            super(BaseCallbackModule, self).v2_playbook_on_setup()
-
-    def v2_playbook_on_task_start(self, task, is_conditional):
-        # FIXME: Flag task path output as vv.
-        task_uuid = str(task._uuid)
-        if task_uuid in self.task_uuids:
-            # FIXME: When this task UUID repeats, it means the play is using the
-            # free strategy, so different hosts may be running different tasks
-            # within a play.
-            return
-        self.task_uuids.add(task_uuid)
-        self.set_task(task)
-        event_data = dict(
-            task=task,
-            name=task.get_name(),
-            is_conditional=is_conditional,
-            uuid=task_uuid,
-        )
-        with self.capture_event_data('playbook_on_task_start', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_task_start(task, is_conditional)
-
-    def v2_playbook_on_cleanup_task_start(self, task):
-        # NOTE: Not used by Ansible 2.x.
-        self.set_task(task)
-        event_data = dict(
-            task=task,
-            name=task.get_name(),
-            uuid=str(task._uuid),
-            is_conditional=True,
-        )
-        with self.capture_event_data('playbook_on_task_start', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_cleanup_task_start(task)
-
-    def v2_playbook_on_handler_task_start(self, task):
-        # NOTE: Re-using playbook_on_task_start event for this v2-specific
-        # event, but setting is_conditional=True, which is how v1 identified a
-        # task run as a handler.
-        self.set_task(task)
-        event_data = dict(
-            task=task,
-            name=task.get_name(),
-            uuid=str(task._uuid),
-            is_conditional=True,
-        )
-        with self.capture_event_data('playbook_on_task_start', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_handler_task_start(task)
-
-    def v2_playbook_on_no_hosts_matched(self):
-        with self.capture_event_data('playbook_on_no_hosts_matched'):
-            super(BaseCallbackModule, self).v2_playbook_on_no_hosts_matched()
-
-    def v2_playbook_on_no_hosts_remaining(self):
-        with self.capture_event_data('playbook_on_no_hosts_remaining'):
-            super(BaseCallbackModule, self).v2_playbook_on_no_hosts_remaining()
-
-    def v2_playbook_on_notify(self, handler, host):
-        # NOTE: Not used by Ansible < 2.5.
-        event_data = dict(
-            host=host.get_name(),
-            handler=handler.get_name(),
-        )
-        with self.capture_event_data('playbook_on_notify', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_notify(handler, host)
-
-    '''
-    ansible_stats is, retoractively, added in 2.2
-    '''
-    def v2_playbook_on_stats(self, stats):
-        self.clear_play()
-        # FIXME: Add count of plays/tasks.
-        event_data = dict(
-            changed=stats.changed,
-            dark=stats.dark,
-            failures=stats.failures,
-            ok=stats.ok,
-            processed=stats.processed,
-            skipped=stats.skipped
-        )
-
-        # write custom set_stat artifact data to the local disk so that it can
-        # be persisted by awx after the process exits
-        custom_artifact_data = stats.custom.get('_run', {}) if hasattr(stats, 'custom') else {}
-        if custom_artifact_data:
-            # create the directory for custom stats artifacts to live in (if it doesn't exist)
-            custom_artifacts_dir = os.path.join(os.getenv('AWX_PRIVATE_DATA_DIR'), 'artifacts')
-            if not os.path.isdir(custom_artifacts_dir):
-                os.makedirs(custom_artifacts_dir, mode=stat.S_IXUSR + stat.S_IWUSR + stat.S_IRUSR)
-
-            custom_artifacts_path = os.path.join(custom_artifacts_dir, 'custom')
-            with codecs.open(custom_artifacts_path, 'w', encoding='utf-8') as f:
-                os.chmod(custom_artifacts_path, stat.S_IRUSR | stat.S_IWUSR)
-                json.dump(custom_artifact_data, f)
-
-        with self.capture_event_data('playbook_on_stats', **event_data):
-            super(BaseCallbackModule, self).v2_playbook_on_stats(stats)
-
-    @staticmethod
-    def _get_event_loop(task):
-        if hasattr(task, 'loop_with'):  # Ansible >=2.5
-            return task.loop_with
-        elif hasattr(task, 'loop'):  # Ansible <2.4
-            return task.loop
-        return None
-
-    def v2_runner_on_ok(self, result):
-        # FIXME: Display detailed results or not based on verbosity.
-
-        # strip environment vars from the job event; it already exists on the
-        # job and sensitive values are filtered there
-        if result._task.action in ('setup', 'gather_facts'):
-            result._result.get('ansible_facts', {}).pop('ansible_env', None)
-
-        event_data = dict(
-            host=result._host.get_name(),
-            remote_addr=result._host.address,
-            task=result._task,
-            res=result._result,
-            event_loop=self._get_event_loop(result._task),
-        )
-        with self.capture_event_data('runner_on_ok', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_ok(result)
-
-    def v2_runner_on_failed(self, result, ignore_errors=False):
-        # FIXME: Add verbosity for exception/results output.
-        event_data = dict(
-            host=result._host.get_name(),
-            remote_addr=result._host.address,
-            res=result._result,
-            task=result._task,
-            ignore_errors=ignore_errors,
-            event_loop=self._get_event_loop(result._task),
-        )
-        with self.capture_event_data('runner_on_failed', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_failed(result, ignore_errors)
-
-    def v2_runner_on_skipped(self, result):
-        event_data = dict(
-            host=result._host.get_name(),
-            remote_addr=result._host.address,
-            task=result._task,
-            event_loop=self._get_event_loop(result._task),
-        )
-        with self.capture_event_data('runner_on_skipped', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_skipped(result)
-
-    def v2_runner_on_unreachable(self, result):
-        event_data = dict(
-            host=result._host.get_name(),
-            remote_addr=result._host.address,
-            task=result._task,
-            res=result._result,
-        )
-        with self.capture_event_data('runner_on_unreachable', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_unreachable(result)
-
-    def v2_runner_on_no_hosts(self, task):
-        # NOTE: Not used by Ansible 2.x.
-        event_data = dict(
-            task=task,
-        )
-        with self.capture_event_data('runner_on_no_hosts', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_no_hosts(task)
-
-    def v2_runner_on_async_poll(self, result):
-        # NOTE: Not used by Ansible 2.x.
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-            jid=result._result.get('ansible_job_id'),
-        )
-        with self.capture_event_data('runner_on_async_poll', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_async_poll(result)
-
-    def v2_runner_on_async_ok(self, result):
-        # NOTE: Not used by Ansible 2.x.
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-            jid=result._result.get('ansible_job_id'),
-        )
-        with self.capture_event_data('runner_on_async_ok', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_async_ok(result)
-
-    def v2_runner_on_async_failed(self, result):
-        # NOTE: Not used by Ansible 2.x.
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-            jid=result._result.get('ansible_job_id'),
-        )
-        with self.capture_event_data('runner_on_async_failed', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_async_failed(result)
-
-    def v2_runner_on_file_diff(self, result, diff):
-        # NOTE: Not used by Ansible 2.x.
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            diff=diff,
-        )
-        with self.capture_event_data('runner_on_file_diff', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_file_diff(result, diff)
-
-    def v2_on_file_diff(self, result):
-        # NOTE: Logged as runner_on_file_diff.
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            diff=result._result.get('diff'),
-        )
-        with self.capture_event_data('runner_on_file_diff', **event_data):
-            super(BaseCallbackModule, self).v2_on_file_diff(result)
-
-    def v2_runner_item_on_ok(self, result):
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-        )
-        with self.capture_event_data('runner_item_on_ok', **event_data):
-            super(BaseCallbackModule, self).v2_runner_item_on_ok(result)
-
-    def v2_runner_item_on_failed(self, result):
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-        )
-        with self.capture_event_data('runner_item_on_failed', **event_data):
-            super(BaseCallbackModule, self).v2_runner_item_on_failed(result)
-
-    def v2_runner_item_on_skipped(self, result):
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-        )
-        with self.capture_event_data('runner_item_on_skipped', **event_data):
-            super(BaseCallbackModule, self).v2_runner_item_on_skipped(result)
-
-    def v2_runner_retry(self, result):
-        event_data = dict(
-            host=result._host.get_name(),
-            task=result._task,
-            res=result._result,
-        )
-        with self.capture_event_data('runner_retry', **event_data):
-            super(BaseCallbackModule, self).v2_runner_retry(result)
-
-    def v2_runner_on_start(self, host, task):
-        event_data = dict(
-            host=host.get_name(),
-            task=task
-        )
-        with self.capture_event_data('runner_on_start', **event_data):
-            super(BaseCallbackModule, self).v2_runner_on_start(host, task)
-            
-
-
-class AWXDefaultCallbackModule(BaseCallbackModule, DefaultCallbackModule):
-
-    CALLBACK_NAME = 'awx_display'
-
-
-class AWXMinimalCallbackModule(BaseCallbackModule, MinimalCallbackModule):
-
-    CALLBACK_NAME = 'minimal'
-
-    def v2_playbook_on_play_start(self, play):
-        pass
-
-    def v2_playbook_on_task_start(self, task, is_conditional):
-        self.set_task(task)

awx/lib/isolated_callbacks/awx_display.py

@@ -1,30 +0,0 @@
-# Copyright (c) 2017 Ansible by Red Hat
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import os
-import sys
-
-# Add awx/lib to sys.path.
-awx_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-if awx_lib_path not in sys.path:
-    sys.path.insert(0, awx_lib_path)
-
-# Tower Display Callback
-from awx_display_callback import AWXDefaultCallbackModule as CallbackModule  # noqa

awx/lib/isolated_callbacks/minimal.py

@@ -1,30 +0,0 @@
-# Copyright (c) 2017 Ansible by Red Hat
-#
-# This file is part of Ansible Tower, but depends on code imported from Ansible.
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import (absolute_import, division, print_function)
-
-# Python
-import os
-import sys
-
-# Add awx/lib to sys.path.
-awx_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-if awx_lib_path not in sys.path:
-    sys.path.insert(0, awx_lib_path)
-
-# Tower Display Callback
-from awx_display_callback import AWXMinimalCallbackModule as CallbackModule  # noqa

awx/lib/sitecustomize.py

@@ -1,26 +0,0 @@
-# Python
-import os
-import sys
-
-# Based on http://stackoverflow.com/a/6879344/131141 -- Initialize awx display
-# callback as early as possible to wrap ansible.display.Display methods.
-
-
-def argv_ready(argv):
-    if argv and os.path.basename(argv[0]) in {'ansible', 'ansible-playbook'}:
-        import awx_display_callback  # noqa
-
-
-class argv_placeholder(object):
-
-    def __del__(self):
-        try:
-            argv_ready(sys.argv)
-        except Exception:
-            pass
-
-
-if hasattr(sys, 'argv'):
-    argv_ready(sys.argv)
-else:
-    sys.argv = argv_placeholder()

awx/lib/tests/pytest.ini

@@ -1,2 +0,0 @@
-[pytest]
-addopts = -v

awx/lib/tests/test_display_callback.py

@@ -1,353 +0,0 @@
-# Copyright (c) 2017 Ansible by Red Hat
-# All Rights Reserved
-
-from __future__ import absolute_import
-
-from collections import OrderedDict
-import json
-import os
-import shutil
-import sys
-import tempfile
-from unittest import mock
-
-import pytest
-
-# ansible uses `ANSIBLE_CALLBACK_PLUGINS` and `ANSIBLE_STDOUT_CALLBACK` to
-# discover callback plugins;  `ANSIBLE_CALLBACK_PLUGINS` is a list of paths to
-# search for a plugin implementation (which should be named `CallbackModule`)
-#
-# this code modifies the Python path to make our
-# `awx.lib.awx_display_callback` callback importable (because `awx.lib`
-# itself is not a package)
-#
-# we use the `awx_display_callback` imports below within this file, but
-# Ansible also uses them when it discovers this file in
-# `ANSIBLE_CALLBACK_PLUGINS`
-CALLBACK = os.path.splitext(os.path.basename(__file__))[0]
-PLUGINS = os.path.dirname(__file__)
-with mock.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': CALLBACK,
-                                  'ANSIBLE_CALLBACK_PLUGINS': PLUGINS}):
-    from ansible import __version__ as ANSIBLE_VERSION
-    from ansible.cli.playbook import PlaybookCLI
-    from ansible.executor.playbook_executor import PlaybookExecutor
-    from ansible.inventory.manager import InventoryManager
-    from ansible.parsing.dataloader import DataLoader
-    from ansible.vars.manager import VariableManager
-
-    # Add awx/lib to sys.path so we can use the plugin
-    path = os.path.abspath(os.path.join(PLUGINS, '..', '..', 'lib'))
-    if path not in sys.path:
-        sys.path.insert(0, path)
-
-    from awx_display_callback import AWXDefaultCallbackModule as CallbackModule  # noqa
-    from awx_display_callback.events import event_context  # noqa
-
-
-@pytest.fixture()
-def cache(request):
-    class Cache(OrderedDict):
-        def set(self, key, value):
-            self[key] = value
-    local_cache = Cache()
-    patch = mock.patch.object(event_context, 'cache', local_cache)
-    patch.start()
-    request.addfinalizer(patch.stop)
-    return local_cache
-
-
-@pytest.fixture()
-def executor(tmpdir_factory, request):
-    playbooks = request.node.callspec.params.get('playbook')
-    playbook_files = []
-    for name, playbook in playbooks.items():
-        filename = str(tmpdir_factory.mktemp('data').join(name))
-        with open(filename, 'w') as f:
-            f.write(playbook)
-        playbook_files.append(filename)
-
-    cli = PlaybookCLI(['', 'playbook.yml'])
-    cli.parse()
-    options = cli.parser.parse_args(['-v'])[0]
-    loader = DataLoader()
-    variable_manager = VariableManager(loader=loader)
-    inventory = InventoryManager(loader=loader, sources='localhost,')
-    variable_manager.set_inventory(inventory)
-
-    return PlaybookExecutor(playbooks=playbook_files, inventory=inventory,
-                            variable_manager=variable_manager, loader=loader,
-                            options=options, passwords={})
-
-
-@pytest.mark.parametrize('event', {'playbook_on_start',
-                                   'playbook_on_play_start',
-                                   'playbook_on_task_start', 'runner_on_ok',
-                                   'playbook_on_stats'})
-@pytest.mark.parametrize('playbook', [
-{'helloworld.yml': '''
-- name: Hello World Sample
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-    - name: Hello Message
-      debug:
-        msg: "Hello World!"
-'''},  # noqa
-{'results_included.yml': '''
-- name: Run module which generates results list
-  connection: local
-  hosts: all
-  gather_facts: no
-  vars:
-    results: ['foo', 'bar']
-  tasks:
-    - name: Generate results list
-      debug:
-        var: results
-'''},  # noqa
-])
-def test_callback_plugin_receives_events(executor, cache, event, playbook):
-    executor.run()
-    assert len(cache)
-    assert event in [task['event'] for task in cache.values()]
-
-
-@pytest.mark.parametrize('playbook', [
-{'no_log_on_ok.yml': '''
-- name: args should not be logged when task-level no_log is set
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-    - shell: echo "SENSITIVE"
-      no_log: true
-'''},  # noqa
-{'no_log_on_fail.yml': '''
-- name: failed args should not be logged when task-level no_log is set
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-    - shell: echo "SENSITIVE"
-      no_log: true
-      failed_when: true
-      ignore_errors: true
-'''},  # noqa
-{'no_log_on_skip.yml': '''
-- name: skipped task args should be suppressed with no_log
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-    - shell: echo "SENSITIVE"
-      no_log: true
-      when: false
-'''},  # noqa
-{'no_log_on_play.yml': '''
-- name: args should not be logged when play-level no_log set
-  connection: local
-  hosts: all
-  gather_facts: no
-  no_log: true
-  tasks:
-      - shell: echo "SENSITIVE"
-'''},  # noqa
-{'async_no_log.yml': '''
-- name: async task args should suppressed with no_log
-  connection: local
-  hosts: all
-  gather_facts: no
-  no_log: true
-  tasks:
-    - async: 10
-      poll: 1
-      shell: echo "SENSITIVE"
-      no_log: true
-'''},  # noqa
-{'with_items.yml': '''
-- name: with_items tasks should be suppressed with no_log
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-      - shell: echo {{ item }}
-        no_log: true
-        with_items: [ "SENSITIVE", "SENSITIVE-SKIPPED", "SENSITIVE-FAILED" ]
-        when: item != "SENSITIVE-SKIPPED"
-        failed_when: item == "SENSITIVE-FAILED"
-        ignore_errors: yes
-'''},  # noqa, NOTE: with_items will be deprecated in 2.9
-{'loop.yml': '''
-- name: loop tasks should be suppressed with no_log
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-      - shell: echo {{ item }}
-        no_log: true
-        loop: [ "SENSITIVE", "SENSITIVE-SKIPPED", "SENSITIVE-FAILED" ]
-        when: item != "SENSITIVE-SKIPPED"
-        failed_when: item == "SENSITIVE-FAILED"
-        ignore_errors: yes
-'''},  # noqa
-])
-def test_callback_plugin_no_log_filters(executor, cache, playbook):
-    executor.run()
-    assert len(cache)
-    assert 'SENSITIVE' not in json.dumps(cache.items())
-
-
-@pytest.mark.parametrize('playbook', [
-{'no_log_on_ok.yml': '''
-- name: args should not be logged when no_log is set at the task or module level
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-    - shell: echo "PUBLIC"
-    - shell: echo "PRIVATE"
-      no_log: true
-    - uri: url=https://example.org username="PUBLIC" password="PRIVATE"
-    - copy: content="PRIVATE" dest="/tmp/tmp_no_log"
-'''},  # noqa
-])
-def test_callback_plugin_task_args_leak(executor, cache, playbook):
-    executor.run()
-    events = cache.values()
-    assert events[0]['event'] == 'playbook_on_start'
-    assert events[1]['event'] == 'playbook_on_play_start'
-
-    # task 1
-    assert events[2]['event'] == 'playbook_on_task_start'
-    assert events[3]['event'] == 'runner_on_ok'
-
-    # task 2 no_log=True
-    assert events[4]['event'] == 'playbook_on_task_start'
-    assert events[5]['event'] == 'runner_on_ok'
-    assert 'PUBLIC' in json.dumps(cache.items())
-    assert 'PRIVATE' not in json.dumps(cache.items())
-    # make sure playbook was successful, so all tasks were hit
-    assert not events[-1]['event_data']['failures'], 'Unexpected playbook execution failure'
-
-
-@pytest.mark.parametrize('playbook', [
-{'loop_with_no_log.yml': '''
-- name: playbook variable should not be overwritten when using no log
-  connection: local
-  hosts: all
-  gather_facts: no
-  tasks:
-    - command: "{{ item }}"
-      register: command_register
-      no_log: True
-      with_items:
-        - "echo helloworld!"
-    - debug: msg="{{ command_register.results|map(attribute='stdout')|list }}"
-'''},  # noqa
-])
-def test_callback_plugin_censoring_does_not_overwrite(executor, cache, playbook):
-    executor.run()
-    events = cache.values()
-    assert events[0]['event'] == 'playbook_on_start'
-    assert events[1]['event'] == 'playbook_on_play_start'
-
-    # task 1
-    assert events[2]['event'] == 'playbook_on_task_start'
-    # Ordering of task and item events may differ randomly
-    assert set(['runner_on_ok', 'runner_item_on_ok']) == set([data['event'] for data in events[3:5]])
-
-    # task 2 no_log=True
-    assert events[5]['event'] == 'playbook_on_task_start'
-    assert events[6]['event'] == 'runner_on_ok'
-    assert 'helloworld!' in events[6]['event_data']['res']['msg']
-
-
-@pytest.mark.parametrize('playbook', [
-{'strip_env_vars.yml': '''
-- name: sensitive environment variables should be stripped from events
-  connection: local
-  hosts: all
-  tasks:
-    - shell: echo "Hello, World!"
-'''},  # noqa
-])
-def test_callback_plugin_strips_task_environ_variables(executor, cache, playbook):
-    executor.run()
-    assert len(cache)
-    for event in cache.values():
-        assert os.environ['PATH'] not in json.dumps(event)
-
-
-@pytest.mark.parametrize('playbook', [
-{'custom_set_stat.yml': '''
-- name: custom set_stat calls should persist to the local disk so awx can save them
-  connection: local
-  hosts: all
-  tasks:
-    - set_stats:
-        data:
-          foo: "bar"
-'''},  # noqa
-])
-def test_callback_plugin_saves_custom_stats(executor, cache, playbook):
-    try:
-        private_data_dir = tempfile.mkdtemp()
-        with mock.patch.dict(os.environ, {'AWX_PRIVATE_DATA_DIR': private_data_dir}):
-            executor.run()
-            artifacts_path = os.path.join(private_data_dir, 'artifacts', 'custom')
-            with open(artifacts_path, 'r') as f:
-                assert json.load(f) == {'foo': 'bar'}
-    finally:
-        shutil.rmtree(os.path.join(private_data_dir))
-
-
-@pytest.mark.parametrize('playbook', [
-{'handle_playbook_on_notify.yml': '''
-- name: handle playbook_on_notify events properly
-  connection: local
-  hosts: all
-  handlers:
-    - name: my_handler
-      debug: msg="My Handler"
-  tasks:
-    - debug: msg="My Task"
-      changed_when: true
-      notify:
-        - my_handler
-'''},  # noqa
-])
-@pytest.mark.skipif(ANSIBLE_VERSION < '2.5', reason="v2_playbook_on_notify doesn't work before ansible 2.5")
-def test_callback_plugin_records_notify_events(executor, cache, playbook):
-    executor.run()
-    assert len(cache)
-    notify_events = [x[1] for x in cache.items() if x[1]['event'] == 'playbook_on_notify']
-    assert len(notify_events) == 1
-    assert notify_events[0]['event_data']['handler'] == 'my_handler'
-    assert notify_events[0]['event_data']['host'] == 'localhost'
-    assert notify_events[0]['event_data']['task'] == 'debug'
-
-
-@pytest.mark.parametrize('playbook', [
-{'no_log_module_with_var.yml': '''
-- name: ensure that module-level secrets are redacted
-  connection: local
-  hosts: all
-  vars:
-    - pw: SENSITIVE
-  tasks:
-    - uri:
-        url: https://example.org
-        user: john-jacob-jingleheimer-schmidt
-        password: "{{ pw }}"
-'''},  # noqa
-])
-def test_module_level_no_log(executor, cache, playbook):
-    # https://github.com/ansible/tower/issues/1101
-    # It's possible for `no_log=True` to be defined at the _module_ level,
-    # e.g., for the URI module password parameter
-    # This test ensures that we properly redact those
-    executor.run()
-    assert len(cache)
-    assert 'john-jacob-jingleheimer-schmidt' in json.dumps(cache.items())
-    assert 'SENSITIVE' not in json.dumps(cache.items())

awx/locale/django.pot

@@ -2020,7 +2020,7 @@ msgstr ""
 
 #: awx/main/conf.py:286
 msgid ""
-"Allows roles to be dynamically downlaoded from a requirements.yml file for "
+"Allows roles to be dynamically downloaded from a requirements.yml file for "
 "SCM projects."
 msgstr ""
 

awx/locale/en-us/LC_MESSAGES/django.po

@@ -2020,7 +2020,7 @@ msgstr ""
 
 #: awx/main/conf.py:286
 msgid ""
-"Allows roles to be dynamically downlaoded from a requirements.yml file for "
+"Allows roles to be dynamically downloaded from a requirements.yml file for "
 "SCM projects."
 msgstr ""
 

awx/main/analytics/__init__.py

@@ -0,0 +1 @@
+from .core import register, gather, ship, table_version  # noqa

awx/main/analytics/collectors.py

@@ -0,0 +1,305 @@
+import os
+import os.path
+import platform
+
+from django.db import connection
+from django.db.models import Count
+from django.conf import settings
+from django.utils.timezone import now
+
+from awx.conf.license import get_license
+from awx.main.utils import (get_awx_version, get_ansible_version,
+                            get_custom_venv_choices, camelcase_to_underscore)
+from awx.main import models
+from django.contrib.sessions.models import Session
+from awx.main.analytics import register, table_version
+
+'''
+This module is used to define metrics collected by awx.main.analytics.gather()
+Each function is decorated with a key name, and should return a data
+structure that can be serialized to JSON
+
+@register('something', '1.0')
+def something(since):
+    # the generated archive will contain a `something.json` w/ this JSON
+    return {'some': 'json'}
+
+All functions - when called - will be passed a datetime.datetime object,
+`since`, which represents the last time analytics were gathered (some metrics
+functions - like those that return metadata about playbook runs, may return
+data _since_ the last report date - i.e., new data in the last 24 hours)
+'''
+
+
+@register('config', '1.0')
+def config(since):
+    license_info = get_license(show_key=False)
+    install_type = 'traditional'
+    if os.environ.get('container') == 'oci':
+        install_type = 'openshift'
+    elif 'KUBERNETES_SERVICE_PORT' in os.environ:
+        install_type = 'k8s'
+    return {
+        'platform': {
+            'system': platform.system(),
+            'dist': platform.dist(),
+            'release': platform.release(),
+            'type': install_type,
+        },
+        'install_uuid': settings.INSTALL_UUID,
+        'instance_uuid': settings.SYSTEM_UUID,
+        'tower_url_base': settings.TOWER_URL_BASE,
+        'tower_version': get_awx_version(),
+        'ansible_version': get_ansible_version(),
+        'license_type': license_info.get('license_type', 'UNLICENSED'),
+        'free_instances': license_info.get('free_instances', 0),
+        'license_expiry': license_info.get('time_remaining', 0),
+        'pendo_tracking': settings.PENDO_TRACKING_STATE,
+        'authentication_backends': settings.AUTHENTICATION_BACKENDS,
+        'logging_aggregators': settings.LOG_AGGREGATOR_LOGGERS,
+        'external_logger_enabled': settings.LOG_AGGREGATOR_ENABLED,
+        'external_logger_type': getattr(settings, 'LOG_AGGREGATOR_TYPE', None),
+    }
+
+
+@register('counts', '1.0')
+def counts(since):
+    counts = {}
+    for cls in (models.Organization, models.Team, models.User,
+                models.Inventory, models.Credential, models.Project,
+                models.JobTemplate, models.WorkflowJobTemplate, 
+                models.Host, models.Schedule, models.CustomInventoryScript, 
+                models.NotificationTemplate):
+        counts[camelcase_to_underscore(cls.__name__)] = cls.objects.count()
+
+    venvs = get_custom_venv_choices()
+    counts['custom_virtualenvs'] = len([
+        v for v in venvs
+        if os.path.basename(v.rstrip('/')) != 'ansible'
+    ])
+
+    inv_counts = dict(models.Inventory.objects.order_by().values_list('kind').annotate(Count('kind')))
+    inv_counts['normal'] = inv_counts.get('', 0)
+    inv_counts.pop('', None)
+    inv_counts['smart'] = inv_counts.get('smart', 0)
+    counts['inventories'] = inv_counts
+    
+    counts['unified_job'] = models.UnifiedJob.objects.exclude(launch_type='sync').count() # excludes implicit project_updates
+    counts['active_host_count'] = models.Host.objects.active_count()   
+    active_sessions = Session.objects.filter(expire_date__gte=now()).count()
+    active_user_sessions = models.UserSessionMembership.objects.select_related('session').filter(session__expire_date__gte=now()).count()
+    active_anonymous_sessions = active_sessions - active_user_sessions
+    counts['active_sessions'] = active_sessions
+    counts['active_user_sessions'] = active_user_sessions
+    counts['active_anonymous_sessions'] = active_anonymous_sessions
+    counts['running_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('running', 'waiting',)).count()
+    counts['pending_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').filter(status__in=('pending',)).count()
+    return counts
+
+    
+@register('org_counts', '1.0')
+def org_counts(since):
+    counts = {}
+    for org in models.Organization.objects.annotate(num_users=Count('member_role__members', distinct=True), 
+                                                    num_teams=Count('teams', distinct=True)).values('name', 'id', 'num_users', 'num_teams'):
+        counts[org['id']] = {'name': org['name'],
+                             'users': org['num_users'],
+                             'teams': org['num_teams']
+                             }
+    return counts
+    
+    
+@register('cred_type_counts', '1.0')
+def cred_type_counts(since):
+    counts = {}
+    for cred_type in models.CredentialType.objects.annotate(num_credentials=Count(
+                                                            'credentials', distinct=True)).values('name', 'id', 'managed_by_tower', 'num_credentials'):  
+        counts[cred_type['id']] = {'name': cred_type['name'],
+                                   'credential_count': cred_type['num_credentials'],
+                                   'managed_by_tower': cred_type['managed_by_tower']
+                                   }
+    return counts
+    
+    
+@register('inventory_counts', '1.0')
+def inventory_counts(since):
+    counts = {}
+    for inv in models.Inventory.objects.filter(kind='').annotate(num_sources=Count('inventory_sources', distinct=True), 
+                                                                 num_hosts=Count('hosts', distinct=True)).only('id', 'name', 'kind'):
+        counts[inv.id] = {'name': inv.name,
+                          'kind': inv.kind,
+                          'hosts': inv.num_hosts,
+                          'sources': inv.num_sources
+                          }
+
+    for smart_inv in models.Inventory.objects.filter(kind='smart'):
+        counts[smart_inv.id] = {'name': smart_inv.name,
+                                'kind': smart_inv.kind,
+                                'num_hosts': smart_inv.hosts.count(),
+                                'num_sources': smart_inv.inventory_sources.count()
+                                }
+    return counts
+
+
+@register('projects_by_scm_type', '1.0')
+def projects_by_scm_type(since):
+    counts = dict(
+        (t[0] or 'manual', 0)
+        for t in models.Project.SCM_TYPE_CHOICES
+    )
+    for result in models.Project.objects.values('scm_type').annotate(
+        count=Count('scm_type')
+    ).order_by('scm_type'):
+        counts[result['scm_type'] or 'manual'] = result['count']
+    return counts
+
+
+def _get_isolated_datetime(last_check):
+    if last_check:
+        return last_check.isoformat()
+    return last_check
+
+
+@register('instance_info', '1.0')
+def instance_info(since, include_hostnames=False):
+    info = {}
+    instances = models.Instance.objects.values_list('hostname').values(
+        'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled')
+    for instance in instances:
+        consumed_capacity = sum(x.task_impact for x in models.UnifiedJob.objects.filter(execution_node=instance['hostname'],
+                                status__in=('running', 'waiting')))
+        instance_info = {
+            'uuid': instance['uuid'],
+            'version': instance['version'],
+            'capacity': instance['capacity'],
+            'cpu': instance['cpu'],
+            'memory': instance['memory'],
+            'managed_by_policy': instance['managed_by_policy'],
+            'last_isolated_check': _get_isolated_datetime(instance['last_isolated_check']),
+            'enabled': instance['enabled'],
+            'consumed_capacity': consumed_capacity,
+            'remaining_capacity': instance['capacity'] - consumed_capacity
+        }
+        if include_hostnames is True:
+            instance_info['hostname'] = instance['hostname']
+        info[instance['uuid']] = instance_info
+    return info
+
+
+@register('job_counts', '1.0')
+def job_counts(since):
+    counts = {}
+    counts['total_jobs'] = models.UnifiedJob.objects.exclude(launch_type='sync').count()
+    counts['status'] = dict(models.UnifiedJob.objects.exclude(launch_type='sync').values_list('status').annotate(Count('status')).order_by())
+    counts['launch_type'] = dict(models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
+        'launch_type').annotate(Count('launch_type')).order_by())
+    return counts
+    
+    
+@register('job_instance_counts', '1.0')
+def job_instance_counts(since):
+    counts = {}
+    job_types = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
+        'execution_node', 'launch_type').annotate(job_launch_type=Count('launch_type')).order_by()
+    for job in job_types:
+        counts.setdefault(job[0], {}).setdefault('launch_type', {})[job[1]] = job[2]
+        
+    job_statuses = models.UnifiedJob.objects.exclude(launch_type='sync').values_list(
+        'execution_node', 'status').annotate(job_status=Count('status')).order_by()
+    for job in job_statuses:
+        counts.setdefault(job[0], {}).setdefault('status', {})[job[1]] = job[2]
+    return counts
+
+
+@register('query_info', '1.0')
+def query_info(since, collection_type):
+    query_info = {}
+    query_info['last_run'] = str(since)
+    query_info['current_time'] = str(now())
+    query_info['collection_type'] = collection_type
+    return query_info
+
+
+# Copies Job Events from db to a .csv to be shipped
+@table_version('events_table.csv', '1.0')
+@table_version('unified_jobs_table.csv', '1.0')
+@table_version('unified_job_template_table.csv', '1.0')
+def copy_tables(since, full_path):
+    def _copy_table(table, query, path):
+        file_path = os.path.join(path, table + '_table.csv')
+        file = open(file_path, 'w', encoding='utf-8')
+        with connection.cursor() as cursor:
+            cursor.copy_expert(query, file)
+            file.close()
+        return file_path
+
+    events_query = '''COPY (SELECT main_jobevent.id, 
+                              main_jobevent.created,
+                              main_jobevent.uuid,
+                              main_jobevent.parent_uuid,
+                              main_jobevent.event, 
+                              main_jobevent.event_data::json->'task_action' AS task_action,
+                              main_jobevent.failed, 
+                              main_jobevent.changed, 
+                              main_jobevent.playbook, 
+                              main_jobevent.play,
+                              main_jobevent.task,
+                              main_jobevent.role, 
+                              main_jobevent.job_id, 
+                              main_jobevent.host_id, 
+                              main_jobevent.host_name
+                              FROM main_jobevent 
+                              WHERE main_jobevent.created > {}
+                              ORDER BY main_jobevent.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))
+    _copy_table(table='events', query=events_query, path=full_path)
+
+    unified_job_query = '''COPY (SELECT main_unifiedjob.id,
+                                 main_unifiedjob.polymorphic_ctype_id,
+                                 django_content_type.model,
+                                 main_project.organization_id,
+                                 main_organization.name as organization_name,
+                                 main_unifiedjob.created,  
+                                 main_unifiedjob.name,  
+                                 main_unifiedjob.unified_job_template_id, 
+                                 main_unifiedjob.launch_type, 
+                                 main_unifiedjob.schedule_id, 
+                                 main_unifiedjob.execution_node, 
+                                 main_unifiedjob.controller_node, 
+                                 main_unifiedjob.cancel_flag, 
+                                 main_unifiedjob.status, 
+                                 main_unifiedjob.failed, 
+                                 main_unifiedjob.started, 
+                                 main_unifiedjob.finished, 
+                                 main_unifiedjob.elapsed, 
+                                 main_unifiedjob.job_explanation, 
+                                 main_unifiedjob.instance_group_id
+                                 FROM main_unifiedjob
+                                 JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id
+                                 JOIN django_content_type ON main_unifiedjob.polymorphic_ctype_id = django_content_type.id
+                                 JOIN main_project ON main_project.unifiedjobtemplate_ptr_id = main_job.project_id
+                                 JOIN main_organization ON main_organization.id = main_project.organization_id
+                                 WHERE main_unifiedjob.created > {} 
+                                 AND main_unifiedjob.launch_type != 'sync'
+                                 ORDER BY main_unifiedjob.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))    
+    _copy_table(table='unified_jobs', query=unified_job_query, path=full_path)
+
+    unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id, 
+                                 main_unifiedjobtemplate.polymorphic_ctype_id,
+                                 django_content_type.model,
+                                 main_unifiedjobtemplate.created, 
+                                 main_unifiedjobtemplate.modified, 
+                                 main_unifiedjobtemplate.created_by_id, 
+                                 main_unifiedjobtemplate.modified_by_id, 
+                                 main_unifiedjobtemplate.name, 
+                                 main_unifiedjobtemplate.current_job_id, 
+                                 main_unifiedjobtemplate.last_job_id, 
+                                 main_unifiedjobtemplate.last_job_failed, 
+                                 main_unifiedjobtemplate.last_job_run, 
+                                 main_unifiedjobtemplate.next_job_run, 
+                                 main_unifiedjobtemplate.next_schedule_id, 
+                                 main_unifiedjobtemplate.status 
+                                 FROM main_unifiedjobtemplate, django_content_type
+                                 WHERE main_unifiedjobtemplate.polymorphic_ctype_id = django_content_type.id
+                                 ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER'''.format(since.strftime("'%Y-%m-%d %H:%M:%S'"))    
+    _copy_table(table='unified_job_template', query=unified_job_template_query, path=full_path)
+    return

awx/main/analytics/core.py

@@ -0,0 +1,187 @@
+import inspect
+import json
+import logging
+import os
+import os.path
+import tempfile
+import shutil
+import requests
+
+from django.conf import settings
+from django.utils.timezone import now, timedelta
+from rest_framework.exceptions import PermissionDenied
+
+from awx.conf.license import get_license
+from awx.main.models import Job
+from awx.main.access import access_registry
+from awx.main.models.ha import TowerAnalyticsState
+from awx.main.utils import get_awx_http_client_headers
+
+
+__all__ = ['register', 'gather', 'ship', 'table_version']
+
+
+logger = logging.getLogger('awx.main.analytics')
+
+manifest = dict()
+
+
+def _valid_license():
+    try:
+        if get_license(show_key=False).get('license_type', 'UNLICENSED') == 'open':
+            return False
+        access_registry[Job](None).check_license()
+    except PermissionDenied:
+        logger.exception("A valid license was not found:")
+        return False
+    return True
+
+
+def register(key, version):
+    """
+    A decorator used to register a function as a metric collector.
+
+    Decorated functions should return JSON-serializable objects.
+
+    @register('projects_by_scm_type', 1)
+    def projects_by_scm_type():
+        return {'git': 5, 'svn': 1, 'hg': 0}
+    """
+
+    def decorate(f):
+        f.__awx_analytics_key__ = key
+        f.__awx_analytics_version__ = version
+        return f
+
+    return decorate
+
+
+def table_version(file_name, version):
+
+    global manifest
+    manifest[file_name] = version
+
+    def decorate(f):
+        return f
+
+    return decorate
+
+
+def gather(dest=None, module=None, collection_type='scheduled'):
+    """
+    Gather all defined metrics and write them as JSON files in a .tgz
+
+    :param dest:    the (optional) absolute path to write a compressed tarball
+    :pararm module: the module to search for registered analytic collector
+                    functions; defaults to awx.main.analytics.collectors
+    """
+
+    run_now = now()
+    state = TowerAnalyticsState.get_solo()
+    last_run = state.last_run
+    logger.debug("Last analytics run was: {}".format(last_run))
+    
+    max_interval = now() - timedelta(weeks=4)
+    if last_run < max_interval or not last_run:
+        last_run = max_interval
+
+    if _valid_license() is False:
+        logger.exception("Invalid License provided, or No License Provided")
+        return "Error: Invalid License provided, or No License Provided"
+
+    if collection_type != 'dry-run' and not settings.INSIGHTS_TRACKING_STATE:
+        logger.error("Automation Analytics not enabled. Use --dry-run to gather locally without sending.")
+        return
+
+    if module is None:
+        from awx.main.analytics import collectors
+        module = collectors
+
+
+    dest = dest or tempfile.mkdtemp(prefix='awx_analytics')
+    for name, func in inspect.getmembers(module):
+        if inspect.isfunction(func) and hasattr(func, '__awx_analytics_key__'):
+            key = func.__awx_analytics_key__
+            manifest['{}.json'.format(key)] = func.__awx_analytics_version__
+            path = '{}.json'.format(os.path.join(dest, key))
+            with open(path, 'w', encoding='utf-8') as f:
+                try:
+                    if func.__name__ == 'query_info':
+                        json.dump(func(last_run, collection_type=collection_type), f)
+                    else:
+                        json.dump(func(last_run), f)
+                except Exception:
+                    logger.exception("Could not generate metric {}.json".format(key))
+                    f.close()
+                    os.remove(f.name)
+    
+    path = os.path.join(dest, 'manifest.json')
+    with open(path, 'w', encoding='utf-8') as f:
+        try:
+            json.dump(manifest, f)
+        except Exception:
+            logger.exception("Could not generate manifest.json")
+            f.close()
+            os.remove(f.name)
+
+    try:
+        collectors.copy_tables(since=last_run, full_path=dest)
+    except Exception:
+        logger.exception("Could not copy tables")
+        
+    # can't use isoformat() since it has colons, which GNU tar doesn't like
+    tarname = '_'.join([
+        settings.SYSTEM_UUID,
+        run_now.strftime('%Y-%m-%d-%H%M%S%z')
+    ])
+    tgz = shutil.make_archive(
+        os.path.join(os.path.dirname(dest), tarname),
+        'gztar',
+        dest
+    )
+    shutil.rmtree(dest)
+    return tgz
+
+
+def ship(path):
+    """
+    Ship gathered metrics to the Insights API
+    """
+    if not path:
+        logger.error('Automation Analytics TAR not found')
+        return
+    if "Error:" in str(path):
+        return
+    try:
+        logger.debug('shipping analytics file: {}'.format(path))
+        url = getattr(settings, 'AUTOMATION_ANALYTICS_URL', None)
+        if not url:
+            logger.error('AUTOMATION_ANALYTICS_URL is not set')
+            return
+        rh_user = getattr(settings, 'REDHAT_USERNAME', None)
+        rh_password = getattr(settings, 'REDHAT_PASSWORD', None)
+        if not rh_user:
+            return logger.error('REDHAT_USERNAME is not set')
+        if not rh_password:
+            return logger.error('REDHAT_PASSWORD is not set')
+        with open(path, 'rb') as f:
+            files = {'file': (os.path.basename(path), f, settings.INSIGHTS_AGENT_MIME)}
+            s = requests.Session()
+            s.headers = get_awx_http_client_headers()
+            s.headers.pop('Content-Type')
+            response = s.post(url, 
+                              files=files,
+                              verify="/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
+                              auth=(rh_user, rh_password),
+                              headers=s.headers,
+                              timeout=(31, 31))
+            if response.status_code != 202:
+                return logger.exception('Upload failed with status {}, {}'.format(response.status_code,
+                                                                                  response.text))
+        run_now = now()
+        state = TowerAnalyticsState.get_solo()
+        state.last_run = run_now
+        state.save()
+    finally:
+        # cleanup tar.gz
+        os.remove(path)

awx/main/analytics/metrics.py

@@ -0,0 +1,134 @@
+from django.conf import settings
+from prometheus_client import (
+    REGISTRY,
+    PROCESS_COLLECTOR,
+    PLATFORM_COLLECTOR,
+    GC_COLLECTOR,
+    Gauge,
+    Info,
+    generate_latest
+)
+
+from awx.conf.license import get_license
+from awx.main.utils import (get_awx_version, get_ansible_version)
+from awx.main.analytics.collectors import (
+    counts, 
+    instance_info,
+    job_instance_counts,
+    job_counts,
+)
+
+
+REGISTRY.unregister(PROCESS_COLLECTOR)
+REGISTRY.unregister(PLATFORM_COLLECTOR)
+REGISTRY.unregister(GC_COLLECTOR)
+
+SYSTEM_INFO = Info('awx_system', 'AWX System Information')
+ORG_COUNT = Gauge('awx_organizations_total', 'Number of organizations')
+USER_COUNT = Gauge('awx_users_total', 'Number of users')
+TEAM_COUNT = Gauge('awx_teams_total', 'Number of teams')
+INV_COUNT = Gauge('awx_inventories_total', 'Number of inventories')
+PROJ_COUNT = Gauge('awx_projects_total', 'Number of projects')
+JT_COUNT = Gauge('awx_job_templates_total', 'Number of job templates')
+WFJT_COUNT = Gauge('awx_workflow_job_templates_total', 'Number of workflow job templates')
+HOST_COUNT = Gauge('awx_hosts_total', 'Number of hosts', ['type',])
+SCHEDULE_COUNT = Gauge('awx_schedules_total', 'Number of schedules')
+INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scripts')
+USER_SESSIONS = Gauge('awx_sessions_total', 'Number of sessions', ['type',])
+CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs')
+RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system')
+PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the Tower system')
+STATUS = Gauge('awx_status_total', 'Status of Job launched', ['status',])
+
+INSTANCE_CAPACITY = Gauge('awx_instance_capacity', 'Capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_CPU = Gauge('awx_instance_cpu', 'CPU cores on each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_MEMORY = Gauge('awx_instance_memory', 'RAM (Kb) on each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_INFO = Info('awx_instance', 'Info about each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_LAUNCH_TYPE = Gauge('awx_instance_launch_type_total', 'Type of Job launched', ['node', 'launch_type',])
+INSTANCE_STATUS = Gauge('awx_instance_status_total', 'Status of Job launched', ['node', 'status',])
+INSTANCE_CONSUMED_CAPACITY = Gauge('awx_instance_consumed_capacity', 'Consumed capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
+INSTANCE_REMAINING_CAPACITY = Gauge('awx_instance_remaining_capacity', 'Remaining capacity of each node in a Tower system', ['hostname', 'instance_uuid',])
+
+LICENSE_INSTANCE_TOTAL = Gauge('awx_license_instance_total', 'Total number of managed hosts provided by your license')
+LICENSE_INSTANCE_FREE = Gauge('awx_license_instance_free', 'Number of remaining managed hosts provided by your license')
+
+
+def metrics():
+    license_info = get_license(show_key=False)
+    SYSTEM_INFO.info({
+        'install_uuid': settings.INSTALL_UUID,
+        'insights_analytics': str(settings.INSIGHTS_TRACKING_STATE),
+        'tower_url_base': settings.TOWER_URL_BASE,
+        'tower_version': get_awx_version(),
+        'ansible_version': get_ansible_version(),
+        'license_type': license_info.get('license_type', 'UNLICENSED'),
+        'license_expiry': str(license_info.get('time_remaining', 0)),
+        'pendo_tracking': settings.PENDO_TRACKING_STATE,
+        'external_logger_enabled': str(settings.LOG_AGGREGATOR_ENABLED),
+        'external_logger_type': getattr(settings, 'LOG_AGGREGATOR_TYPE', 'None')
+    })
+
+    LICENSE_INSTANCE_TOTAL.set(str(license_info.get('available_instances', 0)))
+    LICENSE_INSTANCE_FREE.set(str(license_info.get('free_instances', 0)))
+
+    current_counts = counts(None)
+
+    ORG_COUNT.set(current_counts['organization'])
+    USER_COUNT.set(current_counts['user'])
+    TEAM_COUNT.set(current_counts['team'])
+    INV_COUNT.set(current_counts['inventory'])
+    PROJ_COUNT.set(current_counts['project'])
+    JT_COUNT.set(current_counts['job_template'])
+    WFJT_COUNT.set(current_counts['workflow_job_template'])
+
+    HOST_COUNT.labels(type='all').set(current_counts['host'])
+    HOST_COUNT.labels(type='active').set(current_counts['active_host_count'])
+
+    SCHEDULE_COUNT.set(current_counts['schedule'])
+    INV_SCRIPT_COUNT.set(current_counts['custom_inventory_script'])
+    CUSTOM_VENVS.set(current_counts['custom_virtualenvs'])
+
+    USER_SESSIONS.labels(type='all').set(current_counts['active_sessions'])
+    USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions'])
+    USER_SESSIONS.labels(type='anonymous').set(current_counts['active_anonymous_sessions'])
+
+    all_job_data = job_counts(None)
+    statuses = all_job_data.get('status', {})
+    for status, value in statuses.items():
+        STATUS.labels(status=status).set(value)
+
+    RUNNING_JOBS.set(current_counts['running_jobs'])
+    PENDING_JOBS.set(current_counts['pending_jobs'])
+
+    instance_data = instance_info(None, include_hostnames=True)
+    for uuid, info in instance_data.items():
+        hostname = info['hostname']
+        INSTANCE_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['capacity'])
+        INSTANCE_CPU.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['cpu'])
+        INSTANCE_MEMORY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['memory'])
+        INSTANCE_CONSUMED_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['consumed_capacity'])
+        INSTANCE_REMAINING_CAPACITY.labels(hostname=hostname, instance_uuid=uuid).set(instance_data[uuid]['remaining_capacity'])
+        INSTANCE_INFO.labels(hostname=hostname, instance_uuid=uuid).info({
+            'enabled': str(instance_data[uuid]['enabled']),
+            'last_isolated_check': getattr(instance_data[uuid], 'last_isolated_check', 'None'),
+            'managed_by_policy': str(instance_data[uuid]['managed_by_policy']),
+            'version': instance_data[uuid]['version']
+        })
+
+    instance_data = job_instance_counts(None)
+    for node in instance_data:
+        # skipping internal execution node (for system jobs) 
+        if node == '':
+            continue
+        types = instance_data[node].get('launch_type', {})
+        for launch_type, value in types.items():
+            INSTANCE_LAUNCH_TYPE.labels(node=node, launch_type=launch_type).set(value)
+        statuses = instance_data[node].get('status', {})
+        for status, value in statuses.items():
+            INSTANCE_STATUS.labels(node=node, status=status).set(value)
+
+
+    return generate_latest()
+
+
+__all__ = ['metrics']