Merge pull request #2451 from wenottingham/six-PRs-and-a-movie

Merge assorted PRs from the AWX community

awx/main/tasks.py

@@ -1889,6 +1889,7 @@ class RunInventoryUpdate(BaseTask):
 
             group_patterns = '[]'
             group_prefix = 'foreman_'
+            want_hostcollections = 'False'
             foreman_opts = dict(inventory_update.source_vars_dict.items())
             foreman_opts.setdefault('ssl_verify', 'False')
             for k, v in foreman_opts.items():
@@ -1896,6 +1897,8 @@ class RunInventoryUpdate(BaseTask):
                     group_patterns = v
                 elif k == 'satellite6_group_prefix' and isinstance(v, basestring):
                     group_prefix = v
+                elif k == 'satellite6_want_hostcollections' and isinstance(v, bool):
+                    want_hostcollections = v
                 else:
                     cp.set(section, k, six.text_type(v))
 
@@ -1908,6 +1911,7 @@ class RunInventoryUpdate(BaseTask):
             cp.add_section(section)
             cp.set(section, 'group_patterns', group_patterns)
             cp.set(section, 'want_facts', True)
+            cp.set(section, 'want_hostcollections', want_hostcollections)
             cp.set(section, 'group_prefix', group_prefix)
 
             section = 'cache'
@@ -1946,12 +1950,17 @@ class RunInventoryUpdate(BaseTask):
             cp.set(section, 'group_by_resource_group', 'yes')
             cp.set(section, 'group_by_location', 'yes')
             cp.set(section, 'group_by_tag', 'yes')
+
             if inventory_update.source_regions and 'all' not in inventory_update.source_regions:
                 cp.set(
                     section, 'locations',
                     ','.join([x.strip() for x in inventory_update.source_regions.split(',')])
                 )
 
+            azure_rm_opts = dict(inventory_update.source_vars_dict.items())
+            for k,v in azure_rm_opts.items():
+                cp.set(section, k, six.text_type(v))
+
         # Return INI content.
         if cp.sections():
             f = cStringIO.StringIO()

awx/main/tests/unit/test_tasks.py

@@ -1890,6 +1890,10 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             )
             return cred
         self.instance.get_cloud_credential = get_cred
+        self.instance.source_vars = {
+            'include_powerstate': 'yes',
+            'group_by_resource_group': 'no'
+        }
 
         def run_pexpect_side_effect(*args, **kwargs):
             args, cwd, env, stdout = args
@@ -1902,8 +1906,9 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             config = ConfigParser.ConfigParser()
             config.read(env['AZURE_INI_PATH'])
             assert config.get('azure', 'include_powerstate') == 'yes'
-            assert config.get('azure', 'group_by_resource_group') == 'yes'
+            assert config.get('azure', 'group_by_resource_group') == 'no'
             assert config.get('azure', 'group_by_location') == 'yes'
+            assert 'group_by_security_group' not in config.items('azure')
             assert config.get('azure', 'group_by_tag') == 'yes'
             assert config.get('azure', 'locations') == 'north,south,east,west'
             return ['successful', 0]
@@ -1930,6 +1935,11 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             )
             return cred
         self.instance.get_cloud_credential = get_cred
+        self.instance.source_vars = {
+            'include_powerstate': 'yes',
+            'group_by_resource_group': 'no',
+            'group_by_security_group': 'no'
+        }
 
         def run_pexpect_side_effect(*args, **kwargs):
             args, cwd, env, stdout = args
@@ -1941,8 +1951,9 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             config = ConfigParser.ConfigParser()
             config.read(env['AZURE_INI_PATH'])
             assert config.get('azure', 'include_powerstate') == 'yes'
-            assert config.get('azure', 'group_by_resource_group') == 'yes'
+            assert config.get('azure', 'group_by_resource_group') == 'no'
             assert config.get('azure', 'group_by_location') == 'yes'
+            assert config.get('azure', 'group_by_security_group') == 'no'
             assert config.get('azure', 'group_by_tag') == 'yes'
             assert 'locations' not in config.items('azure')
             return ['successful', 0]
@@ -2055,7 +2066,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             return cred
         self.instance.get_cloud_credential = get_cred
 
-        self.instance.source_vars = '{"satellite6_group_patterns": "[a,b,c]", "satellite6_group_prefix": "hey_"}'
+        self.instance.source_vars = '{"satellite6_group_patterns": "[a,b,c]", "satellite6_group_prefix": "hey_", "satellite6_want_hostcollections": True}'
 
         def run_pexpect_side_effect(*args, **kwargs):
             args, cwd, env, stdout = args
@@ -2066,6 +2077,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             assert config.get('foreman', 'password') == 'secret'
             assert config.get('ansible', 'group_patterns') == '[a,b,c]'
             assert config.get('ansible', 'group_prefix') == 'hey_'
+            assert config.get('ansible', 'want_hostcollections') == 'True'
             return ['successful', 0]
 
         self.run_pexpect.side_effect = run_pexpect_side_effect

awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js

@@ -164,7 +164,7 @@ export default ['$state', '$stateParams', '$scope', 'SourcesFormDefinition',
             else{
                 $scope.credentialBasePath = (source === 'ec2') ? GetBasePath('credentials') + '?kind=aws' : GetBasePath('credentials') + (source === '' ? '' : '?kind=' + (source));
             }
-            if (source === 'ec2' || source === 'custom' || source === 'vmware' || source === 'openstack' || source === 'scm' || source === 'cloudforms' || source === "satellite6") {
+            if (source === 'ec2' || source === 'custom' || source === 'vmware' || source === 'openstack' || source === 'scm' || source === 'cloudforms' || source === "satellite6" || source === "azure_rm") {
                 $scope.envParseType = 'yaml';
 
                 var varName;

awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js

@@ -66,7 +66,7 @@ export default ['$state', '$stateParams', '$scope', 'ParseVariableString',
             if (source === 'ec2' || source === 'custom' ||
                 source === 'vmware' || source === 'openstack' ||
                 source === 'scm' || source === 'cloudforms'  ||
-                source === 'satellite6') {
+                source === 'satellite6' || source === 'azure_rm') {
 
                 var varName;
                 if (source === 'scm') {

awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js

@@ -303,6 +303,30 @@ return {
             dataContainer: 'body',
             subForm: 'sourceSubForm'
         },
+        azure_rm_variables: {
+            id: 'azure_rm_variables',
+            label: i18n._('Source Variables'), //"{{vars_label}}" ,
+            ngShow: "source && source.value == 'azure_rm'",
+            type: 'textarea',
+            class: 'Form-textAreaLabel Form-formGroup--fullWidth',
+            rows: 6,
+            'default': '---',
+            parseTypeName: 'envParseType',
+            dataTitle: i18n._("Source Variables"),
+            dataPlacement: 'right',
+            awPopOver: "<p>" + i18n._("Override variables found in azure_rm.ini and used by the inventory update script. For a detailed description of these variables ") +
+                "<a href=\"https://github.com/ansible/ansible/blob/devel/contrib/inventory/azure_rm.ini\" target=\"_blank\">" +
+                i18n._("view azure_rm.ini in the Ansible github repo.") + "</a></p>" +
+                "<p>" + i18n._("Enter variables using either JSON or YAML syntax. Use the radio button to toggle between the two.") + "</p>" +
+                i18n._("JSON:") + "<br />\n" +
+                "<blockquote>{<br />&emsp;\"somevar\": \"somevalue\",<br />&emsp;\"password\": \"magic\"<br /> }</blockquote>\n" +
+                i18n._("YAML:") + "<br />\n" +
+                "<blockquote>---<br />somevar: somevalue<br />password: magic<br /></blockquote>\n" +
+                "<p>" + i18n._("View JSON examples at ") + '<a href="http://www.json.org" target="_blank">www.json.org</a></p>' +
+                "<p>" + i18n._("View YAML examples at ") + '<a href="http://docs.ansible.com/YAMLSyntax.html" target="_blank">docs.ansible.com</a></p>',
+            dataContainer: 'body',
+            subForm: 'sourceSubForm'
+        },
         verbosity: {
             label: i18n._('Verbosity'),
             type: 'select',

awx/ui/client/src/projects/projects.list.js

@@ -56,8 +56,7 @@ export default ['i18n', function(i18n) {
                 label: i18n._('Last Updated'),
                 filter: "longDate",
                 columnClass: "col-lg-3 hidden-md hidden-sm hidden-xs",
-                excludeModal: true,
+                excludeModal: true
-                nosort: true
             }
         },
 

tools/scripts/request_tower_configuration.ps1

@@ -1,7 +1,8 @@
 Param(
     [string]$tower_url,
     [string]$host_config_key,
-    [string]$job_template_id
+    [string]$job_template_id,
+    [string]$extra_vars
 )
 
 Set-StrictMode -Version 2
@@ -10,24 +11,36 @@ $ErrorActionPreference = "Stop"
 If(-not $tower_url -or -not $host_config_key -or -not $job_template_id)
 {
     Write-Host "Requests server configuration from Ansible Tower"
-    Write-Host "Usage: $($MyInvocation.MyCommand.Name) <server address>[:server port] <host config key> <job template id>"
+    Write-Host "Usage: $($MyInvocation.MyCommand.Name) https://<server address>[:server port] <host config key> <job template id>"
-    Write-Host "Example: $($MyInvocation.MyCommand.Name) example.towerhost.net 44d7507f2ead49af5fca80aa18fd24bc 38"
+    Write-Host "Example: $($MyInvocation.MyCommand.Name) https://example.towerhost.net 44d7507f2ead49af5fca80aa18fd24bc 38"
+    Write-Host "Example with extra_vars: $($MyInvocation.MyCommand.Name) https://example.towerhost.net 44d7507f2ead49af5fca80aa18fd24bc 38 '{ key: value, dict: { key: value }}'"
     Exit 1
 }
 
 $retry_attempts = 10
 $attempt = 0
 
-$data = @{
+If(-not $extra_vars)
-    host_config_key=$host_config_key
+{
+    $data = @{
+        host_config_key=$host_config_key
+    }
+} Else {
+    $data = @{
+        host_config_key=$host_config_key
+        extra_vars=$extra_vars
+    }
 }
 
 While ($attempt -lt $retry_attempts) {
     Try {
-        $resp = Invoke-WebRequest -Method POST -Body $data -Uri http://$tower_url/api/v1/job_templates/$job_template_id/callback/ -UseBasicParsing
+        $resp = Invoke-WebRequest -ContentType application/json -Method POST -Body (ConvertTo-Json $data) -Uri $tower_url/api/v2/job_templates/$job_template_id/callback/
 
-        If($resp.StatusCode -eq 202) {
+        If ($resp.StatusCode -match '^2[0-9]+$') {
             Exit 0
+        } ElseIf ($resp.StatusCode -eq 404) {
+            Write-Host "$resp.StatusCode received... encountered problem, halting"
+            Exit 1
         }
     }
     Catch {

tools/scripts/request_tower_configuration.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 fatal() {
-    if [ -n "${2}" ]; then
+  if [ -n "${2}" ]; then
-        echo -e "Error: ${2}"
+    echo -e "Error: ${2}"
-    fi
+  fi
-    exit ${1}
+  exit ${1}
 }
 
 usage() {
@@ -29,31 +29,31 @@ INSECURE=""
 # Parse arguments
 while getopts “hks:c:t:s:e:” OPTION
 do
-     case ${OPTION} in
+  case ${OPTION} in
-         h)
+    h)
-             usage
+      usage
-             exit 1
+      exit 1
-             ;;
+      ;;
-         s)
+    s)
-             TOWER_SERVER=${OPTARG}
+      TOWER_SERVER=${OPTARG}
-             ;;
+      ;;
-         k)
+    k)
-             INSECURE="-k"
+      INSECURE="-k"
-             ;;
+      ;;
-         c)
+    c)
-             HOST_CFG_KEY=${OPTARG}
+      HOST_CFG_KEY=${OPTARG}
-             ;;
+      ;;
-         t)
+    t)
-             TEMPLATE_ID=${OPTARG}
+      TEMPLATE_ID=${OPTARG}
-             ;;
+      ;;
-         e)
+    e)
-             EXTRA_VARS=${OPTARG}
+      EXTRA_VARS=${OPTARG}
-             ;;
+      ;;
-         ?)
+    ?)
-             usage
+      usage
-             exit
+      exit
-             ;;
+      ;;
-     esac
+  esac
 done
 
 # Validate required arguments
@@ -65,21 +65,32 @@ test -z ${TEMPLATE_ID} && fatal 1 "Missing required -t argument"
 
 # Generate curl --data parameter
 if [ -n "${EXTRA_VARS}" ]; then
-    CURL_DATA="{\"host_config_key\": \"${HOST_CFG_KEY}\", \"extra_vars\": \"${EXTRA_VARS}\"}"
+  CURL_DATA="{\"host_config_key\": \"${HOST_CFG_KEY}\", \"extra_vars\": \"${EXTRA_VARS}\"}"
 else
-    CURL_DATA="{\"host_config_key\": \"${HOST_CFG_KEY}\"}"
+  CURL_DATA="{\"host_config_key\": \"${HOST_CFG_KEY}\"}"
 fi
 
-set -o pipefail
+# Success on any 2xx status received, failure on only 404 status received, retry any other status every min for up to 10 min
-HTTP_STATUS=$(curl ${INSECURE} -s -i -X POST -H 'Content-Type:application/json' --data "$CURL_DATA" ${TOWER_SERVER}/api/v1/job_templates/${TEMPLATE_ID}/callback/ 2>&1 | head -n1 | awk '{print $2}')
+RETRY_ATTEMPTS=10
-CURL_RC=$?
+ATTEMPT=0
-if [ ${CURL_RC} -ne 0 ]; then
+while [[ $ATTEMPT -lt $RETRY_ATTEMPTS ]]
+do
+  set -o pipefail
+  HTTP_STATUS=$(curl ${INSECURE} -s -i -X POST -H 'Content-Type:application/json' --data "$CURL_DATA" ${TOWER_SERVER}/api/v2/job_templates/${TEMPLATE_ID}/callback/ 2>&1 | head -n1 | awk '{print $2}')
+  CURL_RC=$?
+  if [ ${CURL_RC} -ne 0 ]; then
     fatal ${CURL_RC} "curl exited with ${CURL_RC}, halting."
-fi
+  fi
 
-# Extract http status code
+  # Extract http status code
-if [[ ${HTTP_STATUS} -ge 300 ]]; then
+  if [[ ${HTTP_STATUS} =~ ^2[0-9]+$ ]]; then
-    fatal 1 "${HTTP_STATUS} received, encountered problem, halting."
-else
     echo "Success: ${HTTP_STATUS} received."
-fi
+    break
+  elif [[ ${HTTP_STATUS} =~ ^404$ ]]; then
+    fatal 1 "Failed: ${HTTP_STATUS} received, encountered problem, halting."
+  else
+    ATTEMPT=$((ATTEMPT + 1))
+    echo "Failed: ${HTTP_STATUS} received, executing retry #${ATTEMPT} in 1 minute."
+    sleep 60
+  fi
+done