Merge branch 'release_3.2.1' into devel

* release_3.2.1:
  fallback to empty dict when processing extra_data
  fix migration problem from 3.1.1
  move 0005a migration to 0005b
  feedback on ad hoc prohibited vars error msg
  Fix the way we include i18n files in sdist
  Fix migrations to support 3.1.2 -> 3.2.1+ upgrade path
  fix missing parameter to update_capacity method
  fix WARNING log when launching ad hoc command
  Validate against ansible variables on ad hoc launch
  do not allow ansible connection type of local for ad_hoc
  work around an ansible 2.4 inventory caching bug
  fix scan job migration unicode issue
  Assert isolated nodes have capacity set to 0 and restored based on version
  Set capacity to zero if the isolated node has an old version

MANIFEST.in

@@ -1,4 +1,6 @@
 recursive-include awx *.py
+recursive-include awx *.po
+recursive-include awx *.mo
 recursive-include awx/static *
 recursive-include awx/templates *.html
 recursive-include awx/api/templates *.md *.html

awx/api/serializers.py

@@ -42,7 +42,7 @@ from awx.main.fields import ImplicitRoleField
 from awx.main.utils import (
     get_type_for_model, get_model_for_type, timestamp_apiformat,
     camelcase_to_underscore, getattrd, parse_yaml_or_json,
-    has_model_field_prefetched)
+    has_model_field_prefetched, extract_ansible_vars)
 from awx.main.utils.filters import SmartFilter
 
 from awx.main.validators import vars_validate_or_raise
@@ -2749,6 +2749,14 @@ class AdHocCommandSerializer(UnifiedJobSerializer):
             ret['name'] = obj.module_name
         return ret
 
+    def validate_extra_vars(self, value):
+        redacted_extra_vars, removed_vars = extract_ansible_vars(value)
+        if removed_vars:
+            raise serializers.ValidationError(_(
+                "{} are prohibited from use in ad hoc commands."
+            ).format(", ".join(removed_vars)))
+        return vars_validate_or_raise(value)
+
 
 class AdHocCommandCancelSerializer(AdHocCommandSerializer):
 

awx/api/views.py

@@ -68,7 +68,7 @@ from awx.conf.license import get_license, feature_enabled, feature_exists, Licen
 from awx.main.models import * # noqa
 from awx.main.utils import * # noqa
 from awx.main.utils import (
-    callback_filter_out_ansible_extra_vars,
+    extract_ansible_vars,
     decrypt_field,
 )
 from awx.main.utils.filters import SmartFilter
@@ -3112,7 +3112,8 @@ class JobTemplateCallback(GenericAPIView):
         # Everything is fine; actually create the job.
         kv = {"limit": limit, "launch_type": 'callback'}
         if extra_vars is not None and job_template.ask_variables_on_launch:
-            kv['extra_vars'] = callback_filter_out_ansible_extra_vars(extra_vars)
+            extra_vars_redacted, removed = extract_ansible_vars(extra_vars)
+            kv['extra_vars'] = extra_vars_redacted
         with transaction.atomic():
             job = job_template.create_job(**kv)
 

awx/main/expect/isolated_manager.py

@@ -9,6 +9,7 @@ import stat
 import tempfile
 import time
 import logging
+from distutils.version import LooseVersion as Version
 
 from django.conf import settings
 
@@ -370,7 +371,24 @@ class IsolatedManager(object):
             logger.warning('Isolated job {} cleanup error, output:\n{}'.format(self.instance.id, output))
 
     @classmethod
-    def health_check(cls, instance_qs):
+    def update_capacity(cls, instance, task_result, awx_application_version):
+        instance.version = task_result['version']
+
+        isolated_version = instance.version.split("-", 1)[0]
+        cluster_version = awx_application_version.split("-", 1)[0]
+
+        if Version(cluster_version) > Version(isolated_version):
+            err_template = "Isolated instance {} reports version {}, cluster node is at {}, setting capacity to zero."
+            logger.error(err_template.format(instance.hostname, instance.version, awx_application_version))
+            instance.capacity = 0
+        else:
+            if instance.capacity == 0 and task_result['capacity']:
+                logger.warning('Isolated instance {} has re-joined.'.format(instance.hostname))
+            instance.capacity = int(task_result['capacity'])
+        instance.save(update_fields=['capacity', 'version', 'modified'])
+
+    @classmethod
+    def health_check(cls, instance_qs, awx_application_version):
         '''
         :param instance_qs:         List of Django objects representing the
                                     isolated instances to manage
@@ -412,11 +430,7 @@ class IsolatedManager(object):
             except (KeyError, IndexError):
                 task_result = {}
             if 'capacity' in task_result:
-                instance.version = task_result['version']
+                cls.update_capacity(instance, task_result, awx_application_version)
-                if instance.capacity == 0 and task_result['capacity']:
-                    logger.warning('Isolated instance {} has re-joined.'.format(instance.hostname))
-                instance.capacity = int(task_result['capacity'])
-                instance.save(update_fields=['capacity', 'version', 'modified'])
             elif instance.capacity == 0:
                 logger.debug('Isolated instance {} previously marked as lost, could not re-join.'.format(
                     instance.hostname))

awx/main/migrations/0005_squashed_v310_v313_updates.py

@@ -12,8 +12,6 @@ class Migration(migrations.Migration):
 
     replaces = [
         (b'main', '0035_v310_remove_tower_settings'),
-        (b'main', '0036_v311_insights'),
-        (b'main', '0037_v313_instance_version'),
     ]
 
     operations = [
@@ -36,11 +34,4 @@ class Migration(migrations.Migration):
             name='scm_type',
             field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'),
         ),
-
-        migrations.AddField(
-            model_name='instance',
-            name='version',
-            field=models.CharField(max_length=24, blank=True),
-        ),
-
     ]

awx/main/migrations/0005a_squashed_v310_v313_updates.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0005_squashed_v310_v313_updates'),
+    ]
+
+    replaces = [
+        (b'main', '0036_v311_insights'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='scm_type',
+            field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'),
+        ),
+        migrations.AlterField(
+            model_name='projectupdate',
+            name='scm_type',
+            field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'),
+        ),
+    ]

awx/main/migrations/0005b_squashed_v310_v313_updates.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0005a_squashed_v310_v313_updates'),
+    ]
+
+    replaces = [
+        (b'main', '0037_v313_instance_version'),
+    ]
+
+    operations = [
+        # Remove Tower settings, these settings are now in separate awx.conf app.
+        migrations.AddField(
+            model_name='instance',
+            name='version',
+            field=models.CharField(max_length=24, blank=True),
+        ),
+    ]

awx/main/migrations/0006_v320_release.py

@@ -18,7 +18,7 @@ from awx.main.models import Host
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0005_squashed_v310_v313_updates'),
+        ('main', '0005b_squashed_v310_v313_updates'),
     ]
 
     operations = [

awx/main/migrations/_scan_jobs.py

@@ -12,7 +12,7 @@ logger = logging.getLogger('awx.main.migrations')
 
 def _create_fact_scan_project(ContentType, Project, org):
     ct = ContentType.objects.get_for_model(Project)
-    name = "Tower Fact Scan - {}".format(org.name if org else "No Organization")
+    name = u"Tower Fact Scan - {}".format(org.name if org else "No Organization")
     proj = Project(name=name, 
                    scm_url='https://github.com/ansible/awx-facts-playbooks',
                    scm_type='git',

awx/main/models/unified_jobs.py

@@ -34,7 +34,7 @@ from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin
 from awx.main.utils import (
     decrypt_field, _inventory_updates,
     copy_model_by_class, copy_m2m_relationships,
-    get_type_for_model
+    get_type_for_model, parse_yaml_or_json
 )
 from awx.main.redact import UriCleaner, REPLACE_STR
 from awx.main.consumers import emit_channel_notification
@@ -878,21 +878,14 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         return []
 
     def handle_extra_data(self, extra_data):
-        if hasattr(self, 'extra_vars'):
+        if hasattr(self, 'extra_vars') and extra_data:
-            extra_vars = {}
+            extra_data_dict = {}
-            if isinstance(extra_data, dict):
+            try:
-                extra_vars = extra_data
+                extra_data_dict = parse_yaml_or_json(extra_data, silent_failure=False)
-            elif extra_data is None:
+            except Exception as e:
-                return
+                logger.warn("Exception deserializing extra vars: " + str(e))
-            else:
-                if extra_data == "":
-                    return
-                try:
-                    extra_vars = json.loads(extra_data)
-                except Exception as e:
-                    logger.warn("Exception deserializing extra vars: " + str(e))
             evars = self.extra_vars_dict
-            evars.update(extra_vars)
+            evars.update(extra_data_dict)
             self.update_fields(extra_vars=json.dumps(evars))
 
     @property

awx/main/tasks.py

@@ -56,7 +56,7 @@ from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field,
                             check_proot_installed, build_proot_temp_dir, get_licenser,
                             wrap_args_with_proot, get_system_task_capacity, OutputEventFilter,
                             parse_yaml_or_json, ignore_inventory_computed_fields, ignore_inventory_group_removal,
-                            get_type_for_model)
+                            get_type_for_model, extract_ansible_vars)
 from awx.main.utils.reload import restart_local_services, stop_local_services
 from awx.main.utils.handlers import configure_external_logger
 from awx.main.consumers import emit_channel_notification
@@ -283,7 +283,7 @@ def awx_isolated_heartbeat(self):
     # Slow pass looping over isolated IGs and their isolated instances
     if len(isolated_instance_qs) > 0:
         logger.debug("Managing isolated instances {}.".format(','.join([inst.hostname for inst in isolated_instance_qs])))
-        isolated_manager.IsolatedManager.health_check(isolated_instance_qs)
+        isolated_manager.IsolatedManager.health_check(isolated_instance_qs, awx_application_version)
 
 
 @task(bind=True, queue='tower', base=LogErrorsTask)
@@ -683,7 +683,14 @@ class BaseTask(LogErrorsTask):
                 json_data = json.dumps(instance.inventory.get_script_data(hostvars=True))
                 f.write('#! /usr/bin/env python\n# -*- coding: utf-8 -*-\nprint """%s"""\n' % json_data)
                 os.chmod(path, stat.S_IRUSR | stat.S_IXUSR)
-        return path
+            return path
+        else:
+            # work around an inventory caching bug in Ansible 2.4.0
+            # see: https://github.com/ansible/ansible/pull/30817
+            # see: https://github.com/ansible/awx/issues/246
+            inventory_script = tempfile.mktemp(suffix='.awxrest.py', dir=kwargs['private_data_dir'])
+            shutil.copy(plugin, inventory_script)
+            return inventory_script
 
     def build_args(self, instance, **kwargs):
         raise NotImplementedError
@@ -2108,6 +2115,12 @@ class RunAdHocCommand(BaseTask):
             args.append('-%s' % ('v' * min(5, ad_hoc_command.verbosity)))
 
         if ad_hoc_command.extra_vars_dict:
+            redacted_extra_vars, removed_vars = extract_ansible_vars(ad_hoc_command.extra_vars_dict)
+            if removed_vars:
+                raise ValueError(_(
+                    "{} are prohibited from use in ad hoc commands."
+                ).format(", ".join(removed_vars)))
+
             args.extend(['-e', json.dumps(ad_hoc_command.extra_vars_dict)])
 
         args.extend(['-m', ad_hoc_command.module_name])

awx/main/tests/functional/test_scan_jobs_migration.py

@@ -19,46 +19,46 @@ from awx.main.migrations._scan_jobs import _migrate_scan_job_templates
 
 @pytest.fixture
 def organizations():
-    return [Organization.objects.create(name="org-{}".format(x)) for x in range(3)]
+    return [Organization.objects.create(name=u"org-\xe9-{}".format(x)) for x in range(3)]
 
 
 @pytest.fixture
 def inventories(organizations):
-    return [Inventory.objects.create(name="inv-{}".format(x), 
+    return [Inventory.objects.create(name=u"inv-\xe9-{}".format(x), 
                                      organization=organizations[x]) for x in range(3)]
 
 
 @pytest.fixture
 def job_templates_scan(inventories):
-    return [JobTemplate.objects.create(name="jt-scan-{}".format(x), 
+    return [JobTemplate.objects.create(name=u"jt-\xe9-scan-{}".format(x), 
                                        job_type=PERM_INVENTORY_SCAN,
                                        inventory=inventories[x]) for x in range(3)]
 
 
 @pytest.fixture
 def job_templates_deploy(inventories):
-    return [JobTemplate.objects.create(name="jt-deploy-{}".format(x), 
+    return [JobTemplate.objects.create(name=u"jt-\xe9-deploy-{}".format(x), 
                                        job_type=PERM_INVENTORY_DEPLOY,
                                        inventory=inventories[x]) for x in range(3)]
 
 
 @pytest.fixture
 def project_custom(organizations):
-    return Project.objects.create(name="proj-scan_custom", 
+    return Project.objects.create(name=u"proj-\xe9-scan_custom", 
                                   scm_url='https://giggity.com',
                                   organization=organizations[0])
 
 
 @pytest.fixture
 def job_templates_custom_scan_project(project_custom):
-    return [JobTemplate.objects.create(name="jt-scan-custom-{}".format(x), 
+    return [JobTemplate.objects.create(name=u"jt-\xe9-scan-custom-{}".format(x), 
                                        project=project_custom, 
                                        job_type=PERM_INVENTORY_SCAN) for x in range(3)]
 
 
 @pytest.fixture
 def job_template_scan_no_org():
-    return JobTemplate.objects.create(name="jt-scan-no-org",
+    return JobTemplate.objects.create(name=u"jt-\xe9-scan-no-org",
                                       job_type=PERM_INVENTORY_SCAN)
 
 

awx/main/tests/functional/test_tasks.py

@@ -117,6 +117,28 @@ class TestIsolatedManagementTask:
         inst.save()
         return inst
 
+    @pytest.fixture
+    def old_version(self, control_group):
+        ig = InstanceGroup.objects.create(name='thepentagon', controller=control_group)
+        inst = ig.instances.create(hostname='isolated-old', capacity=103)
+        inst.save()
+        return inst
+
+    def test_old_version(self, control_instance, old_version):
+        update_capacity = isolated_manager.IsolatedManager.update_capacity
+
+        assert old_version.capacity == 103
+        with mock.patch('awx.main.tasks.settings', MockSettings()):
+            # Isolated node is reporting an older version than the cluster
+            # instance that issued the health check, set capacity to zero.
+            update_capacity(old_version, {'version': '1.0.0'}, '3.0.0')
+            assert old_version.capacity == 0
+
+            # Upgrade was completed, health check playbook now reports matching
+            # version, make sure capacity is set.
+            update_capacity(old_version, {'version': '5.0.0-things', 'capacity':103}, '5.0.0-stuff')
+            assert old_version.capacity == 103
+
     def test_takes_action(self, control_instance, needs_updating):
         original_isolated_instance = needs_updating.instances.all().first()
         with mock.patch('awx.main.tasks.settings', MockSettings()):

awx/main/tests/unit/models/test_survey_models.py

@@ -1,3 +1,5 @@
+import tempfile
+import pytest
 import json
 import tempfile
 
@@ -73,7 +75,7 @@ def test_job_safe_args_redacted_passwords(job):
     assert extra_vars['secret_key'] == '$encrypted$'
 
 
-def test_job_args_unredacted_passwords(job):
+def test_job_args_unredacted_passwords(job, tmpdir_factory):
     kwargs = {'ansible_version': '2.1', 'private_data_dir': tempfile.mkdtemp()}
     run_job = RunJob()
     args = run_job.build_args(job, **kwargs)

awx/main/tests/unit/utils/test_common.py

@@ -155,3 +155,12 @@ def test_memoize_parameter_error():
     with pytest.raises(common.IllegalArgumentError):
         fn()
 
+
+def test_extract_ansible_vars():
+    my_dict = {
+        "foobar": "baz",
+        "ansible_connetion_setting": "1928"
+    }
+    redacted, var_list = common.extract_ansible_vars(json.dumps(my_dict))
+    assert var_list == set(['ansible_connetion_setting'])
+    assert redacted == {"foobar": "baz"}

awx/main/utils/common.py

@@ -42,7 +42,7 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore',
            'ignore_inventory_computed_fields', 'ignore_inventory_group_removal',
            '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'NoDefaultProvided',
            'get_current_apps', 'set_current_apps', 'OutputEventFilter',
-           'callback_filter_out_ansible_extra_vars', 'get_search_fields', 'get_system_task_capacity',
+           'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity',
            'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict',
            'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest',
            'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError',]
@@ -904,13 +904,18 @@ class OutputEventFilter(object):
             self._current_event_data = None
 
 
-def callback_filter_out_ansible_extra_vars(extra_vars):
+def is_ansible_variable(key):
-    extra_vars_redacted = {}
+    return key.startswith('ansible_')
+
+
+def extract_ansible_vars(extra_vars):
     extra_vars = parse_yaml_or_json(extra_vars)
-    for key, value in extra_vars.iteritems():
+    ansible_vars = set([])
-        if not key.startswith('ansible_'):
+    for key in extra_vars.keys():
-            extra_vars_redacted[key] = value
+        if is_ansible_variable(key):
-    return extra_vars_redacted
+            extra_vars.pop(key)
+            ansible_vars.add(key)
+    return (extra_vars, ansible_vars)
 
 
 def get_search_fields(model):

setup.py

@@ -155,9 +155,7 @@ setup(
     },
     data_files = proc_data_files([
         ("%s" % homedir,        ["config/wsgi.py",
-                                 "awx/static/favicon.ico",
+                                 "awx/static/favicon.ico"]),
-                                 "awx/locale/*/LC_MESSAGES/*.po",
-                                 "awx/locale/*/LC_MESSAGES/*.mo"]),
         ("%s" % siteconfig,      ["config/awx-nginx.conf"]),
         #        ("%s" % webconfig,      ["config/uwsgi_params"]),
         ("%s" % sharedir,       ["tools/scripts/request_tower_configuration.sh","tools/scripts/request_tower_configuration.ps1"]),