Refactors the project form

Adding ppc64le support parameters

.github/workflows/update_dependabot_prs.yml

@@ -19,8 +19,11 @@ jobs:
             GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
             OWNER: ${{ github.repository_owner }}
             REPO: ${{ github.event.repository.name }}
-            BRANCH: ${{github.event.pull_request.head.ref}}
-            PR: ${{github.event.pull_request}}
+            PR: ${{github.event.pull_request.number}}
+            PR_BODY: ${{github.event.pull_request.body}}
         run: |
-          gh pr checkout ${{ env.BRANCH }}
-          gh pr edit --body "${{ env.PR }}\nBug, Docs Fix or other nominal change"
+          gh pr checkout ${{ env.PR }}
+          echo "${{ env.PR_BODY }}" > my_pr_body.txt
+          echo "" >> my_pr_body.txt
+          echo "Bug, Docs Fix or other nominal change" >> my_pr_body.txt
+          gh pr edit ${{env.PR}} --body-file my_pr_body.txt

.yamllint

@@ -8,6 +8,8 @@ ignore: |
   awx/ui/test/e2e/tests/smoke-vars.yml
   awx/ui/node_modules
   tools/docker-compose/_sources
+  # django template files
+  awx/api/templates/instance_install_bundle/**
 
 extends: default
 

MANIFEST.in

@@ -3,7 +3,7 @@ recursive-include awx *.po
 recursive-include awx *.mo
 recursive-include awx/static *
 recursive-include awx/templates *.html
-recursive-include awx/api/templates *.md *.html
+recursive-include awx/api/templates *.md *.html *.yml
 recursive-include awx/ui/build *.html
 recursive-include awx/ui/build *
 recursive-include awx/playbooks *.yml

Makefile

@@ -379,7 +379,7 @@ clean-ui:
 	rm -rf $(UI_BUILD_FLAG_FILE)
 
 awx/ui/node_modules:
-	NODE_OPTIONS=--max-old-space-size=6144 $(NPM_BIN) --prefix awx/ui --loglevel warn ci
+	NODE_OPTIONS=--max-old-space-size=6144 $(NPM_BIN) --prefix awx/ui --loglevel warn --force ci
 
 $(UI_BUILD_FLAG_FILE):
 	$(MAKE) awx/ui/node_modules

awx/api/generics.py

@@ -63,7 +63,6 @@ __all__ = [
     'SubDetailAPIView',
     'ResourceAccessList',
     'ParentMixin',
-    'DeleteLastUnattachLabelMixin',
     'SubListAttachDetachAPIView',
     'CopyAPIView',
     'BaseUsersList',
@@ -98,7 +97,6 @@ class LoggedLoginView(auth_views.LoginView):
             current_user = UserSerializer(self.request.user)
             current_user = smart_str(JSONRenderer().render(current_user.data))
             current_user = urllib.parse.quote('%s' % current_user, '')
-            ret.set_cookie('current_user', current_user, secure=settings.SESSION_COOKIE_SECURE or None)
             ret.setdefault('X-API-Session-Cookie-Name', getattr(settings, 'SESSION_COOKIE_NAME', 'awx_sessionid'))
 
             return ret
@@ -775,28 +773,6 @@ class SubListAttachDetachAPIView(SubListCreateAttachDetachAPIView):
         return {'id': None}
 
 
-class DeleteLastUnattachLabelMixin(object):
-    """
-    Models for which you want the last instance to be deleted from the database
-    when the last disassociate is called should inherit from this class. Further,
-    the model should implement is_detached()
-    """
-
-    def unattach(self, request, *args, **kwargs):
-        (sub_id, res) = super(DeleteLastUnattachLabelMixin, self).unattach_validate(request)
-        if res:
-            return res
-
-        res = super(DeleteLastUnattachLabelMixin, self).unattach_by_id(request, sub_id)
-
-        obj = self.model.objects.get(id=sub_id)
-
-        if obj.is_detached():
-            obj.delete()
-
-        return res
-
-
 class SubDetailAPIView(ParentMixin, generics.RetrieveAPIView, GenericAPIView):
     pass
 

awx/api/serializers.py

@@ -615,7 +615,7 @@ class BaseSerializer(serializers.ModelSerializer, metaclass=BaseSerializerMetacl
     def validate(self, attrs):
         attrs = super(BaseSerializer, self).validate(attrs)
         try:
-            # Create/update a model instance and run it's full_clean() method to
+            # Create/update a model instance and run its full_clean() method to
             # do any validation implemented on the model class.
             exclusions = self.get_validation_exclusions(self.instance)
             obj = self.instance or self.Meta.model()
@@ -1680,6 +1680,7 @@ class InventorySerializer(LabelsListMixin, BaseSerializerWithVariables):
             'total_inventory_sources',
             'inventory_sources_with_failures',
             'pending_deletion',
+            'prevent_instance_group_fallback',
         )
 
     def get_related(self, obj):
@@ -2232,6 +2233,7 @@ class InventoryUpdateSerializer(UnifiedJobSerializer, InventorySourceOptionsSeri
             'source_project_update',
             'custom_virtualenv',
             'instance_group',
+            'scm_revision',
         )
 
     def get_related(self, obj):
@@ -2922,6 +2924,12 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
             'ask_verbosity_on_launch',
             'ask_inventory_on_launch',
             'ask_credential_on_launch',
+            'ask_execution_environment_on_launch',
+            'ask_labels_on_launch',
+            'ask_forks_on_launch',
+            'ask_job_slice_count_on_launch',
+            'ask_timeout_on_launch',
+            'ask_instance_groups_on_launch',
             'survey_enabled',
             'become_enabled',
             'diff_mode',
@@ -2930,6 +2938,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
             'job_slice_count',
             'webhook_service',
             'webhook_credential',
+            'prevent_instance_group_fallback',
         )
         read_only_fields = ('*', 'custom_virtualenv')
 
@@ -3184,7 +3193,7 @@ class JobRelaunchSerializer(BaseSerializer):
         return attrs
 
 
-class JobCreateScheduleSerializer(BaseSerializer):
+class JobCreateScheduleSerializer(LabelsListMixin, BaseSerializer):
 
     can_schedule = serializers.SerializerMethodField()
     prompts = serializers.SerializerMethodField()
@@ -3210,14 +3219,17 @@ class JobCreateScheduleSerializer(BaseSerializer):
         try:
             config = obj.launch_config
             ret = config.prompts_dict(display=True)
-            if 'inventory' in ret:
-                ret['inventory'] = self._summarize('inventory', ret['inventory'])
-            if 'credentials' in ret:
-                all_creds = [self._summarize('credential', cred) for cred in ret['credentials']]
-                ret['credentials'] = all_creds
+            for field_name in ('inventory', 'execution_environment'):
+                if field_name in ret:
+                    ret[field_name] = self._summarize(field_name, ret[field_name])
+            for field_name, singular in (('credentials', 'credential'), ('instance_groups', 'instance_group')):
+                if field_name in ret:
+                    ret[field_name] = [self._summarize(singular, obj) for obj in ret[field_name]]
+            if 'labels' in ret:
+                ret['labels'] = self._summary_field_labels(config)
             return ret
         except JobLaunchConfig.DoesNotExist:
-            return {'all': _('Unknown, job may have been ran before launch configurations were saved.')}
+            return {'all': _('Unknown, job may have been run before launch configurations were saved.')}
 
 
 class AdHocCommandSerializer(UnifiedJobSerializer):
@@ -3387,6 +3399,9 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
     limit = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
     scm_branch = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
 
+    skip_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+    job_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+
     class Meta:
         model = WorkflowJobTemplate
         fields = (
@@ -3405,6 +3420,11 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
             'webhook_service',
             'webhook_credential',
             '-execution_environment',
+            'ask_labels_on_launch',
+            'ask_skip_tags_on_launch',
+            'ask_tags_on_launch',
+            'skip_tags',
+            'job_tags',
         )
 
     def get_related(self, obj):
@@ -3448,7 +3468,7 @@ class WorkflowJobTemplateSerializer(JobTemplateMixin, LabelsListMixin, UnifiedJo
 
         # process char_prompts, these are not direct fields on the model
         mock_obj = self.Meta.model()
-        for field_name in ('scm_branch', 'limit'):
+        for field_name in ('scm_branch', 'limit', 'skip_tags', 'job_tags'):
             if field_name in attrs:
                 setattr(mock_obj, field_name, attrs[field_name])
                 attrs.pop(field_name)
@@ -3474,6 +3494,9 @@ class WorkflowJobSerializer(LabelsListMixin, UnifiedJobSerializer):
     limit = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
     scm_branch = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
 
+    skip_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+    job_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
+
     class Meta:
         model = WorkflowJob
         fields = (
@@ -3493,6 +3516,8 @@ class WorkflowJobSerializer(LabelsListMixin, UnifiedJobSerializer):
             'webhook_service',
             'webhook_credential',
             'webhook_guid',
+            'skip_tags',
+            'job_tags',
         )
 
     def get_related(self, obj):
@@ -3609,6 +3634,9 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
     skip_tags = serializers.CharField(allow_blank=True, allow_null=True, required=False, default=None)
     diff_mode = serializers.BooleanField(required=False, allow_null=True, default=None)
     verbosity = serializers.ChoiceField(allow_null=True, required=False, default=None, choices=VERBOSITY_CHOICES)
+    forks = serializers.IntegerField(required=False, allow_null=True, min_value=0, default=None)
+    job_slice_count = serializers.IntegerField(required=False, allow_null=True, min_value=0, default=None)
+    timeout = serializers.IntegerField(required=False, allow_null=True, default=None)
     exclude_errors = ()
 
     class Meta:
@@ -3624,13 +3652,21 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
             'skip_tags',
             'diff_mode',
             'verbosity',
+            'execution_environment',
+            'forks',
+            'job_slice_count',
+            'timeout',
         )
 
     def get_related(self, obj):
         res = super(LaunchConfigurationBaseSerializer, self).get_related(obj)
         if obj.inventory_id:
             res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory_id})
+        if obj.execution_environment_id:
+            res['execution_environment'] = self.reverse('api:execution_environment_detail', kwargs={'pk': obj.execution_environment_id})
+        res['labels'] = self.reverse('api:{}_labels_list'.format(get_type_for_model(self.Meta.model)), kwargs={'pk': obj.pk})
         res['credentials'] = self.reverse('api:{}_credentials_list'.format(get_type_for_model(self.Meta.model)), kwargs={'pk': obj.pk})
+        res['instance_groups'] = self.reverse('api:{}_instance_groups_list'.format(get_type_for_model(self.Meta.model)), kwargs={'pk': obj.pk})
         return res
 
     def _build_mock_obj(self, attrs):
@@ -4082,7 +4118,6 @@ class SystemJobEventSerializer(AdHocCommandEventSerializer):
 
 
 class JobLaunchSerializer(BaseSerializer):
-
     # Representational fields
     passwords_needed_to_start = serializers.ReadOnlyField()
     can_start_without_user_input = serializers.BooleanField(read_only=True)
@@ -4105,6 +4140,12 @@ class JobLaunchSerializer(BaseSerializer):
     skip_tags = serializers.CharField(required=False, write_only=True, allow_blank=True)
     limit = serializers.CharField(required=False, write_only=True, allow_blank=True)
     verbosity = serializers.ChoiceField(required=False, choices=VERBOSITY_CHOICES, write_only=True)
+    execution_environment = serializers.PrimaryKeyRelatedField(queryset=ExecutionEnvironment.objects.all(), required=False, write_only=True)
+    labels = serializers.PrimaryKeyRelatedField(many=True, queryset=Label.objects.all(), required=False, write_only=True)
+    forks = serializers.IntegerField(required=False, write_only=True, min_value=0)
+    job_slice_count = serializers.IntegerField(required=False, write_only=True, min_value=0)
+    timeout = serializers.IntegerField(required=False, write_only=True)
+    instance_groups = serializers.PrimaryKeyRelatedField(many=True, queryset=InstanceGroup.objects.all(), required=False, write_only=True)
 
     class Meta:
         model = JobTemplate
@@ -4132,6 +4173,12 @@ class JobLaunchSerializer(BaseSerializer):
             'ask_verbosity_on_launch',
             'ask_inventory_on_launch',
             'ask_credential_on_launch',
+            'ask_execution_environment_on_launch',
+            'ask_labels_on_launch',
+            'ask_forks_on_launch',
+            'ask_job_slice_count_on_launch',
+            'ask_timeout_on_launch',
+            'ask_instance_groups_on_launch',
             'survey_enabled',
             'variables_needed_to_start',
             'credential_needed_to_start',
@@ -4139,6 +4186,12 @@ class JobLaunchSerializer(BaseSerializer):
             'job_template_data',
             'defaults',
             'verbosity',
+            'execution_environment',
+            'labels',
+            'forks',
+            'job_slice_count',
+            'timeout',
+            'instance_groups',
         )
         read_only_fields = (
             'ask_scm_branch_on_launch',
@@ -4151,6 +4204,12 @@ class JobLaunchSerializer(BaseSerializer):
             'ask_verbosity_on_launch',
             'ask_inventory_on_launch',
             'ask_credential_on_launch',
+            'ask_execution_environment_on_launch',
+            'ask_labels_on_launch',
+            'ask_forks_on_launch',
+            'ask_job_slice_count_on_launch',
+            'ask_timeout_on_launch',
+            'ask_instance_groups_on_launch',
         )
 
     def get_credential_needed_to_start(self, obj):
@@ -4175,6 +4234,17 @@ class JobLaunchSerializer(BaseSerializer):
                     if cred.credential_type.managed and 'vault_id' in cred.credential_type.defined_fields:
                         cred_dict['vault_id'] = cred.get_input('vault_id', default=None)
                     defaults_dict.setdefault(field_name, []).append(cred_dict)
+            elif field_name == 'execution_environment':
+                if obj.execution_environment_id:
+                    defaults_dict[field_name] = {'id': obj.execution_environment.id, 'name': obj.execution_environment.name}
+                else:
+                    defaults_dict[field_name] = {}
+            elif field_name == 'labels':
+                for label in obj.labels.all():
+                    label_dict = {'id': label.id, 'name': label.name}
+                    defaults_dict.setdefault(field_name, []).append(label_dict)
+            elif field_name == 'instance_groups':
+                defaults_dict[field_name] = []
             else:
                 defaults_dict[field_name] = getattr(obj, field_name)
         return defaults_dict
@@ -4282,6 +4352,10 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
     scm_branch = serializers.CharField(required=False, write_only=True, allow_blank=True)
     workflow_job_template_data = serializers.SerializerMethodField()
 
+    labels = serializers.PrimaryKeyRelatedField(many=True, queryset=Label.objects.all(), required=False, write_only=True)
+    skip_tags = serializers.CharField(required=False, write_only=True, allow_blank=True)
+    job_tags = serializers.CharField(required=False, write_only=True, allow_blank=True)
+
     class Meta:
         model = WorkflowJobTemplate
         fields = (
@@ -4301,8 +4375,22 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
             'workflow_job_template_data',
             'survey_enabled',
             'ask_variables_on_launch',
+            'ask_labels_on_launch',
+            'labels',
+            'ask_skip_tags_on_launch',
+            'ask_tags_on_launch',
+            'skip_tags',
+            'job_tags',
+        )
+        read_only_fields = (
+            'ask_inventory_on_launch',
+            'ask_variables_on_launch',
+            'ask_skip_tags_on_launch',
+            'ask_labels_on_launch',
+            'ask_limit_on_launch',
+            'ask_scm_branch_on_launch',
+            'ask_tags_on_launch',
         )
-        read_only_fields = ('ask_inventory_on_launch', 'ask_variables_on_launch')
 
     def get_survey_enabled(self, obj):
         if obj:
@@ -4310,10 +4398,15 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
         return False
 
     def get_defaults(self, obj):
+
         defaults_dict = {}
         for field_name in WorkflowJobTemplate.get_ask_mapping().keys():
             if field_name == 'inventory':
                 defaults_dict[field_name] = dict(name=getattrd(obj, '%s.name' % field_name, None), id=getattrd(obj, '%s.pk' % field_name, None))
+            elif field_name == 'labels':
+                for label in obj.labels.all():
+                    label_dict = {"id": label.id, "name": label.name}
+                    defaults_dict.setdefault(field_name, []).append(label_dict)
             else:
                 defaults_dict[field_name] = getattr(obj, field_name)
         return defaults_dict
@@ -4322,6 +4415,7 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
         return dict(name=obj.name, id=obj.id, description=obj.description)
 
     def validate(self, attrs):
+
         template = self.instance
 
         accepted, rejected, errors = template._accept_or_ignore_job_kwargs(**attrs)
@@ -4339,6 +4433,7 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
         WFJT_inventory = template.inventory
         WFJT_limit = template.limit
         WFJT_scm_branch = template.scm_branch
+
         super(WorkflowJobLaunchSerializer, self).validate(attrs)
         template.extra_vars = WFJT_extra_vars
         template.inventory = WFJT_inventory
@@ -4730,6 +4825,8 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
         if isinstance(obj.unified_job_template, SystemJobTemplate):
             summary_fields['unified_job_template']['job_type'] = obj.unified_job_template.job_type
 
+        # We are not showing instance groups on summary fields because JTs don't either
+
         if 'inventory' in summary_fields:
             return summary_fields
 
@@ -4764,7 +4861,7 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
 class InstanceLinkSerializer(BaseSerializer):
     class Meta:
         model = InstanceLink
-        fields = ('source', 'target')
+        fields = ('source', 'target', 'link_state')
 
     source = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
     target = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
@@ -4773,63 +4870,80 @@ class InstanceLinkSerializer(BaseSerializer):
 class InstanceNodeSerializer(BaseSerializer):
     class Meta:
         model = Instance
-        fields = ('id', 'hostname', 'node_type', 'node_state')
-
-    node_state = serializers.SerializerMethodField()
-
-    def get_node_state(self, obj):
-        if not obj.enabled:
-            return "disabled"
-        return "error" if obj.errors else "healthy"
+        fields = ('id', 'hostname', 'node_type', 'node_state', 'enabled')
 
 
 class InstanceSerializer(BaseSerializer):
+    show_capabilities = ['edit']
 
     consumed_capacity = serializers.SerializerMethodField()
     percent_capacity_remaining = serializers.SerializerMethodField()
-    jobs_running = serializers.IntegerField(help_text=_('Count of jobs in the running or waiting state that ' 'are targeted for this instance'), read_only=True)
+    jobs_running = serializers.IntegerField(help_text=_('Count of jobs in the running or waiting state that are targeted for this instance'), read_only=True)
     jobs_total = serializers.IntegerField(help_text=_('Count of all jobs that target this instance'), read_only=True)
+    health_check_pending = serializers.SerializerMethodField()
 
     class Meta:
         model = Instance
-        read_only_fields = ('uuid', 'hostname', 'version', 'node_type')
+        read_only_fields = ('ip_address', 'uuid', 'version')
         fields = (
-            "id",
-            "type",
-            "url",
-            "related",
-            "uuid",
-            "hostname",
-            "created",
-            "modified",
-            "last_seen",
-            "last_health_check",
-            "errors",
+            'id',
+            'hostname',
+            'type',
+            'url',
+            'related',
+            'summary_fields',
+            'uuid',
+            'created',
+            'modified',
+            'last_seen',
+            'health_check_started',
+            'health_check_pending',
+            'last_health_check',
+            'errors',
             'capacity_adjustment',
-            "version",
-            "capacity",
-            "consumed_capacity",
-            "percent_capacity_remaining",
-            "jobs_running",
-            "jobs_total",
-            "cpu",
-            "memory",
-            "cpu_capacity",
-            "mem_capacity",
-            "enabled",
-            "managed_by_policy",
-            "node_type",
+            'version',
+            'capacity',
+            'consumed_capacity',
+            'percent_capacity_remaining',
+            'jobs_running',
+            'jobs_total',
+            'cpu',
+            'memory',
+            'cpu_capacity',
+            'mem_capacity',
+            'enabled',
+            'managed_by_policy',
+            'node_type',
+            'node_state',
+            'ip_address',
+            'listener_port',
         )
+        extra_kwargs = {
+            'node_type': {'initial': Instance.Types.EXECUTION, 'default': Instance.Types.EXECUTION},
+            'node_state': {'initial': Instance.States.INSTALLED, 'default': Instance.States.INSTALLED},
+        }
 
     def get_related(self, obj):
         res = super(InstanceSerializer, self).get_related(obj)
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
+        if settings.IS_K8S and obj.node_type in (Instance.Types.EXECUTION,):
+            res['install_bundle'] = self.reverse('api:instance_install_bundle', kwargs={'pk': obj.pk})
+        res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
             if obj.node_type != 'hop':
                 res['health_check'] = self.reverse('api:instance_health_check', kwargs={'pk': obj.pk})
         return res
 
+    def get_summary_fields(self, obj):
+        summary = super().get_summary_fields(obj)
+
+        # use this handle to distinguish between a listView and a detailView
+        if self.is_detail_view:
+            summary['links'] = InstanceLinkSerializer(InstanceLink.objects.select_related('target', 'source').filter(source=obj), many=True).data
+
+        return summary
+
     def get_consumed_capacity(self, obj):
         return obj.consumed_capacity
 
@@ -4839,10 +4953,54 @@ class InstanceSerializer(BaseSerializer):
         else:
             return float("{0:.2f}".format(((float(obj.capacity) - float(obj.consumed_capacity)) / (float(obj.capacity))) * 100))
 
-    def validate(self, attrs):
-        if self.instance.node_type == 'hop':
-            raise serializers.ValidationError(_('Hop node instances may not be changed.'))
-        return attrs
+    def get_health_check_pending(self, obj):
+        return obj.health_check_pending
+
+    def validate(self, data):
+        if self.instance:
+            if self.instance.node_type == Instance.Types.HOP:
+                raise serializers.ValidationError("Hop node instances may not be changed.")
+        else:
+            if not settings.IS_K8S:
+                raise serializers.ValidationError("Can only create instances on Kubernetes or OpenShift.")
+        return data
+
+    def validate_node_type(self, value):
+        if not self.instance:
+            if value not in (Instance.Types.EXECUTION,):
+                raise serializers.ValidationError("Can only create execution nodes.")
+        else:
+            if self.instance.node_type != value:
+                raise serializers.ValidationError("Cannot change node type.")
+
+        return value
+
+    def validate_node_state(self, value):
+        if self.instance:
+            if value != self.instance.node_state:
+                if not settings.IS_K8S:
+                    raise serializers.ValidationError("Can only change the state on Kubernetes or OpenShift.")
+                if value != Instance.States.DEPROVISIONING:
+                    raise serializers.ValidationError("Can only change instances to the 'deprovisioning' state.")
+                if self.instance.node_type not in (Instance.Types.EXECUTION,):
+                    raise serializers.ValidationError("Can only deprovision execution nodes.")
+        else:
+            if value and value != Instance.States.INSTALLED:
+                raise serializers.ValidationError("Can only create instances in the 'installed' state.")
+
+        return value
+
+    def validate_hostname(self, value):
+        if self.instance and self.instance.hostname != value:
+            raise serializers.ValidationError("Cannot change hostname.")
+
+        return value
+
+    def validate_listener_port(self, value):
+        if self.instance and self.instance.listener_port != value:
+            raise serializers.ValidationError("Cannot change listener port.")
+
+        return value
 
 
 class InstanceHealthCheckSerializer(BaseSerializer):

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -0,0 +1,21 @@
+receptor_verify: true
+receptor_tls: true
+receptor_work_commands:
+  ansible-runner:
+    command: ansible-runner
+    params: worker
+    allowruntimeparams: true
+    verifysignature: true
+custom_worksign_public_keyfile: receptor/work-public-key.pem
+custom_tls_certfile: receptor/tls/receptor.crt
+custom_tls_keyfile: receptor/tls/receptor.key
+custom_ca_certfile: receptor/tls/ca/receptor-ca.crt
+receptor_user: awx
+receptor_group: awx
+receptor_protocol: 'tcp'
+receptor_listener: true
+receptor_port: {{ instance.listener_port }}
+receptor_dependencies:
+  - podman
+  - crun
+  - python39-pip

awx/api/templates/instance_install_bundle/install_receptor.yml

@@ -0,0 +1,18 @@
+{% verbatim %}
+---
+- hosts: all
+  become: yes
+  tasks:
+    - name: Create the receptor user
+      user:
+        name: "{{ receptor_user }}"
+        shell: /bin/bash
+    - name: Enable Copr repo for Receptor
+      command: dnf copr enable ansible-awx/receptor -y
+    - import_role:
+        name: ansible.receptor.setup
+    - name: Install ansible-runner
+      pip:
+        name: ansible-runner
+        executable: pip3.9
+{% endverbatim %}

awx/api/templates/instance_install_bundle/inventory.yml

@@ -0,0 +1,7 @@
+---
+all:
+  hosts:
+    remote-execution:
+      ansible_host: {{ instance.hostname }}
+      ansible_user: <username> # user provided
+      ansible_ssh_private_key_file: ~/.ssh/id_rsa

awx/api/templates/instance_install_bundle/requirements.yml

@@ -0,0 +1,6 @@
+---
+collections:
+  - name: ansible.receptor
+    source: https://github.com/ansible/receptor-collection/
+    type: git
+    version: 0.1.1

awx/api/urls/instance.py

@@ -3,7 +3,15 @@
 
 from django.urls import re_path
 
-from awx.api.views import InstanceList, InstanceDetail, InstanceUnifiedJobsList, InstanceInstanceGroupsList, InstanceHealthCheck
+from awx.api.views import (
+    InstanceList,
+    InstanceDetail,
+    InstanceUnifiedJobsList,
+    InstanceInstanceGroupsList,
+    InstanceHealthCheck,
+    InstanceInstallBundle,
+    InstancePeersList,
+)
 
 
 urls = [
@@ -12,6 +20,8 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/jobs/$', InstanceUnifiedJobsList.as_view(), name='instance_unified_jobs_list'),
     re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', InstanceInstanceGroupsList.as_view(), name='instance_instance_groups_list'),
     re_path(r'^(?P<pk>[0-9]+)/health_check/$', InstanceHealthCheck.as_view(), name='instance_health_check'),
+    re_path(r'^(?P<pk>[0-9]+)/peers/$', InstancePeersList.as_view(), name='instance_peers_list'),
+    re_path(r'^(?P<pk>[0-9]+)/install_bundle/$', InstanceInstallBundle.as_view(), name='instance_install_bundle'),
 ]
 
 __all__ = ['urls']

awx/api/urls/label.py

@@ -3,7 +3,7 @@
 
 from django.urls import re_path
 
-from awx.api.views import LabelList, LabelDetail
+from awx.api.views.labels import LabelList, LabelDetail
 
 
 urls = [re_path(r'^$', LabelList.as_view(), name='label_list'), re_path(r'^(?P<pk>[0-9]+)/$', LabelDetail.as_view(), name='label_detail')]

awx/api/urls/schedule.py

@@ -3,7 +3,7 @@
 
 from django.urls import re_path
 
-from awx.api.views import ScheduleList, ScheduleDetail, ScheduleUnifiedJobsList, ScheduleCredentialsList
+from awx.api.views import ScheduleList, ScheduleDetail, ScheduleUnifiedJobsList, ScheduleCredentialsList, ScheduleLabelsList, ScheduleInstanceGroupList
 
 
 urls = [
@@ -11,6 +11,8 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/$', ScheduleDetail.as_view(), name='schedule_detail'),
     re_path(r'^(?P<pk>[0-9]+)/jobs/$', ScheduleUnifiedJobsList.as_view(), name='schedule_unified_jobs_list'),
     re_path(r'^(?P<pk>[0-9]+)/credentials/$', ScheduleCredentialsList.as_view(), name='schedule_credentials_list'),
+    re_path(r'^(?P<pk>[0-9]+)/labels/$', ScheduleLabelsList.as_view(), name='schedule_labels_list'),
+    re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', ScheduleInstanceGroupList.as_view(), name='schedule_instance_groups_list'),
 ]
 
 __all__ = ['urls']

awx/api/urls/workflow_job_node.py

@@ -10,6 +10,8 @@ from awx.api.views import (
     WorkflowJobNodeFailureNodesList,
     WorkflowJobNodeAlwaysNodesList,
     WorkflowJobNodeCredentialsList,
+    WorkflowJobNodeLabelsList,
+    WorkflowJobNodeInstanceGroupsList,
 )
 
 
@@ -20,6 +22,8 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/failure_nodes/$', WorkflowJobNodeFailureNodesList.as_view(), name='workflow_job_node_failure_nodes_list'),
     re_path(r'^(?P<pk>[0-9]+)/always_nodes/$', WorkflowJobNodeAlwaysNodesList.as_view(), name='workflow_job_node_always_nodes_list'),
     re_path(r'^(?P<pk>[0-9]+)/credentials/$', WorkflowJobNodeCredentialsList.as_view(), name='workflow_job_node_credentials_list'),
+    re_path(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobNodeLabelsList.as_view(), name='workflow_job_node_labels_list'),
+    re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', WorkflowJobNodeInstanceGroupsList.as_view(), name='workflow_job_node_instance_groups_list'),
 ]
 
 __all__ = ['urls']

awx/api/urls/workflow_job_template_node.py

@@ -11,6 +11,8 @@ from awx.api.views import (
     WorkflowJobTemplateNodeAlwaysNodesList,
     WorkflowJobTemplateNodeCredentialsList,
     WorkflowJobTemplateNodeCreateApproval,
+    WorkflowJobTemplateNodeLabelsList,
+    WorkflowJobTemplateNodeInstanceGroupsList,
 )
 
 
@@ -21,6 +23,8 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/failure_nodes/$', WorkflowJobTemplateNodeFailureNodesList.as_view(), name='workflow_job_template_node_failure_nodes_list'),
     re_path(r'^(?P<pk>[0-9]+)/always_nodes/$', WorkflowJobTemplateNodeAlwaysNodesList.as_view(), name='workflow_job_template_node_always_nodes_list'),
     re_path(r'^(?P<pk>[0-9]+)/credentials/$', WorkflowJobTemplateNodeCredentialsList.as_view(), name='workflow_job_template_node_credentials_list'),
+    re_path(r'^(?P<pk>[0-9]+)/labels/$', WorkflowJobTemplateNodeLabelsList.as_view(), name='workflow_job_template_node_labels_list'),
+    re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', WorkflowJobTemplateNodeInstanceGroupsList.as_view(), name='workflow_job_template_node_instance_groups_list'),
     re_path(r'^(?P<pk>[0-9]+)/create_approval_template/$', WorkflowJobTemplateNodeCreateApproval.as_view(), name='workflow_job_template_node_create_approval'),
 ]
 

awx/api/views/__init__.py

@@ -22,6 +22,7 @@ from django.conf import settings
 from django.core.exceptions import FieldError, ObjectDoesNotExist
 from django.db.models import Q, Sum
 from django.db import IntegrityError, ProgrammingError, transaction, connection
+from django.db.models.fields.related import ManyToManyField, ForeignKey
 from django.shortcuts import get_object_or_404
 from django.utils.safestring import mark_safe
 from django.utils.timezone import now
@@ -68,7 +69,6 @@ from awx.api.generics import (
     APIView,
     BaseUsersList,
     CopyAPIView,
-    DeleteLastUnattachLabelMixin,
     GenericAPIView,
     ListAPIView,
     ListCreateAPIView,
@@ -85,6 +85,7 @@ from awx.api.generics import (
     SubListCreateAttachDetachAPIView,
     SubListDestroyAPIView,
 )
+from awx.api.views.labels import LabelSubListCreateAttachDetachView
 from awx.api.versioning import reverse
 from awx.main import models
 from awx.main.utils import (
@@ -121,6 +122,22 @@ from awx.api.views.mixin import (
     UnifiedJobDeletionMixin,
     NoTruncateMixin,
 )
+from awx.api.views.instance_install_bundle import InstanceInstallBundle  # noqa
+from awx.api.views.inventory import (  # noqa
+    InventoryList,
+    InventoryDetail,
+    InventoryUpdateEventsList,
+    InventoryList,
+    InventoryDetail,
+    InventoryActivityStreamList,
+    InventoryInstanceGroupsList,
+    InventoryAccessList,
+    InventoryObjectRolesList,
+    InventoryJobTemplateList,
+    InventoryLabelList,
+    InventoryCopy,
+)
+from awx.api.views.mesh_visualizer import MeshVisualizer  # noqa
 from awx.api.views.organization import (  # noqa
     OrganizationList,
     OrganizationDetail,
@@ -144,21 +161,6 @@ from awx.api.views.organization import (  # noqa
     OrganizationAccessList,
     OrganizationObjectRolesList,
 )
-from awx.api.views.inventory import (  # noqa
-    InventoryList,
-    InventoryDetail,
-    InventoryUpdateEventsList,
-    InventoryList,
-    InventoryDetail,
-    InventoryActivityStreamList,
-    InventoryInstanceGroupsList,
-    InventoryAccessList,
-    InventoryObjectRolesList,
-    InventoryJobTemplateList,
-    InventoryLabelList,
-    InventoryCopy,
-)
-from awx.api.views.mesh_visualizer import MeshVisualizer  # noqa
 from awx.api.views.root import (  # noqa
     ApiRootView,
     ApiOAuthAuthorizationRootView,
@@ -173,7 +175,6 @@ from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, Gitlab
 from awx.api.pagination import UnifiedJobEventPagination
 from awx.main.utils import set_environ
 
-
 logger = logging.getLogger('awx.api.views')
 
 
@@ -358,7 +359,7 @@ class DashboardJobsGraphView(APIView):
         return Response(dashboard_data)
 
 
-class InstanceList(ListAPIView):
+class InstanceList(ListCreateAPIView):
 
     name = _("Instances")
     model = models.Instance
@@ -397,6 +398,17 @@ class InstanceUnifiedJobsList(SubListAPIView):
         return qs
 
 
+class InstancePeersList(SubListAPIView):
+
+    name = _("Instance Peers")
+    parent_model = models.Instance
+    model = models.Instance
+    serializer_class = serializers.InstanceSerializer
+    parent_access = 'read'
+    search_fields = {'hostname'}
+    relationship = 'peers'
+
+
 class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAttachDetachAPIView):
 
     name = _("Instance's Instance Groups")
@@ -439,40 +451,21 @@ class InstanceHealthCheck(GenericAPIView):
 
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
+        if obj.health_check_pending:
+            return Response({'msg': f"Health check was already in progress for {obj.hostname}."}, status=status.HTTP_200_OK)
 
-        if obj.node_type == 'execution':
+        # Note: hop nodes are already excluded by the get_queryset method
+        obj.health_check_started = now()
+        obj.save(update_fields=['health_check_started'])
+        if obj.node_type == models.Instance.Types.EXECUTION:
             from awx.main.tasks.system import execution_node_health_check
 
-            runner_data = execution_node_health_check(obj.hostname)
-            obj.refresh_from_db()
-            data = self.get_serializer(data=request.data).to_representation(obj)
-            # Add in some extra unsaved fields
-            for extra_field in ('transmit_timing', 'run_timing'):
-                if extra_field in runner_data:
-                    data[extra_field] = runner_data[extra_field]
+            execution_node_health_check.apply_async([obj.hostname])
         else:
             from awx.main.tasks.system import cluster_node_health_check
 
-            if settings.CLUSTER_HOST_ID == obj.hostname:
-                cluster_node_health_check(obj.hostname)
-            else:
-                cluster_node_health_check.apply_async([obj.hostname], queue=obj.hostname)
-                start_time = time.time()
-                prior_check_time = obj.last_health_check
-                while time.time() - start_time < 50.0:
-                    obj.refresh_from_db(fields=['last_health_check'])
-                    if obj.last_health_check != prior_check_time:
-                        break
-                    if time.time() - start_time < 1.0:
-                        time.sleep(0.1)
-                    else:
-                        time.sleep(1.0)
-                else:
-                    obj.mark_offline(errors=_('Health check initiated by user determined this instance to be unresponsive'))
-            obj.refresh_from_db()
-            data = self.get_serializer(data=request.data).to_representation(obj)
-
-        return Response(data, status=status.HTTP_200_OK)
+            cluster_node_health_check.apply_async([obj.hostname], queue=obj.hostname)
+        return Response({'msg': f"Health check is running for {obj.hostname}."}, status=status.HTTP_200_OK)
 
 
 class InstanceGroupList(ListCreateAPIView):
@@ -617,6 +610,19 @@ class ScheduleCredentialsList(LaunchConfigCredentialsBase):
     parent_model = models.Schedule
 
 
+class ScheduleLabelsList(LabelSubListCreateAttachDetachView):
+
+    parent_model = models.Schedule
+
+
+class ScheduleInstanceGroupList(SubListAttachDetachAPIView):
+
+    model = models.InstanceGroup
+    serializer_class = serializers.InstanceGroupSerializer
+    parent_model = models.Schedule
+    relationship = 'instance_groups'
+
+
 class ScheduleUnifiedJobsList(SubListAPIView):
 
     model = models.UnifiedJob
@@ -2381,10 +2387,13 @@ class JobTemplateLaunch(RetrieveAPIView):
             for field, ask_field_name in modified_ask_mapping.items():
                 if not getattr(obj, ask_field_name):
                     data.pop(field, None)
-                elif field == 'inventory':
+                elif isinstance(getattr(obj.__class__, field).field, ForeignKey):
                     data[field] = getattrd(obj, "%s.%s" % (field, 'id'), None)
-                elif field == 'credentials':
-                    data[field] = [cred.id for cred in obj.credentials.all()]
+                elif isinstance(getattr(obj.__class__, field).field, ManyToManyField):
+                    if field == 'instance_groups':
+                        data[field] = []
+                        continue
+                    data[field] = [item.id for item in getattr(obj, field).all()]
                 else:
                     data[field] = getattr(obj, field)
         return data
@@ -2397,9 +2406,8 @@ class JobTemplateLaunch(RetrieveAPIView):
         """
         modern_data = data.copy()
 
-        id_fd = '{}_id'.format('inventory')
-        if 'inventory' not in modern_data and id_fd in modern_data:
-            modern_data['inventory'] = modern_data[id_fd]
+        if 'inventory' not in modern_data and 'inventory_id' in modern_data:
+            modern_data['inventory'] = modern_data['inventory_id']
 
         # credential passwords were historically provided as top-level attributes
         if 'credential_passwords' not in modern_data:
@@ -2719,28 +2727,9 @@ class JobTemplateCredentialsList(SubListCreateAttachDetachAPIView):
         return super(JobTemplateCredentialsList, self).is_valid_relation(parent, sub, created)
 
 
-class JobTemplateLabelList(DeleteLastUnattachLabelMixin, SubListCreateAttachDetachAPIView):
+class JobTemplateLabelList(LabelSubListCreateAttachDetachView):
 
-    model = models.Label
-    serializer_class = serializers.LabelSerializer
     parent_model = models.JobTemplate
-    relationship = 'labels'
-
-    def post(self, request, *args, **kwargs):
-        # If a label already exists in the database, attach it instead of erroring out
-        # that it already exists
-        if 'id' not in request.data and 'name' in request.data and 'organization' in request.data:
-            existing = models.Label.objects.filter(name=request.data['name'], organization_id=request.data['organization'])
-            if existing.exists():
-                existing = existing[0]
-                request.data['id'] = existing.id
-                del request.data['name']
-                del request.data['organization']
-        if models.Label.objects.filter(unifiedjobtemplate_labels=self.kwargs['pk']).count() > 100:
-            return Response(
-                dict(msg=_('Maximum number of labels for {} reached.'.format(self.parent_model._meta.verbose_name_raw))), status=status.HTTP_400_BAD_REQUEST
-            )
-        return super(JobTemplateLabelList, self).post(request, *args, **kwargs)
 
 
 class JobTemplateCallback(GenericAPIView):
@@ -2966,6 +2955,22 @@ class WorkflowJobNodeCredentialsList(SubListAPIView):
     relationship = 'credentials'
 
 
+class WorkflowJobNodeLabelsList(SubListAPIView):
+
+    model = models.Label
+    serializer_class = serializers.LabelSerializer
+    parent_model = models.WorkflowJobNode
+    relationship = 'labels'
+
+
+class WorkflowJobNodeInstanceGroupsList(SubListAttachDetachAPIView):
+
+    model = models.InstanceGroup
+    serializer_class = serializers.InstanceGroupSerializer
+    parent_model = models.WorkflowJobNode
+    relationship = 'instance_groups'
+
+
 class WorkflowJobTemplateNodeList(ListCreateAPIView):
 
     model = models.WorkflowJobTemplateNode
@@ -2984,6 +2989,19 @@ class WorkflowJobTemplateNodeCredentialsList(LaunchConfigCredentialsBase):
     parent_model = models.WorkflowJobTemplateNode
 
 
+class WorkflowJobTemplateNodeLabelsList(LabelSubListCreateAttachDetachView):
+
+    parent_model = models.WorkflowJobTemplateNode
+
+
+class WorkflowJobTemplateNodeInstanceGroupsList(SubListAttachDetachAPIView):
+
+    model = models.InstanceGroup
+    serializer_class = serializers.InstanceGroupSerializer
+    parent_model = models.WorkflowJobTemplateNode
+    relationship = 'instance_groups'
+
+
 class WorkflowJobTemplateNodeChildrenBaseList(EnforceParentRelationshipMixin, SubListCreateAttachDetachAPIView):
 
     model = models.WorkflowJobTemplateNode
@@ -3196,13 +3214,17 @@ class WorkflowJobTemplateLaunch(RetrieveAPIView):
                 data['extra_vars'] = extra_vars
             modified_ask_mapping = models.WorkflowJobTemplate.get_ask_mapping()
             modified_ask_mapping.pop('extra_vars')
-            for field_name, ask_field_name in obj.get_ask_mapping().items():
+
+            for field, ask_field_name in modified_ask_mapping.items():
                 if not getattr(obj, ask_field_name):
-                    data.pop(field_name, None)
-                elif field_name == 'inventory':
-                    data[field_name] = getattrd(obj, "%s.%s" % (field_name, 'id'), None)
+                    data.pop(field, None)
+                elif isinstance(getattr(obj.__class__, field).field, ForeignKey):
+                    data[field] = getattrd(obj, "%s.%s" % (field, 'id'), None)
+                elif isinstance(getattr(obj.__class__, field).field, ManyToManyField):
+                    data[field] = [item.id for item in getattr(obj, field).all()]
                 else:
-                    data[field_name] = getattr(obj, field_name)
+                    data[field] = getattr(obj, field)
+
         return data
 
     def post(self, request, *args, **kwargs):
@@ -3689,15 +3711,21 @@ class JobCreateSchedule(RetrieveAPIView):
             extra_data=config.extra_data,
             survey_passwords=config.survey_passwords,
             inventory=config.inventory,
+            execution_environment=config.execution_environment,
             char_prompts=config.char_prompts,
             credentials=set(config.credentials.all()),
+            labels=set(config.labels.all()),
+            instance_groups=list(config.instance_groups.all()),
         )
         if not request.user.can_access(models.Schedule, 'add', schedule_data):
             raise PermissionDenied()
 
-        creds_list = schedule_data.pop('credentials')
+        related_fields = ('credentials', 'labels', 'instance_groups')
+        related = [schedule_data.pop(relationship) for relationship in related_fields]
         schedule = models.Schedule.objects.create(**schedule_data)
-        schedule.credentials.add(*creds_list)
+        for relationship, items in zip(related_fields, related):
+            for item in items:
+                getattr(schedule, relationship).add(item)
 
         data = serializers.ScheduleSerializer(schedule, context=self.get_serializer_context()).data
         data.serializer.instance = None  # hack to avoid permissions.py assuming this is Job model
@@ -4428,18 +4456,6 @@ class NotificationDetail(RetrieveAPIView):
     serializer_class = serializers.NotificationSerializer
 
 
-class LabelList(ListCreateAPIView):
-
-    model = models.Label
-    serializer_class = serializers.LabelSerializer
-
-
-class LabelDetail(RetrieveUpdateAPIView):
-
-    model = models.Label
-    serializer_class = serializers.LabelSerializer
-
-
 class ActivityStreamList(SimpleListAPIView):
 
     model = models.ActivityStream

awx/api/views/instance_install_bundle.py

@@ -0,0 +1,199 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+import datetime
+import io
+import ipaddress
+import os
+import tarfile
+
+import asn1
+from awx.api import serializers
+from awx.api.generics import GenericAPIView, Response
+from awx.api.permissions import IsSystemAdminOrAuditor
+from awx.main import models
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509 import DNSName, IPAddress, ObjectIdentifier, OtherName
+from cryptography.x509.oid import NameOID
+from django.http import HttpResponse
+from django.template.loader import render_to_string
+from django.utils.translation import gettext_lazy as _
+from rest_framework import status
+
+# Red Hat has an OID namespace (RHANANA). Receptor has its own designation under that.
+RECEPTOR_OID = "1.3.6.1.4.1.2312.19.1"
+
+# generate install bundle for the instance
+# install bundle directory structure
+# ├── install_receptor.yml (playbook)
+# ├── inventory.yml
+# ├── group_vars
+# │   └── all.yml
+# ├── receptor
+# │   ├── tls
+# │   │   ├── ca
+# │   │   │   └── receptor-ca.crt
+# │   │   ├── receptor.crt
+# │   │   └── receptor.key
+# │   └── work-public-key.pem
+# └── requirements.yml
+class InstanceInstallBundle(GenericAPIView):
+
+    name = _('Install Bundle')
+    model = models.Instance
+    serializer_class = serializers.InstanceSerializer
+    permission_classes = (IsSystemAdminOrAuditor,)
+
+    def get(self, request, *args, **kwargs):
+        instance_obj = self.get_object()
+
+        if instance_obj.node_type not in ('execution',):
+            return Response(
+                data=dict(msg=_('Install bundle can only be generated for execution nodes.')),
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        with io.BytesIO() as f:
+            with tarfile.open(fileobj=f, mode='w:gz') as tar:
+                # copy /etc/receptor/tls/ca/receptor-ca.crt to receptor/tls/ca in the tar file
+                tar.add(
+                    os.path.realpath('/etc/receptor/tls/ca/receptor-ca.crt'), arcname=f"{instance_obj.hostname}_install_bundle/receptor/tls/ca/receptor-ca.crt"
+                )
+
+                # copy /etc/receptor/signing/work-public-key.pem to receptor/work-public-key.pem
+                tar.add('/etc/receptor/signing/work-public-key.pem', arcname=f"{instance_obj.hostname}_install_bundle/receptor/work-public-key.pem")
+
+                # generate and write the receptor key to receptor/tls/receptor.key in the tar file
+                key, cert = generate_receptor_tls(instance_obj)
+
+                key_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/receptor/tls/receptor.key")
+                key_tarinfo.size = len(key)
+                tar.addfile(key_tarinfo, io.BytesIO(key))
+
+                cert_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/receptor/tls/receptor.crt")
+                cert_tarinfo.size = len(cert)
+                tar.addfile(cert_tarinfo, io.BytesIO(cert))
+
+                # generate and write install_receptor.yml to the tar file
+                playbook = generate_playbook().encode('utf-8')
+                playbook_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/install_receptor.yml")
+                playbook_tarinfo.size = len(playbook)
+                tar.addfile(playbook_tarinfo, io.BytesIO(playbook))
+
+                # generate and write inventory.yml to the tar file
+                inventory_yml = generate_inventory_yml(instance_obj).encode('utf-8')
+                inventory_yml_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/inventory.yml")
+                inventory_yml_tarinfo.size = len(inventory_yml)
+                tar.addfile(inventory_yml_tarinfo, io.BytesIO(inventory_yml))
+
+                # generate and write group_vars/all.yml to the tar file
+                group_vars = generate_group_vars_all_yml(instance_obj).encode('utf-8')
+                group_vars_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/group_vars/all.yml")
+                group_vars_tarinfo.size = len(group_vars)
+                tar.addfile(group_vars_tarinfo, io.BytesIO(group_vars))
+
+                # generate and write requirements.yml to the tar file
+                requirements_yml = generate_requirements_yml().encode('utf-8')
+                requirements_yml_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/requirements.yml")
+                requirements_yml_tarinfo.size = len(requirements_yml)
+                tar.addfile(requirements_yml_tarinfo, io.BytesIO(requirements_yml))
+
+            # respond with the tarfile
+            f.seek(0)
+            response = HttpResponse(f.read(), status=status.HTTP_200_OK)
+            response['Content-Disposition'] = f"attachment; filename={instance_obj.hostname}_install_bundle.tar.gz"
+            return response
+
+
+def generate_playbook():
+    return render_to_string("instance_install_bundle/install_receptor.yml")
+
+
+def generate_requirements_yml():
+    return render_to_string("instance_install_bundle/requirements.yml")
+
+
+def generate_inventory_yml(instance_obj):
+    return render_to_string("instance_install_bundle/inventory.yml", context=dict(instance=instance_obj))
+
+
+def generate_group_vars_all_yml(instance_obj):
+    return render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj))
+
+
+def generate_receptor_tls(instance_obj):
+    # generate private key for the receptor
+    key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+
+    # encode receptor hostname to asn1
+    hostname = instance_obj.hostname
+    encoder = asn1.Encoder()
+    encoder.start()
+    encoder.write(hostname.encode(), nr=asn1.Numbers.UTF8String)
+    hostname_asn1 = encoder.output()
+
+    san_params = [
+        DNSName(hostname),
+        OtherName(ObjectIdentifier(RECEPTOR_OID), hostname_asn1),
+    ]
+
+    try:
+        san_params.append(IPAddress(ipaddress.IPv4Address(hostname)))
+    except ipaddress.AddressValueError:
+        pass
+
+    # generate certificate for the receptor
+    csr = (
+        x509.CertificateSigningRequestBuilder()
+        .subject_name(
+            x509.Name(
+                [
+                    x509.NameAttribute(NameOID.COMMON_NAME, hostname),
+                ]
+            )
+        )
+        .add_extension(
+            x509.SubjectAlternativeName(san_params),
+            critical=False,
+        )
+        .sign(key, hashes.SHA256())
+    )
+
+    # sign csr with the receptor ca key from /etc/receptor/ca/receptor-ca.key
+    with open('/etc/receptor/tls/ca/receptor-ca.key', 'rb') as f:
+        ca_key = serialization.load_pem_private_key(
+            f.read(),
+            password=None,
+        )
+
+    with open('/etc/receptor/tls/ca/receptor-ca.crt', 'rb') as f:
+        ca_cert = x509.load_pem_x509_certificate(f.read())
+
+    cert = (
+        x509.CertificateBuilder()
+        .subject_name(csr.subject)
+        .issuer_name(ca_cert.issuer)
+        .public_key(csr.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(datetime.datetime.utcnow())
+        .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=10))
+        .add_extension(
+            csr.extensions.get_extension_for_class(x509.SubjectAlternativeName).value,
+            critical=csr.extensions.get_extension_for_class(x509.SubjectAlternativeName).critical,
+        )
+        .sign(ca_key, hashes.SHA256())
+    )
+
+    key = key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+
+    cert = cert.public_bytes(
+        encoding=serialization.Encoding.PEM,
+    )
+
+    return key, cert

awx/api/views/inventory.py

@@ -18,8 +18,6 @@ from rest_framework import status
 # AWX
 from awx.main.models import ActivityStream, Inventory, JobTemplate, Role, User, InstanceGroup, InventoryUpdateEvent, InventoryUpdate
 
-from awx.main.models.label import Label
-
 from awx.api.generics import (
     ListCreateAPIView,
     RetrieveUpdateDestroyAPIView,
@@ -27,9 +25,8 @@ from awx.api.generics import (
     SubListAttachDetachAPIView,
     ResourceAccessList,
     CopyAPIView,
-    DeleteLastUnattachLabelMixin,
-    SubListCreateAttachDetachAPIView,
 )
+from awx.api.views.labels import LabelSubListCreateAttachDetachView
 
 
 from awx.api.serializers import (
@@ -39,7 +36,6 @@ from awx.api.serializers import (
     InstanceGroupSerializer,
     InventoryUpdateEventSerializer,
     JobTemplateSerializer,
-    LabelSerializer,
 )
 from awx.api.views.mixin import RelatedJobsPreventDeleteMixin
 
@@ -157,28 +153,9 @@ class InventoryJobTemplateList(SubListAPIView):
         return qs.filter(inventory=parent)
 
 
-class InventoryLabelList(DeleteLastUnattachLabelMixin, SubListCreateAttachDetachAPIView, SubListAPIView):
+class InventoryLabelList(LabelSubListCreateAttachDetachView):
 
-    model = Label
-    serializer_class = LabelSerializer
     parent_model = Inventory
-    relationship = 'labels'
-
-    def post(self, request, *args, **kwargs):
-        # If a label already exists in the database, attach it instead of erroring out
-        # that it already exists
-        if 'id' not in request.data and 'name' in request.data and 'organization' in request.data:
-            existing = Label.objects.filter(name=request.data['name'], organization_id=request.data['organization'])
-            if existing.exists():
-                existing = existing[0]
-                request.data['id'] = existing.id
-                del request.data['name']
-                del request.data['organization']
-        if Label.objects.filter(inventory_labels=self.kwargs['pk']).count() > 100:
-            return Response(
-                dict(msg=_('Maximum number of labels for {} reached.'.format(self.parent_model._meta.verbose_name_raw))), status=status.HTTP_400_BAD_REQUEST
-            )
-        return super(InventoryLabelList, self).post(request, *args, **kwargs)
 
 
 class InventoryCopy(CopyAPIView):

awx/api/views/labels.py

@@ -0,0 +1,71 @@
+# AWX
+from awx.api.generics import SubListCreateAttachDetachAPIView, RetrieveUpdateAPIView, ListCreateAPIView
+from awx.main.models import Label
+from awx.api.serializers import LabelSerializer
+
+# Django
+from django.utils.translation import gettext_lazy as _
+
+# Django REST Framework
+from rest_framework.response import Response
+from rest_framework.status import HTTP_400_BAD_REQUEST
+
+
+class LabelSubListCreateAttachDetachView(SubListCreateAttachDetachAPIView):
+    """
+    For related labels lists like /api/v2/inventories/N/labels/
+
+    We want want the last instance to be deleted from the database
+    when the last disassociate happens.
+
+    Subclasses need to define parent_model
+    """
+
+    model = Label
+    serializer_class = LabelSerializer
+    relationship = 'labels'
+
+    def unattach(self, request, *args, **kwargs):
+        (sub_id, res) = super().unattach_validate(request)
+        if res:
+            return res
+
+        res = super().unattach_by_id(request, sub_id)
+
+        obj = self.model.objects.get(id=sub_id)
+
+        if obj.is_detached():
+            obj.delete()
+
+        return res
+
+    def post(self, request, *args, **kwargs):
+        # If a label already exists in the database, attach it instead of erroring out
+        # that it already exists
+        if 'id' not in request.data and 'name' in request.data and 'organization' in request.data:
+            existing = Label.objects.filter(name=request.data['name'], organization_id=request.data['organization'])
+            if existing.exists():
+                existing = existing[0]
+                request.data['id'] = existing.id
+                del request.data['name']
+                del request.data['organization']
+
+        # Give a 400 error if we have attached too many labels to this object
+        label_filter = self.parent_model._meta.get_field(self.relationship).remote_field.name
+        if Label.objects.filter(**{label_filter: self.kwargs['pk']}).count() > 100:
+            return Response(dict(msg=_(f'Maximum number of labels for {self.parent_model._meta.verbose_name_raw} reached.')), status=HTTP_400_BAD_REQUEST)
+
+        return super().post(request, *args, **kwargs)
+
+
+class LabelDetail(RetrieveUpdateAPIView):
+
+    model = Label
+    serializer_class = LabelSerializer
+
+
+class LabelList(ListCreateAPIView):
+
+    name = _("Labels")
+    model = Label
+    serializer_class = LabelSerializer

awx/main/access.py

@@ -12,7 +12,7 @@ from django.conf import settings
 from django.db.models import Q, Prefetch
 from django.contrib.auth.models import User
 from django.utils.translation import gettext_lazy as _
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, FieldDoesNotExist
 
 # Django REST Framework
 from rest_framework.exceptions import ParseError, PermissionDenied
@@ -281,13 +281,23 @@ class BaseAccess(object):
         """
         return True
 
+    def assure_relationship_exists(self, obj, relationship):
+        if '.' in relationship:
+            return  # not attempting validation for complex relationships now
+        try:
+            obj._meta.get_field(relationship)
+        except FieldDoesNotExist:
+            raise NotImplementedError(f'The relationship {relationship} does not exist for model {type(obj)}')
+
     def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
+        self.assure_relationship_exists(obj, relationship)
         if skip_sub_obj_read_check:
             return self.can_change(obj, None)
         else:
             return bool(self.can_change(obj, None) and self.user.can_access(type(sub_obj), 'read', sub_obj))
 
     def can_unattach(self, obj, sub_obj, relationship, data=None):
+        self.assure_relationship_exists(obj, relationship)
         return self.can_change(obj, data)
 
     def check_related(self, field, Model, data, role_field='admin_role', obj=None, mandatory=False):
@@ -328,6 +338,8 @@ class BaseAccess(object):
             role = getattr(resource, role_field, None)
             if role is None:
                 # Handle special case where resource does not have direct roles
+                if role_field == 'read_role':
+                    return self.user.can_access(type(resource), 'read', resource)
                 access_method_type = {'admin_role': 'change', 'execute_role': 'start'}[role_field]
                 return self.user.can_access(type(resource), access_method_type, resource, None)
             return self.user in role
@@ -499,6 +511,21 @@ class BaseAccess(object):
         return False
 
 
+class UnifiedCredentialsMixin(BaseAccess):
+    """
+    The credentials many-to-many is a standard relationship for JT, jobs, and others
+    Permission to attach is always use permission, and permission to unattach is admin to the parent object
+    """
+
+    @check_superuser
+    def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
+        if relationship == 'credentials':
+            if not isinstance(sub_obj, Credential):
+                raise RuntimeError(f'Can only attach credentials to credentials relationship, got {type(sub_obj)}')
+            return self.can_change(obj, None) and (self.user in sub_obj.use_role)
+        return super().can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
+
+
 class NotificationAttachMixin(BaseAccess):
     """For models that can have notifications attached
 
@@ -552,7 +579,8 @@ class InstanceAccess(BaseAccess):
         return super(InstanceAccess, self).can_unattach(obj, sub_obj, relationship, relationship, data=data)
 
     def can_add(self, data):
-        return False
+
+        return self.user.is_superuser
 
     def can_change(self, obj, data):
         return False
@@ -1031,7 +1059,7 @@ class GroupAccess(BaseAccess):
         return bool(obj and self.user in obj.inventory.admin_role)
 
 
-class InventorySourceAccess(NotificationAttachMixin, BaseAccess):
+class InventorySourceAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAccess):
     """
     I can see inventory sources whenever I can see their inventory.
     I can change inventory sources whenever I can change their inventory.
@@ -1075,18 +1103,6 @@ class InventorySourceAccess(NotificationAttachMixin, BaseAccess):
             return self.user in obj.inventory.update_role
         return False
 
-    @check_superuser
-    def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        if relationship == 'credentials' and isinstance(sub_obj, Credential):
-            return obj and obj.inventory and self.user in obj.inventory.admin_role and self.user in sub_obj.use_role
-        return super(InventorySourceAccess, self).can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
-
-    @check_superuser
-    def can_unattach(self, obj, sub_obj, relationship, *args, **kwargs):
-        if relationship == 'credentials' and isinstance(sub_obj, Credential):
-            return obj and obj.inventory and self.user in obj.inventory.admin_role
-        return super(InventorySourceAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs)
-
 
 class InventoryUpdateAccess(BaseAccess):
     """
@@ -1485,7 +1501,7 @@ class ProjectUpdateAccess(BaseAccess):
         return obj and self.user in obj.project.admin_role
 
 
-class JobTemplateAccess(NotificationAttachMixin, BaseAccess):
+class JobTemplateAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAccess):
     """
     I can see job templates when:
      - I have read role for the job template.
@@ -1549,8 +1565,7 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess):
             if self.user not in inventory.use_role:
                 return False
 
-        ee = get_value(ExecutionEnvironment, 'execution_environment')
-        if ee and not self.user.can_access(ExecutionEnvironment, 'read', ee):
+        if not self.check_related('execution_environment', ExecutionEnvironment, data, role_field='read_role'):
             return False
 
         project = get_value(Project, 'project')
@@ -1600,10 +1615,8 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess):
         if self.changes_are_non_sensitive(obj, data):
             return True
 
-        if data.get('execution_environment'):
-            ee = get_object_from_data('execution_environment', ExecutionEnvironment, data)
-            if not self.user.can_access(ExecutionEnvironment, 'read', ee):
-                return False
+        if not self.check_related('execution_environment', ExecutionEnvironment, data, obj=obj, role_field='read_role'):
+            return False
 
         for required_field, cls in (('inventory', Inventory), ('project', Project)):
             is_mandatory = True
@@ -1667,17 +1680,13 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess):
             if not obj.organization:
                 return False
             return self.user.can_access(type(sub_obj), "read", sub_obj) and self.user in obj.organization.admin_role
-        if relationship == 'credentials' and isinstance(sub_obj, Credential):
-            return self.user in obj.admin_role and self.user in sub_obj.use_role
         return super(JobTemplateAccess, self).can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
 
     @check_superuser
     def can_unattach(self, obj, sub_obj, relationship, *args, **kwargs):
         if relationship == "instance_groups":
             return self.can_attach(obj, sub_obj, relationship, *args, **kwargs)
-        if relationship == 'credentials' and isinstance(sub_obj, Credential):
-            return self.user in obj.admin_role
-        return super(JobTemplateAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs)
+        return super(JobTemplateAccess, self).can_unattach(obj, sub_obj, relationship, *args, **kwargs)
 
 
 class JobAccess(BaseAccess):
@@ -1824,7 +1833,7 @@ class SystemJobAccess(BaseAccess):
         return False  # no relaunching of system jobs
 
 
-class JobLaunchConfigAccess(BaseAccess):
+class JobLaunchConfigAccess(UnifiedCredentialsMixin, BaseAccess):
     """
     Launch configs must have permissions checked for
      - relaunching
@@ -1832,63 +1841,69 @@ class JobLaunchConfigAccess(BaseAccess):
 
     In order to create a new object with a copy of this launch config, I need:
      - use access to related inventory (if present)
+     - read access to Execution Environment (if present), unless the specified ee is already in the template
      - use role to many-related credentials (if any present)
+     - read access to many-related labels (if any present), unless the specified label is already in the template
+     - read access to many-related instance groups (if any present), unless the specified instance group is already in the template
     """
 
     model = JobLaunchConfig
     select_related = 'job'
     prefetch_related = ('credentials', 'inventory')
 
-    def _unusable_creds_exist(self, qs):
-        return qs.exclude(pk__in=Credential._accessible_pk_qs(Credential, self.user, 'use_role')).exists()
+    M2M_CHECKS = {'credentials': Credential, 'labels': Label, 'instance_groups': InstanceGroup}
 
-    def has_credentials_access(self, obj):
-        # user has access if no related credentials exist that the user lacks use role for
-        return not self._unusable_creds_exist(obj.credentials)
+    def _related_filtered_queryset(self, cls):
+        if cls is Label:
+            return LabelAccess(self.user).filtered_queryset()
+        elif cls is InstanceGroup:
+            return InstanceGroupAccess(self.user).filtered_queryset()
+        else:
+            return cls._accessible_pk_qs(cls, self.user, 'use_role')
+
+    def has_obj_m2m_access(self, obj):
+        for relationship, cls in self.M2M_CHECKS.items():
+            if getattr(obj, relationship).exclude(pk__in=self._related_filtered_queryset(cls)).exists():
+                return False
+        return True
 
     @check_superuser
     def can_add(self, data, template=None):
         # This is a special case, we don't check related many-to-many elsewhere
         # launch RBAC checks use this
-        if 'credentials' in data and data['credentials'] or 'reference_obj' in data:
-            if 'reference_obj' in data:
-                prompted_cred_qs = data['reference_obj'].credentials.all()
-            else:
-                # If given model objects, only use the primary key from them
-                cred_pks = [cred.pk for cred in data['credentials']]
-                if template:
-                    for cred in template.credentials.all():
-                        if cred.pk in cred_pks:
-                            cred_pks.remove(cred.pk)
-                prompted_cred_qs = Credential.objects.filter(pk__in=cred_pks)
-            if self._unusable_creds_exist(prompted_cred_qs):
+        if 'reference_obj' in data:
+            if not self.has_obj_m2m_access(data['reference_obj']):
                 return False
-        return self.check_related('inventory', Inventory, data, role_field='use_role')
+        else:
+            for relationship, cls in self.M2M_CHECKS.items():
+                if relationship in data and data[relationship]:
+                    # If given model objects, only use the primary key from them
+                    sub_obj_pks = [sub_obj.pk for sub_obj in data[relationship]]
+                    if template:
+                        for sub_obj in getattr(template, relationship).all():
+                            if sub_obj.pk in sub_obj_pks:
+                                sub_obj_pks.remove(sub_obj.pk)
+                    if cls.objects.filter(pk__in=sub_obj_pks).exclude(pk__in=self._related_filtered_queryset(cls)).exists():
+                        return False
+        return self.check_related('inventory', Inventory, data, role_field='use_role') and self.check_related(
+            'execution_environment', ExecutionEnvironment, data, role_field='read_role'
+        )
 
     @check_superuser
     def can_use(self, obj):
-        return self.check_related('inventory', Inventory, {}, obj=obj, role_field='use_role', mandatory=True) and self.has_credentials_access(obj)
+        return (
+            self.has_obj_m2m_access(obj)
+            and self.check_related('inventory', Inventory, {}, obj=obj, role_field='use_role', mandatory=True)
+            and self.check_related('execution_environment', ExecutionEnvironment, {}, obj=obj, role_field='read_role')
+        )
 
     def can_change(self, obj, data):
-        return self.check_related('inventory', Inventory, data, obj=obj, role_field='use_role')
-
-    def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        if isinstance(sub_obj, Credential) and relationship == 'credentials':
-            return self.user in sub_obj.use_role
-        else:
-            raise NotImplementedError('Only credentials can be attached to launch configurations.')
-
-    def can_unattach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        if isinstance(sub_obj, Credential) and relationship == 'credentials':
-            if skip_sub_obj_read_check:
-                return True
-            else:
-                return self.user in sub_obj.read_role
-        else:
-            raise NotImplementedError('Only credentials can be attached to launch configurations.')
+        return self.check_related('inventory', Inventory, data, obj=obj, role_field='use_role') and self.check_related(
+            'execution_environment', ExecutionEnvironment, data, obj=obj, role_field='read_role'
+        )
 
 
-class WorkflowJobTemplateNodeAccess(BaseAccess):
+class WorkflowJobTemplateNodeAccess(UnifiedCredentialsMixin, BaseAccess):
     """
     I can see/use a WorkflowJobTemplateNode if I have read permission
         to associated Workflow Job Template
@@ -1911,7 +1926,7 @@ class WorkflowJobTemplateNodeAccess(BaseAccess):
     """
 
     model = WorkflowJobTemplateNode
-    prefetch_related = ('success_nodes', 'failure_nodes', 'always_nodes', 'unified_job_template', 'credentials', 'workflow_job_template')
+    prefetch_related = ('success_nodes', 'failure_nodes', 'always_nodes', 'unified_job_template', 'workflow_job_template')
 
     def filtered_queryset(self):
         return self.model.objects.filter(workflow_job_template__in=WorkflowJobTemplate.accessible_objects(self.user, 'read_role'))
@@ -1923,7 +1938,8 @@ class WorkflowJobTemplateNodeAccess(BaseAccess):
         return (
             self.check_related('workflow_job_template', WorkflowJobTemplate, data, mandatory=True)
             and self.check_related('unified_job_template', UnifiedJobTemplate, data, role_field='execute_role')
-            and JobLaunchConfigAccess(self.user).can_add(data)
+            and self.check_related('inventory', Inventory, data, role_field='use_role')
+            and self.check_related('execution_environment', ExecutionEnvironment, data, role_field='read_role')
         )
 
     def wfjt_admin(self, obj):
@@ -1932,17 +1948,14 @@ class WorkflowJobTemplateNodeAccess(BaseAccess):
         else:
             return self.user in obj.workflow_job_template.admin_role
 
-    def ujt_execute(self, obj):
+    def ujt_execute(self, obj, data=None):
         if not obj.unified_job_template:
             return True
-        return self.check_related('unified_job_template', UnifiedJobTemplate, {}, obj=obj, role_field='execute_role', mandatory=True)
+        return self.check_related('unified_job_template', UnifiedJobTemplate, data, obj=obj, role_field='execute_role', mandatory=True)
 
     def can_change(self, obj, data):
-        if not data:
-            return True
-
         # should not be able to edit the prompts if lacking access to UJT or WFJT
-        return self.ujt_execute(obj) and self.wfjt_admin(obj) and JobLaunchConfigAccess(self.user).can_change(obj, data)
+        return self.ujt_execute(obj, data=data) and self.wfjt_admin(obj) and JobLaunchConfigAccess(self.user).can_change(obj, data)
 
     def can_delete(self, obj):
         return self.wfjt_admin(obj)
@@ -1955,29 +1968,14 @@ class WorkflowJobTemplateNodeAccess(BaseAccess):
         return True
 
     def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        if not self.wfjt_admin(obj):
-            return False
-        if relationship == 'credentials':
-            # Need permission to related template to attach a credential
-            if not self.ujt_execute(obj):
-                return False
-            return JobLaunchConfigAccess(self.user).can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
-        elif relationship in ('success_nodes', 'failure_nodes', 'always_nodes'):
-            return self.check_same_WFJT(obj, sub_obj)
-        else:
-            raise NotImplementedError('Relationship {} not understood for WFJT nodes.'.format(relationship))
+        if relationship in ('success_nodes', 'failure_nodes', 'always_nodes'):
+            return self.wfjt_admin(obj) and self.check_same_WFJT(obj, sub_obj)
+        return super().can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
 
-    def can_unattach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        if not self.wfjt_admin(obj):
-            return False
-        if relationship == 'credentials':
-            if not self.ujt_execute(obj):
-                return False
-            return JobLaunchConfigAccess(self.user).can_unattach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
-        elif relationship in ('success_nodes', 'failure_nodes', 'always_nodes'):
-            return self.check_same_WFJT(obj, sub_obj)
-        else:
-            raise NotImplementedError('Relationship {} not understood for WFJT nodes.'.format(relationship))
+    def can_unattach(self, obj, sub_obj, relationship, data=None):
+        if relationship in ('success_nodes', 'failure_nodes', 'always_nodes'):
+            return self.wfjt_admin(obj)
+        return super().can_unattach(obj, sub_obj, relationship, data=None)
 
 
 class WorkflowJobNodeAccess(BaseAccess):
@@ -2052,13 +2050,10 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
         if not data:  # So the browseable API will work
             return Organization.accessible_objects(self.user, 'workflow_admin_role').exists()
 
-        if data.get('execution_environment'):
-            ee = get_object_from_data('execution_environment', ExecutionEnvironment, data)
-            if not self.user.can_access(ExecutionEnvironment, 'read', ee):
-                return False
-
-        return self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True) and self.check_related(
-            'inventory', Inventory, data, role_field='use_role'
+        return bool(
+            self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True)
+            and self.check_related('inventory', Inventory, data, role_field='use_role')
+            and self.check_related('execution_environment', ExecutionEnvironment, data, role_field='read_role')
         )
 
     def can_copy(self, obj):
@@ -2104,14 +2099,10 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
         if self.user.is_superuser:
             return True
 
-        if data and data.get('execution_environment'):
-            ee = get_object_from_data('execution_environment', ExecutionEnvironment, data)
-            if not self.user.can_access(ExecutionEnvironment, 'read', ee):
-                return False
-
         return (
             self.check_related('organization', Organization, data, role_field='workflow_admin_role', obj=obj)
             and self.check_related('inventory', Inventory, data, role_field='use_role', obj=obj)
+            and self.check_related('execution_environment', ExecutionEnvironment, data, obj=obj, role_field='read_role')
             and self.user in obj.admin_role
         )
 
@@ -2518,7 +2509,7 @@ class UnifiedJobAccess(BaseAccess):
         return super(UnifiedJobAccess, self).get_queryset().filter(workflowapproval__isnull=True)
 
 
-class ScheduleAccess(BaseAccess):
+class ScheduleAccess(UnifiedCredentialsMixin, BaseAccess):
     """
     I can see a schedule if I can see it's related unified job, I can create them or update them if I have write access
     """
@@ -2559,12 +2550,6 @@ class ScheduleAccess(BaseAccess):
     def can_delete(self, obj):
         return self.can_change(obj, {})
 
-    def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        return JobLaunchConfigAccess(self.user).can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
-
-    def can_unattach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
-        return JobLaunchConfigAccess(self.user).can_unattach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
-
 
 class NotificationTemplateAccess(BaseAccess):
     """

awx/main/analytics/metrics.py

@@ -3,6 +3,7 @@ from prometheus_client import CollectorRegistry, Gauge, Info, generate_latest
 
 from awx.conf.license import get_license
 from awx.main.utils import get_awx_version
+from awx.main.models import UnifiedJob
 from awx.main.analytics.collectors import (
     counts,
     instance_info,
@@ -169,8 +170,9 @@ def metrics():
 
     all_job_data = job_counts(None)
     statuses = all_job_data.get('status', {})
-    for status, value in statuses.items():
-        STATUS.labels(status=status).set(value)
+    states = set(dict(UnifiedJob.STATUS_CHOICES).keys()) - set(['new'])
+    for state in states:
+        STATUS.labels(status=state).set(statuses.get(state, 0))
 
     RUNNING_JOBS.set(current_counts['running_jobs'])
     PENDING_JOBS.set(current_counts['pending_jobs'])

awx/main/management/commands/register_peers.py

@@ -27,7 +27,9 @@ class Command(BaseCommand):
         )
 
     def handle(self, **options):
+        # provides a mapping of hostname to Instance objects
         nodes = Instance.objects.in_bulk(field_name='hostname')
+
         if options['source'] not in nodes:
             raise CommandError(f"Host {options['source']} is not a registered instance.")
         if not (options['peers'] or options['disconnect'] or options['exact'] is not None):
@@ -57,7 +59,9 @@ class Command(BaseCommand):
 
             results = 0
             for target in options['peers']:
-                _, created = InstanceLink.objects.get_or_create(source=nodes[options['source']], target=nodes[target])
+                _, created = InstanceLink.objects.update_or_create(
+                    source=nodes[options['source']], target=nodes[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
+                )
                 if created:
                     results += 1
 
@@ -80,7 +84,9 @@ class Command(BaseCommand):
                 links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__hostname', flat=True))
                 removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__hostname__in=links - peers).delete()
                 for target in peers - links:
-                    _, created = InstanceLink.objects.get_or_create(source=nodes[options['source']], target=nodes[target])
+                    _, created = InstanceLink.objects.update_or_create(
+                        source=nodes[options['source']], target=nodes[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
+                    )
                     if created:
                         additions += 1
 

awx/main/managers.py

@@ -129,10 +129,13 @@ class InstanceManager(models.Manager):
                 # if instance was not retrieved by uuid and hostname was, use the hostname
                 instance = self.filter(hostname=hostname)
 
+            from awx.main.models import Instance
+
             # Return existing instance
             if instance.exists():
                 instance = instance.first()  # in the unusual occasion that there is more than one, only get one
-                update_fields = []
+                instance.node_state = Instance.States.INSTALLED  # Wait for it to show up on the mesh
+                update_fields = ['node_state']
                 # if instance was retrieved by uuid and hostname has changed, update hostname
                 if instance.hostname != hostname:
                     logger.warning("passed in hostname {0} is different from the original hostname {1}, updating to {0}".format(hostname, instance.hostname))
@@ -141,6 +144,7 @@ class InstanceManager(models.Manager):
                 # if any other fields are to be updated
                 if instance.ip_address != ip_address:
                     instance.ip_address = ip_address
+                    update_fields.append('ip_address')
                 if instance.node_type != node_type:
                     instance.node_type = node_type
                     update_fields.append('node_type')
@@ -151,12 +155,12 @@ class InstanceManager(models.Manager):
                     return (False, instance)
 
             # Create new instance, and fill in default values
-            create_defaults = dict(capacity=0)
+            create_defaults = {'node_state': Instance.States.INSTALLED, 'capacity': 0}
             if defaults is not None:
                 create_defaults.update(defaults)
             uuid_option = {}
             if uuid is not None:
-                uuid_option = dict(uuid=uuid)
+                uuid_option = {'uuid': uuid}
             if node_type == 'execution' and 'version' not in create_defaults:
                 create_defaults['version'] = RECEPTOR_PENDING
             instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)

awx/main/migrations/0168_inventoryupdate_scm_revision.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.13 on 2022-09-08 16:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0167_project_signature_validation_credential'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='inventoryupdate',
+            name='scm_revision',
+            field=models.CharField(
+                blank=True,
+                default='',
+                editable=False,
+                help_text='The SCM Revision from the Project used for this inventory update.  Only applicable to inventories source from scm',
+                max_length=1024,
+                verbose_name='SCM Revision',
+            ),
+        ),
+    ]

awx/main/migrations/0169_jt_prompt_everything_on_launch.py

@@ -0,0 +1,225 @@
+# Generated by Django 3.2.13 on 2022-09-15 14:07
+
+import awx.main.fields
+import awx.main.utils.polymorphic
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0168_inventoryupdate_scm_revision'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='execution_environment',
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                help_text='The container image to be used for execution.',
+                null=True,
+                on_delete=awx.main.utils.polymorphic.SET_NULL,
+                related_name='joblaunchconfig_as_prompt',
+                to='main.executionenvironment',
+            ),
+        ),
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='labels',
+            field=models.ManyToManyField(related_name='joblaunchconfig_labels', to='main.Label'),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_execution_environment_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_forks_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_instance_groups_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_job_slice_count_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_labels_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='ask_timeout_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='schedule',
+            name='execution_environment',
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                help_text='The container image to be used for execution.',
+                null=True,
+                on_delete=awx.main.utils.polymorphic.SET_NULL,
+                related_name='schedule_as_prompt',
+                to='main.executionenvironment',
+            ),
+        ),
+        migrations.AddField(
+            model_name='schedule',
+            name='labels',
+            field=models.ManyToManyField(related_name='schedule_labels', to='main.Label'),
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='execution_environment',
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                help_text='The container image to be used for execution.',
+                null=True,
+                on_delete=awx.main.utils.polymorphic.SET_NULL,
+                related_name='workflowjobnode_as_prompt',
+                to='main.executionenvironment',
+            ),
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='labels',
+            field=models.ManyToManyField(related_name='workflowjobnode_labels', to='main.Label'),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_labels_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_skip_tags_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplate',
+            name='ask_tags_on_launch',
+            field=awx.main.fields.AskForField(blank=True, default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplatenode',
+            name='execution_environment',
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                help_text='The container image to be used for execution.',
+                null=True,
+                on_delete=awx.main.utils.polymorphic.SET_NULL,
+                related_name='workflowjobtemplatenode_as_prompt',
+                to='main.executionenvironment',
+            ),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplatenode',
+            name='labels',
+            field=models.ManyToManyField(related_name='workflowjobtemplatenode_labels', to='main.Label'),
+        ),
+        migrations.CreateModel(
+            name='WorkflowJobTemplateNodeBaseInstanceGroupMembership',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)),
+                ('instancegroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.instancegroup')),
+                ('workflowjobtemplatenode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.workflowjobtemplatenode')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='WorkflowJobNodeBaseInstanceGroupMembership',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)),
+                ('instancegroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.instancegroup')),
+                ('workflowjobnode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.workflowjobnode')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='WorkflowJobInstanceGroupMembership',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)),
+                ('instancegroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.instancegroup')),
+                ('workflowjobnode', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.workflowjob')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ScheduleInstanceGroupMembership',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)),
+                ('instancegroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.instancegroup')),
+                ('schedule', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.schedule')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='JobLaunchConfigInstanceGroupMembership',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)),
+                ('instancegroup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.instancegroup')),
+                ('joblaunchconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.joblaunchconfig')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='instance_groups',
+            field=awx.main.fields.OrderedManyToManyField(
+                blank=True, editable=False, related_name='joblaunchconfigs', through='main.JobLaunchConfigInstanceGroupMembership', to='main.InstanceGroup'
+            ),
+        ),
+        migrations.AddField(
+            model_name='schedule',
+            name='instance_groups',
+            field=awx.main.fields.OrderedManyToManyField(
+                blank=True, editable=False, related_name='schedule_instance_groups', through='main.ScheduleInstanceGroupMembership', to='main.InstanceGroup'
+            ),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='instance_groups',
+            field=awx.main.fields.OrderedManyToManyField(
+                blank=True,
+                editable=False,
+                related_name='workflow_job_instance_groups',
+                through='main.WorkflowJobInstanceGroupMembership',
+                to='main.InstanceGroup',
+            ),
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='instance_groups',
+            field=awx.main.fields.OrderedManyToManyField(
+                blank=True,
+                editable=False,
+                related_name='workflow_job_node_instance_groups',
+                through='main.WorkflowJobNodeBaseInstanceGroupMembership',
+                to='main.InstanceGroup',
+            ),
+        ),
+        migrations.AddField(
+            model_name='workflowjobtemplatenode',
+            name='instance_groups',
+            field=awx.main.fields.OrderedManyToManyField(
+                blank=True,
+                editable=False,
+                related_name='workflow_job_template_node_instance_groups',
+                through='main.WorkflowJobTemplateNodeBaseInstanceGroupMembership',
+                to='main.InstanceGroup',
+            ),
+        ),
+    ]

awx/main/migrations/0170_node_and_link_state.py

@@ -0,0 +1,79 @@
+# Generated by Django 3.2.13 on 2022-08-02 17:53
+
+import django.core.validators
+from django.db import migrations, models
+
+
+def forwards(apps, schema_editor):
+    # All existing InstanceLink objects need to be in the state
+    # 'Established', which is the default, so nothing needs to be done
+    # for that.
+
+    Instance = apps.get_model('main', 'Instance')
+    for instance in Instance.objects.all():
+        instance.node_state = 'ready' if not instance.errors else 'unavailable'
+        instance.save(update_fields=['node_state'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0169_jt_prompt_everything_on_launch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instance',
+            name='listener_port',
+            field=models.PositiveIntegerField(
+                blank=True,
+                default=27199,
+                help_text='Port that Receptor will listen for incoming connections on.',
+                validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(65535)],
+            ),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='node_state',
+            field=models.CharField(
+                choices=[
+                    ('provisioning', 'Provisioning'),
+                    ('provision-fail', 'Provisioning Failure'),
+                    ('installed', 'Installed'),
+                    ('ready', 'Ready'),
+                    ('unavailable', 'Unavailable'),
+                    ('deprovisioning', 'De-provisioning'),
+                    ('deprovision-fail', 'De-provisioning Failure'),
+                ],
+                default='ready',
+                help_text='Indicates the current life cycle stage of this instance.',
+                max_length=16,
+            ),
+        ),
+        migrations.AddField(
+            model_name='instancelink',
+            name='link_state',
+            field=models.CharField(
+                choices=[('adding', 'Adding'), ('established', 'Established'), ('removing', 'Removing')],
+                default='established',
+                help_text='Indicates the current life cycle stage of this peer link.',
+                max_length=16,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='node_type',
+            field=models.CharField(
+                choices=[
+                    ('control', 'Control plane node'),
+                    ('execution', 'Execution plane node'),
+                    ('hybrid', 'Controller and execution'),
+                    ('hop', 'Message-passing node, no execution capability'),
+                ],
+                default='hybrid',
+                help_text='Role that this node plays in the mesh.',
+                max_length=16,
+            ),
+        ),
+        migrations.RunPython(forwards, reverse_code=migrations.RunPython.noop),
+    ]

awx/main/migrations/0171_add_health_check_started.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.13 on 2022-09-26 20:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0170_node_and_link_state'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instance',
+            name='health_check_started',
+            field=models.DateTimeField(editable=False, help_text='The last time a health check was initiated on this instance.', null=True),
+        ),
+    ]

awx/main/migrations/0172_prevent_instance_fallback.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.13 on 2022-09-29 18:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0171_add_health_check_started'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='inventory',
+            name='prevent_instance_group_fallback',
+            field=models.BooleanField(
+                default=False,
+                help_text='If enabled, the inventory will prevent adding any organization instance groups to the list of preferred instances groups to run associated job templates on.If this setting is enabled and you provided an empty list, the global instance groups will be applied.',
+            ),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='prevent_instance_group_fallback',
+            field=models.BooleanField(
+                default=False,
+                help_text='If enabled, the job template will prevent adding any inventory or organization instance groups to the list of preferred instances groups to run on.If this setting is enabled and you provided an empty list, the global instance groups will be applied.',
+            ),
+        ),
+    ]

awx/main/models/ad_hoc_commands.py

@@ -228,15 +228,14 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
 
     @property
     def preferred_instance_groups(self):
-        if self.inventory is not None and self.inventory.organization is not None:
-            organization_groups = [x for x in self.inventory.organization.instance_groups.all()]
-        else:
-            organization_groups = []
+        selected_groups = []
         if self.inventory is not None:
-            inventory_groups = [x for x in self.inventory.instance_groups.all()]
-        else:
-            inventory_groups = []
-        selected_groups = inventory_groups + organization_groups
+            for instance_group in self.inventory.instance_groups.all():
+                selected_groups.append(instance_group)
+            if not self.inventory.prevent_instance_group_fallback and self.inventory.organization is not None:
+                for instance_group in self.inventory.organization.instance_groups.all():
+                    selected_groups.append(instance_group)
+
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups

awx/main/models/ha.py

@@ -5,7 +5,7 @@ from decimal import Decimal
 import logging
 import os
 
-from django.core.validators import MinValueValidator
+from django.core.validators import MinValueValidator, MaxValueValidator
 from django.db import models, connection
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
@@ -59,6 +59,15 @@ class InstanceLink(BaseModel):
     source = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='+')
     target = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='reverse_peers')
 
+    class States(models.TextChoices):
+        ADDING = 'adding', _('Adding')
+        ESTABLISHED = 'established', _('Established')
+        REMOVING = 'removing', _('Removing')
+
+    link_state = models.CharField(
+        choices=States.choices, default=States.ESTABLISHED, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
+    )
+
     class Meta:
         unique_together = ('source', 'target')
 
@@ -105,6 +114,11 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         editable=False,
         help_text=_('Last time instance ran its heartbeat task for main cluster nodes. Last known connection to receptor mesh for execution nodes.'),
     )
+    health_check_started = models.DateTimeField(
+        null=True,
+        editable=False,
+        help_text=_("The last time a health check was initiated on this instance."),
+    )
     last_health_check = models.DateTimeField(
         null=True,
         editable=False,
@@ -127,13 +141,33 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         default=0,
         editable=False,
     )
-    NODE_TYPE_CHOICES = [
-        ("control", "Control plane node"),
-        ("execution", "Execution plane node"),
-        ("hybrid", "Controller and execution"),
-        ("hop", "Message-passing node, no execution capability"),
-    ]
-    node_type = models.CharField(default='hybrid', choices=NODE_TYPE_CHOICES, max_length=16)
+
+    class Types(models.TextChoices):
+        CONTROL = 'control', _("Control plane node")
+        EXECUTION = 'execution', _("Execution plane node")
+        HYBRID = 'hybrid', _("Controller and execution")
+        HOP = 'hop', _("Message-passing node, no execution capability")
+
+    node_type = models.CharField(default=Types.HYBRID, choices=Types.choices, max_length=16, help_text=_("Role that this node plays in the mesh."))
+
+    class States(models.TextChoices):
+        PROVISIONING = 'provisioning', _('Provisioning')
+        PROVISION_FAIL = 'provision-fail', _('Provisioning Failure')
+        INSTALLED = 'installed', _('Installed')
+        READY = 'ready', _('Ready')
+        UNAVAILABLE = 'unavailable', _('Unavailable')
+        DEPROVISIONING = 'deprovisioning', _('De-provisioning')
+        DEPROVISION_FAIL = 'deprovision-fail', _('De-provisioning Failure')
+
+    node_state = models.CharField(
+        choices=States.choices, default=States.READY, max_length=16, help_text=_("Indicates the current life cycle stage of this instance.")
+    )
+    listener_port = models.PositiveIntegerField(
+        blank=True,
+        default=27199,
+        validators=[MinValueValidator(1), MaxValueValidator(65535)],
+        help_text=_("Port that Receptor will listen for incoming connections on."),
+    )
 
     peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'))
 
@@ -178,6 +212,14 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     def jobs_total(self):
         return UnifiedJob.objects.filter(execution_node=self.hostname).count()
 
+    @property
+    def health_check_pending(self):
+        if self.health_check_started is None:
+            return False
+        if self.last_health_check is None:
+            return True
+        return self.health_check_started > self.last_health_check
+
     def get_cleanup_task_kwargs(self, **kwargs):
         """
         Produce options to use for the command: ansible-runner worker cleanup
@@ -213,18 +255,22 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         return self.last_seen < ref_time - timedelta(seconds=grace_period)
 
     def mark_offline(self, update_last_seen=False, perform_save=True, errors=''):
-        if self.cpu_capacity == 0 and self.mem_capacity == 0 and self.capacity == 0 and self.errors == errors and (not update_last_seen):
-            return
+        if self.node_state not in (Instance.States.READY, Instance.States.UNAVAILABLE, Instance.States.INSTALLED):
+            return []
+        if self.node_state == Instance.States.UNAVAILABLE and self.errors == errors and (not update_last_seen):
+            return []
+        self.node_state = Instance.States.UNAVAILABLE
         self.cpu_capacity = self.mem_capacity = self.capacity = 0
         self.errors = errors
         if update_last_seen:
             self.last_seen = now()
 
+        update_fields = ['node_state', 'capacity', 'cpu_capacity', 'mem_capacity', 'errors']
+        if update_last_seen:
+            update_fields += ['last_seen']
         if perform_save:
-            update_fields = ['capacity', 'cpu_capacity', 'mem_capacity', 'errors']
-            if update_last_seen:
-                update_fields += ['last_seen']
             self.save(update_fields=update_fields)
+        return update_fields
 
     def set_capacity_value(self):
         """Sets capacity according to capacity adjustment rule (no save)"""
@@ -278,8 +324,12 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         if not errors:
             self.refresh_capacity_fields()
             self.errors = ''
+            if self.node_state in (Instance.States.UNAVAILABLE, Instance.States.INSTALLED):
+                self.node_state = Instance.States.READY
+                update_fields.append('node_state')
         else:
-            self.mark_offline(perform_save=False, errors=errors)
+            fields_to_update = self.mark_offline(perform_save=False, errors=errors)
+            update_fields.extend(fields_to_update)
         update_fields.extend(['cpu_capacity', 'mem_capacity', 'capacity'])
 
         # disabling activity stream will avoid extra queries, which is important for heatbeat actions
@@ -296,7 +346,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             # playbook event data; we should consider this a zero capacity event
             redis.Redis.from_url(settings.BROKER_URL).ping()
         except redis.ConnectionError:
-            errors = _('Failed to connect ot Redis')
+            errors = _('Failed to connect to Redis')
 
         self.save_health_data(awx_application_version, get_cpu_count(), get_mem_in_bytes(), update_last_seen=True, errors=errors)
 
@@ -388,6 +438,20 @@ def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs
 
 @receiver(post_save, sender=Instance)
 def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
+    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION,):
+        if instance.node_state == Instance.States.DEPROVISIONING:
+            from awx.main.tasks.receptor import remove_deprovisioned_node  # prevents circular import
+
+            # wait for jobs on the node to complete, then delete the
+            # node and kick off write_receptor_config
+            connection.on_commit(lambda: remove_deprovisioned_node.apply_async([instance.hostname]))
+
+        if instance.node_state == Instance.States.INSTALLED:
+            from awx.main.tasks.receptor import write_receptor_config  # prevents circular import
+
+            # broadcast to all control instances to update their receptor configs
+            connection.on_commit(lambda: write_receptor_config.apply_async(queue='tower_broadcast_all'))
+
     if created or instance.has_policy_changes():
         schedule_policy_task()
 
@@ -434,3 +498,58 @@ class InventoryInstanceGroupMembership(models.Model):
         default=None,
         db_index=True,
     )
+
+
+class JobLaunchConfigInstanceGroupMembership(models.Model):
+
+    joblaunchconfig = models.ForeignKey('JobLaunchConfig', on_delete=models.CASCADE)
+    instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
+    position = models.PositiveIntegerField(
+        null=True,
+        default=None,
+        db_index=True,
+    )
+
+
+class ScheduleInstanceGroupMembership(models.Model):
+
+    schedule = models.ForeignKey('Schedule', on_delete=models.CASCADE)
+    instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
+    position = models.PositiveIntegerField(
+        null=True,
+        default=None,
+        db_index=True,
+    )
+
+
+class WorkflowJobTemplateNodeBaseInstanceGroupMembership(models.Model):
+
+    workflowjobtemplatenode = models.ForeignKey('WorkflowJobTemplateNode', on_delete=models.CASCADE)
+    instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
+    position = models.PositiveIntegerField(
+        null=True,
+        default=None,
+        db_index=True,
+    )
+
+
+class WorkflowJobNodeBaseInstanceGroupMembership(models.Model):
+
+    workflowjobnode = models.ForeignKey('WorkflowJobNode', on_delete=models.CASCADE)
+    instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
+    position = models.PositiveIntegerField(
+        null=True,
+        default=None,
+        db_index=True,
+    )
+
+
+class WorkflowJobInstanceGroupMembership(models.Model):
+
+    workflowjobnode = models.ForeignKey('WorkflowJob', on_delete=models.CASCADE)
+    instancegroup = models.ForeignKey('InstanceGroup', on_delete=models.CASCADE)
+    position = models.PositiveIntegerField(
+        null=True,
+        default=None,
+        db_index=True,
+    )

awx/main/models/inventory.py

@@ -63,7 +63,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
     an inventory source contains lists and hosts.
     """
 
-    FIELDS_TO_PRESERVE_AT_COPY = ['hosts', 'groups', 'instance_groups']
+    FIELDS_TO_PRESERVE_AT_COPY = ['hosts', 'groups', 'instance_groups', 'prevent_instance_group_fallback']
     KIND_CHOICES = [
         ('', _('Hosts have a direct link to this inventory.')),
         ('smart', _('Hosts for inventory generated using the host_filter property.')),
@@ -175,6 +175,16 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         related_name='inventory_labels',
         help_text=_('Labels associated with this inventory.'),
     )
+    prevent_instance_group_fallback = models.BooleanField(
+        default=False,
+        help_text=(
+            "If enabled, the inventory will prevent adding any organization "
+            "instance groups to the list of preferred instances groups to run "
+            "associated job templates on."
+            "If this setting is enabled and you provided an empty list, the global instance "
+            "groups will be applied."
+        ),
+    )
 
     def get_absolute_url(self, request=None):
         return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request)
@@ -1191,6 +1201,14 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
         default=None,
         null=True,
     )
+    scm_revision = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        editable=False,
+        verbose_name=_('SCM Revision'),
+        help_text=_('The SCM Revision from the Project used for this inventory update.  Only applicable to inventories source from scm'),
+    )
 
     @property
     def is_container_group_task(self):
@@ -1260,15 +1278,19 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
 
     @property
     def preferred_instance_groups(self):
-        if self.inventory_source.inventory is not None and self.inventory_source.inventory.organization is not None:
-            organization_groups = [x for x in self.inventory_source.inventory.organization.instance_groups.all()]
-        else:
-            organization_groups = []
+        selected_groups = []
         if self.inventory_source.inventory is not None:
-            inventory_groups = [x for x in self.inventory_source.inventory.instance_groups.all()]
-        else:
-            inventory_groups = []
-        selected_groups = inventory_groups + organization_groups
+            # Add the inventory sources IG to the selected IGs first
+            for instance_group in self.inventory_source.inventory.instance_groups.all():
+                selected_groups.append(instance_group)
+            # If the inventory allows for fallback and we have an organization then also append the orgs IGs to the end of the list
+            if (
+                not getattr(self.inventory_source.inventory, 'prevent_instance_group_fallback', False)
+                and self.inventory_source.inventory.organization is not None
+            ):
+                for instance_group in self.inventory_source.inventory.organization.instance_groups.all():
+                    selected_groups.append(instance_group)
+
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups

awx/main/models/jobs.py

@@ -43,8 +43,8 @@ from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin,
 )
-from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
-from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob
+from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField, polymorphic
+from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob, OrderedManyToManyField
 from awx.main.models.mixins import (
     ResourceMixin,
     SurveyJobTemplateMixin,
@@ -203,7 +203,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     playbook) to an inventory source with a given credential.
     """
 
-    FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials', 'survey_spec']
+    FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials', 'survey_spec', 'prevent_instance_group_fallback']
     FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential']
     SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
 
@@ -227,15 +227,6 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         blank=True,
         default=False,
     )
-    ask_limit_on_launch = AskForField(
-        blank=True,
-        default=False,
-    )
-    ask_tags_on_launch = AskForField(blank=True, default=False, allows_field='job_tags')
-    ask_skip_tags_on_launch = AskForField(
-        blank=True,
-        default=False,
-    )
     ask_job_type_on_launch = AskForField(
         blank=True,
         default=False,
@@ -244,12 +235,27 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         blank=True,
         default=False,
     )
-    ask_inventory_on_launch = AskForField(
+    ask_credential_on_launch = AskForField(blank=True, default=False, allows_field='credentials')
+    ask_execution_environment_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_forks_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_job_slice_count_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_timeout_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_instance_groups_on_launch = AskForField(
         blank=True,
         default=False,
     )
-    ask_credential_on_launch = AskForField(blank=True, default=False, allows_field='credentials')
-    ask_scm_branch_on_launch = AskForField(blank=True, default=False, allows_field='scm_branch')
     job_slice_count = models.PositiveIntegerField(
         blank=True,
         default=1,
@@ -268,6 +274,15 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
             'admin_role',
         ],
     )
+    prevent_instance_group_fallback = models.BooleanField(
+        default=False,
+        help_text=(
+            "If enabled, the job template will prevent adding any inventory or organization "
+            "instance groups to the list of preferred instances groups to run on."
+            "If this setting is enabled and you provided an empty list, the global instance "
+            "groups will be applied."
+        ),
+    )
 
     @classmethod
     def _get_unified_job_class(cls):
@@ -276,7 +291,17 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in JobOptions._meta.fields) | set(
-            ['name', 'description', 'organization', 'survey_passwords', 'labels', 'credentials', 'job_slice_number', 'job_slice_count', 'execution_environment']
+            [
+                'name',
+                'description',
+                'organization',
+                'survey_passwords',
+                'labels',
+                'credentials',
+                'job_slice_number',
+                'job_slice_count',
+                'execution_environment',
+            ]
         )
 
     @property
@@ -314,10 +339,13 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
         actual_inventory = self.inventory
         if self.ask_inventory_on_launch and 'inventory' in kwargs:
             actual_inventory = kwargs['inventory']
+        actual_slice_count = self.job_slice_count
+        if self.ask_job_slice_count_on_launch and 'job_slice_count' in kwargs:
+            actual_slice_count = kwargs['job_slice_count']
         if actual_inventory:
-            return min(self.job_slice_count, actual_inventory.hosts.count())
+            return min(actual_slice_count, actual_inventory.hosts.count())
         else:
-            return self.job_slice_count
+            return actual_slice_count
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get('update_fields', [])
@@ -425,10 +453,15 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
 
             field = self._meta.get_field(field_name)
             if isinstance(field, models.ManyToManyField):
-                old_value = set(old_value.all())
-                new_value = set(kwargs[field_name]) - old_value
-                if not new_value:
-                    continue
+                if field_name == 'instance_groups':
+                    # Instance groups are ordered so we can't make a set out of them
+                    old_value = old_value.all()
+                elif field_name == 'credentials':
+                    # Credentials have a weird pattern because of how they are layered
+                    old_value = set(old_value.all())
+                    new_value = set(kwargs[field_name]) - old_value
+                    if not new_value:
+                        continue
 
             if new_value == old_value:
                 # no-op case: Fields the same as template's value
@@ -449,6 +482,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
                         rejected_data[field_name] = new_value
                         errors_dict[field_name] = _('Project does not allow override of branch.')
                         continue
+                elif field_name == 'job_slice_count' and (new_value > 1) and (self.get_effective_slice_ct(kwargs) <= 1):
+                    rejected_data[field_name] = new_value
+                    errors_dict[field_name] = _('Job inventory does not have enough hosts for slicing')
+                    continue
                 # accepted prompt
                 prompted_data[field_name] = new_value
             else:
@@ -767,19 +804,15 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
 
     @property
     def preferred_instance_groups(self):
-        if self.organization is not None:
-            organization_groups = [x for x in self.organization.instance_groups.all()]
-        else:
-            organization_groups = []
-        if self.inventory is not None:
-            inventory_groups = [x for x in self.inventory.instance_groups.all()]
-        else:
-            inventory_groups = []
-        if self.job_template is not None:
-            template_groups = [x for x in self.job_template.instance_groups.all()]
-        else:
-            template_groups = []
-        selected_groups = template_groups + inventory_groups + organization_groups
+        # If the user specified instance groups those will be handled by the unified_job.create_unified_job
+        # This function handles only the defaults for a template w/o user specification
+        selected_groups = []
+        for obj_type in ['job_template', 'inventory', 'organization']:
+            if getattr(self, obj_type) is not None:
+                for instance_group in getattr(self, obj_type).instance_groups.all():
+                    selected_groups.append(instance_group)
+                if getattr(getattr(self, obj_type), 'prevent_instance_group_fallback', False):
+                    break
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups
@@ -906,10 +939,36 @@ class LaunchTimeConfigBase(BaseModel):
     # This is a solution to the nullable CharField problem, specific to prompting
     char_prompts = JSONBlob(default=dict, blank=True)
 
-    def prompts_dict(self, display=False):
+    # Define fields that are not really fields, but alias to char_prompts lookups
+    limit = NullablePromptPseudoField('limit')
+    scm_branch = NullablePromptPseudoField('scm_branch')
+    job_tags = NullablePromptPseudoField('job_tags')
+    skip_tags = NullablePromptPseudoField('skip_tags')
+    diff_mode = NullablePromptPseudoField('diff_mode')
+    job_type = NullablePromptPseudoField('job_type')
+    verbosity = NullablePromptPseudoField('verbosity')
+    forks = NullablePromptPseudoField('forks')
+    job_slice_count = NullablePromptPseudoField('job_slice_count')
+    timeout = NullablePromptPseudoField('timeout')
+
+    # NOTE: additional fields are assumed to exist but must be defined in subclasses
+    # due to technical limitations
+    SUBCLASS_FIELDS = (
+        'instance_groups',  # needs a through model defined
+        'extra_vars',  # alternates between extra_vars and extra_data
+        'credentials',  # already a unified job and unified JT field
+        'labels',  # already a unified job and unified JT field
+        'execution_environment',  # already a unified job and unified JT field
+    )
+
+    def prompts_dict(self, display=False, for_cls=None):
         data = {}
+        if for_cls:
+            cls = for_cls
+        else:
+            cls = JobTemplate
         # Some types may have different prompts, but always subset of JT prompts
-        for prompt_name in JobTemplate.get_ask_mapping().keys():
+        for prompt_name in cls.get_ask_mapping().keys():
             try:
                 field = self._meta.get_field(prompt_name)
             except FieldDoesNotExist:
@@ -917,18 +976,23 @@ class LaunchTimeConfigBase(BaseModel):
             if isinstance(field, models.ManyToManyField):
                 if not self.pk:
                     continue  # unsaved object can't have related many-to-many
-                prompt_val = set(getattr(self, prompt_name).all())
-                if len(prompt_val) > 0:
-                    data[prompt_name] = prompt_val
+                prompt_values = list(getattr(self, prompt_name).all())
+                # Many to manys can't distinguish between None and []
+                # Because of this, from a config perspective, we assume [] is none and we don't save [] into the config
+                if len(prompt_values) > 0:
+                    data[prompt_name] = prompt_values
             elif prompt_name == 'extra_vars':
                 if self.extra_vars:
+                    extra_vars = {}
                     if display:
-                        data[prompt_name] = self.display_extra_vars()
+                        extra_vars = self.display_extra_vars()
                     else:
-                        data[prompt_name] = self.extra_vars
+                        extra_vars = self.extra_vars
                     # Depending on model, field type may save and return as string
-                    if isinstance(data[prompt_name], str):
-                        data[prompt_name] = parse_yaml_or_json(data[prompt_name])
+                    if isinstance(extra_vars, str):
+                        extra_vars = parse_yaml_or_json(extra_vars)
+                    if extra_vars:
+                        data['extra_vars'] = extra_vars
                 if self.survey_passwords and not display:
                     data['survey_passwords'] = self.survey_passwords
             else:
@@ -938,15 +1002,6 @@ class LaunchTimeConfigBase(BaseModel):
         return data
 
 
-for field_name in JobTemplate.get_ask_mapping().keys():
-    if field_name == 'extra_vars':
-        continue
-    try:
-        LaunchTimeConfigBase._meta.get_field(field_name)
-    except FieldDoesNotExist:
-        setattr(LaunchTimeConfigBase, field_name, NullablePromptPseudoField(field_name))
-
-
 class LaunchTimeConfig(LaunchTimeConfigBase):
     """
     Common model for all objects that save details of a saved launch config
@@ -965,8 +1020,18 @@ class LaunchTimeConfig(LaunchTimeConfigBase):
             blank=True,
         )
     )
-    # Credentials needed for non-unified job / unified JT models
+    # Fields needed for non-unified job / unified JT models, because they are defined on unified models
     credentials = models.ManyToManyField('Credential', related_name='%(class)ss')
+    labels = models.ManyToManyField('Label', related_name='%(class)s_labels')
+    execution_environment = models.ForeignKey(
+        'ExecutionEnvironment',
+        null=True,
+        blank=True,
+        default=None,
+        on_delete=polymorphic.SET_NULL,
+        related_name='%(class)s_as_prompt',
+        help_text="The container image to be used for execution.",
+    )
 
     @property
     def extra_vars(self):
@@ -1010,6 +1075,11 @@ class JobLaunchConfig(LaunchTimeConfig):
         editable=False,
     )
 
+    # Instance Groups needed for non-unified job / unified JT models
+    instance_groups = OrderedManyToManyField(
+        'InstanceGroup', related_name='%(class)ss', blank=True, editable=False, through='JobLaunchConfigInstanceGroupMembership'
+    )
+
     def has_user_prompts(self, template):
         """
         Returns True if any fields exist in the launch config that are

awx/main/models/label.py

@@ -10,6 +10,8 @@ from awx.api.versioning import reverse
 from awx.main.models.base import CommonModelNameNotUnique
 from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
 from awx.main.models.inventory import Inventory
+from awx.main.models.schedules import Schedule
+from awx.main.models.workflow import WorkflowJobTemplateNode, WorkflowJobNode
 
 __all__ = ('Label',)
 
@@ -34,16 +36,22 @@ class Label(CommonModelNameNotUnique):
     def get_absolute_url(self, request=None):
         return reverse('api:label_detail', kwargs={'pk': self.pk}, request=request)
 
-    @staticmethod
-    def get_orphaned_labels():
-        return Label.objects.filter(organization=None, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
-
     def is_detached(self):
-        return Label.objects.filter(id=self.id, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True).exists()
+        return Label.objects.filter(
+            id=self.id,
+            unifiedjob_labels__isnull=True,
+            unifiedjobtemplate_labels__isnull=True,
+            inventory_labels__isnull=True,
+            schedule_labels__isnull=True,
+            workflowjobtemplatenode_labels__isnull=True,
+            workflowjobnode_labels__isnull=True,
+        ).exists()
 
     def is_candidate_for_detach(self):
-
-        c1 = UnifiedJob.objects.filter(labels__in=[self.id]).count()
-        c2 = UnifiedJobTemplate.objects.filter(labels__in=[self.id]).count()
-        c3 = Inventory.objects.filter(labels__in=[self.id]).count()
-        return (c1 + c2 + c3 - 1) == 0
+        count = UnifiedJob.objects.filter(labels__in=[self.id]).count()  # Both Jobs and WFJobs
+        count += UnifiedJobTemplate.objects.filter(labels__in=[self.id]).count()  # Both JTs and WFJT
+        count += Inventory.objects.filter(labels__in=[self.id]).count()
+        count += Schedule.objects.filter(labels__in=[self.id]).count()
+        count += WorkflowJobTemplateNode.objects.filter(labels__in=[self.id]).count()
+        count += WorkflowJobNode.objects.filter(labels__in=[self.id]).count()
+        return (count - 1) == 0

awx/main/models/mixins.py

@@ -104,6 +104,33 @@ class SurveyJobTemplateMixin(models.Model):
         default=False,
     )
     survey_spec = prevent_search(JSONBlob(default=dict, blank=True))
+
+    ask_inventory_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_limit_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_scm_branch_on_launch = AskForField(
+        blank=True,
+        default=False,
+        allows_field='scm_branch',
+    )
+    ask_labels_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
+    ask_tags_on_launch = AskForField(
+        blank=True,
+        default=False,
+        allows_field='job_tags',
+    )
+    ask_skip_tags_on_launch = AskForField(
+        blank=True,
+        default=False,
+    )
     ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars')
 
     def survey_password_variables(self):

awx/main/models/schedules.py

@@ -18,6 +18,7 @@ from django.utils.translation import gettext_lazy as _
 
 # AWX
 from awx.api.versioning import reverse
+from awx.main.fields import OrderedManyToManyField
 from awx.main.models.base import PrimordialModel
 from awx.main.models.jobs import LaunchTimeConfig
 from awx.main.utils import ignore_inventory_computed_fields
@@ -83,6 +84,13 @@ class Schedule(PrimordialModel, LaunchTimeConfig):
     )
     rrule = models.TextField(help_text=_("A value representing the schedules iCal recurrence rule."))
     next_run = models.DateTimeField(null=True, default=None, editable=False, help_text=_("The next time that the scheduled action will run."))
+    instance_groups = OrderedManyToManyField(
+        'InstanceGroup',
+        related_name='schedule_instance_groups',
+        blank=True,
+        editable=False,
+        through='ScheduleInstanceGroupMembership',
+    )
 
     @classmethod
     def get_zoneinfo(cls):

awx/main/models/unified_jobs.py

@@ -332,10 +332,11 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
 
         return NotificationTemplate.objects.none()
 
-    def create_unified_job(self, **kwargs):
+    def create_unified_job(self, instance_groups=None, **kwargs):
         """
         Create a new unified job based on this unified job template.
         """
+        # TODO: rename kwargs to prompts, to set expectation that these are runtime values
         new_job_passwords = kwargs.pop('survey_passwords', {})
         eager_fields = kwargs.pop('_eager_fields', None)
 
@@ -382,7 +383,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
             unified_job.survey_passwords = new_job_passwords
             kwargs['survey_passwords'] = new_job_passwords  # saved in config object for relaunch
 
-        unified_job.preferred_instance_groups_cache = unified_job._get_preferred_instance_group_cache()
+        if instance_groups:
+            unified_job.preferred_instance_groups_cache = [ig.id for ig in instance_groups]
+        else:
+            unified_job.preferred_instance_groups_cache = unified_job._get_preferred_instance_group_cache()
 
         unified_job._set_default_dependencies_processed()
         unified_job.task_impact = unified_job._get_task_impact()
@@ -412,13 +416,17 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
             unified_job.handle_extra_data(validated_kwargs['extra_vars'])
 
         # Create record of provided prompts for relaunch and rescheduling
-        unified_job.create_config_from_prompts(kwargs, parent=self)
+        config = unified_job.create_config_from_prompts(kwargs, parent=self)
+        if instance_groups:
+            for ig in instance_groups:
+                config.instance_groups.add(ig)
 
         # manually issue the create activity stream entry _after_ M2M relations
         # have been associated to the UJ
         if unified_job.__class__ in activity_stream_registrar.models:
             activity_stream_create(None, unified_job, True)
         unified_job.log_lifecycle("created")
+
         return unified_job
 
     @classmethod
@@ -973,22 +981,38 @@ class UnifiedJob(
             valid_fields.extend(['survey_passwords', 'extra_vars'])
         else:
             kwargs.pop('survey_passwords', None)
+        many_to_many_fields = []
         for field_name, value in kwargs.items():
             if field_name not in valid_fields:
                 raise Exception('Unrecognized launch config field {}.'.format(field_name))
-            if field_name == 'credentials':
+            field = None
+            # may use extra_data as a proxy for extra_vars
+            if field_name in config.SUBCLASS_FIELDS and field_name != 'extra_vars':
+                field = config._meta.get_field(field_name)
+            if isinstance(field, models.ManyToManyField):
+                many_to_many_fields.append(field_name)
                 continue
-            key = field_name
-            if key == 'extra_vars':
-                key = 'extra_data'
-            setattr(config, key, value)
+            if isinstance(field, (models.ForeignKey)) and (value is None):
+                continue  # the null value indicates not-provided for ForeignKey case
+            setattr(config, field_name, value)
         config.save()
 
-        job_creds = set(kwargs.get('credentials', []))
-        if 'credentials' in [field.name for field in parent._meta.get_fields()]:
-            job_creds = job_creds - set(parent.credentials.all())
-        if job_creds:
-            config.credentials.add(*job_creds)
+        for field_name in many_to_many_fields:
+            prompted_items = kwargs.get(field_name, [])
+            if not prompted_items:
+                continue
+            if field_name == 'instance_groups':
+                # Here we are doing a loop to make sure we preserve order for this Ordered field
+                # also do not merge IGs with parent, so this saves the literal list
+                for item in prompted_items:
+                    getattr(config, field_name).add(item)
+            else:
+                # Assuming this field merges prompts with parent, save just the diff
+                if field_name in [field.name for field in parent._meta.get_fields()]:
+                    prompted_items = set(prompted_items) - set(getattr(parent, field_name).all())
+                if prompted_items:
+                    getattr(config, field_name).add(*prompted_items)
+
         return config
 
     @property

awx/main/models/workflow.py

@@ -29,7 +29,7 @@ from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, Un
 from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
 from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
 from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
-from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob
+from awx.main.fields import ImplicitRoleField, JSONBlob, OrderedManyToManyField
 from awx.main.models.mixins import (
     ResourceMixin,
     SurveyJobTemplateMixin,
@@ -114,6 +114,9 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
             'credentials',
             'char_prompts',
             'all_parents_must_converge',
+            'labels',
+            'instance_groups',
+            'execution_environment',
         ]
 
     def create_workflow_job_node(self, **kwargs):
@@ -122,7 +125,7 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
         """
         create_kwargs = {}
         for field_name in self._get_workflow_job_field_names():
-            if field_name == 'credentials':
+            if field_name in ['credentials', 'labels', 'instance_groups']:
                 continue
             if field_name in kwargs:
                 create_kwargs[field_name] = kwargs[field_name]
@@ -132,10 +135,20 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
         new_node = WorkflowJobNode.objects.create(**create_kwargs)
         if self.pk:
             allowed_creds = self.credentials.all()
+            allowed_labels = self.labels.all()
+            allowed_instance_groups = self.instance_groups.all()
         else:
             allowed_creds = []
+            allowed_labels = []
+            allowed_instance_groups = []
         for cred in allowed_creds:
             new_node.credentials.add(cred)
+
+        for label in allowed_labels:
+            new_node.labels.add(label)
+        for instance_group in allowed_instance_groups:
+            new_node.instance_groups.add(instance_group)
+
         return new_node
 
 
@@ -153,6 +166,9 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
         'char_prompts',
         'all_parents_must_converge',
         'identifier',
+        'labels',
+        'execution_environment',
+        'instance_groups',
     ]
     REENCRYPTION_BLOCKLIST_AT_COPY = ['extra_data', 'survey_passwords']
 
@@ -167,6 +183,13 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
         blank=False,
         help_text=_('An identifier for this node that is unique within its workflow. ' 'It is copied to workflow job nodes corresponding to this node.'),
     )
+    instance_groups = OrderedManyToManyField(
+        'InstanceGroup',
+        related_name='workflow_job_template_node_instance_groups',
+        blank=True,
+        editable=False,
+        through='WorkflowJobTemplateNodeBaseInstanceGroupMembership',
+    )
 
     class Meta:
         app_label = 'main'
@@ -211,7 +234,7 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
         approval_template = WorkflowApprovalTemplate(**kwargs)
         approval_template.save()
         self.unified_job_template = approval_template
-        self.save()
+        self.save(update_fields=['unified_job_template'])
         return approval_template
 
 
@@ -250,6 +273,9 @@ class WorkflowJobNode(WorkflowNodeBase):
         blank=True,  # blank denotes pre-migration job nodes
         help_text=_('An identifier coresponding to the workflow job template node that this node was created from.'),
     )
+    instance_groups = OrderedManyToManyField(
+        'InstanceGroup', related_name='workflow_job_node_instance_groups', blank=True, editable=False, through='WorkflowJobNodeBaseInstanceGroupMembership'
+    )
 
     class Meta:
         app_label = 'main'
@@ -265,19 +291,6 @@ class WorkflowJobNode(WorkflowNodeBase):
     def get_absolute_url(self, request=None):
         return reverse('api:workflow_job_node_detail', kwargs={'pk': self.pk}, request=request)
 
-    def prompts_dict(self, *args, **kwargs):
-        r = super(WorkflowJobNode, self).prompts_dict(*args, **kwargs)
-        # Explanation - WFJT extra_vars still break pattern, so they are not
-        # put through prompts processing, but inventory and others are only accepted
-        # if JT prompts for it, so it goes through this mechanism
-        if self.workflow_job:
-            if self.workflow_job.inventory_id:
-                # workflow job inventory takes precedence
-                r['inventory'] = self.workflow_job.inventory
-            if self.workflow_job.char_prompts:
-                r.update(self.workflow_job.char_prompts)
-        return r
-
     def get_job_kwargs(self):
         """
         In advance of creating a new unified job as part of a workflow,
@@ -287,16 +300,38 @@ class WorkflowJobNode(WorkflowNodeBase):
         """
         # reject/accept prompted fields
         data = {}
+        wj_special_vars = {}
+        wj_special_passwords = {}
         ujt_obj = self.unified_job_template
         if ujt_obj is not None:
-            # MERGE note: move this to prompts_dict method on node when merging
-            # with the workflow inventory branch
-            prompts_data = self.prompts_dict()
-            if isinstance(ujt_obj, WorkflowJobTemplate):
-                if self.workflow_job.extra_vars:
-                    prompts_data.setdefault('extra_vars', {})
-                    prompts_data['extra_vars'].update(self.workflow_job.extra_vars_dict)
-            accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs(**prompts_data)
+            node_prompts_data = self.prompts_dict(for_cls=ujt_obj.__class__)
+            wj_prompts_data = self.workflow_job.prompts_dict(for_cls=ujt_obj.__class__)
+            # Explanation - special historical case
+            # WFJT extra_vars ignored JobTemplate.ask_variables_on_launch, bypassing _accept_or_ignore_job_kwargs
+            # inventory and others are only accepted if JT prompts for it with related ask_ field
+            # this is inconsistent, but maintained
+            if not isinstance(ujt_obj, WorkflowJobTemplate):
+                wj_special_vars = wj_prompts_data.pop('extra_vars', {})
+                wj_special_passwords = wj_prompts_data.pop('survey_passwords', {})
+            elif 'extra_vars' in node_prompts_data:
+                # Follow the vars combination rules
+                node_prompts_data['extra_vars'].update(wj_prompts_data.pop('extra_vars', {}))
+            elif 'survey_passwords' in node_prompts_data:
+                node_prompts_data['survey_passwords'].update(wj_prompts_data.pop('survey_passwords', {}))
+
+            # Follow the credential combination rules
+            if ('credentials' in wj_prompts_data) and ('credentials' in node_prompts_data):
+                wj_pivoted_creds = Credential.unique_dict(wj_prompts_data['credentials'])
+                node_pivoted_creds = Credential.unique_dict(node_prompts_data['credentials'])
+                node_pivoted_creds.update(wj_pivoted_creds)
+                wj_prompts_data['credentials'] = [cred for cred in node_pivoted_creds.values()]
+
+            # NOTE: no special rules for instance_groups, because they do not merge
+            # or labels, because they do not propogate WFJT-->node at all
+
+            # Combine WFJT prompts with node here, WFJT at higher level
+            node_prompts_data.update(wj_prompts_data)
+            accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs(**node_prompts_data)
             if errors:
                 logger.info(
                     _('Bad launch configuration starting template {template_pk} as part of ' 'workflow {workflow_pk}. Errors:\n{error_text}').format(
@@ -304,15 +339,6 @@ class WorkflowJobNode(WorkflowNodeBase):
                     )
                 )
             data.update(accepted_fields)  # missing fields are handled in the scheduler
-            try:
-                # config saved on the workflow job itself
-                wj_config = self.workflow_job.launch_config
-            except ObjectDoesNotExist:
-                wj_config = None
-            if wj_config:
-                accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs(**wj_config.prompts_dict())
-                accepted_fields.pop('extra_vars', None)  # merge handled with other extra_vars later
-                data.update(accepted_fields)
         # build ancestor artifacts, save them to node model for later
         aa_dict = {}
         is_root_node = True
@@ -325,15 +351,12 @@ class WorkflowJobNode(WorkflowNodeBase):
             self.ancestor_artifacts = aa_dict
             self.save(update_fields=['ancestor_artifacts'])
         # process password list
-        password_dict = {}
+        password_dict = data.get('survey_passwords', {})
         if '_ansible_no_log' in aa_dict:
             for key in aa_dict:
                 if key != '_ansible_no_log':
                     password_dict[key] = REPLACE_STR
-        if self.workflow_job.survey_passwords:
-            password_dict.update(self.workflow_job.survey_passwords)
-        if self.survey_passwords:
-            password_dict.update(self.survey_passwords)
+        password_dict.update(wj_special_passwords)
         if password_dict:
             data['survey_passwords'] = password_dict
         # process extra_vars
@@ -343,12 +366,12 @@ class WorkflowJobNode(WorkflowNodeBase):
                 functional_aa_dict = copy(aa_dict)
                 functional_aa_dict.pop('_ansible_no_log', None)
                 extra_vars.update(functional_aa_dict)
-        if ujt_obj and isinstance(ujt_obj, JobTemplate):
-            # Workflow Job extra_vars higher precedence than ancestor artifacts
-            if self.workflow_job and self.workflow_job.extra_vars:
-                extra_vars.update(self.workflow_job.extra_vars_dict)
+
+        # Workflow Job extra_vars higher precedence than ancestor artifacts
+        extra_vars.update(wj_special_vars)
         if extra_vars:
             data['extra_vars'] = extra_vars
+
         # ensure that unified jobs created by WorkflowJobs are marked
         data['_eager_fields'] = {'launch_type': 'workflow'}
         if self.workflow_job and self.workflow_job.created_by:
@@ -374,6 +397,10 @@ class WorkflowJobOptions(LaunchTimeConfigBase):
             )
         )
     )
+    # Workflow jobs are used for sliced jobs, and thus, must be a conduit for any JT prompts
+    instance_groups = OrderedManyToManyField(
+        'InstanceGroup', related_name='workflow_job_instance_groups', blank=True, editable=False, through='WorkflowJobInstanceGroupMembership'
+    )
     allow_simultaneous = models.BooleanField(default=False)
 
     extra_vars_dict = VarsDictProperty('extra_vars', True)
@@ -385,7 +412,7 @@ class WorkflowJobOptions(LaunchTimeConfigBase):
     @classmethod
     def _get_unified_job_field_names(cls):
         r = set(f.name for f in WorkflowJobOptions._meta.fields) | set(
-            ['name', 'description', 'organization', 'survey_passwords', 'labels', 'limit', 'scm_branch']
+            ['name', 'description', 'organization', 'survey_passwords', 'labels', 'limit', 'scm_branch', 'job_tags', 'skip_tags']
         )
         r.remove('char_prompts')  # needed due to copying launch config to launch config
         return r
@@ -425,26 +452,29 @@ class WorkflowJobOptions(LaunchTimeConfigBase):
 class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin, RelatedJobsMixin, WebhookTemplateMixin):
 
     SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
-    FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'organization', 'instance_groups', 'workflow_job_template_nodes', 'credentials', 'survey_spec']
+    FIELDS_TO_PRESERVE_AT_COPY = [
+        'labels',
+        'organization',
+        'instance_groups',
+        'workflow_job_template_nodes',
+        'credentials',
+        'survey_spec',
+        'skip_tags',
+        'job_tags',
+        'execution_environment',
+    ]
 
     class Meta:
         app_label = 'main'
 
-    ask_inventory_on_launch = AskForField(
+    notification_templates_approvals = models.ManyToManyField(
+        "NotificationTemplate",
         blank=True,
-        default=False,
+        related_name='%(class)s_notification_templates_for_approvals',
     )
-    ask_limit_on_launch = AskForField(
-        blank=True,
-        default=False,
+    admin_role = ImplicitRoleField(
+        parent_role=['singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, 'organization.workflow_admin_role'],
     )
-    ask_scm_branch_on_launch = AskForField(
-        blank=True,
-        default=False,
-    )
-    notification_templates_approvals = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_approvals')
-
-    admin_role = ImplicitRoleField(parent_role=['singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, 'organization.workflow_admin_role'])
     execute_role = ImplicitRoleField(
         parent_role=[
             'admin_role',
@@ -713,6 +743,25 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
             artifacts.update(job.get_effective_artifacts(parents_set=new_parents_set))
         return artifacts
 
+    def prompts_dict(self, *args, **kwargs):
+        if self.job_template_id:
+            # HACK: Exception for sliced jobs here, this is bad
+            # when sliced jobs were introduced, workflows did not have all the prompted JT fields
+            # so to support prompting with slicing, we abused the workflow job launch config
+            # these would be more properly saved on the workflow job, but it gets the wrong fields now
+            try:
+                wj_config = self.launch_config
+                r = wj_config.prompts_dict(*args, **kwargs)
+            except ObjectDoesNotExist:
+                r = {}
+        else:
+            r = super().prompts_dict(*args, **kwargs)
+            # Workflow labels and job labels are treated separately
+            # that means that they do not propogate from WFJT / workflow job to jobs in workflow
+            r.pop('labels', None)
+
+        return r
+
     def get_notification_templates(self):
         return self.workflow_job_template.notification_templates
 

awx/main/scheduler/task_manager.py

@@ -609,8 +609,6 @@ class TaskManager(TaskBase):
 
             found_acceptable_queue = False
 
-            preferred_instance_groups = self.instance_groups.get_instance_groups_from_task_cache(task)
-
             # Determine if there is control capacity for the task
             if task.capacity_type == 'control':
                 control_impact = task.task_impact + settings.AWX_CONTROL_NODE_TASK_IMPACT
@@ -636,16 +634,12 @@ class TaskManager(TaskBase):
                 found_acceptable_queue = True
                 continue
 
-            for instance_group in preferred_instance_groups:
+            for instance_group in self.instance_groups.get_instance_groups_from_task_cache(task):
                 if instance_group.is_container_group:
                     self.start_task(task, instance_group, task.get_jobs_fail_chain(), None)
                     found_acceptable_queue = True
                     break
 
-                # TODO: remove this after we have confidence that OCP control nodes are reporting node_type=control
-                if settings.IS_K8S and task.capacity_type == 'execution':
-                    logger.debug("Skipping group {}, task cannot run on control plane".format(instance_group.name))
-                    continue
                 # at this point we know the instance group is NOT a container group
                 # because if it was, it would have started the task and broke out of the loop.
                 execution_instance = self.instance_groups.fit_task_to_most_remaining_capacity_instance(

awx/main/scheduler/task_manager_models.py

@@ -37,7 +37,11 @@ class TaskManagerInstances:
     def __init__(self, active_tasks, instances=None, instance_fields=('node_type', 'capacity', 'hostname', 'enabled')):
         self.instances_by_hostname = dict()
         if instances is None:
-            instances = Instance.objects.filter(hostname__isnull=False, enabled=True).exclude(node_type='hop').only(*instance_fields)
+            instances = (
+                Instance.objects.filter(hostname__isnull=False, node_state=Instance.States.READY, enabled=True)
+                .exclude(node_type='hop')
+                .only('node_type', 'node_state', 'capacity', 'hostname', 'enabled')
+            )
         for instance in instances:
             self.instances_by_hostname[instance.hostname] = TaskManagerInstance(instance)
 

awx/main/tasks/jobs.py

@@ -145,7 +145,7 @@ class BaseTask(object):
         """
         Return params structure to be executed by the container runtime
         """
-        if settings.IS_K8S:
+        if settings.IS_K8S and instance.instance_group.is_container_group:
             return {}
 
         image = instance.execution_environment.image
@@ -739,8 +739,7 @@ class SourceControlMixin(BaseTask):
                 sync_task = RunProjectUpdate(job_private_data_dir=private_data_dir)
                 sync_task.run(local_project_sync.id)
                 local_project_sync.refresh_from_db()
-                if isinstance(self.instance, Job):
-                    self.instance = self.update_model(self.instance.pk, scm_revision=local_project_sync.scm_revision)
+                self.instance = self.update_model(self.instance.pk, scm_revision=local_project_sync.scm_revision)
             except Exception:
                 local_project_sync.refresh_from_db()
                 if local_project_sync.status != 'canceled':
@@ -759,8 +758,7 @@ class SourceControlMixin(BaseTask):
         else:
             # Case where a local sync is not needed, meaning that local tree is
             # up-to-date with project, job is running project current version
-            if isinstance(self.instance, Job):
-                self.instance = self.update_model(self.instance.pk, scm_revision=project.scm_revision)
+            self.instance = self.update_model(self.instance.pk, scm_revision=project.scm_revision)
             # Project update does not copy the folder, so copy here
             RunProjectUpdate.make_local_copy(project, private_data_dir)
 

awx/main/tasks/receptor.py

@@ -27,12 +27,18 @@ from awx.main.utils.common import (
 )
 from awx.main.constants import MAX_ISOLATED_PATH_COLON_DELIMITER
 from awx.main.tasks.signals import signal_state, signal_callback, SignalExit
+from awx.main.models import Instance, InstanceLink, UnifiedJob
+from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch.publish import task
 
 # Receptorctl
 from receptorctl.socket_interface import ReceptorControl
 
+from filelock import FileLock
+
 logger = logging.getLogger('awx.main.tasks.receptor')
 __RECEPTOR_CONF = '/etc/receptor/receptor.conf'
+__RECEPTOR_CONF_LOCKFILE = f'{__RECEPTOR_CONF}.lock'
 RECEPTOR_ACTIVE_STATES = ('Pending', 'Running')
 
 
@@ -42,9 +48,22 @@ class ReceptorConnectionType(Enum):
     STREAMTLS = 2
 
 
+def read_receptor_config():
+    # for K8S deployments, getting a lock is necessary as another process
+    # may be re-writing the config at this time
+    if settings.IS_K8S:
+        lock = FileLock(__RECEPTOR_CONF_LOCKFILE)
+        with lock:
+            with open(__RECEPTOR_CONF, 'r') as f:
+                return yaml.safe_load(f)
+    else:
+        with open(__RECEPTOR_CONF, 'r') as f:
+            return yaml.safe_load(f)
+
+
 def get_receptor_sockfile():
-    with open(__RECEPTOR_CONF, 'r') as f:
-        data = yaml.safe_load(f)
+    data = read_receptor_config()
+
     for section in data:
         for entry_name, entry_data in section.items():
             if entry_name == 'control-service':
@@ -60,8 +79,7 @@ def get_tls_client(use_stream_tls=None):
     if not use_stream_tls:
         return None
 
-    with open(__RECEPTOR_CONF, 'r') as f:
-        data = yaml.safe_load(f)
+    data = read_receptor_config()
     for section in data:
         for entry_name, entry_data in section.items():
             if entry_name == 'tls-client':
@@ -78,12 +96,25 @@ def get_receptor_ctl():
         return ReceptorControl(receptor_sockfile)
 
 
+def find_node_in_mesh(node_name, receptor_ctl):
+    attempts = 10
+    backoff = 1
+    for attempt in range(attempts):
+        all_nodes = receptor_ctl.simple_command("status").get('Advertisements', None)
+        for node in all_nodes:
+            if node.get('NodeID') == node_name:
+                return node
+        else:
+            logger.warning(f"Instance {node_name} is not in the receptor mesh. {attempts-attempt} attempts left.")
+            time.sleep(backoff)
+            backoff += 1
+    else:
+        raise ReceptorNodeNotFound(f'Instance {node_name} is not in the receptor mesh')
+
+
 def get_conn_type(node_name, receptor_ctl):
-    all_nodes = receptor_ctl.simple_command("status").get('Advertisements', None)
-    for node in all_nodes:
-        if node.get('NodeID') == node_name:
-            return ReceptorConnectionType(node.get('ConnType'))
-    raise ReceptorNodeNotFound(f'Instance {node_name} is not in the receptor mesh')
+    node = find_node_in_mesh(node_name, receptor_ctl)
+    return ReceptorConnectionType(node.get('ConnType'))
 
 
 def administrative_workunit_reaper(work_list=None):
@@ -136,8 +167,7 @@ def run_until_complete(node, timing_data=None, **kwargs):
     kwargs.setdefault('payload', '')
 
     transmit_start = time.time()
-    sign_work = False if settings.IS_K8S else True
-    result = receptor_ctl.submit_work(worktype='ansible-runner', node=node, signwork=sign_work, **kwargs)
+    result = receptor_ctl.submit_work(worktype='ansible-runner', node=node, signwork=True, **kwargs)
 
     unit_id = result['unitid']
     run_start = time.time()
@@ -212,7 +242,7 @@ def worker_info(node_name, work_type='ansible-runner'):
         else:
             error_list.append(details)
 
-    except (ReceptorNodeNotFound, RuntimeError) as exc:
+    except Exception as exc:
         error_list.append(str(exc))
 
     # If we have a connection error, missing keys would be trivial consequence of that
@@ -283,10 +313,6 @@ class AWXReceptorJob:
                 except Exception:
                     logger.exception(f"Error releasing work unit {self.unit_id}.")
 
-    @property
-    def sign_work(self):
-        return False if settings.IS_K8S else True
-
     def _run_internal(self, receptor_ctl):
         # Create a socketpair. Where the left side will be used for writing our payload
         # (private data dir, kwargs). The right side will be passed to Receptor for
@@ -446,6 +472,10 @@ class AWXReceptorJob:
 
         return receptor_params
 
+    @property
+    def sign_work(self):
+        return True if self.work_type in ('ansible-runner', 'local') else False
+
     @property
     def work_type(self):
         if self.task.instance.is_container_group_task:
@@ -574,3 +604,105 @@ class AWXReceptorJob:
         else:
             config["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = True
         return config
+
+
+# TODO: receptor reload expects ordering within config items to be preserved
+# if python dictionary is not preserving order properly, may need to find a
+# solution. yaml.dump does not seem to work well with OrderedDict. below line may help
+# yaml.add_representer(OrderedDict, lambda dumper, data: dumper.represent_mapping('tag:yaml.org,2002:map', data.items()))
+#
+RECEPTOR_CONFIG_STARTER = (
+    {'local-only': None},
+    {'log-level': 'debug'},
+    {'node': {'firewallrules': [{'action': 'reject', 'tonode': settings.CLUSTER_HOST_ID, 'toservice': 'control'}]}},
+    {'control-service': {'service': 'control', 'filename': '/var/run/receptor/receptor.sock', 'permissions': '0660'}},
+    {'work-command': {'worktype': 'local', 'command': 'ansible-runner', 'params': 'worker', 'allowruntimeparams': True}},
+    {'work-signing': {'privatekey': '/etc/receptor/signing/work-private-key.pem', 'tokenexpiration': '1m'}},
+    {
+        'work-kubernetes': {
+            'worktype': 'kubernetes-runtime-auth',
+            'authmethod': 'runtime',
+            'allowruntimeauth': True,
+            'allowruntimepod': True,
+            'allowruntimeparams': True,
+        }
+    },
+    {
+        'work-kubernetes': {
+            'worktype': 'kubernetes-incluster-auth',
+            'authmethod': 'incluster',
+            'allowruntimeauth': True,
+            'allowruntimepod': True,
+            'allowruntimeparams': True,
+        }
+    },
+    {
+        'tls-client': {
+            'name': 'tlsclient',
+            'rootcas': '/etc/receptor/tls/ca/receptor-ca.crt',
+            'cert': '/etc/receptor/tls/receptor.crt',
+            'key': '/etc/receptor/tls/receptor.key',
+        }
+    },
+)
+
+
+@task()
+def write_receptor_config():
+    lock = FileLock(__RECEPTOR_CONF_LOCKFILE)
+    with lock:
+        receptor_config = list(RECEPTOR_CONFIG_STARTER)
+
+        this_inst = Instance.objects.me()
+        instances = Instance.objects.filter(node_type=Instance.Types.EXECUTION)
+        existing_peers = {link.target_id for link in InstanceLink.objects.filter(source=this_inst)}
+        new_links = []
+        for instance in instances:
+            peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
+            receptor_config.append(peer)
+            if instance.id not in existing_peers:
+                new_links.append(InstanceLink(source=this_inst, target=instance, link_state=InstanceLink.States.ADDING))
+
+        InstanceLink.objects.bulk_create(new_links)
+
+        with open(__RECEPTOR_CONF, 'w') as file:
+            yaml.dump(receptor_config, file, default_flow_style=False)
+
+    # This needs to be outside of the lock because this function itself will acquire the lock.
+    receptor_ctl = get_receptor_ctl()
+
+    attempts = 10
+    for backoff in range(1, attempts + 1):
+        try:
+            receptor_ctl.simple_command("reload")
+            break
+        except ValueError:
+            logger.warning(f"Unable to reload Receptor configuration. {attempts-backoff} attempts left.")
+            time.sleep(backoff)
+    else:
+        raise RuntimeError("Receptor reload failed")
+
+    links = InstanceLink.objects.filter(source=this_inst, target__in=instances, link_state=InstanceLink.States.ADDING)
+    links.update(link_state=InstanceLink.States.ESTABLISHED)
+
+
+@task(queue=get_local_queuename)
+def remove_deprovisioned_node(hostname):
+    InstanceLink.objects.filter(source__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
+    InstanceLink.objects.filter(target__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
+
+    node_jobs = UnifiedJob.objects.filter(
+        execution_node=hostname,
+        status__in=(
+            'running',
+            'waiting',
+        ),
+    )
+    while node_jobs.exists():
+        time.sleep(60)
+
+    # This will as a side effect also delete the InstanceLinks that are tied to it.
+    Instance.objects.filter(hostname=hostname).delete()
+
+    # Update the receptor configs for all of the control-plane.
+    write_receptor_config.apply_async(queue='tower_broadcast_all')

awx/main/tasks/system.py

@@ -61,7 +61,7 @@ from awx.main.utils.common import (
 from awx.main.utils.external_logging import reconfigure_rsyslog
 from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
-from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanup, administrative_workunit_reaper
+from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanup, administrative_workunit_reaper, write_receptor_config
 from awx.main.consumers import emit_channel_notification
 from awx.main import analytics
 from awx.conf import settings_registry
@@ -81,6 +81,10 @@ Try upgrading OpenSSH or providing your private key in an different format. \
 def dispatch_startup():
     startup_logger = logging.getLogger('awx.main.tasks')
 
+    # TODO: Enable this on VM installs
+    if settings.IS_K8S:
+        write_receptor_config()
+
     startup_logger.debug("Syncing Schedules")
     for sch in Schedule.objects.all():
         try:
@@ -122,7 +126,7 @@ def inform_cluster_of_shutdown():
             reaper.reap_waiting(this_inst, grace_period=0)
         except Exception:
             logger.exception('failed to reap waiting jobs for {}'.format(this_inst.hostname))
-        logger.warning('Normal shutdown signal for instance {}, ' 'removed self from capacity pool.'.format(this_inst.hostname))
+        logger.warning('Normal shutdown signal for instance {}, removed self from capacity pool.'.format(this_inst.hostname))
     except Exception:
         logger.exception('Encountered problem with normal shutdown signal.')
 
@@ -349,9 +353,13 @@ def _cleanup_images_and_files(**kwargs):
             logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}')
 
     # if we are the first instance alphabetically, then run cleanup on execution nodes
-    checker_instance = Instance.objects.filter(node_type__in=['hybrid', 'control'], enabled=True, capacity__gt=0).order_by('-hostname').first()
+    checker_instance = (
+        Instance.objects.filter(node_type__in=['hybrid', 'control'], node_state=Instance.States.READY, enabled=True, capacity__gt=0)
+        .order_by('-hostname')
+        .first()
+    )
     if checker_instance and this_inst.hostname == checker_instance.hostname:
-        for inst in Instance.objects.filter(node_type='execution', enabled=True, capacity__gt=0):
+        for inst in Instance.objects.filter(node_type='execution', node_state=Instance.States.READY, enabled=True, capacity__gt=0):
             runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs)
             if not runner_cleanup_kwargs:
                 continue
@@ -405,7 +413,12 @@ def execution_node_health_check(node):
         return
 
     if instance.node_type != 'execution':
-        raise RuntimeError(f'Execution node health check ran against {instance.node_type} node {instance.hostname}')
+        logger.warning(f'Execution node health check ran against {instance.node_type} node {instance.hostname}')
+        return
+
+    if instance.node_state not in (Instance.States.READY, Instance.States.UNAVAILABLE, Instance.States.INSTALLED):
+        logger.warning(f"Execution node health check ran against node {instance.hostname} in state {instance.node_state}")
+        return
 
     data = worker_info(node)
 
@@ -440,6 +453,7 @@ def inspect_execution_nodes(instance_list):
 
         nowtime = now()
         workers = mesh_status['Advertisements']
+
         for ad in workers:
             hostname = ad['NodeID']
 
@@ -450,25 +464,23 @@ def inspect_execution_nodes(instance_list):
                 continue
 
             # Control-plane nodes are dealt with via local_health_check instead.
-            if instance.node_type in ('control', 'hybrid'):
+            if instance.node_type in (Instance.Types.CONTROL, Instance.Types.HYBRID):
                 continue
 
-            was_lost = instance.is_lost(ref_time=nowtime)
             last_seen = parse_date(ad['Time'])
-
             if instance.last_seen and instance.last_seen >= last_seen:
                 continue
             instance.last_seen = last_seen
             instance.save(update_fields=['last_seen'])
 
             # Only execution nodes should be dealt with by execution_node_health_check
-            if instance.node_type == 'hop':
-                if was_lost and (not instance.is_lost(ref_time=nowtime)):
+            if instance.node_type == Instance.Types.HOP:
+                if instance.node_state in (Instance.States.UNAVAILABLE, Instance.States.INSTALLED):
                     logger.warning(f'Hop node {hostname}, has rejoined the receptor mesh')
                     instance.save_health_data(errors='')
                 continue
 
-            if was_lost:
+            if instance.node_state in (Instance.States.UNAVAILABLE, Instance.States.INSTALLED):
                 # if the instance *was* lost, but has appeared again,
                 # attempt to re-establish the initial capacity and version
                 # check
@@ -487,7 +499,7 @@ def inspect_execution_nodes(instance_list):
 def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
     logger.debug("Cluster node heartbeat task.")
     nowtime = now()
-    instance_list = list(Instance.objects.all())
+    instance_list = list(Instance.objects.filter(node_state__in=(Instance.States.READY, Instance.States.UNAVAILABLE, Instance.States.INSTALLED)))
     this_inst = None
     lost_instances = []
 
@@ -549,11 +561,11 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
         except Exception:
             logger.exception('failed to reap jobs for {}'.format(other_inst.hostname))
         try:
-            if settings.AWX_AUTO_DEPROVISION_INSTANCES:
+            if settings.AWX_AUTO_DEPROVISION_INSTANCES and other_inst.node_type == "control":
                 deprovision_hostname = other_inst.hostname
-                other_inst.delete()
+                other_inst.delete()  # FIXME: what about associated inbound links?
                 logger.info("Host {} Automatically Deprovisioned.".format(deprovision_hostname))
-            elif other_inst.capacity != 0 or (not other_inst.errors):
+            elif other_inst.node_state == Instance.States.READY:
                 other_inst.mark_offline(errors=_('Another cluster node has determined this instance to be unresponsive'))
                 logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
 

awx/main/tests/factories/fixtures.py

@@ -210,7 +210,7 @@ def mk_workflow_job_template(name, extra_vars='', spec=None, organization=None,
     if extra_vars:
         extra_vars = json.dumps(extra_vars)
 
-    wfjt = WorkflowJobTemplate(name=name, extra_vars=extra_vars, organization=organization, webhook_service=webhook_service)
+    wfjt = WorkflowJobTemplate.objects.create(name=name, extra_vars=extra_vars, organization=organization, webhook_service=webhook_service)
 
     if spec:
         wfjt.survey_spec = spec

awx/main/tests/functional/analytics/test_metrics.py

@@ -19,8 +19,7 @@ EXPECTED_VALUES = {
     'awx_hosts_total': 1.0,
     'awx_schedules_total': 1.0,
     'awx_sessions_total': 0.0,
-    'awx_sessions_total': 0.0,
-    'awx_sessions_total': 0.0,
+    'awx_status_total': 0.0,
     'awx_running_jobs_total': 0.0,
     'awx_instance_capacity': 100.0,
     'awx_instance_consumed_capacity': 0.0,

awx/main/tests/functional/api/test_instance.py

@@ -1,16 +1,9 @@
 import pytest
 
-from unittest import mock
-
 from awx.api.versioning import reverse
 from awx.main.models.activity_stream import ActivityStream
 from awx.main.models.ha import Instance
 
-import redis
-
-# Django
-from django.test.utils import override_settings
-
 
 INSTANCE_KWARGS = dict(hostname='example-host', cpu=6, memory=36000000000, cpu_capacity=6, mem_capacity=42)
 
@@ -50,33 +43,14 @@ def test_enabled_sets_capacity(patch, admin_user):
 def test_auditor_user_health_check(get, post, system_auditor):
     instance = Instance.objects.create(**INSTANCE_KWARGS)
     url = reverse('api:instance_health_check', kwargs={'pk': instance.pk})
-    r = get(url=url, user=system_auditor, expect=200)
-    assert r.data['cpu_capacity'] == instance.cpu_capacity
+    get(url=url, user=system_auditor, expect=200)
     post(url=url, user=system_auditor, expect=403)
 
 
 @pytest.mark.django_db
-def test_health_check_throws_error(post, admin_user):
-    instance = Instance.objects.create(node_type='execution', **INSTANCE_KWARGS)
-    url = reverse('api:instance_health_check', kwargs={'pk': instance.pk})
-    # we will simulate a receptor error, similar to this one
-    # https://github.com/ansible/receptor/blob/156e6e24a49fbf868734507f9943ac96208ed8f5/receptorctl/receptorctl/socket_interface.py#L204
-    # related to issue https://github.com/ansible/tower/issues/5315
-    with mock.patch('awx.main.tasks.receptor.run_until_complete', side_effect=RuntimeError('Remote error: foobar')):
-        post(url=url, user=admin_user, expect=200)
-    instance.refresh_from_db()
-    assert 'Remote error: foobar' in instance.errors
-    assert instance.capacity == 0
-
-
-@pytest.mark.django_db
-@mock.patch.object(redis.client.Redis, 'ping', lambda self: True)
 def test_health_check_usage(get, post, admin_user):
     instance = Instance.objects.create(**INSTANCE_KWARGS)
     url = reverse('api:instance_health_check', kwargs={'pk': instance.pk})
-    r = get(url=url, user=admin_user, expect=200)
-    assert r.data['cpu_capacity'] == instance.cpu_capacity
-    assert r.data['last_health_check'] is None
-    with override_settings(CLUSTER_HOST_ID=instance.hostname):  # force direct call of cluster_node_health_check
-        r = post(url=url, user=admin_user, expect=200)
-        assert r.data['last_health_check'] is not None
+    get(url=url, user=admin_user, expect=200)
+    r = post(url=url, user=admin_user, expect=200)
+    assert r.data['msg'] == f"Health check is running for {instance.hostname}."

awx/main/tests/functional/api/test_job.py

@@ -13,17 +13,11 @@ from django.utils import timezone
 # AWX
 from awx.api.versioning import reverse
 from awx.api.views import RelatedJobsPreventDeleteMixin, UnifiedJobDeletionMixin
-from awx.main.models import (
-    JobTemplate,
-    User,
-    Job,
-    AdHocCommand,
-    ProjectUpdate,
-)
+from awx.main.models import JobTemplate, User, Job, AdHocCommand, ProjectUpdate, InstanceGroup, Label, Organization
 
 
 @pytest.mark.django_db
-def test_job_relaunch_permission_denied_response(post, get, inventory, project, credential, net_credential, machine_credential):
+def test_job_relaunch_permission_denied_response(post, get, inventory, project, net_credential, machine_credential):
     jt = JobTemplate.objects.create(name='testjt', inventory=inventory, project=project, ask_credential_on_launch=True)
     jt.credentials.add(machine_credential)
     jt_user = User.objects.create(username='jobtemplateuser')
@@ -39,6 +33,22 @@ def test_job_relaunch_permission_denied_response(post, get, inventory, project,
     job.launch_config.credentials.add(net_credential)
     r = post(reverse('api:job_relaunch', kwargs={'pk': job.pk}), {}, jt_user, expect=403)
     assert 'launched with prompted fields you do not have access to' in r.data['detail']
+    job.launch_config.credentials.clear()
+
+    # Job has prompted instance group that user cannot see
+    job.launch_config.instance_groups.add(InstanceGroup.objects.create())
+    r = post(reverse('api:job_relaunch', kwargs={'pk': job.pk}), {}, jt_user, expect=403)
+    assert 'launched with prompted fields you do not have access to' in r.data['detail']
+    job.launch_config.instance_groups.clear()
+
+    # Job has prompted label that user cannot see
+    job.launch_config.labels.add(Label.objects.create(organization=Organization.objects.create()))
+    r = post(reverse('api:job_relaunch', kwargs={'pk': job.pk}), {}, jt_user, expect=403)
+    assert 'launched with prompted fields you do not have access to' in r.data['detail']
+    job.launch_config.labels.clear()
+
+    # without any of those prompts, user can launch
+    r = post(reverse('api:job_relaunch', kwargs={'pk': job.pk}), {}, jt_user, expect=201)
 
 
 @pytest.mark.django_db

awx/main/tests/functional/api/test_job_runtime_params.py

@@ -4,8 +4,7 @@ import yaml
 import json
 
 from awx.api.serializers import JobLaunchSerializer
-from awx.main.models.credential import Credential
-from awx.main.models.inventory import Inventory, Host
+from awx.main.models import Credential, Inventory, Host, ExecutionEnvironment, Label, InstanceGroup
 from awx.main.models.jobs import Job, JobTemplate, UnifiedJobTemplate
 
 from awx.api.versioning import reverse
@@ -15,6 +14,11 @@ from awx.api.versioning import reverse
 def runtime_data(organization, credentialtype_ssh):
     cred_obj = Credential.objects.create(name='runtime-cred', credential_type=credentialtype_ssh, inputs={'username': 'test_user2', 'password': 'pas4word2'})
     inv_obj = organization.inventories.create(name="runtime-inv")
+    inv_obj.hosts.create(name='foo1')
+    inv_obj.hosts.create(name='foo2')
+    ee_obj = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    ig_obj = InstanceGroup.objects.create(name='bar', policy_instance_percentage=100, policy_instance_minimum=2)
+    labels_obj = Label.objects.create(name='foo', description='bar', organization=organization)
     return dict(
         extra_vars='{"job_launch_var": 4}',
         limit='test-servers',
@@ -25,6 +29,12 @@ def runtime_data(organization, credentialtype_ssh):
         credentials=[cred_obj.pk],
         diff_mode=True,
         verbosity=2,
+        execution_environment=ee_obj.pk,
+        labels=[labels_obj.pk],
+        forks=7,
+        job_slice_count=2,
+        timeout=10,
+        instance_groups=[ig_obj.pk],
     )
 
 
@@ -54,6 +64,12 @@ def job_template_prompts(project, inventory, machine_credential):
             ask_credential_on_launch=on_off,
             ask_diff_mode_on_launch=on_off,
             ask_verbosity_on_launch=on_off,
+            ask_execution_environment_on_launch=on_off,
+            ask_labels_on_launch=on_off,
+            ask_forks_on_launch=on_off,
+            ask_job_slice_count_on_launch=on_off,
+            ask_timeout_on_launch=on_off,
+            ask_instance_groups_on_launch=on_off,
         )
         jt.credentials.add(machine_credential)
         return jt
@@ -77,6 +93,12 @@ def job_template_prompts_null(project):
         ask_credential_on_launch=True,
         ask_diff_mode_on_launch=True,
         ask_verbosity_on_launch=True,
+        ask_execution_environment_on_launch=True,
+        ask_labels_on_launch=True,
+        ask_forks_on_launch=True,
+        ask_job_slice_count_on_launch=True,
+        ask_timeout_on_launch=True,
+        ask_instance_groups_on_launch=True,
     )
 
 
@@ -92,6 +114,12 @@ def data_to_internal(data):
         internal['credentials'] = set(Credential.objects.get(pk=_id) for _id in data['credentials'])
     if 'inventory' in data:
         internal['inventory'] = Inventory.objects.get(pk=data['inventory'])
+    if 'execution_environment' in data:
+        internal['execution_environment'] = ExecutionEnvironment.objects.get(pk=data['execution_environment'])
+    if 'labels' in data:
+        internal['labels'] = [Label.objects.get(pk=_id) for _id in data['labels']]
+    if 'instance_groups' in data:
+        internal['instance_groups'] = [InstanceGroup.objects.get(pk=_id) for _id in data['instance_groups']]
     return internal
 
 
@@ -124,6 +152,12 @@ def test_job_ignore_unprompted_vars(runtime_data, job_template_prompts, post, ad
     assert 'credentials' in response.data['ignored_fields']
     assert 'job_tags' in response.data['ignored_fields']
     assert 'skip_tags' in response.data['ignored_fields']
+    assert 'execution_environment' in response.data['ignored_fields']
+    assert 'labels' in response.data['ignored_fields']
+    assert 'forks' in response.data['ignored_fields']
+    assert 'job_slice_count' in response.data['ignored_fields']
+    assert 'timeout' in response.data['ignored_fields']
+    assert 'instance_groups' in response.data['ignored_fields']
 
 
 @pytest.mark.django_db
@@ -162,6 +196,34 @@ def test_job_accept_empty_tags(job_template_prompts, post, admin_user, mocker):
     mock_job.signal_start.assert_called_once()
 
 
+@pytest.mark.django_db
+@pytest.mark.job_runtime_vars
+def test_slice_timeout_forks_need_int(job_template_prompts, post, admin_user, mocker):
+    job_template = job_template_prompts(True)
+
+    mock_job = mocker.MagicMock(spec=Job, id=968)
+
+    with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job):
+        with mocker.patch('awx.api.serializers.JobSerializer.to_representation'):
+            response = post(
+                reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), {'timeout': '', 'job_slice_count': '', 'forks': ''}, admin_user, expect=400
+            )
+            assert 'forks' in response.data and response.data['forks'][0] == 'A valid integer is required.'
+            assert 'job_slice_count' in response.data and response.data['job_slice_count'][0] == 'A valid integer is required.'
+            assert 'timeout' in response.data and response.data['timeout'][0] == 'A valid integer is required.'
+
+
+@pytest.mark.django_db
+@pytest.mark.job_runtime_vars
+def test_slice_count_not_supported(job_template_prompts, post, admin_user):
+    job_template = job_template_prompts(True)
+    assert job_template.inventory.hosts.count() == 0
+    job_template.inventory.hosts.create(name='foo')
+
+    response = post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), {'job_slice_count': 8}, admin_user, expect=400)
+    assert response.data['job_slice_count'][0] == 'Job inventory does not have enough hosts for slicing'
+
+
 @pytest.mark.django_db
 @pytest.mark.job_runtime_vars
 def test_job_accept_prompted_vars_null(runtime_data, job_template_prompts_null, post, rando, mocker):
@@ -176,6 +238,10 @@ def test_job_accept_prompted_vars_null(runtime_data, job_template_prompts_null,
     inventory = Inventory.objects.get(pk=runtime_data['inventory'])
     inventory.use_role.members.add(rando)
 
+    # Instance Groups and label can not currently easily be used by rando so we need to remove the instance groups from the runtime data
+    runtime_data.pop('instance_groups')
+    runtime_data.pop('labels')
+
     mock_job = mocker.MagicMock(spec=Job, id=968, **runtime_data)
 
     with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job):
@@ -243,12 +309,59 @@ def test_job_launch_fails_without_inventory_access(job_template_prompts, runtime
 
 @pytest.mark.django_db
 @pytest.mark.job_runtime_vars
-def test_job_launch_fails_without_credential_access(job_template_prompts, runtime_data, post, rando):
+def test_job_launch_works_without_access_to_ig_if_ig_in_template(job_template_prompts, runtime_data, post, rando, mocker):
+    job_template = job_template_prompts(True)
+    job_template.instance_groups.add(InstanceGroup.objects.get(id=runtime_data['instance_groups'][0]))
+    job_template.instance_groups.add(InstanceGroup.objects.create(name='foo'))
+    job_template.save()
+    job_template.execute_role.members.add(rando)
+
+    # Make sure we get a 201 instead of a 403 since we are providing an override of just a subset of the instance gorup that was already added
+    post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), dict(instance_groups=runtime_data['instance_groups']), rando, expect=201)
+
+
+@pytest.mark.django_db
+@pytest.mark.job_runtime_vars
+def test_job_launch_works_without_access_to_label_if_label_in_template(job_template_prompts, runtime_data, post, rando, mocker, organization):
+    job_template = job_template_prompts(True)
+    job_template.labels.add(Label.objects.get(id=runtime_data['labels'][0]))
+    job_template.labels.add(Label.objects.create(name='baz', description='faz', organization=organization))
+    job_template.save()
+    job_template.execute_role.members.add(rando)
+
+    # Make sure we get a 201 instead of a 403 since we are providing an override of just a subset of the instance gorup that was already added
+    post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), dict(labels=runtime_data['labels']), rando, expect=201)
+
+
+@pytest.mark.django_db
+@pytest.mark.job_runtime_vars
+def test_job_launch_works_without_access_to_ee_if_ee_in_template(job_template_prompts, runtime_data, post, rando, mocker, organization):
+    job_template = job_template_prompts(True)
+    job_template.execute_role.members.add(rando)
+
+    # Make sure we get a 201 instead of a 403 since we are providing an override that is already in the template
+    post(
+        reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), dict(execution_environment=runtime_data['execution_environment']), rando, expect=201
+    )
+
+
+@pytest.mark.parametrize(
+    'item_type',
+    [
+        ('credentials'),
+        ('labels'),
+        ('instance_groups'),
+    ],
+)
+@pytest.mark.django_db
+@pytest.mark.job_runtime_vars
+def test_job_launch_fails_without_access(job_template_prompts, runtime_data, post, rando, item_type):
     job_template = job_template_prompts(True)
     job_template.execute_role.members.add(rando)
 
     # Assure that giving a credential without access blocks the launch
-    post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), dict(credentials=runtime_data['credentials']), rando, expect=403)
+    data = {item_type: runtime_data[item_type]}
+    post(reverse('api:job_template_launch', kwargs={'pk': job_template.pk}), data, rando, expect=403)
 
 
 @pytest.mark.django_db

awx/main/tests/functional/api/test_workflow_node.py

@@ -77,6 +77,18 @@ class TestApprovalNodes:
         assert approval_node.unified_job_template.description == 'Approval Node'
         assert approval_node.unified_job_template.timeout == 0
 
+    def test_approval_node_creation_with_timeout(self, post, approval_node, admin_user):
+        assert approval_node.timeout is None
+
+        url = reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': approval_node.pk, 'version': 'v2'})
+        post(url, {'name': 'Test', 'description': 'Approval Node', 'timeout': 10}, user=admin_user, expect=201)
+
+        approval_node = WorkflowJobTemplateNode.objects.get(pk=approval_node.pk)
+        approval_node.refresh_from_db()
+        assert approval_node.timeout is None
+        assert isinstance(approval_node.unified_job_template, WorkflowApprovalTemplate)
+        assert approval_node.unified_job_template.timeout == 10
+
     def test_approval_node_creation_failure(self, post, approval_node, admin_user):
         # This test leaves off a required param to assert that user will get a 400.
         url = reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': approval_node.pk, 'version': 'v2'})

awx/main/tests/functional/conftest.py

@@ -706,7 +706,7 @@ def jt_linked(organization, project, inventory, machine_credential, credential,
 
 @pytest.fixture
 def workflow_job_template(organization):
-    wjt = WorkflowJobTemplate(name='test-workflow_job_template', organization=organization)
+    wjt = WorkflowJobTemplate.objects.create(name='test-workflow_job_template', organization=organization)
     wjt.save()
 
     return wjt

awx/main/tests/functional/models/test_job.py

@@ -64,3 +64,26 @@ class TestSlicingModels:
         inventory2 = Inventory.objects.create(organization=organization, name='fooinv')
         [inventory2.hosts.create(name='foo{}'.format(i)) for i in range(3)]
         assert job_template.get_effective_slice_ct({'inventory': inventory2})
+
+    def test_effective_slice_count_prompt(self, job_template, inventory, organization):
+        job_template.inventory = inventory
+        # Add our prompt fields to the JT to allow overrides
+        job_template.ask_job_slice_count_on_launch = True
+        job_template.ask_inventory_on_launch = True
+        # Set a default value of the slice count to something low
+        job_template.job_slice_count = 2
+        # Create an inventory with 4 nodes
+        inventory2 = Inventory.objects.create(organization=organization, name='fooinv')
+        [inventory2.hosts.create(name='foo{}'.format(i)) for i in range(4)]
+        # The inventory slice count will be the min of the number of nodes (4) or the job slice (2)
+        assert job_template.get_effective_slice_ct({'inventory': inventory2}) == 2
+        # Now we are going to pass in an override (like the prompt would) and as long as that is < host count we expect that back
+        assert job_template.get_effective_slice_ct({'inventory': inventory2, 'job_slice_count': 3}) == 3
+
+    def test_slice_count_prompt_limited_by_inventory(self, job_template, inventory, organization):
+        assert inventory.hosts.count() == 0
+        job_template.inventory = inventory
+        inventory.hosts.create(name='foo')
+
+        unified_job = job_template.create_unified_job(job_slice_count=2)
+        assert isinstance(unified_job, Job)

awx/main/tests/functional/models/test_job_launch_config.py

@@ -1,7 +1,8 @@
 import pytest
 
 # AWX
-from awx.main.models import JobTemplate, JobLaunchConfig
+from awx.main.models.jobs import JobTemplate, LaunchTimeConfigBase
+from awx.main.models.execution_environments import ExecutionEnvironment
 
 
 @pytest.fixture
@@ -11,18 +12,6 @@ def full_jt(inventory, project, machine_credential):
     return jt
 
 
-@pytest.fixture
-def config_factory(full_jt):
-    def return_config(data):
-        job = full_jt.create_unified_job(**data)
-        try:
-            return job.launch_config
-        except JobLaunchConfig.DoesNotExist:
-            return None
-
-    return return_config
-
-
 @pytest.mark.django_db
 class TestConfigCreation:
     """
@@ -40,28 +29,73 @@ class TestConfigCreation:
         assert config.limit == 'foobar'
         assert config.char_prompts == {'limit': 'foobar'}
 
-    def test_added_credential(self, full_jt, credential):
-        job = full_jt.create_unified_job(credentials=[credential])
+    def test_added_related(self, full_jt, credential, default_instance_group, label):
+        job = full_jt.create_unified_job(credentials=[credential], instance_groups=[default_instance_group], labels=[label])
         config = job.launch_config
         assert set(config.credentials.all()) == set([credential])
+        assert set(config.labels.all()) == set([label])
+        assert set(config.instance_groups.all()) == set([default_instance_group])
 
     def test_survey_passwords_ignored(self, inventory_source):
         iu = inventory_source.create_unified_job(survey_passwords={'foo': '$encrypted$'})
         assert iu.launch_config.prompts_dict() == {}
 
 
+@pytest.fixture
+def full_prompts_dict(inventory, credential, label, default_instance_group):
+    ee = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    r = {
+        'limit': 'foobar',
+        'inventory': inventory,
+        'credentials': [credential],
+        'execution_environment': ee,
+        'labels': [label],
+        'instance_groups': [default_instance_group],
+        'verbosity': 3,
+        'scm_branch': 'non_dev',
+        'diff_mode': True,
+        'skip_tags': 'foobar',
+        'job_tags': 'untagged',
+        'forks': 26,
+        'job_slice_count': 2,
+        'timeout': 200,
+        'extra_vars': {'prompted_key': 'prompted_val'},
+        'job_type': 'check',
+    }
+    assert set(JobTemplate.get_ask_mapping().keys()) - set(r.keys()) == set()  # make fixture comprehensive
+    return r
+
+
 @pytest.mark.django_db
-class TestConfigReversibility:
+def test_config_reversibility(full_jt, full_prompts_dict):
     """
     Checks that a blob of saved prompts will be re-created in the
     prompts_dict for launching new jobs
     """
+    config = full_jt.create_unified_job(**full_prompts_dict).launch_config
+    assert config.prompts_dict() == full_prompts_dict
 
-    def test_char_field_only(self, config_factory):
-        config = config_factory({'limit': 'foobar'})
-        assert config.prompts_dict() == {'limit': 'foobar'}
 
-    def test_related_objects(self, config_factory, inventory, credential):
-        prompts = {'limit': 'foobar', 'inventory': inventory, 'credentials': set([credential])}
-        config = config_factory(prompts)
-        assert config.prompts_dict() == prompts
+@pytest.mark.django_db
+class TestLaunchConfigModels:
+    def get_concrete_subclasses(self, cls):
+        r = []
+        for c in cls.__subclasses__():
+            if c._meta.abstract:
+                r.extend(self.get_concrete_subclasses(c))
+            else:
+                r.append(c)
+        return r
+
+    def test_non_job_config_complete(self):
+        """This performs model validation which replaces code that used run on import."""
+        for field_name in JobTemplate.get_ask_mapping().keys():
+            if field_name in LaunchTimeConfigBase.SUBCLASS_FIELDS:
+                assert not hasattr(LaunchTimeConfigBase, field_name)
+            else:
+                assert hasattr(LaunchTimeConfigBase, field_name)
+
+    def test_subclass_fields_complete(self):
+        for cls in self.get_concrete_subclasses(LaunchTimeConfigBase):
+            for field_name in LaunchTimeConfigBase.SUBCLASS_FIELDS:
+                assert hasattr(cls, field_name)

awx/main/tests/functional/models/test_workflow.py

@@ -12,6 +12,9 @@ from awx.main.models.workflow import (
 )
 from awx.main.models.jobs import JobTemplate, Job
 from awx.main.models.projects import ProjectUpdate
+from awx.main.models.credential import Credential, CredentialType
+from awx.main.models.label import Label
+from awx.main.models.ha import InstanceGroup
 from awx.main.scheduler.dag_workflow import WorkflowDAG
 from awx.api.versioning import reverse
 from awx.api.views import WorkflowJobTemplateNodeSuccessNodesList
@@ -229,6 +232,65 @@ class TestWorkflowJob:
         assert queued_node.get_job_kwargs()['extra_vars'] == {'a': 42, 'b': 43}
         assert queued_node.ancestor_artifacts == {'a': 42, 'b': 43}
 
+    def test_combine_prompts_WFJT_to_node(self, project, inventory, organization):
+        """
+        Test that complex prompts like variables, credentials, labels, etc
+        are properly combined from the workflow-level with the node-level
+        """
+        jt = JobTemplate.objects.create(
+            project=project,
+            inventory=inventory,
+            ask_variables_on_launch=True,
+            ask_credential_on_launch=True,
+            ask_instance_groups_on_launch=True,
+            ask_labels_on_launch=True,
+            ask_limit_on_launch=True,
+        )
+        wj = WorkflowJob.objects.create(name='test-wf-job', extra_vars='{}')
+
+        common_ig = InstanceGroup.objects.create(name='common')
+        common_ct = CredentialType.objects.create(name='common')
+
+        node = WorkflowJobNode.objects.create(workflow_job=wj, unified_job_template=jt, extra_vars={'node_key': 'node_val'})
+        node.limit = 'node_limit'
+        node.save()
+        node_cred_unique = Credential.objects.create(credential_type=CredentialType.objects.create(name='node'))
+        node_cred_conflicting = Credential.objects.create(credential_type=common_ct)
+        node.credentials.add(node_cred_unique, node_cred_conflicting)
+        node_labels = [Label.objects.create(name='node1', organization=organization), Label.objects.create(name='node2', organization=organization)]
+        node.labels.add(*node_labels)
+        node_igs = [common_ig, InstanceGroup.objects.create(name='node')]
+        for ig in node_igs:
+            node.instance_groups.add(ig)
+
+        # assertions for where node has prompts but workflow job does not
+        data = node.get_job_kwargs()
+        assert data['extra_vars'] == {'node_key': 'node_val'}
+        assert set(data['credentials']) == set([node_cred_conflicting, node_cred_unique])
+        assert data['instance_groups'] == node_igs
+        assert set(data['labels']) == set(node_labels)
+        assert data['limit'] == 'node_limit'
+
+        # add prompts to the WorkflowJob
+        wj.limit = 'wj_limit'
+        wj.extra_vars = {'wj_key': 'wj_val'}
+        wj.save()
+        wj_cred_unique = Credential.objects.create(credential_type=CredentialType.objects.create(name='wj'))
+        wj_cred_conflicting = Credential.objects.create(credential_type=common_ct)
+        wj.credentials.add(wj_cred_unique, wj_cred_conflicting)
+        wj.labels.add(Label.objects.create(name='wj1', organization=organization), Label.objects.create(name='wj2', organization=organization))
+        wj_igs = [InstanceGroup.objects.create(name='wj'), common_ig]
+        for ig in wj_igs:
+            wj.instance_groups.add(ig)
+
+        # assertions for behavior where node and workflow jobs have prompts
+        data = node.get_job_kwargs()
+        assert data['extra_vars'] == {'node_key': 'node_val', 'wj_key': 'wj_val'}
+        assert set(data['credentials']) == set([wj_cred_unique, wj_cred_conflicting, node_cred_unique])
+        assert data['instance_groups'] == wj_igs
+        assert set(data['labels']) == set(node_labels)  # as exception, WFJT labels not applied
+        assert data['limit'] == 'wj_limit'
+
 
 @pytest.mark.django_db
 class TestWorkflowJobTemplate:
@@ -287,12 +349,25 @@ class TestWorkflowJobTemplatePrompts:
     @pytest.fixture
     def wfjt_prompts(self):
         return WorkflowJobTemplate.objects.create(
-            ask_inventory_on_launch=True, ask_variables_on_launch=True, ask_limit_on_launch=True, ask_scm_branch_on_launch=True
+            ask_variables_on_launch=True,
+            ask_inventory_on_launch=True,
+            ask_tags_on_launch=True,
+            ask_labels_on_launch=True,
+            ask_limit_on_launch=True,
+            ask_scm_branch_on_launch=True,
+            ask_skip_tags_on_launch=True,
         )
 
     @pytest.fixture
     def prompts_data(self, inventory):
-        return dict(inventory=inventory, extra_vars={'foo': 'bar'}, limit='webservers', scm_branch='release-3.3')
+        return dict(
+            inventory=inventory,
+            extra_vars={'foo': 'bar'},
+            limit='webservers',
+            scm_branch='release-3.3',
+            job_tags='foo',
+            skip_tags='bar',
+        )
 
     def test_apply_workflow_job_prompts(self, workflow_job_template, wfjt_prompts, prompts_data, inventory):
         # null or empty fields used
@@ -300,6 +375,9 @@ class TestWorkflowJobTemplatePrompts:
         assert workflow_job.limit is None
         assert workflow_job.inventory is None
         assert workflow_job.scm_branch is None
+        assert workflow_job.job_tags is None
+        assert workflow_job.skip_tags is None
+        assert len(workflow_job.labels.all()) is 0
 
         # fields from prompts used
         workflow_job = workflow_job_template.create_unified_job(**prompts_data)
@@ -307,15 +385,21 @@ class TestWorkflowJobTemplatePrompts:
         assert workflow_job.limit == 'webservers'
         assert workflow_job.inventory == inventory
         assert workflow_job.scm_branch == 'release-3.3'
+        assert workflow_job.job_tags == 'foo'
+        assert workflow_job.skip_tags == 'bar'
 
         # non-null fields from WFJT used
         workflow_job_template.inventory = inventory
         workflow_job_template.limit = 'fooo'
         workflow_job_template.scm_branch = 'bar'
+        workflow_job_template.job_tags = 'baz'
+        workflow_job_template.skip_tags = 'dinosaur'
         workflow_job = workflow_job_template.create_unified_job()
         assert workflow_job.limit == 'fooo'
         assert workflow_job.inventory == inventory
         assert workflow_job.scm_branch == 'bar'
+        assert workflow_job.job_tags == 'baz'
+        assert workflow_job.skip_tags == 'dinosaur'
 
     @pytest.mark.django_db
     def test_process_workflow_job_prompts(self, inventory, workflow_job_template, wfjt_prompts, prompts_data):
@@ -340,12 +424,19 @@ class TestWorkflowJobTemplatePrompts:
                 ask_limit_on_launch=True,
                 scm_branch='bar',
                 ask_scm_branch_on_launch=True,
+                job_tags='foo',
+                skip_tags='bar',
             ),
             user=org_admin,
             expect=201,
         )
         wfjt = WorkflowJobTemplate.objects.get(id=r.data['id'])
-        assert wfjt.char_prompts == {'limit': 'foooo', 'scm_branch': 'bar'}
+        assert wfjt.char_prompts == {
+            'limit': 'foooo',
+            'scm_branch': 'bar',
+            'job_tags': 'foo',
+            'skip_tags': 'bar',
+        }
         assert wfjt.ask_scm_branch_on_launch is True
         assert wfjt.ask_limit_on_launch is True
 
@@ -355,6 +446,67 @@ class TestWorkflowJobTemplatePrompts:
         assert r.data['limit'] == 'prompt_limit'
         assert r.data['scm_branch'] == 'prompt_branch'
 
+    @pytest.mark.django_db
+    def test_set_all_ask_for_prompts_false_from_post(self, post, organization, inventory, org_admin):
+        '''
+        Tests default behaviour and values of ask_for_* fields on WFJT via POST
+        '''
+        r = post(
+            url=reverse('api:workflow_job_template_list'),
+            data=dict(
+                name='workflow that tests ask_for prompts',
+                organization=organization.id,
+                inventory=inventory.id,
+                job_tags='',
+                skip_tags='',
+            ),
+            user=org_admin,
+            expect=201,
+        )
+        wfjt = WorkflowJobTemplate.objects.get(id=r.data['id'])
+
+        assert wfjt.ask_inventory_on_launch is False
+        assert wfjt.ask_labels_on_launch is False
+        assert wfjt.ask_limit_on_launch is False
+        assert wfjt.ask_scm_branch_on_launch is False
+        assert wfjt.ask_skip_tags_on_launch is False
+        assert wfjt.ask_tags_on_launch is False
+        assert wfjt.ask_variables_on_launch is False
+
+    @pytest.mark.django_db
+    def test_set_all_ask_for_prompts_true_from_post(self, post, organization, inventory, org_admin):
+        '''
+        Tests behaviour and values of ask_for_* fields on WFJT via POST
+        '''
+        r = post(
+            url=reverse('api:workflow_job_template_list'),
+            data=dict(
+                name='workflow that tests ask_for prompts',
+                organization=organization.id,
+                inventory=inventory.id,
+                job_tags='',
+                skip_tags='',
+                ask_inventory_on_launch=True,
+                ask_labels_on_launch=True,
+                ask_limit_on_launch=True,
+                ask_scm_branch_on_launch=True,
+                ask_skip_tags_on_launch=True,
+                ask_tags_on_launch=True,
+                ask_variables_on_launch=True,
+            ),
+            user=org_admin,
+            expect=201,
+        )
+        wfjt = WorkflowJobTemplate.objects.get(id=r.data['id'])
+
+        assert wfjt.ask_inventory_on_launch is True
+        assert wfjt.ask_labels_on_launch is True
+        assert wfjt.ask_limit_on_launch is True
+        assert wfjt.ask_scm_branch_on_launch is True
+        assert wfjt.ask_skip_tags_on_launch is True
+        assert wfjt.ask_tags_on_launch is True
+        assert wfjt.ask_variables_on_launch is True
+
 
 @pytest.mark.django_db
 def test_workflow_ancestors(organization):

awx/main/tests/functional/test_copy.py

@@ -6,12 +6,20 @@ from awx.main.utils import decrypt_field
 from awx.main.models.workflow import WorkflowJobTemplate, WorkflowJobTemplateNode, WorkflowApprovalTemplate
 from awx.main.models.jobs import JobTemplate
 from awx.main.tasks.system import deep_copy_model_obj
+from awx.main.models import Label, ExecutionEnvironment, InstanceGroup
 
 
 @pytest.mark.django_db
-def test_job_template_copy(post, get, project, inventory, machine_credential, vault_credential, credential, alice, job_template_with_survey_passwords, admin):
+def test_job_template_copy(
+    post, get, project, inventory, machine_credential, vault_credential, credential, alice, job_template_with_survey_passwords, admin, organization
+):
+    label = Label.objects.create(name="foobar", organization=organization)
+    ig = InstanceGroup.objects.create(name="bazbar", organization=organization)
     job_template_with_survey_passwords.project = project
     job_template_with_survey_passwords.inventory = inventory
+    job_template_with_survey_passwords.labels.add(label)
+    job_template_with_survey_passwords.instance_groups.add(ig)
+    job_template_with_survey_passwords.prevent_instance_group_fallback = True
     job_template_with_survey_passwords.save()
     job_template_with_survey_passwords.credentials.add(credential)
     job_template_with_survey_passwords.credentials.add(machine_credential)
@@ -54,6 +62,11 @@ def test_job_template_copy(post, get, project, inventory, machine_credential, va
         assert vault_credential in jt_copy.credentials.all()
         assert machine_credential in jt_copy.credentials.all()
         assert job_template_with_survey_passwords.survey_spec == jt_copy.survey_spec
+        assert jt_copy.labels.count() != 0
+        assert jt_copy.labels.get(pk=label.pk) == label
+        assert jt_copy.instance_groups.count() != 0
+        assert jt_copy.instance_groups.get(pk=ig.pk) == ig
+        assert jt_copy.prevent_instance_group_fallback == True
 
 
 @pytest.mark.django_db
@@ -84,6 +97,8 @@ def test_inventory_copy(inventory, group_factory, post, get, alice, organization
     host = group_1_1.hosts.create(name='host', inventory=inventory)
     group_2_1.hosts.add(host)
     inventory.admin_role.members.add(alice)
+    inventory.prevent_instance_group_fallback = True
+    inventory.save()
     assert get(reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200).data['can_copy'] is False
     inventory.organization.admin_role.members.add(alice)
     assert get(reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200).data['can_copy'] is True
@@ -99,6 +114,7 @@ def test_inventory_copy(inventory, group_factory, post, get, alice, organization
     assert inventory_copy.organization == organization
     assert inventory_copy.created_by == alice
     assert inventory_copy.name == 'new inv name'
+    assert inventory_copy.prevent_instance_group_fallback == True
     assert set(group_1_1_copy.parents.all()) == set()
     assert set(group_2_1_copy.parents.all()) == set([group_1_1_copy])
     assert set(group_2_2_copy.parents.all()) == set([group_1_1_copy, group_2_1_copy])
@@ -109,8 +125,22 @@ def test_inventory_copy(inventory, group_factory, post, get, alice, organization
 
 @pytest.mark.django_db
 def test_workflow_job_template_copy(workflow_job_template, post, get, admin, organization):
+    '''
+    Tests the FIELDS_TO_PRESERVE_AT_COPY attribute on WFJTs
+    '''
     workflow_job_template.organization = organization
+
+    label = Label.objects.create(name="foobar", organization=organization)
+    workflow_job_template.labels.add(label)
+
+    ee = ExecutionEnvironment.objects.create(name="barfoo", organization=organization)
+    workflow_job_template.execution_environment = ee
+
+    ig = InstanceGroup.objects.create(name="bazbar", organization=organization)
+    workflow_job_template.instance_groups.add(ig)
+
     workflow_job_template.save()
+
     jts = [JobTemplate.objects.create(name='test-jt-{}'.format(i)) for i in range(0, 5)]
     nodes = [WorkflowJobTemplateNode.objects.create(workflow_job_template=workflow_job_template, unified_job_template=jts[i]) for i in range(0, 5)]
     nodes[0].success_nodes.add(nodes[1])
@@ -124,9 +154,16 @@ def test_workflow_job_template_copy(workflow_job_template, post, get, admin, org
         wfjt_copy = type(workflow_job_template).objects.get(pk=wfjt_copy_id)
         args, kwargs = deep_copy_mock.call_args
         deep_copy_model_obj(*args, **kwargs)
+
     assert wfjt_copy.organization == organization
     assert wfjt_copy.created_by == admin
     assert wfjt_copy.name == 'new wfjt name'
+    assert wfjt_copy.labels.count() != 0
+    assert wfjt_copy.labels.get(pk=label.pk) == label
+    assert wfjt_copy.execution_environment == ee
+    assert wfjt_copy.instance_groups.count() != 0
+    assert wfjt_copy.instance_groups.get(pk=ig.pk) == ig
+
     copied_node_list = [x for x in wfjt_copy.workflow_job_template_nodes.all()]
     copied_node_list.sort(key=lambda x: int(x.unified_job_template.name[-1]))
     for node, success_count, failure_count, always_count in zip(copied_node_list, [1, 1, 0, 0, 0], [1, 0, 0, 1, 0], [0, 0, 0, 0, 0]):

awx/main/tests/functional/test_instances.py

@@ -391,6 +391,8 @@ class TestInstanceGroupOrdering:
         assert ad_hoc.preferred_instance_groups == [ig_org]
         inventory.instance_groups.add(ig_inv)
         assert ad_hoc.preferred_instance_groups == [ig_inv, ig_org]
+        inventory.prevent_instance_group_fallback = True
+        assert ad_hoc.preferred_instance_groups == [ig_inv]
 
     def test_inventory_update_instance_groups(self, instance_group_factory, inventory_source, default_instance_group):
         iu = InventoryUpdate.objects.create(inventory_source=inventory_source, source=inventory_source.source)
@@ -404,6 +406,8 @@ class TestInstanceGroupOrdering:
         inventory_source.instance_groups.add(ig_tmp)
         # API does not allow setting IGs on inventory source, so ignore those
         assert iu.preferred_instance_groups == [ig_inv, ig_org]
+        inventory_source.inventory.prevent_instance_group_fallback = True
+        assert iu.preferred_instance_groups == [ig_inv]
 
     def test_job_instance_groups(self, instance_group_factory, inventory, project, default_instance_group):
         jt = JobTemplate.objects.create(inventory=inventory, project=project)
@@ -417,3 +421,31 @@ class TestInstanceGroupOrdering:
         assert job.preferred_instance_groups == [ig_inv, ig_org]
         job.job_template.instance_groups.add(ig_tmp)
         assert job.preferred_instance_groups == [ig_tmp, ig_inv, ig_org]
+
+    def test_job_instance_groups_cache_default(self, instance_group_factory, inventory, project, default_instance_group):
+        jt = JobTemplate.objects.create(inventory=inventory, project=project)
+        job = jt.create_unified_job()
+        print(job.preferred_instance_groups_cache)
+        print(default_instance_group)
+        assert job.preferred_instance_groups_cache == [default_instance_group.id]
+
+    def test_job_instance_groups_cache_default_additional_items(self, instance_group_factory, inventory, project, default_instance_group):
+        ig_org = instance_group_factory("OrgIstGrp", [default_instance_group.instances.first()])
+        ig_inv = instance_group_factory("InvIstGrp", [default_instance_group.instances.first()])
+        ig_tmp = instance_group_factory("TmpIstGrp", [default_instance_group.instances.first()])
+        project.organization.instance_groups.add(ig_org)
+        inventory.instance_groups.add(ig_inv)
+        jt = JobTemplate.objects.create(inventory=inventory, project=project)
+        jt.instance_groups.add(ig_tmp)
+        job = jt.create_unified_job()
+        assert job.preferred_instance_groups_cache == [ig_tmp.id, ig_inv.id, ig_org.id]
+
+    def test_job_instance_groups_cache_prompt(self, instance_group_factory, inventory, project, default_instance_group):
+        ig_org = instance_group_factory("OrgIstGrp", [default_instance_group.instances.first()])
+        ig_inv = instance_group_factory("InvIstGrp", [default_instance_group.instances.first()])
+        ig_tmp = instance_group_factory("TmpIstGrp", [default_instance_group.instances.first()])
+        project.organization.instance_groups.add(ig_org)
+        inventory.instance_groups.add(ig_inv)
+        jt = JobTemplate.objects.create(inventory=inventory, project=project)
+        job = jt.create_unified_job(instance_groups=[ig_tmp])
+        assert job.preferred_instance_groups_cache == [ig_tmp.id]

awx/main/tests/functional/test_jobs.py

@@ -3,7 +3,20 @@ import pytest
 from unittest import mock
 import json
 
-from awx.main.models import Job, Instance, JobHostSummary, InventoryUpdate, InventorySource, Project, ProjectUpdate, SystemJob, AdHocCommand
+from awx.main.models import (
+    Job,
+    Instance,
+    JobHostSummary,
+    InventoryUpdate,
+    InventorySource,
+    Project,
+    ProjectUpdate,
+    SystemJob,
+    AdHocCommand,
+    InstanceGroup,
+    Label,
+    ExecutionEnvironment,
+)
 from awx.main.tasks.system import cluster_node_heartbeat
 from django.test.utils import override_settings
 
@@ -103,14 +116,88 @@ def test_job_notification_host_data(inventory, machine_credential, project, job_
 class TestLaunchConfig:
     def test_null_creation_from_prompts(self):
         job = Job.objects.create()
-        data = {"credentials": [], "extra_vars": {}, "limit": None, "job_type": None}
+        data = {
+            "credentials": [],
+            "extra_vars": {},
+            "limit": None,
+            "job_type": None,
+            "execution_environment": None,
+            "instance_groups": None,
+            "labels": None,
+            "forks": None,
+            "timeout": None,
+            "job_slice_count": None,
+        }
         config = job.create_config_from_prompts(data)
         assert config is None
 
     def test_only_limit_defined(self, job_template):
         job = Job.objects.create(job_template=job_template)
-        data = {"credentials": [], "extra_vars": {}, "job_tags": None, "limit": ""}
+        data = {
+            "credentials": [],
+            "extra_vars": {},
+            "job_tags": None,
+            "limit": "",
+            "execution_environment": None,
+            "instance_groups": None,
+            "labels": None,
+            "forks": None,
+            "timeout": None,
+            "job_slice_count": None,
+        }
         config = job.create_config_from_prompts(data)
         assert config.char_prompts == {"limit": ""}
         assert not config.credentials.exists()
         assert config.prompts_dict() == {"limit": ""}
+
+    def test_many_to_many_fields(self, job_template, organization):
+        job = Job.objects.create(job_template=job_template)
+        ig1 = InstanceGroup.objects.create(name='bar')
+        ig2 = InstanceGroup.objects.create(name='foo')
+        job_template.instance_groups.add(ig2)
+        label1 = Label.objects.create(name='foo', description='bar', organization=organization)
+        label2 = Label.objects.create(name='faz', description='baz', organization=organization)
+        # Order should matter here which is why we do 2 and then 1
+        data = {
+            "credentials": [],
+            "extra_vars": {},
+            "job_tags": None,
+            "limit": None,
+            "execution_environment": None,
+            "instance_groups": [ig2, ig1],
+            "labels": [label2, label1],
+            "forks": None,
+            "timeout": None,
+            "job_slice_count": None,
+        }
+        config = job.create_config_from_prompts(data)
+
+        assert config.instance_groups.exists()
+        config_instance_group_ids = [item.id for item in config.instance_groups.all()]
+        assert config_instance_group_ids == [ig2.id, ig1.id]
+
+        assert config.labels.exists()
+        config_label_ids = [item.id for item in config.labels.all()]
+        assert config_label_ids == [label2.id, label1.id]
+
+    def test_pk_field(self, job_template, organization):
+        job = Job.objects.create(job_template=job_template)
+        ee = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+        # Order should matter here which is why we do 2 and then 1
+        data = {
+            "credentials": [],
+            "extra_vars": {},
+            "job_tags": None,
+            "limit": None,
+            "execution_environment": ee,
+            "instance_groups": [],
+            "labels": [],
+            "forks": None,
+            "timeout": None,
+            "job_slice_count": None,
+        }
+        config = job.create_config_from_prompts(data)
+
+        assert config.execution_environment
+        # We just write the PK instead of trying to assign an item, that happens on the save
+        assert config.execution_environment_id == ee.id

awx/main/tests/functional/test_rbac_job.py

@@ -3,7 +3,20 @@ import pytest
 from rest_framework.exceptions import PermissionDenied
 
 from awx.main.access import JobAccess, JobLaunchConfigAccess, AdHocCommandAccess, InventoryUpdateAccess, ProjectUpdateAccess
-from awx.main.models import Job, JobLaunchConfig, JobTemplate, AdHocCommand, InventoryUpdate, InventorySource, ProjectUpdate, User, Credential
+from awx.main.models import (
+    Job,
+    JobLaunchConfig,
+    JobTemplate,
+    AdHocCommand,
+    InventoryUpdate,
+    InventorySource,
+    ProjectUpdate,
+    User,
+    Credential,
+    ExecutionEnvironment,
+    InstanceGroup,
+    Label,
+)
 
 from crum import impersonate
 
@@ -302,13 +315,33 @@ class TestLaunchConfigAccess:
         access = JobLaunchConfigAccess(rando)
         cred1, cred2 = self._make_two_credentials(credentialtype_ssh)
 
-        assert access.has_credentials_access(config)  # has access if 0 creds
+        assert access.has_obj_m2m_access(config)  # has access if 0 creds
         config.credentials.add(cred1, cred2)
-        assert not access.has_credentials_access(config)  # lacks access to both
+        assert not access.has_obj_m2m_access(config)  # lacks access to both
         cred1.use_role.members.add(rando)
-        assert not access.has_credentials_access(config)  # lacks access to 1
+        assert not access.has_obj_m2m_access(config)  # lacks access to 1
         cred2.use_role.members.add(rando)
-        assert access.has_credentials_access(config)  # has access to both
+        assert access.has_obj_m2m_access(config)  # has access to both
+
+    def test_new_execution_environment_access(self, rando):
+        ee = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+        access = JobLaunchConfigAccess(rando)
+
+        assert access.can_add({'execution_environment': ee})  # can add because access to ee will be granted
+
+    def test_new_label_access(self, rando, organization):
+        label = Label.objects.create(name='foo', description='bar', organization=organization)
+        access = JobLaunchConfigAccess(rando)
+
+        assert not access.can_add({'labels': [label]})  # can't add because no access to label
+        # We assert in JT unit tests that the access will be granted if label is in JT
+
+    def test_new_instance_group_access(self, rando):
+        ig = InstanceGroup.objects.create(name='bar', policy_instance_percentage=100, policy_instance_minimum=2)
+        access = JobLaunchConfigAccess(rando)
+
+        assert not access.can_add({'instance_groups': [ig]})  # can't add because no access to ig
+        # We assert in JT unit tests that the access will be granted if instance group is in JT
 
     def test_can_use_minor(self, rando):
         # Config object only has flat-field overrides, no RBAC restrictions

awx/main/tests/functional/test_rbac_workflow.py

@@ -6,6 +6,7 @@ from awx.main.access import (
     WorkflowJobAccess,
     # WorkflowJobNodeAccess
 )
+from awx.main.models import JobTemplate, WorkflowJobTemplateNode
 
 from rest_framework.exceptions import PermissionDenied
 
@@ -87,6 +88,16 @@ class TestWorkflowJobTemplateNodeAccess:
         job_template.read_role.members.add(rando)
         assert not access.can_add({'workflow_job_template': wfjt, 'unified_job_template': job_template})
 
+    def test_change_JT_no_start_perm(self, wfjt, rando):
+        wfjt.admin_role.members.add(rando)
+        access = WorkflowJobTemplateNodeAccess(rando)
+        jt1 = JobTemplate.objects.create()
+        jt1.execute_role.members.add(rando)
+        assert access.can_add({'workflow_job_template': wfjt, 'unified_job_template': jt1})
+        node = WorkflowJobTemplateNode.objects.create(workflow_job_template=wfjt, unified_job_template=jt1)
+        jt2 = JobTemplate.objects.create()
+        assert not access.can_change(node, {'unified_job_template': jt2.id})
+
     def test_add_node_with_minimum_permissions(self, wfjt, job_template, inventory, rando):
         wfjt.admin_role.members.add(rando)
         access = WorkflowJobTemplateNodeAccess(rando)
@@ -101,6 +112,92 @@ class TestWorkflowJobTemplateNodeAccess:
         access = WorkflowJobTemplateNodeAccess(rando)
         assert access.can_delete(wfjt_node)
 
+    @pytest.mark.parametrize(
+        "add_wfjt_admin, add_jt_admin, permission_type, expected_result, method_type",
+        [
+            (True, False, 'credentials', False, 'can_attach'),
+            (True, True, 'credentials', True, 'can_attach'),
+            (True, False, 'labels', False, 'can_attach'),
+            (True, True, 'labels', True, 'can_attach'),
+            (True, False, 'instance_groups', False, 'can_attach'),
+            (True, True, 'instance_groups', True, 'can_attach'),
+            (True, False, 'credentials', False, 'can_unattach'),
+            (True, True, 'credentials', True, 'can_unattach'),
+            (True, False, 'labels', False, 'can_unattach'),
+            (True, True, 'labels', True, 'can_unattach'),
+            (True, False, 'instance_groups', False, 'can_unattach'),
+            (True, True, 'instance_groups', True, 'can_unattach'),
+        ],
+    )
+    def test_attacher_permissions(self, wfjt_node, job_template, rando, add_wfjt_admin, permission_type, add_jt_admin, expected_result, mocker, method_type):
+        wfjt = wfjt_node.workflow_job_template
+        if add_wfjt_admin:
+            wfjt.admin_role.members.add(rando)
+            wfjt.unified_job_template = job_template
+        if add_jt_admin:
+            job_template.execute_role.members.add(rando)
+
+        from awx.main.models import Credential, Label, InstanceGroup, Organization, CredentialType
+
+        if permission_type == 'credentials':
+            sub_obj = Credential.objects.create(credential_type=CredentialType.objects.create())
+            sub_obj.use_role.members.add(rando)
+        elif permission_type == 'labels':
+            sub_obj = Label.objects.create(organization=Organization.objects.create())
+            sub_obj.organization.member_role.members.add(rando)
+        elif permission_type == 'instance_groups':
+            sub_obj = InstanceGroup.objects.create()
+            org = Organization.objects.create()
+            org.admin_role.members.add(rando)  # only admins can see IGs
+            org.instance_groups.add(sub_obj)
+
+        access = WorkflowJobTemplateNodeAccess(rando)
+        if method_type == 'can_unattach':
+            assert getattr(access, method_type)(wfjt_node, sub_obj, permission_type) == expected_result
+        else:
+            assert getattr(access, method_type)(wfjt_node, sub_obj, permission_type, {}) == expected_result
+
+    # The actual attachment of labels, credentials and instance groups are tested from JobLaunchConfigAccess
+
+    @pytest.mark.parametrize(
+        "attachment_type, expect_exception, method_type",
+        [
+            ("credentials", False, 'can_attach'),
+            ("labels", False, 'can_attach'),
+            ("instance_groups", False, 'can_attach'),
+            ("success_nodes", False, 'can_attach'),
+            ("failure_nodes", False, 'can_attach'),
+            ("always_nodes", False, 'can_attach'),
+            ("junk", True, 'can_attach'),
+            ("credentials", False, 'can_unattach'),
+            ("labels", False, 'can_unattach'),
+            ("instance_groups", False, 'can_unattach'),
+            ("success_nodes", False, 'can_unattach'),
+            ("failure_nodes", False, 'can_unattach'),
+            ("always_nodes", False, 'can_unattach'),
+            ("junk", True, 'can_unattach'),
+        ],
+    )
+    def test_attacher_raise_not_implemented(self, wfjt_node, rando, attachment_type, expect_exception, method_type):
+        wfjt = wfjt_node.workflow_job_template
+        wfjt.admin_role.members.add(rando)
+        access = WorkflowJobTemplateNodeAccess(rando)
+        if expect_exception:
+            with pytest.raises(NotImplementedError):
+                access.can_attach(wfjt_node, None, attachment_type, None)
+        else:
+            try:
+                getattr(access, method_type)(wfjt_node, None, attachment_type, None)
+            except NotImplementedError:
+                # We explicitly catch NotImplemented because the _nodes type will raise a different exception
+                assert False, "Exception was raised when it should not have been"
+            except Exception:
+                #  File "/awx_devel/awx/main/access.py", line 2074, in check_same_WFJT
+                #    raise Exception('Attaching workflow nodes only allowed for other nodes')
+                pass
+
+    # TODO: Implement additional tests for _nodes attachments here
+
 
 @pytest.mark.django_db
 class TestWorkflowJobAccess:

awx/main/tests/functional/test_tasks.py

@@ -19,12 +19,11 @@ def scm_revision_file(tmpdir_factory):
 
 
 @pytest.mark.django_db
-@pytest.mark.parametrize('node_type', ('control', 'hybrid'))
+@pytest.mark.parametrize('node_type', ('control. hybrid'))
 def test_no_worker_info_on_AWX_nodes(node_type):
     hostname = 'us-south-3-compute.invalid'
     Instance.objects.create(hostname=hostname, node_type=node_type)
-    with pytest.raises(RuntimeError):
-        execution_node_health_check(hostname)
+    assert execution_node_health_check(hostname) is None
 
 
 @pytest.fixture

awx/main/tests/unit/api/serializers/test_primary_key_related_field.py

@@ -8,9 +8,17 @@ from rest_framework.exceptions import ValidationError
 from awx.api.serializers import JobLaunchSerializer
 
 
-def test_primary_key_related_field():
+@pytest.mark.parametrize(
+    "param",
+    [
+        ('credentials'),
+        ('instance_groups'),
+        ('labels'),
+    ],
+)
+def test_primary_key_related_field(param):
     # We are testing if the PrimaryKeyRelatedField in this serializer can take dictionary.
     # PrimaryKeyRelatedField should not be able to take dictionary as input, and should raise a ValidationError.
-    data = {'credentials': {'1': '2', '3': '4'}}
+    data = {param: {'1': '2', '3': '4'}}
     with pytest.raises(ValidationError):
         JobLaunchSerializer(data=data)

awx/main/tests/unit/api/serializers/test_workflow_serializers.py

@@ -11,6 +11,7 @@ from awx.api.serializers import (
 from awx.main.models import Job, WorkflowJobTemplateNode, WorkflowJob, WorkflowJobNode, WorkflowJobTemplate, Project, Inventory, JobTemplate
 
 
+@pytest.mark.django_db
 @mock.patch('awx.api.serializers.UnifiedJobTemplateSerializer.get_related', lambda x, y: {})
 class TestWorkflowJobTemplateSerializerGetRelated:
     @pytest.fixture
@@ -26,6 +27,7 @@ class TestWorkflowJobTemplateSerializerGetRelated:
             'launch',
             'workflow_nodes',
             'webhook_key',
+            'labels',
         ],
     )
     def test_get_related(self, mocker, test_get_related, workflow_job_template, related_resource_name):
@@ -58,6 +60,7 @@ class TestWorkflowNodeBaseSerializerGetRelated:
         assert 'unified_job_template' not in related
 
 
+@pytest.mark.django_db
 @mock.patch('awx.api.serializers.BaseSerializer.get_related', lambda x, y: {})
 class TestWorkflowJobTemplateNodeSerializerGetRelated:
     @pytest.fixture
@@ -87,6 +90,8 @@ class TestWorkflowJobTemplateNodeSerializerGetRelated:
             'success_nodes',
             'failure_nodes',
             'always_nodes',
+            'labels',
+            'instance_groups',
         ],
     )
     def test_get_related(self, test_get_related, workflow_job_template_node, related_resource_name):
@@ -146,6 +151,7 @@ class TestWorkflowJobTemplateNodeSerializerCharPrompts:
         assert WFJT_serializer.instance.limit == 'webservers'
 
 
+@pytest.mark.django_db
 @mock.patch('awx.api.serializers.BaseSerializer.validate', lambda self, attrs: attrs)
 class TestWorkflowJobTemplateNodeSerializerSurveyPasswords:
     @pytest.fixture
@@ -162,7 +168,7 @@ class TestWorkflowJobTemplateNodeSerializerSurveyPasswords:
 
     def test_set_survey_passwords_create(self, jt):
         serializer = WorkflowJobTemplateNodeSerializer()
-        wfjt = WorkflowJobTemplate(name='fake-wfjt')
+        wfjt = WorkflowJobTemplate.objects.create(name='fake-wfjt')
         attrs = serializer.validate({'unified_job_template': jt, 'workflow_job_template': wfjt, 'extra_data': {'var1': 'secret_answer'}})
         assert 'survey_passwords' in attrs
         assert 'var1' in attrs['survey_passwords']
@@ -171,7 +177,7 @@ class TestWorkflowJobTemplateNodeSerializerSurveyPasswords:
 
     def test_set_survey_passwords_modify(self, jt):
         serializer = WorkflowJobTemplateNodeSerializer()
-        wfjt = WorkflowJobTemplate(name='fake-wfjt')
+        wfjt = WorkflowJobTemplate.objects.create(name='fake-wfjt')
         serializer.instance = WorkflowJobTemplateNode(workflow_job_template=wfjt, unified_job_template=jt)
         attrs = serializer.validate({'unified_job_template': jt, 'workflow_job_template': wfjt, 'extra_data': {'var1': 'secret_answer'}})
         assert 'survey_passwords' in attrs
@@ -181,7 +187,7 @@ class TestWorkflowJobTemplateNodeSerializerSurveyPasswords:
 
     def test_use_db_answer(self, jt, mocker):
         serializer = WorkflowJobTemplateNodeSerializer()
-        wfjt = WorkflowJobTemplate(name='fake-wfjt')
+        wfjt = WorkflowJobTemplate.objects.create(name='fake-wfjt')
         serializer.instance = WorkflowJobTemplateNode(workflow_job_template=wfjt, unified_job_template=jt, extra_data={'var1': '$encrypted$foooooo'})
         with mocker.patch('awx.main.models.mixins.decrypt_value', return_value='foo'):
             attrs = serializer.validate({'unified_job_template': jt, 'workflow_job_template': wfjt, 'extra_data': {'var1': '$encrypted$'}})
@@ -196,7 +202,7 @@ class TestWorkflowJobTemplateNodeSerializerSurveyPasswords:
         with that particular var omitted so on launch time the default takes effect
         """
         serializer = WorkflowJobTemplateNodeSerializer()
-        wfjt = WorkflowJobTemplate(name='fake-wfjt')
+        wfjt = WorkflowJobTemplate.objects.create(name='fake-wfjt')
         jt.survey_spec['spec'][0]['default'] = '$encrypted$bar'
         attrs = serializer.validate({'unified_job_template': jt, 'workflow_job_template': wfjt, 'extra_data': {'var1': '$encrypted$'}})
         assert 'survey_passwords' in attrs
@@ -230,6 +236,8 @@ class TestWorkflowJobNodeSerializerGetRelated:
             'success_nodes',
             'failure_nodes',
             'always_nodes',
+            'labels',
+            'instance_groups',
         ],
     )
     def test_get_related(self, test_get_related, workflow_job_node, related_resource_name):

awx/main/tests/unit/api/test_views.py

@@ -59,7 +59,7 @@ class TestApiRootView:
 
 class TestJobTemplateLabelList:
     def test_inherited_mixin_unattach(self):
-        with mock.patch('awx.api.generics.DeleteLastUnattachLabelMixin.unattach') as mixin_unattach:
+        with mock.patch('awx.api.views.labels.LabelSubListCreateAttachDetachView.unattach') as mixin_unattach:
             view = JobTemplateLabelList()
             mock_request = mock.MagicMock()
 

awx/main/tests/unit/models/test_label.py

@@ -1,9 +1,15 @@
 import pytest
 from unittest import mock
 
-from awx.main.models.label import Label
-from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
-from awx.main.models.inventory import Inventory
+from awx.main.models import (
+    Label,
+    UnifiedJobTemplate,
+    UnifiedJob,
+    Inventory,
+    Schedule,
+    WorkflowJobTemplateNode,
+    WorkflowJobNode,
+)
 
 
 mock_query_set = mock.MagicMock()
@@ -14,12 +20,6 @@ mock_objects = mock.MagicMock(filter=mock.MagicMock(return_value=mock_query_set)
 @pytest.mark.django_db
 @mock.patch('awx.main.models.label.Label.objects', mock_objects)
 class TestLabelFilterMocked:
-    def test_get_orphaned_labels(self, mocker):
-        ret = Label.get_orphaned_labels()
-
-        assert mock_query_set == ret
-        Label.objects.filter.assert_called_with(organization=None, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
-
     def test_is_detached(self, mocker):
         mock_query_set.exists.return_value = True
 
@@ -27,7 +27,15 @@ class TestLabelFilterMocked:
         ret = label.is_detached()
 
         assert ret is True
-        Label.objects.filter.assert_called_with(id=37, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
+        Label.objects.filter.assert_called_with(
+            id=37,
+            unifiedjob_labels__isnull=True,
+            unifiedjobtemplate_labels__isnull=True,
+            inventory_labels__isnull=True,
+            schedule_labels__isnull=True,
+            workflowjobtemplatenode_labels__isnull=True,
+            workflowjobnode_labels__isnull=True,
+        )
         mock_query_set.exists.assert_called_with()
 
     def test_is_detached_not(self, mocker):
@@ -37,39 +45,102 @@ class TestLabelFilterMocked:
         ret = label.is_detached()
 
         assert ret is False
-        Label.objects.filter.assert_called_with(id=37, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
+        Label.objects.filter.assert_called_with(
+            id=37,
+            unifiedjob_labels__isnull=True,
+            unifiedjobtemplate_labels__isnull=True,
+            inventory_labels__isnull=True,
+            schedule_labels__isnull=True,
+            workflowjobtemplatenode_labels__isnull=True,
+            workflowjobnode_labels__isnull=True,
+        )
+
         mock_query_set.exists.assert_called_with()
 
     @pytest.mark.parametrize(
-        "jt_count,j_count,inv_count,expected",
+        "jt_count,j_count,inv_count,sched_count,wfnode_count,wfnodej_count,expected",
         [
-            (1, 0, 0, True),
-            (0, 1, 0, True),
-            (0, 0, 1, True),
-            (1, 1, 1, False),
+            (1, 0, 0, 0, 0, 0, True),
+            (0, 1, 0, 0, 0, 0, True),
+            (1, 1, 0, 0, 0, 0, False),
+            (0, 0, 1, 0, 0, 0, True),
+            (1, 0, 1, 0, 0, 0, False),
+            (0, 1, 1, 0, 0, 0, False),
+            (1, 1, 1, 0, 0, 0, False),
+            (0, 0, 0, 1, 0, 0, True),
+            (1, 0, 0, 1, 0, 0, False),
+            (0, 1, 0, 1, 0, 0, False),
+            (1, 1, 0, 1, 0, 0, False),
+            (0, 0, 1, 1, 0, 0, False),
+            (1, 0, 1, 1, 0, 0, False),
+            (0, 1, 1, 1, 0, 0, False),
+            (1, 1, 1, 1, 0, 0, False),
+            (0, 0, 0, 0, 1, 0, True),
+            (1, 0, 0, 0, 1, 0, False),
+            (0, 1, 0, 0, 1, 0, False),
+            (1, 1, 0, 0, 1, 0, False),
+            (0, 0, 1, 0, 1, 0, False),
+            (1, 0, 1, 0, 1, 0, False),
+            (0, 1, 1, 0, 1, 0, False),
+            (1, 1, 1, 0, 1, 0, False),
+            (0, 0, 0, 1, 1, 0, False),
+            (1, 0, 0, 1, 1, 0, False),
+            (0, 1, 0, 1, 1, 0, False),
+            (1, 1, 0, 1, 1, 0, False),
+            (0, 0, 1, 1, 1, 0, False),
+            (1, 0, 1, 1, 1, 0, False),
+            (0, 1, 1, 1, 1, 0, False),
+            (1, 1, 1, 1, 1, 0, False),
+            (0, 0, 0, 0, 0, 1, True),
+            (1, 0, 0, 0, 0, 1, False),
+            (0, 1, 0, 0, 0, 1, False),
+            (1, 1, 0, 0, 0, 1, False),
+            (0, 0, 1, 0, 0, 1, False),
+            (1, 0, 1, 0, 0, 1, False),
+            (0, 1, 1, 0, 0, 1, False),
+            (1, 1, 1, 0, 0, 1, False),
+            (0, 0, 0, 1, 0, 1, False),
+            (1, 0, 0, 1, 0, 1, False),
+            (0, 1, 0, 1, 0, 1, False),
+            (1, 1, 0, 1, 0, 1, False),
+            (0, 0, 1, 1, 0, 1, False),
+            (1, 0, 1, 1, 0, 1, False),
+            (0, 1, 1, 1, 0, 1, False),
+            (1, 1, 1, 1, 0, 1, False),
+            (0, 0, 0, 0, 1, 1, False),
+            (1, 0, 0, 0, 1, 1, False),
+            (0, 1, 0, 0, 1, 1, False),
+            (1, 1, 0, 0, 1, 1, False),
+            (0, 0, 1, 0, 1, 1, False),
+            (1, 0, 1, 0, 1, 1, False),
+            (0, 1, 1, 0, 1, 1, False),
+            (1, 1, 1, 0, 1, 1, False),
+            (0, 0, 0, 1, 1, 1, False),
+            (1, 0, 0, 1, 1, 1, False),
+            (0, 1, 0, 1, 1, 1, False),
+            (1, 1, 0, 1, 1, 1, False),
+            (0, 0, 1, 1, 1, 1, False),
+            (1, 0, 1, 1, 1, 1, False),
+            (0, 1, 1, 1, 1, 1, False),
+            (1, 1, 1, 1, 1, 1, False),
         ],
     )
-    def test_is_candidate_for_detach(self, mocker, jt_count, j_count, inv_count, expected):
-        mock_job_qs = mocker.MagicMock()
-        mock_job_qs.count = mocker.MagicMock(return_value=j_count)
-        mocker.patch.object(UnifiedJob, 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=mock_job_qs)))
-
-        mock_jt_qs = mocker.MagicMock()
-        mock_jt_qs.count = mocker.MagicMock(return_value=jt_count)
-        mocker.patch.object(UnifiedJobTemplate, 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=mock_jt_qs)))
-
-        mock_inv_qs = mocker.MagicMock()
-        mock_inv_qs.count = mocker.MagicMock(return_value=inv_count)
-        mocker.patch.object(Inventory, 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=mock_inv_qs)))
+    def test_is_candidate_for_detach(self, mocker, jt_count, j_count, inv_count, sched_count, wfnode_count, wfnodej_count, expected):
+        counts = [jt_count, j_count, inv_count, sched_count, wfnode_count, wfnodej_count]
+        models = [UnifiedJobTemplate, UnifiedJob, Inventory, Schedule, WorkflowJobTemplateNode, WorkflowJobNode]
+        mockers = []
+        for index in range(0, len(models)):
+            a_mocker = mocker.MagicMock()
+            a_mocker.count = mocker.MagicMock(return_value=counts[index])
+            mocker.patch.object(models[index], 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=a_mocker)))
+            mockers.append(a_mocker)
 
         label = Label(id=37)
         ret = label.is_candidate_for_detach()
 
-        UnifiedJob.objects.filter.assert_called_with(labels__in=[label.id])
-        UnifiedJobTemplate.objects.filter.assert_called_with(labels__in=[label.id])
-        Inventory.objects.filter.assert_called_with(labels__in=[label.id])
-        mock_job_qs.count.assert_called_with()
-        mock_jt_qs.count.assert_called_with()
-        mock_inv_qs.count.assert_called_with()
+        for index in range(0, len(models)):
+            models[index].objects.filter.assert_called_with(labels__in=[label.id])
+        for index in range(0, len(mockers)):
+            mockers[index].count.assert_called_with()
 
         assert ret is expected

awx/main/tests/unit/models/test_survey_models.py

@@ -259,13 +259,14 @@ def test_survey_encryption_defaults(survey_spec_factory, question_type, default,
 
 
 @pytest.mark.survey
+@pytest.mark.django_db
 class TestWorkflowSurveys:
     def test_update_kwargs_survey_defaults(self, survey_spec_factory):
         "Assure that the survey default over-rides a JT variable"
         spec = survey_spec_factory('var1')
         spec['spec'][0]['default'] = 3
         spec['spec'][0]['required'] = False
-        wfjt = WorkflowJobTemplate(name="test-wfjt", survey_spec=spec, survey_enabled=True, extra_vars="var1: 5")
+        wfjt = WorkflowJobTemplate.objects.create(name="test-wfjt", survey_spec=spec, survey_enabled=True, extra_vars="var1: 5")
         updated_extra_vars = wfjt._update_unified_job_kwargs({}, {})
         assert 'extra_vars' in updated_extra_vars
         assert json.loads(updated_extra_vars['extra_vars'])['var1'] == 3
@@ -277,7 +278,7 @@ class TestWorkflowSurveys:
         spec['spec'][0]['required'] = False
         spec['spec'][1]['required'] = True
         spec['spec'][2]['required'] = False
-        wfjt = WorkflowJobTemplate(name="test-wfjt", survey_spec=spec, survey_enabled=True, extra_vars="question2: hiworld")
+        wfjt = WorkflowJobTemplate.objects.create(name="test-wfjt", survey_spec=spec, survey_enabled=True, extra_vars="question2: hiworld")
         assert wfjt.variables_needed_to_start == ['question2']
         assert not wfjt.can_start_without_user_input()
 
@@ -311,6 +312,6 @@ class TestExtraVarsNoPrompt:
         self.process_vars_and_assert(jt, provided_vars, valid)
 
     def test_wfjt_extra_vars_counting(self, provided_vars, valid):
-        wfjt = WorkflowJobTemplate(name='foo', extra_vars={'tmpl_var': 'bar'})
+        wfjt = WorkflowJobTemplate.objects.create(name='foo', extra_vars={'tmpl_var': 'bar'})
         prompted_fields, ignored_fields, errors = wfjt._accept_or_ignore_job_kwargs(extra_vars=provided_vars)
         self.process_vars_and_assert(wfjt, provided_vars, valid)

awx/main/tests/unit/models/test_workflow_unit.py

@@ -94,7 +94,7 @@ def workflow_job_unit():
 
 @pytest.fixture
 def workflow_job_template_unit():
-    return WorkflowJobTemplate(name='workflow')
+    return WorkflowJobTemplate.objects.create(name='workflow')
 
 
 @pytest.fixture
@@ -151,6 +151,7 @@ def test_node_getter_and_setters():
     assert node.job_type == 'check'
 
 
+@pytest.mark.django_db
 class TestWorkflowJobCreate:
     def test_create_no_prompts(self, wfjt_node_no_prompts, workflow_job_unit, mocker):
         mock_create = mocker.MagicMock()
@@ -165,6 +166,7 @@ class TestWorkflowJobCreate:
                 unified_job_template=wfjt_node_no_prompts.unified_job_template,
                 workflow_job=workflow_job_unit,
                 identifier=mocker.ANY,
+                execution_environment=None,
             )
 
     def test_create_with_prompts(self, wfjt_node_with_prompts, workflow_job_unit, credential, mocker):
@@ -180,9 +182,11 @@ class TestWorkflowJobCreate:
                 unified_job_template=wfjt_node_with_prompts.unified_job_template,
                 workflow_job=workflow_job_unit,
                 identifier=mocker.ANY,
+                execution_environment=None,
             )
 
 
+@pytest.mark.django_db
 @mock.patch('awx.main.models.workflow.WorkflowNodeBase.get_parent_nodes', lambda self: [])
 class TestWorkflowJobNodeJobKWARGS:
     """
@@ -231,4 +235,12 @@ class TestWorkflowJobNodeJobKWARGS:
 
 
 def test_get_ask_mapping_integrity():
-    assert list(WorkflowJobTemplate.get_ask_mapping().keys()) == ['extra_vars', 'inventory', 'limit', 'scm_branch']
+    assert list(WorkflowJobTemplate.get_ask_mapping().keys()) == [
+        'inventory',
+        'limit',
+        'scm_branch',
+        'labels',
+        'job_tags',
+        'skip_tags',
+        'extra_vars',
+    ]

awx/main/tests/unit/test_access.py

@@ -196,6 +196,7 @@ def test_jt_can_add_bad_data(user_unit):
     assert not access.can_add({'asdf': 'asdf'})
 
 
+@pytest.mark.django_db
 class TestWorkflowAccessMethods:
     @pytest.fixture
     def workflow(self, workflow_job_template_factory):

awx/main/utils/common.py

@@ -532,6 +532,10 @@ def copy_m2m_relationships(obj1, obj2, fields, kwargs=None):
                 if kwargs and field_name in kwargs:
                     override_field_val = kwargs[field_name]
                     if isinstance(override_field_val, (set, list, QuerySet)):
+                        # Labels are additive so we are going to add any src labels in addition to the override labels
+                        if field_name == 'labels':
+                            for jt_label in src_field_value.all():
+                                getattr(obj2, field_name).add(jt_label.id)
                         getattr(obj2, field_name).add(*override_field_val)
                         continue
                     if override_field_val.__class__.__name__ == 'ManyRelatedManager':

awx/playbooks/action_plugins/verify_project.py

@@ -88,7 +88,7 @@ class ActionModule(ActionBase):
         if not os.path.exists(manifest_file):
             return {
                 "failed": True,
-                "msg": f"Expected file not found: {path}",
+                "msg": f"Expected file not found: {manifest_file}",
             }
 
         checksum_file_contents = open(manifest_file, "r").read()

awx/playbooks/project_update.yml

@@ -160,19 +160,17 @@
   name: Perform project signature/checksum verification
   tasks:
     - name: Verify project content using GPG signature
-      playbook_integrity:
+      verify_project:
         project_path: "{{ project_path | quote }}"
         validation_type: gpg
         gpg_pubkey: "{{ gpg_pubkey }}"
-      register: gpg_result
       tags:
         - validation_gpg_public_key
 
     - name: Verify project content against checksum manifest
-      playbook_integrity:
+      verify_project:
         project_path: "{{ project_path | quote }}"
         validation_type: checksum_manifest
-      register: checksum_result
       tags:
         - validation_checksum_manifest
 

awx/sso/views.py

@@ -45,7 +45,6 @@ class CompleteView(BaseRedirectView):
             current_user = UserSerializer(self.request.user)
             current_user = smart_str(JSONRenderer().render(current_user.data))
             current_user = urllib.parse.quote('%s' % current_user, '')
-            response.set_cookie('current_user', current_user, secure=settings.SESSION_COOKIE_SECURE or None)
             response.setdefault('X-API-Session-Cookie-Name', getattr(settings, 'SESSION_COOKIE_NAME', 'awx_sessionid'))
         return response
 

awx/ui/package-lock.json

@@ -47,6 +47,7 @@
         "@nteract/mockument": "^1.0.4",
         "@testing-library/jest-dom": "^5.16.2",
         "@testing-library/react": "^12.1.5",
+        "@testing-library/user-event": "14.4.3",
         "@wojtekmaj/enzyme-adapter-react-17": "0.6.5",
         "babel-plugin-macros": "3.1.0",
         "enzyme": "^3.10.0",
@@ -4514,6 +4515,19 @@
         "react-dom": "<18.0.0"
       }
     },
+    "node_modules/@testing-library/user-event": {
+      "version": "14.4.3",
+      "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.4.3.tgz",
+      "integrity": "sha512-kCUc5MEwaEMakkO5x7aoD+DLi02ehmEM2QCGWvNqAS1dV/fAvORWEjnjsEIvml59M7Y5kCkWN6fCCyPOe8OL6Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=12",
+        "npm": ">=6"
+      },
+      "peerDependencies": {
+        "@testing-library/dom": ">=7.21.4"
+      }
+    },
     "node_modules/@tootallnate/once": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
@@ -25653,6 +25667,13 @@
         "@types/react-dom": "<18.0.0"
       }
     },
+    "@testing-library/user-event": {
+      "version": "14.4.3",
+      "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.4.3.tgz",
+      "integrity": "sha512-kCUc5MEwaEMakkO5x7aoD+DLi02ehmEM2QCGWvNqAS1dV/fAvORWEjnjsEIvml59M7Y5kCkWN6fCCyPOe8OL6Q==",
+      "dev": true,
+      "requires": {}
+    },
     "@tootallnate/once": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",

awx/ui/package.json

@@ -47,6 +47,7 @@
     "@nteract/mockument": "^1.0.4",
     "@testing-library/jest-dom": "^5.16.2",
     "@testing-library/react": "^12.1.5",
+    "@testing-library/user-event": "14.4.3",
     "@wojtekmaj/enzyme-adapter-react-17": "0.6.5",
     "babel-plugin-macros": "3.1.0",
     "enzyme": "^3.10.0",

awx/ui/src/api/mixins/Labels.mixin.js

@@ -0,0 +1,49 @@
+const LabelsMixin = (parent) =>
+  class extends parent {
+    readLabels(id, params) {
+      return this.http.get(`${this.baseUrl}${id}/labels/`, {
+        params,
+      });
+    }
+
+    readAllLabels(id) {
+      const fetchLabels = async (pageNo = 1, labels = []) => {
+        try {
+          const { data } = await this.http.get(`${this.baseUrl}${id}/labels/`, {
+            params: {
+              page: pageNo,
+              page_size: 200,
+            },
+          });
+          if (data?.next) {
+            return fetchLabels(pageNo + 1, labels.concat(data.results));
+          }
+          return Promise.resolve({
+            data: {
+              results: labels.concat(data.results),
+            },
+          });
+        } catch (error) {
+          return Promise.reject(error);
+        }
+      };
+
+      return fetchLabels();
+    }
+
+    associateLabel(id, label, orgId) {
+      return this.http.post(`${this.baseUrl}${id}/labels/`, {
+        name: label.name,
+        organization: orgId,
+      });
+    }
+
+    disassociateLabel(id, label) {
+      return this.http.post(`${this.baseUrl}${id}/labels/`, {
+        id: label.id,
+        disassociate: true,
+      });
+    }
+  };
+
+export default LabelsMixin;

awx/ui/src/api/models/Instances.js

@@ -8,6 +8,7 @@ class Instances extends Base {
     this.readHealthCheckDetail = this.readHealthCheckDetail.bind(this);
     this.healthCheck = this.healthCheck.bind(this);
     this.readInstanceGroup = this.readInstanceGroup.bind(this);
+    this.deprovisionInstance = this.deprovisionInstance.bind(this);
   }
 
   healthCheck(instanceId) {
@@ -18,9 +19,19 @@ class Instances extends Base {
     return this.http.get(`${this.baseUrl}${instanceId}/health_check/`);
   }
 
+  readPeers(instanceId, params) {
+    return this.http.get(`${this.baseUrl}${instanceId}/peers/`, { params });
+  }
+
   readInstanceGroup(instanceId) {
     return this.http.get(`${this.baseUrl}${instanceId}/instance_groups/`);
   }
+
+  deprovisionInstance(instanceId) {
+    return this.http.patch(`${this.baseUrl}${instanceId}/`, {
+      node_state: 'deprovisioning',
+    });
+  }
 }
 
 export default Instances;

awx/ui/src/api/models/JobTemplates.js

@@ -1,10 +1,11 @@
 import Base from '../Base';
 import NotificationsMixin from '../mixins/Notifications.mixin';
 import InstanceGroupsMixin from '../mixins/InstanceGroups.mixin';
+import LabelsMixin from '../mixins/Labels.mixin';
 import SchedulesMixin from '../mixins/Schedules.mixin';
 
 class JobTemplates extends SchedulesMixin(
-  InstanceGroupsMixin(NotificationsMixin(Base))
+  InstanceGroupsMixin(NotificationsMixin(LabelsMixin(Base)))
 ) {
   constructor(http) {
     super(http);
@@ -33,20 +34,6 @@ class JobTemplates extends SchedulesMixin(
     return this.http.get(`${this.baseUrl}${id}/launch/`);
   }
 
-  associateLabel(id, label, orgId) {
-    return this.http.post(`${this.baseUrl}${id}/labels/`, {
-      name: label.name,
-      organization: orgId,
-    });
-  }
-
-  disassociateLabel(id, label) {
-    return this.http.post(`${this.baseUrl}${id}/labels/`, {
-      id: label.id,
-      disassociate: true,
-    });
-  }
-
   readCredentials(id, params) {
     return this.http.get(`${this.baseUrl}${id}/credentials/`, {
       params,

awx/ui/src/api/models/Schedules.js

@@ -1,6 +1,8 @@
 import Base from '../Base';
+import InstanceGroupsMixin from '../mixins/InstanceGroups.mixin';
+import LabelsMixin from '../mixins/Labels.mixin';
 
-class Schedules extends Base {
+class Schedules extends InstanceGroupsMixin(LabelsMixin(Base)) {
   constructor(http) {
     super(http);
     this.baseUrl = 'api/v2/schedules/';

awx/ui/src/api/models/WorkflowJobTemplateNodes.js

@@ -1,6 +1,8 @@
 import Base from '../Base';
+import InstanceGroupsMixin from '../mixins/InstanceGroups.mixin';
+import LabelsMixin from '../mixins/Labels.mixin';
 
-class WorkflowJobTemplateNodes extends Base {
+class WorkflowJobTemplateNodes extends LabelsMixin(InstanceGroupsMixin(Base)) {
   constructor(http) {
     super(http);
     this.baseUrl = 'api/v2/workflow_job_template_nodes/';

awx/ui/src/api/models/WorkflowJobTemplates.js

@@ -1,8 +1,11 @@
 import Base from '../Base';
 import SchedulesMixin from '../mixins/Schedules.mixin';
 import NotificationsMixin from '../mixins/Notifications.mixin';
+import LabelsMixin from '../mixins/Labels.mixin';
 
-class WorkflowJobTemplates extends SchedulesMixin(NotificationsMixin(Base)) {
+class WorkflowJobTemplates extends SchedulesMixin(
+  NotificationsMixin(LabelsMixin(Base))
+) {
   constructor(http) {
     super(http);
     this.baseUrl = 'api/v2/workflow_job_templates/';

awx/ui/src/components/AppContainer/PageHeaderToolbar.js

@@ -5,21 +5,18 @@ import { t } from '@lingui/macro';
 import { Link } from 'react-router-dom';
 import styled from 'styled-components';
 import {
-  Badge,
   Dropdown,
   DropdownItem,
   DropdownToggle,
   DropdownPosition,
+  NotificationBadge,
+  NotificationBadgeVariant,
   PageHeaderTools,
   PageHeaderToolsGroup,
   PageHeaderToolsItem,
   Tooltip,
 } from '@patternfly/react-core';
-import {
-  BellIcon,
-  QuestionCircleIcon,
-  UserIcon,
-} from '@patternfly/react-icons';
+import { QuestionCircleIcon, UserIcon } from '@patternfly/react-icons';
 import { WorkflowApprovalsAPI } from 'api';
 import useRequest from 'hooks/useRequest';
 import getDocsBaseUrl from 'util/getDocsBaseUrl';
@@ -33,10 +30,6 @@ const PendingWorkflowApprovals = styled.div`
   margin-right: 10px;
 `;
 
-const PendingWorkflowApprovalBadge = styled(Badge)`
-  margin-left: 10px;
-`;
-
 function PageHeaderToolbar({
   isAboutDisabled,
   onAboutClick,
@@ -84,13 +77,15 @@ function PageHeaderToolbar({
           <PageHeaderToolsItem>
             <Link to="/workflow_approvals?workflow_approvals.status=pending">
               <PendingWorkflowApprovals>
-                <BellIcon color="white" />
-                <PendingWorkflowApprovalBadge
+                <NotificationBadge
                   id="toolbar-workflow-approval-badge"
-                  isRead
-                >
-                  {pendingApprovalsCount}
-                </PendingWorkflowApprovalBadge>
+                  count={pendingApprovalsCount}
+                  variant={
+                    pendingApprovalsCount === 0
+                      ? NotificationBadgeVariant.read
+                      : NotificationBadgeVariant.unread
+                  }
+                />
               </PendingWorkflowApprovals>
             </Link>
           </PageHeaderToolsItem>

awx/ui/src/components/AppContainer/PageHeaderToolbar.test.js

@@ -80,7 +80,7 @@ describe('PageHeaderToolbar', () => {
     });
 
     expect(
-      wrapper.find('Badge#toolbar-workflow-approval-badge').text()
+      wrapper.find('NotificationBadge#toolbar-workflow-approval-badge').text()
     ).toEqual('20');
   });
 });

awx/ui/src/components/HealthCheckAlert/HealthCheckAlert.js

@@ -0,0 +1,34 @@
+import React from 'react';
+import { t } from '@lingui/macro';
+import {
+  Alert as PFAlert,
+  Button,
+  AlertActionCloseButton,
+} from '@patternfly/react-core';
+import styled from 'styled-components';
+
+const Alert = styled(PFAlert)`
+  z-index: 1;
+`;
+function HealthCheckAlert({ onSetHealthCheckAlert }) {
+  return (
+    <Alert
+      variant="default"
+      actionClose={
+        <AlertActionCloseButton onClose={() => onSetHealthCheckAlert(false)} />
+      }
+      title={
+        <>
+          {t`Health check request(s) submitted. Please wait and reload the page.`}{' '}
+          <Button
+            variant="link"
+            isInline
+            onClick={() => window.location.reload()}
+          >{t`Reload`}</Button>
+        </>
+      }
+    />
+  );
+}
+
+export default HealthCheckAlert;

awx/ui/src/components/HealthCheckAlert/index.js

@@ -0,0 +1 @@
+export { default } from './HealthCheckAlert';

awx/ui/src/components/LabelSelect/LabelSelect.js

@@ -1,6 +1,12 @@
 import React, { useState, useEffect } from 'react';
 import { func, arrayOf, number, shape, string, oneOfType } from 'prop-types';
-import { Select, SelectOption, SelectVariant } from '@patternfly/react-core';
+import {
+  Chip,
+  ChipGroup,
+  Select,
+  SelectOption,
+  SelectVariant,
+} from '@patternfly/react-core';
 import { t } from '@lingui/macro';
 import { LabelsAPI } from 'api';
 import useIsMounted from 'hooks/useIsMounted';
@@ -60,7 +66,12 @@ function LabelSelect({ value, placeholder, onChange, onError, createText }) {
 
   const renderOptions = (opts) =>
     opts.map((option) => (
-      <SelectOption key={option.id} aria-label={option.name} value={option}>
+      <SelectOption
+        key={option.id}
+        aria-label={option.name}
+        value={option}
+        isDisabled={option.isReadOnly}
+      >
         {option.name}
       </SelectOption>
     ));
@@ -73,6 +84,23 @@ function LabelSelect({ value, placeholder, onChange, onError, createText }) {
     }
     return null;
   };
+
+  const chipGroupComponent = () => (
+    <ChipGroup>
+      {(selections || []).map((currentChip) => (
+        <Chip
+          isReadOnly={currentChip.isReadOnly}
+          key={currentChip.name}
+          onClick={(e) => {
+            onSelect(e, currentChip);
+          }}
+        >
+          {currentChip.name}
+        </Chip>
+      ))}
+    </ChipGroup>
+  );
+
   return (
     <Select
       variant={SelectVariant.typeaheadMulti}
@@ -83,7 +111,7 @@ function LabelSelect({ value, placeholder, onChange, onError, createText }) {
         }
         onSelect(e, item);
       }}
-      onClear={() => onChange([])}
+      onClear={() => onChange(selections.filter((label) => label.isReadOnly))}
       onFilter={onFilter}
       isCreatable
       onCreateOption={(label) => {
@@ -101,6 +129,7 @@ function LabelSelect({ value, placeholder, onChange, onError, createText }) {
       createText={createText}
       noResultsFoundText={t`No results found`}
       ouiaId="template-label-select"
+      chipGroupComponent={chipGroupComponent()}
     >
       {renderOptions(options)}
     </Select>

awx/ui/src/components/LabelSelect/LabelSelect.test.js

@@ -63,7 +63,7 @@ describe('<LabelSelect />', () => {
     const selectOptions = wrapper.find('SelectOption');
     expect(selectOptions).toHaveLength(4);
   });
-  test('Generate a label  ', async () => {
+  test('Generate a label', async () => {
     let wrapper;
     const onChange = jest.fn();
     LabelsAPI.read.mockReturnValue({
@@ -79,4 +79,33 @@ describe('<LabelSelect />', () => {
     await wrapper.find('Select').invoke('onSelect')({}, 'foo');
     expect(onChange).toBeCalledWith([{ id: 'foo', name: 'foo' }]);
   });
+  test('should handle read-only labels', async () => {
+    let wrapper;
+    const onChange = jest.fn();
+    LabelsAPI.read.mockReturnValue({
+      data: {
+        results: [
+          { id: 1, name: 'read only' },
+          { id: 2, name: 'not read only' },
+        ],
+      },
+    });
+    await act(async () => {
+      wrapper = mount(
+        <LabelSelect
+          value={[
+            { id: 1, name: 'read only', isReadOnly: true },
+            { id: 2, name: 'not read only' },
+          ]}
+          onError={() => {}}
+          onChange={onChange}
+        />
+      );
+    });
+    wrapper.find('SelectToggle').simulate('click');
+    const selectOptions = wrapper.find('SelectOption');
+    expect(selectOptions).toHaveLength(2);
+    expect(selectOptions.at(0).prop('isDisabled')).toBe(true);
+    expect(selectOptions.at(1).prop('isDisabled')).toBe(false);
+  });
 });

awx/ui/src/components/LaunchButton/LaunchButton.js

@@ -1,9 +1,7 @@
 import React, { useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import { number, shape } from 'prop-types';
-
 import { t } from '@lingui/macro';
-
 import {
   AdHocCommandsAPI,
   InventorySourcesAPI,
@@ -24,6 +22,12 @@ function canLaunchWithoutPrompt(launchData) {
     !launchData.ask_variables_on_launch &&
     !launchData.ask_limit_on_launch &&
     !launchData.ask_scm_branch_on_launch &&
+    !launchData.ask_execution_environment_on_launch &&
+    !launchData.ask_labels_on_launch &&
+    !launchData.ask_forks_on_launch &&
+    !launchData.ask_job_slice_count_on_launch &&
+    !launchData.ask_timeout_on_launch &&
+    !launchData.ask_instance_groups_on_launch &&
     !launchData.survey_enabled &&
     (!launchData.passwords_needed_to_start ||
       launchData.passwords_needed_to_start.length === 0) &&
@@ -37,6 +41,7 @@ function LaunchButton({ resource, children }) {
   const [showLaunchPrompt, setShowLaunchPrompt] = useState(false);
   const [launchConfig, setLaunchConfig] = useState(null);
   const [surveyConfig, setSurveyConfig] = useState(null);
+  const [labels, setLabels] = useState([]);
   const [isLaunching, setIsLaunching] = useState(false);
   const [error, setError] = useState(null);
 
@@ -50,6 +55,11 @@ function LaunchButton({ resource, children }) {
       resource.type === 'workflow_job_template'
         ? WorkflowJobTemplatesAPI.readSurvey(resource.id)
         : JobTemplatesAPI.readSurvey(resource.id);
+    const readLabels =
+      resource.type === 'workflow_job_template'
+        ? WorkflowJobTemplatesAPI.readAllLabels(resource.id)
+        : JobTemplatesAPI.readAllLabels(resource.id);
+
     try {
       const { data: launch } = await readLaunch;
       setLaunchConfig(launch);
@@ -60,6 +70,19 @@ function LaunchButton({ resource, children }) {
         setSurveyConfig(data);
       }
 
+      if (launch.ask_labels_on_launch) {
+        const {
+          data: { results },
+        } = await readLabels;
+
+        const allLabels = results.map((label) => ({
+          ...label,
+          isReadOnly: true,
+        }));
+
+        setLabels(allLabels);
+      }
+
       if (canLaunchWithoutPrompt(launch)) {
         await launchWithParams({});
       } else {
@@ -171,6 +194,7 @@ function LaunchButton({ resource, children }) {
           launchConfig={launchConfig}
           surveyConfig={surveyConfig}
           resource={resource}
+          labels={labels}
           onLaunch={launchWithParams}
           onCancel={() => setShowLaunchPrompt(false)}
         />

awx/ui/src/components/LaunchButton/LaunchButton.test.js

@@ -37,6 +37,12 @@ describe('LaunchButton', () => {
         ask_variables_on_launch: false,
         ask_limit_on_launch: false,
         ask_scm_branch_on_launch: false,
+        ask_execution_environment_on_launch: false,
+        ask_labels_on_launch: false,
+        ask_forks_on_launch: false,
+        ask_job_slice_count_on_launch: false,
+        ask_timeout_on_launch: false,
+        ask_instance_groups_on_launch: false,
         survey_enabled: false,
         variables_needed_to_start: [],
       },

awx/ui/src/components/LaunchPrompt/LaunchPrompt.js

@@ -5,6 +5,7 @@ import { Formik, useFormikContext } from 'formik';
 import { useDismissableError } from 'hooks/useRequest';
 import mergeExtraVars from 'util/prompt/mergeExtraVars';
 import getSurveyValues from 'util/prompt/getSurveyValues';
+import createNewLabels from 'util/labels';
 import ContentLoading from '../ContentLoading';
 import ContentError from '../ContentError';
 import useLaunchSteps from './useLaunchSteps';
@@ -15,7 +16,9 @@ function PromptModalForm({
   onCancel,
   onSubmit,
   resource,
+  labels,
   surveyConfig,
+  instanceGroups,
 }) {
   const { setFieldTouched, values } = useFormikContext();
   const [showDescription, setShowDescription] = useState(false);
@@ -27,9 +30,15 @@ function PromptModalForm({
     visitStep,
     visitAllSteps,
     contentError,
-  } = useLaunchSteps(launchConfig, surveyConfig, resource);
+  } = useLaunchSteps(
+    launchConfig,
+    surveyConfig,
+    resource,
+    labels,
+    instanceGroups
+  );
 
-  const handleSubmit = () => {
+  const handleSubmit = async () => {
     const postValues = {};
     const setValue = (key, value) => {
       if (typeof value !== 'undefined' && value !== null) {
@@ -53,6 +62,27 @@ function PromptModalForm({
     setValue('extra_vars', mergeExtraVars(extraVars, surveyValues));
     setValue('scm_branch', values.scm_branch);
     setValue('verbosity', values.verbosity);
+    setValue('timeout', values.timeout);
+    setValue('forks', values.forks);
+    setValue('job_slice_count', values.job_slice_count);
+    setValue('execution_environment', values.execution_environment?.id);
+
+    if (launchConfig.ask_instance_groups_on_launch) {
+      const instanceGroupIds = [];
+      values.instance_groups.forEach((instance_group) => {
+        instanceGroupIds.push(instance_group.id);
+      });
+      setValue('instance_groups', instanceGroupIds);
+    }
+
+    if (launchConfig.ask_labels_on_launch) {
+      const { labelIds } = createNewLabels(
+        values.labels,
+        resource.organization
+      );
+
+      setValue('labels', labelIds);
+    }
 
     onSubmit(postValues);
   };
@@ -137,6 +167,7 @@ function LaunchPrompt({
   onCancel,
   onLaunch,
   resource = {},
+  labels = [],
   surveyConfig,
   resourceDefaultCredentials = [],
 }) {
@@ -148,7 +179,9 @@ function LaunchPrompt({
         launchConfig={launchConfig}
         surveyConfig={surveyConfig}
         resource={resource}
+        labels={labels}
         resourceDefaultCredentials={resourceDefaultCredentials}
+        instanceGroups={[]}
       />
     </Formik>
   );

awx/ui/src/components/LaunchPrompt/LaunchPrompt.test.js

@@ -1,6 +1,8 @@
 import React from 'react';
 import { act, isElementOfType } from 'react-dom/test-utils';
 import {
+  ExecutionEnvironmentsAPI,
+  InstanceGroupsAPI,
   InventoriesAPI,
   CredentialsAPI,
   CredentialTypesAPI,
@@ -16,11 +18,16 @@ import CredentialsStep from './steps/CredentialsStep';
 import CredentialPasswordsStep from './steps/CredentialPasswordsStep';
 import OtherPromptsStep from './steps/OtherPromptsStep';
 import PreviewStep from './steps/PreviewStep';
+import ExecutionEnvironmentStep from './steps/ExecutionEnvironmentStep';
+import InstanceGroupsStep from './steps/InstanceGroupsStep';
+import SurveyStep from './steps/SurveyStep';
 
 jest.mock('../../api/models/Inventories');
+jest.mock('../../api/models/ExecutionEnvironments');
 jest.mock('../../api/models/CredentialTypes');
 jest.mock('../../api/models/Credentials');
 jest.mock('../../api/models/JobTemplates');
+jest.mock('../../api/models/InstanceGroups');
 
 let config;
 const resource = {
@@ -62,6 +69,79 @@ describe('LaunchPrompt', () => {
         spec: [{ type: 'text', variable: 'foo' }],
       },
     });
+    InstanceGroupsAPI.read.mockResolvedValue({
+      data: {
+        results: [
+          {
+            id: 2,
+            type: 'instance_group',
+            url: '/api/v2/instance_groups/2/',
+            related: {
+              jobs: '/api/v2/instance_groups/2/jobs/',
+              instances: '/api/v2/instance_groups/2/instances/',
+            },
+            name: 'default',
+            created: '2022-08-30T20:35:05.747132Z',
+            modified: '2022-08-30T20:35:05.756690Z',
+            capacity: 177,
+            consumed_capacity: 0,
+            percent_capacity_remaining: 100.0,
+            jobs_running: 0,
+            jobs_total: 2,
+            instances: 3,
+            is_container_group: false,
+            credential: null,
+            policy_instance_percentage: 100,
+            policy_instance_minimum: 0,
+            policy_instance_list: [],
+            pod_spec_override: '',
+            summary_fields: {
+              user_capabilities: {
+                edit: true,
+                delete: false,
+              },
+            },
+          },
+        ],
+        count: 1,
+      },
+    });
+    ExecutionEnvironmentsAPI.read.mockResolvedValue({
+      data: {
+        results: [
+          {
+            id: 1,
+            type: 'execution_environment',
+            url: '/api/v2/execution_environments/1/',
+            related: {
+              activity_stream:
+                '/api/v2/execution_environments/1/activity_stream/',
+              unified_job_templates:
+                '/api/v2/execution_environments/1/unified_job_templates/',
+              copy: '/api/v2/execution_environments/1/copy/',
+            },
+            summary_fields: {
+              execution_environment: {},
+              user_capabilities: {
+                edit: true,
+                delete: true,
+                copy: true,
+              },
+            },
+            created: '2022-08-30T20:34:55.842997Z',
+            modified: '2022-08-30T20:34:55.859874Z',
+            name: 'AWX EE (latest)',
+            description: '',
+            organization: null,
+            image: 'quay.io/ansible/awx-ee:latest',
+            managed: false,
+            credential: null,
+            pull: '',
+          },
+        ],
+        count: 1,
+      },
+    });
 
     config = {
       can_start_without_user_input: false,
@@ -76,6 +156,12 @@ describe('LaunchPrompt', () => {
       ask_verbosity_on_launch: false,
       ask_inventory_on_launch: false,
       ask_credential_on_launch: false,
+      ask_execution_environment_on_launch: false,
+      ask_labels_on_launch: false,
+      ask_forks_on_launch: false,
+      ask_job_slice_count_on_launch: false,
+      ask_timeout_on_launch: false,
+      ask_instance_groups_on_launch: false,
       survey_enabled: false,
       variables_needed_to_start: [],
       credential_needed_to_start: false,
@@ -96,6 +182,8 @@ describe('LaunchPrompt', () => {
             ask_inventory_on_launch: true,
             ask_credential_on_launch: true,
             ask_scm_branch_on_launch: true,
+            ask_execution_environment_on_launch: true,
+            ask_instance_groups_on_launch: true,
             survey_enabled: true,
             passwords_needed_to_start: ['ssh_password'],
             defaults: {
@@ -150,13 +238,15 @@ describe('LaunchPrompt', () => {
     const wizard = await waitForElement(wrapper, 'Wizard');
     const steps = wizard.prop('steps');
 
-    expect(steps).toHaveLength(6);
+    expect(steps).toHaveLength(8);
     expect(steps[0].name.props.children).toEqual('Inventory');
     expect(steps[1].name.props.children).toEqual('Credentials');
     expect(steps[2].name.props.children).toEqual('Credential passwords');
-    expect(steps[3].name.props.children).toEqual('Other prompts');
-    expect(steps[4].name.props.children).toEqual('Survey');
-    expect(steps[5].name.props.children).toEqual('Preview');
+    expect(steps[3].name.props.children).toEqual('Execution Environment');
+    expect(steps[4].name.props.children).toEqual('Instance Groups');
+    expect(steps[5].name.props.children).toEqual('Other prompts');
+    expect(steps[6].name.props.children).toEqual('Survey');
+    expect(steps[7].name.props.children).toEqual('Preview');
     expect(wizard.find('WizardHeader').prop('title')).toBe('Launch | Foobar');
     expect(wizard.find('WizardHeader').prop('description')).toBe(
       'Foo Description'
@@ -214,6 +304,58 @@ describe('LaunchPrompt', () => {
     expect(isElementOfType(steps[2].component, PreviewStep)).toEqual(true);
   });
 
+  test('should add execution environment step', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <LaunchPrompt
+          launchConfig={{
+            ...config,
+            ask_execution_environment_on_launch: true,
+          }}
+          resource={resource}
+          onLaunch={noop}
+          onCancel={noop}
+        />
+      );
+    });
+    const wizard = await waitForElement(wrapper, 'Wizard');
+    const steps = wizard.prop('steps');
+
+    expect(steps).toHaveLength(2);
+    expect(steps[0].name.props.children).toEqual('Execution Environment');
+    expect(
+      isElementOfType(steps[0].component, ExecutionEnvironmentStep)
+    ).toEqual(true);
+    expect(isElementOfType(steps[1].component, PreviewStep)).toEqual(true);
+  });
+
+  test('should add instance groups step', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <LaunchPrompt
+          launchConfig={{
+            ...config,
+            ask_instance_groups_on_launch: true,
+          }}
+          resource={resource}
+          onLaunch={noop}
+          onCancel={noop}
+        />
+      );
+    });
+    const wizard = await waitForElement(wrapper, 'Wizard');
+    const steps = wizard.prop('steps');
+
+    expect(steps).toHaveLength(2);
+    expect(steps[0].name.props.children).toEqual('Instance Groups');
+    expect(isElementOfType(steps[0].component, InstanceGroupsStep)).toEqual(
+      true
+    );
+    expect(isElementOfType(steps[1].component, PreviewStep)).toEqual(true);
+  });
+
   test('should add other prompts step', async () => {
     let wrapper;
     await act(async () => {
@@ -237,4 +379,46 @@ describe('LaunchPrompt', () => {
     expect(isElementOfType(steps[0].component, OtherPromptsStep)).toEqual(true);
     expect(isElementOfType(steps[1].component, PreviewStep)).toEqual(true);
   });
+
+  test('should add survey step', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <LaunchPrompt
+          launchConfig={{
+            ...config,
+            survey_enabled: true,
+          }}
+          resource={resource}
+          onLaunch={noop}
+          onCancel={noop}
+          surveyConfig={{
+            name: '',
+            description: '',
+            spec: [
+              {
+                choices: '',
+                default: '',
+                max: 1024,
+                min: 0,
+                new_question: false,
+                question_description: '',
+                question_name: 'foo',
+                required: true,
+                type: 'text',
+                variable: 'foo',
+              },
+            ],
+          }}
+        />
+      );
+    });
+    const wizard = await waitForElement(wrapper, 'Wizard');
+    const steps = wizard.prop('steps');
+
+    expect(steps).toHaveLength(2);
+    expect(steps[0].name.props.children).toEqual('Survey');
+    expect(isElementOfType(steps[0].component, SurveyStep)).toEqual(true);
+    expect(isElementOfType(steps[1].component, PreviewStep)).toEqual(true);
+  });
 });

awx/ui/src/components/LaunchPrompt/steps/CredentialsStep.js

@@ -132,7 +132,7 @@ function CredentialsStep({
   );
 
   return (
-    <>
+    <div data-cy="credentials-prompt">
       {meta.error && (
         <CredentialErrorAlert variant="danger" isInline title={meta.error} />
       )}
@@ -208,7 +208,7 @@ function CredentialsStep({
         }}
         renderItemChip={renderChip}
       />
-    </>
+    </div>
   );
 }
 

awx/ui/src/components/LaunchPrompt/steps/ExecutionEnvironmentStep.js

@@ -0,0 +1,118 @@
+import React, { useCallback, useEffect } from 'react';
+import { useHistory } from 'react-router-dom';
+import { t } from '@lingui/macro';
+import { useField } from 'formik';
+import { ExecutionEnvironmentsAPI } from 'api';
+import { getSearchableKeys } from 'components/PaginatedTable';
+import { getQSConfig, parseQueryString } from 'util/qs';
+import useRequest from 'hooks/useRequest';
+import OptionsList from '../../OptionsList';
+import ContentLoading from '../../ContentLoading';
+import ContentError from '../../ContentError';
+
+const QS_CONFIG = getQSConfig('execution_environment', {
+  page: 1,
+  page_size: 5,
+});
+
+function ExecutionEnvironmentStep() {
+  const [field, , helpers] = useField('execution_environment');
+
+  const history = useHistory();
+
+  const {
+    isLoading,
+    error,
+    result: {
+      execution_environments,
+      count,
+      relatedSearchableKeys,
+      searchableKeys,
+    },
+    request: fetchExecutionEnvironments,
+  } = useRequest(
+    useCallback(async () => {
+      const params = parseQueryString(QS_CONFIG, history.location.search);
+      const [{ data }, actionsResponse] = await Promise.all([
+        ExecutionEnvironmentsAPI.read(params),
+        ExecutionEnvironmentsAPI.readOptions(),
+      ]);
+      return {
+        execution_environments: data.results,
+        count: data.count,
+        relatedSearchableKeys: (
+          actionsResponse?.data?.related_search_fields || []
+        ).map((val) => val.slice(0, -8)),
+        searchableKeys: getSearchableKeys(actionsResponse.data.actions?.GET),
+      };
+    }, [history.location]),
+    {
+      count: 0,
+      execution_environments: [],
+      relatedSearchableKeys: [],
+      searchableKeys: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchExecutionEnvironments();
+  }, [fetchExecutionEnvironments]);
+
+  if (isLoading) {
+    return <ContentLoading />;
+  }
+  if (error) {
+    return <ContentError error={error} />;
+  }
+
+  return (
+    <div data-cy="execution-environment-prompt">
+      <OptionsList
+        value={field.value ? [field.value] : []}
+        options={execution_environments}
+        optionCount={count}
+        columns={[
+          {
+            name: t`Name`,
+            key: 'name',
+          },
+          {
+            name: t`Image`,
+            key: 'image',
+          },
+        ]}
+        searchColumns={[
+          {
+            name: t`Name`,
+            key: 'name__icontains',
+            isDefault: true,
+          },
+          {
+            name: t`Image`,
+            key: 'image__icontains',
+          },
+        ]}
+        sortColumns={[
+          {
+            name: t`Name`,
+            key: 'name',
+          },
+          {
+            name: t`Image`,
+            key: 'image',
+          },
+        ]}
+        searchableKeys={searchableKeys}
+        relatedSearchableKeys={relatedSearchableKeys}
+        header={t`Execution Environments`}
+        name="execution_environment"
+        qsConfig={QS_CONFIG}
+        readOnly
+        selectItem={helpers.setValue}
+        deselectItem={() => helpers.setValue(null)}
+      />
+    </div>
+  );
+}
+
+export default ExecutionEnvironmentStep;

awx/ui/src/components/LaunchPrompt/steps/ExecutionEnvironmentStep.test.js

@@ -0,0 +1,52 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { Formik } from 'formik';
+import { ExecutionEnvironmentsAPI } from 'api';
+import { mountWithContexts } from '../../../../testUtils/enzymeHelpers';
+import ExecutionEnvironmentStep from './ExecutionEnvironmentStep';
+
+jest.mock('../../../api/models/ExecutionEnvironments');
+
+const execution_environments = [
+  { id: 1, name: 'ee one', url: '/execution_environments/1' },
+  { id: 2, name: 'ee two', url: '/execution_environments/2' },
+  { id: 3, name: 'ee three', url: '/execution_environments/3' },
+];
+
+describe('ExecutionEnvironmentStep', () => {
+  beforeEach(() => {
+    ExecutionEnvironmentsAPI.read.mockResolvedValue({
+      data: {
+        results: execution_environments,
+        count: 3,
+      },
+    });
+
+    ExecutionEnvironmentsAPI.readOptions.mockResolvedValue({
+      data: {
+        actions: {
+          GET: {},
+          POST: {},
+        },
+        related_search_fields: [],
+      },
+    });
+  });
+
+  test('should load execution environments', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <ExecutionEnvironmentStep />
+        </Formik>
+      );
+    });
+    wrapper.update();
+
+    expect(ExecutionEnvironmentsAPI.read).toHaveBeenCalled();
+    expect(wrapper.find('OptionsList').prop('options')).toEqual(
+      execution_environments
+    );
+  });
+});

awx/ui/src/components/LaunchPrompt/steps/InstanceGroupsStep.js

@@ -0,0 +1,108 @@
+import React, { useCallback, useEffect } from 'react';
+import { useHistory } from 'react-router-dom';
+import { t } from '@lingui/macro';
+import { useField } from 'formik';
+import { InstanceGroupsAPI } from 'api';
+import { getSearchableKeys } from 'components/PaginatedTable';
+import { getQSConfig, parseQueryString } from 'util/qs';
+import useRequest from 'hooks/useRequest';
+import useSelected from 'hooks/useSelected';
+import OptionsList from '../../OptionsList';
+import ContentLoading from '../../ContentLoading';
+import ContentError from '../../ContentError';
+
+const QS_CONFIG = getQSConfig('instance-groups', {
+  page: 1,
+  page_size: 5,
+  order_by: 'name',
+});
+
+function InstanceGroupsStep() {
+  const [field, , helpers] = useField('instance_groups');
+  const { selected, handleSelect, setSelected } = useSelected([], field.value);
+
+  const history = useHistory();
+
+  const {
+    result: { instance_groups, count, relatedSearchableKeys, searchableKeys },
+    request: fetchInstanceGroups,
+    error,
+    isLoading,
+  } = useRequest(
+    useCallback(async () => {
+      const params = parseQueryString(QS_CONFIG, history.location.search);
+      const [{ data }, actionsResponse] = await Promise.all([
+        InstanceGroupsAPI.read(params),
+        InstanceGroupsAPI.readOptions(),
+      ]);
+      return {
+        instance_groups: data.results,
+        count: data.count,
+        relatedSearchableKeys: (
+          actionsResponse?.data?.related_search_fields || []
+        ).map((val) => val.slice(0, -8)),
+        searchableKeys: getSearchableKeys(actionsResponse.data.actions?.GET),
+      };
+    }, [history.location]),
+    {
+      instance_groups: [],
+      count: 0,
+      relatedSearchableKeys: [],
+      searchableKeys: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchInstanceGroups();
+  }, [fetchInstanceGroups]);
+
+  useEffect(() => {
+    helpers.setValue(selected);
+  }, [selected]); // eslint-disable-line react-hooks/exhaustive-deps
+
+  if (isLoading) {
+    return <ContentLoading />;
+  }
+  if (error) {
+    return <ContentError error={error} />;
+  }
+
+  return (
+    <div data-cy="instance-groups-prompt">
+      <OptionsList
+        value={selected}
+        options={instance_groups}
+        optionCount={count}
+        searchColumns={[
+          {
+            name: t`Name`,
+            key: 'name__icontains',
+            isDefault: true,
+          },
+          {
+            name: t`Credential Name`,
+            key: 'credential__name__icontains',
+          },
+        ]}
+        sortColumns={[
+          {
+            name: t`Name`,
+            key: 'name',
+          },
+        ]}
+        searchableKeys={searchableKeys}
+        relatedSearchableKeys={relatedSearchableKeys}
+        multiple
+        header={t`Instance Groups`}
+        name="instanceGroups"
+        qsConfig={QS_CONFIG}
+        selectItem={handleSelect}
+        deselectItem={handleSelect}
+        sortSelectedItems={(selectedItems) => setSelected(selectedItems)}
+        isSelectedDraggable
+      />
+    </div>
+  );
+}
+
+export default InstanceGroupsStep;

awx/ui/src/components/LaunchPrompt/steps/InstanceGroupsStep.test.js

@@ -0,0 +1,52 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { Formik } from 'formik';
+import { InstanceGroupsAPI } from 'api';
+import { mountWithContexts } from '../../../../testUtils/enzymeHelpers';
+import InstanceGroupsStep from './InstanceGroupsStep';
+
+jest.mock('../../../api/models/InstanceGroups');
+
+const instance_groups = [
+  { id: 1, name: 'ig one', url: '/instance_groups/1' },
+  { id: 2, name: 'ig two', url: '/instance_groups/2' },
+  { id: 3, name: 'ig three', url: '/instance_groups/3' },
+];
+
+describe('InstanceGroupsStep', () => {
+  beforeEach(() => {
+    InstanceGroupsAPI.read.mockResolvedValue({
+      data: {
+        results: instance_groups,
+        count: 3,
+      },
+    });
+
+    InstanceGroupsAPI.readOptions.mockResolvedValue({
+      data: {
+        actions: {
+          GET: {},
+          POST: {},
+        },
+        related_search_fields: [],
+      },
+    });
+  });
+
+  test('should load instance groups', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik initialValues={{ instance_groups: [] }}>
+          <InstanceGroupsStep />
+        </Formik>
+      );
+    });
+    wrapper.update();
+
+    expect(InstanceGroupsAPI.read).toHaveBeenCalled();
+    expect(wrapper.find('OptionsList').prop('options')).toEqual(
+      instance_groups
+    );
+  });
+});

awx/ui/src/components/LaunchPrompt/steps/InventoryStep.js

@@ -70,7 +70,7 @@ function InventoryStep({ warningMessage = null }) {
   }
 
   return (
-    <>
+    <div data-cy="inventory-prompt">
       {meta.touched && meta.error && (
         <InventoryErrorAlert variant="danger" isInline title={meta.error} />
       )}
@@ -109,7 +109,7 @@ function InventoryStep({ warningMessage = null }) {
         selectItem={helpers.setValue}
         deselectItem={() => field.onChange(null)}
       />
-    </>
+    </div>
   );
 }
 

awx/ui/src/components/LaunchPrompt/steps/OtherPromptsStep.js

@@ -1,15 +1,17 @@
 import React from 'react';
-
 import { t } from '@lingui/macro';
 import { useField } from 'formik';
 import { Form, FormGroup, Switch } from '@patternfly/react-core';
 import styled from 'styled-components';
+import LabelSelect from '../../LabelSelect';
 import FormField from '../../FormField';
 import { TagMultiSelect } from '../../MultiSelect';
 import AnsibleSelect from '../../AnsibleSelect';
 import { VariablesField } from '../../CodeEditor';
 import Popover from '../../Popover';
 import { VerbositySelectField } from '../../VerbositySelectField';
+import jobHelpText from '../../../screens/Job/Job.helptext';
+import workflowHelpText from '../../../screens/Template/shared/WorkflowJobTemplate.helptext';
 
 const FieldHeader = styled.div`
   display: flex;
@@ -22,72 +24,105 @@ const FieldHeader = styled.div`
 `;
 
 function OtherPromptsStep({ launchConfig, variablesMode, onVarModeChange }) {
+  const helpTextSource = launchConfig.job_template_data
+    ? jobHelpText
+    : workflowHelpText;
   return (
-    <Form
-      onSubmit={(e) => {
-        e.preventDefault();
-      }}
-    >
-      {launchConfig.ask_job_type_on_launch && <JobTypeField />}
-      {launchConfig.ask_limit_on_launch && (
-        <FormField
-          id="prompt-limit"
-          name="limit"
-          label={t`Limit`}
-          tooltip={t`Provide a host pattern to further constrain the list
-          of hosts that will be managed or affected by the playbook. Multiple
-          patterns are allowed. Refer to Ansible documentation for more
-          information and examples on patterns.`}
-        />
-      )}
-      {launchConfig.ask_scm_branch_on_launch && (
-        <FormField
-          id="prompt-scm-branch"
-          name="scm_branch"
-          label={t`Source Control Branch`}
-          tooltip={t`Select a branch for the workflow. This branch is applied to all job template nodes that prompt for a branch`}
-        />
-      )}
-      {launchConfig.ask_verbosity_on_launch && <VerbosityField />}
-      {launchConfig.ask_diff_mode_on_launch && <ShowChangesToggle />}
-      {launchConfig.ask_tags_on_launch && (
-        <TagField
-          id="prompt-job-tags"
-          name="job_tags"
-          label={t`Job Tags`}
-          aria-label={t`Job Tags`}
-          tooltip={t`Tags are useful when you have a large
-            playbook, and you want to run a specific part of a play or task.
-            Use commas to separate multiple tags. Refer to Ansible Controller
-            documentation for details on the usage of tags.`}
-        />
-      )}
-      {launchConfig.ask_skip_tags_on_launch && (
-        <TagField
-          id="prompt-skip-tags"
-          name="skip_tags"
-          label={t`Skip Tags`}
-          aria-label={t`Skip Tags`}
-          tooltip={t`Skip tags are useful when you have a large
-            playbook, and you want to skip specific parts of a play or task.
-            Use commas to separate multiple tags. Refer to Ansible Controller
-            documentation for details on the usage of tags.`}
-        />
-      )}
-      {launchConfig.ask_variables_on_launch && (
-        <VariablesField
-          id="prompt-variables"
-          name="extra_vars"
-          label={t`Variables`}
-          initialMode={variablesMode}
-          onModeChange={onVarModeChange}
-        />
-      )}
-    </Form>
+    <div data-cy="other-prompts">
+      <Form
+        onSubmit={(e) => {
+          e.preventDefault();
+        }}
+      >
+        {launchConfig.ask_job_type_on_launch && (
+          <JobTypeField helpTextSource={helpTextSource} />
+        )}
+        {launchConfig.ask_scm_branch_on_launch && (
+          <FormField
+            id="prompt-scm-branch"
+            name="scm_branch"
+            label={t`Source Control Branch`}
+            tooltip={helpTextSource.sourceControlBranch}
+          />
+        )}
+        {launchConfig.ask_labels_on_launch && (
+          <LabelsField helpTextSource={helpTextSource} />
+        )}
+        {launchConfig.ask_forks_on_launch && (
+          <FormField
+            id="prompt-forks"
+            name="forks"
+            label={t`Forks`}
+            type="number"
+            min="0"
+            tooltip={helpTextSource.forks}
+          />
+        )}
+        {launchConfig.ask_limit_on_launch && (
+          <FormField
+            id="prompt-limit"
+            name="limit"
+            label={t`Limit`}
+            tooltip={helpTextSource.limit}
+          />
+        )}
+        {launchConfig.ask_verbosity_on_launch && (
+          <VerbosityField helpTextSource={helpTextSource} />
+        )}
+        {launchConfig.ask_job_slice_count_on_launch && (
+          <FormField
+            id="prompt-job-slicing"
+            name="job_slice_count"
+            label={t`Job Slicing`}
+            type="number"
+            min="1"
+            tooltip={helpTextSource.jobSlicing}
+          />
+        )}
+        {launchConfig.ask_timeout_on_launch && (
+          <FormField
+            id="prompt-timeout"
+            name="timeout"
+            label={t`Timeout`}
+            type="number"
+            min="0"
+            tooltip={helpTextSource.timeout}
+          />
+        )}
+        {launchConfig.ask_diff_mode_on_launch && <ShowChangesToggle />}
+        {launchConfig.ask_tags_on_launch && (
+          <TagField
+            id="prompt-job-tags"
+            name="job_tags"
+            label={t`Job Tags`}
+            aria-label={t`Job Tags`}
+            tooltip={helpTextSource.jobTags}
+          />
+        )}
+        {launchConfig.ask_skip_tags_on_launch && (
+          <TagField
+            id="prompt-skip-tags"
+            name="skip_tags"
+            label={t`Skip Tags`}
+            aria-label={t`Skip Tags`}
+            tooltip={helpTextSource.skipTags}
+          />
+        )}
+        {launchConfig.ask_variables_on_launch && (
+          <VariablesField
+            id="prompt-variables"
+            name="extra_vars"
+            label={t`Variables`}
+            initialMode={variablesMode}
+            onModeChange={onVarModeChange}
+          />
+        )}
+      </Form>
+    </div>
   );
 }
 
-function JobTypeField() {
+function JobTypeField({ helpTextSource }) {
   const [field, meta, helpers] = useField('job_type');
   const options = [
     {
@@ -107,15 +142,9 @@ function JobTypeField() {
   const isValid = !(meta.touched && meta.error);
   return (
     <FormGroup
-      fieldId="propmt-job-type"
+      fieldId="prompt-job-type"
       label={t`Job Type`}
-      labelIcon={
-        <Popover
-          content={t`For job templates, select run to execute the playbook.
-      Select check to only check playbook syntax, test environment setup,
-      and report problems without executing the playbook.`}
-        />
-      }
+      labelIcon={<Popover content={helpTextSource.jobType} />}
       isRequired
       validated={isValid ? 'default' : 'error'}
     >
@@ -129,15 +158,14 @@ function JobTypeField() {
   );
 }
 
-function VerbosityField() {
+function VerbosityField({ helpTextSource }) {
   const [, meta] = useField('verbosity');
   const isValid = !(meta.touched && meta.error);
 
   return (
     <VerbositySelectField
       fieldId="prompt-verbosity"
-      tooltip={t`Control the level of output ansible
-          will produce as the playbook executes.`}
+      tooltip={helpTextSource.verbosity}
       isValid={isValid ? 'default' : 'error'}
     />
   );
@@ -186,4 +214,25 @@ function TagField({ id, name, label, tooltip }) {
   );
 }
 
+function LabelsField({ helpTextSource }) {
+  const [field, meta, helpers] = useField('labels');
+
+  return (
+    <FormGroup
+      fieldId="prompt-labels"
+      label={t`Labels`}
+      labelIcon={<Popover content={helpTextSource.labels} />}
+      validated={!meta.touched || !meta.error ? 'default' : 'error'}
+      helperTextInvalid={meta.error}
+    >
+      <LabelSelect
+        value={field.value}
+        onChange={(labels) => helpers.setValue(labels)}
+        createText={t`Create`}
+        onError={(err) => helpers.setError(err)}
+      />
+    </FormGroup>
+  );
+}
+
 export default OtherPromptsStep;

awx/ui/src/components/LaunchPrompt/steps/OtherPromptsStep.test.js

@@ -13,6 +13,11 @@ describe('OtherPromptsStep', () => {
           <OtherPromptsStep
             launchConfig={{
               ask_job_type_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
             }}
           />
         </Formik>
@@ -36,6 +41,11 @@ describe('OtherPromptsStep', () => {
           <OtherPromptsStep
             launchConfig={{
               ask_limit_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
             }}
           />
         </Formik>
@@ -48,6 +58,81 @@ describe('OtherPromptsStep', () => {
     );
   });
 
+  test('should render timeout field', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <OtherPromptsStep
+            launchConfig={{
+              ask_timeout_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
+            }}
+          />
+        </Formik>
+      );
+    });
+
+    expect(wrapper.find('FormField#prompt-timeout')).toHaveLength(1);
+    expect(wrapper.find('FormField#prompt-timeout input').prop('name')).toEqual(
+      'timeout'
+    );
+  });
+
+  test('should render forks field', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <OtherPromptsStep
+            launchConfig={{
+              ask_forks_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
+            }}
+          />
+        </Formik>
+      );
+    });
+
+    expect(wrapper.find('FormField#prompt-forks')).toHaveLength(1);
+    expect(wrapper.find('FormField#prompt-forks input').prop('name')).toEqual(
+      'forks'
+    );
+  });
+
+  test('should render job slicing field', async () => {
+    let wrapper;
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <OtherPromptsStep
+            launchConfig={{
+              ask_job_slice_count_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
+            }}
+          />
+        </Formik>
+      );
+    });
+
+    expect(wrapper.find('FormField#prompt-job-slicing')).toHaveLength(1);
+    expect(
+      wrapper.find('FormField#prompt-job-slicing input').prop('name')
+    ).toEqual('job_slice_count');
+  });
+
   test('should render source control branch field', async () => {
     let wrapper;
     await act(async () => {
@@ -56,6 +141,11 @@ describe('OtherPromptsStep', () => {
           <OtherPromptsStep
             launchConfig={{
               ask_scm_branch_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
             }}
           />
         </Formik>
@@ -76,6 +166,11 @@ describe('OtherPromptsStep', () => {
           <OtherPromptsStep
             launchConfig={{
               ask_verbosity_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
             }}
           />
         </Formik>
@@ -96,6 +191,11 @@ describe('OtherPromptsStep', () => {
           <OtherPromptsStep
             launchConfig={{
               ask_diff_mode_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
             }}
           />
         </Formik>
@@ -119,6 +219,11 @@ describe('OtherPromptsStep', () => {
             onVarModeChange={onModeChange}
             launchConfig={{
               ask_variables_on_launch: true,
+              job_template_data: {
+                name: 'Demo Job Template',
+                id: 1,
+                description: '',
+              },
             }}
           />
         </Formik>

awx/ui/src/components/LaunchPrompt/steps/PreviewStep.js

@@ -52,7 +52,7 @@ function PreviewStep({ resource, launchConfig, surveyConfig, formErrors }) {
   }
 
   return (
-    <>
+    <div data-cy="prompt-preview">
       {formErrors && (
         <ErrorMessageWrapper>
           {t`Some of the previous step(s) have errors`}
@@ -70,7 +70,7 @@ function PreviewStep({ resource, launchConfig, surveyConfig, formErrors }) {
         launchConfig={launchConfig}
         overrides={overrides}
       />
-    </>
+    </div>
   );
 }
 

awx/ui/src/components/LaunchPrompt/steps/SurveyStep.js

@@ -31,16 +31,18 @@ function SurveyStep({ surveyConfig }) {
     float: NumberField,
   };
   return (
-    <Form
-      onSubmit={(e) => {
-        e.preventDefault();
-      }}
-    >
-      {surveyConfig.spec.map((question) => {
-        const Field = fieldTypes[question.type];
-        return <Field key={question.variable} question={question} />;
-      })}
-    </Form>
+    <div data-cy="survey-prompts">
+      <Form
+        onSubmit={(e) => {
+          e.preventDefault();
+        }}
+      >
+        {surveyConfig.spec.map((question) => {
+          const Field = fieldTypes[question.type];
+          return <Field key={question.variable} question={question} />;
+        })}
+      </Form>
+    </div>
   );
 }
 SurveyStep.propTypes = {

awx/ui/src/components/LaunchPrompt/steps/useCredentialsStep.js

@@ -10,7 +10,7 @@ const STEP_ID = 'credentials';
 export default function useCredentialsStep(
   launchConfig,
   resource,
-  resourceDefaultCredentials,
+  resourceDefaultCredentials = [],
   allowCredentialsWithPasswords = false
 ) {
   const [field, meta, helpers] = useField('credentials');
@@ -78,6 +78,6 @@ function getInitialValues(launchConfig, resourceDefaultCredentials) {
   }
 
   return {
-    credentials: resourceDefaultCredentials || [],
+    credentials: resourceDefaultCredentials,
   };
 }

awx/ui/src/components/LaunchPrompt/steps/useExecutionEnvironmentStep.js

@@ -0,0 +1,46 @@
+import React from 'react';
+import { t } from '@lingui/macro';
+import ExecutionEnvironmentStep from './ExecutionEnvironmentStep';
+import StepName from './StepName';
+
+const STEP_ID = 'executionEnvironment';
+
+export default function useExecutionEnvironmentStep(launchConfig, resource) {
+  return {
+    step: getStep(launchConfig, resource),
+    initialValues: getInitialValues(launchConfig, resource),
+    isReady: true,
+    contentError: null,
+    hasError: false,
+    setTouched: (setFieldTouched) => {
+      setFieldTouched('execution_environment', true, false);
+    },
+    validate: () => {},
+  };
+}
+function getStep(launchConfig) {
+  if (!launchConfig.ask_execution_environment_on_launch) {
+    return null;
+  }
+  return {
+    id: STEP_ID,
+    name: (
+      <StepName id="execution-environment-step">
+        {t`Execution Environment`}
+      </StepName>
+    ),
+    component: <ExecutionEnvironmentStep />,
+    enableNext: true,
+  };
+}
+
+function getInitialValues(launchConfig, resource) {
+  if (!launchConfig.ask_execution_environment_on_launch) {
+    return {};
+  }
+
+  return {
+    execution_environment:
+      resource?.summary_fields?.execution_environment || null,
+  };
+}