From 41613ff54487cb88a4efa9b7a0e1077c47bc2b81 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 26 Jun 2020 17:01:05 -0400 Subject: [PATCH 001/157] Add a new ExecutionEnvironment model --- .../migrations/0124_execution_environments.py | 69 +++++++++++++++++++ awx/main/models/__init__.py | 3 +- awx/main/models/activity_stream.py | 2 + awx/main/models/execution_environments.py | 37 ++++++++++ awx/main/models/mixins.py | 18 ++++- awx/main/models/organization.py | 9 +++ awx/main/models/unified_jobs.py | 6 +- 7 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 awx/main/migrations/0124_execution_environments.py create mode 100644 awx/main/models/execution_environments.py diff --git a/awx/main/migrations/0124_execution_environments.py b/awx/main/migrations/0124_execution_environments.py new file mode 100644 index 0000000000..982b1b21e8 --- /dev/null +++ b/awx/main/migrations/0124_execution_environments.py @@ -0,0 +1,69 @@ +# Generated by Django 2.2.11 on 2020-07-08 18:42 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.db.models.expressions +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('taggit', '0003_taggeditem_add_unique_index'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('main', '0123_drop_hg_support'), + ] + + operations = [ + migrations.AddField( + model_name='unifiedjob', + name='pull', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='unifiedjobtemplate', + name='pull', + field=models.BooleanField(default=True), + ), + migrations.CreateModel( + name='ExecutionEnvironment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=None, editable=False)), + ('modified', models.DateTimeField(default=None, editable=False)), + ('description', models.TextField(blank=True, default='')), + ('image', models.CharField(help_text='The registry location where the container is stored.', max_length=1024, verbose_name='image location')), + ('managed_by_tower', models.BooleanField(default=False, editable=False)), + ('created_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'executionenvironment', 'model_name': 'executionenvironment', 'app_label': 'main'}(class)s_created+", to=settings.AUTH_USER_MODEL)), + ('credential', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='executionenvironments', to='main.Credential')), + ('modified_by', models.ForeignKey(default=None, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="{'class': 'executionenvironment', 'model_name': 'executionenvironment', 'app_label': 'main'}(class)s_modified+", to=settings.AUTH_USER_MODEL)), + ('organization', models.ForeignKey(blank=True, default=None, help_text='The organization used to determine access to this execution environment.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='executionenvironments', to='main.Organization')), + ('tags', taggit.managers.TaggableManager(blank=True, help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), + ], + options={ + 'ordering': (django.db.models.expressions.OrderBy(django.db.models.expressions.F('organization_id'), nulls_first=True), 'image'), + 'unique_together': {('organization', 'image')}, + }, + ), + migrations.AddField( + model_name='activitystream', + name='execution_environment', + field=models.ManyToManyField(blank=True, to='main.ExecutionEnvironment'), + ), + migrations.AddField( + model_name='organization', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + migrations.AddField( + model_name='unifiedjob', + name='execution_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='unifiedjobs', to='main.ExecutionEnvironment'), + ), + migrations.AddField( + model_name='unifiedjobtemplate', + name='execution_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='unifiedjobtemplates', to='main.ExecutionEnvironment'), + ), + ] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 87fa5d791f..7e3209bd6d 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -35,6 +35,7 @@ from awx.main.models.events import ( # noqa ) from awx.main.models.ad_hoc_commands import AdHocCommand # noqa from awx.main.models.schedules import Schedule # noqa +from awx.main.models.execution_environments import ExecutionEnvironment # noqa from awx.main.models.activity_stream import ActivityStream # noqa from awx.main.models.ha import ( # noqa Instance, InstanceGroup, TowerScheduleState, @@ -45,7 +46,7 @@ from awx.main.models.rbac import ( # noqa ROLE_SINGLETON_SYSTEM_AUDITOR, ) from awx.main.models.mixins import ( # noqa - CustomVirtualEnvMixin, ResourceMixin, SurveyJobMixin, + CustomVirtualEnvMixin, ExecutionEnvironmentMixin, ResourceMixin, SurveyJobMixin, SurveyJobTemplateMixin, TaskManagerInventoryUpdateMixin, TaskManagerJobMixin, TaskManagerProjectUpdateMixin, TaskManagerUnifiedJobMixin, diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index 85666e49d2..1c344692d6 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -61,6 +61,7 @@ class ActivityStream(models.Model): team = models.ManyToManyField("Team", blank=True) project = models.ManyToManyField("Project", blank=True) project_update = models.ManyToManyField("ProjectUpdate", blank=True) + execution_environment = models.ManyToManyField("ExecutionEnvironment", blank=True) job_template = models.ManyToManyField("JobTemplate", blank=True) job = models.ManyToManyField("Job", blank=True) workflow_job_template_node = models.ManyToManyField("WorkflowJobTemplateNode", blank=True) @@ -74,6 +75,7 @@ class ActivityStream(models.Model): ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) custom_inventory_script = models.ManyToManyField("CustomInventoryScript", blank=True) + execution_environment = models.ManyToManyField("ExecutionEnvironment", blank=True) notification_template = models.ManyToManyField("NotificationTemplate", blank=True) notification = models.ManyToManyField("Notification", blank=True) label = models.ManyToManyField("Label", blank=True) diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py new file mode 100644 index 0000000000..c4d6fcb155 --- /dev/null +++ b/awx/main/models/execution_environments.py @@ -0,0 +1,37 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from awx.main.models.base import PrimordialModel + + +__all__ = ['ExecutionEnvironment'] + + +class ExecutionEnvironment(PrimordialModel): + class Meta: + unique_together = ('organization', 'image') + ordering = (models.F('organization_id').asc(nulls_first=True), 'image') + + organization = models.ForeignKey( + 'Organization', + null=True, + default=None, + blank=True, + on_delete=models.CASCADE, + related_name='%(class)ss', + help_text=_('The organization used to determine access to this execution environment.'), + ) + image = models.CharField( + max_length=1024, + verbose_name=_('image location'), + help_text=_("The registry location where the container is stored."), + ) + managed_by_tower = models.BooleanField(default=False, editable=False) + credential = models.ForeignKey( + 'Credential', + related_name='%(class)ss', + blank=True, + null=True, + default=None, + on_delete=models.SET_NULL, + ) diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index ce6d3717a7..54c8a000a4 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -34,7 +34,7 @@ logger = logging.getLogger('awx.main.models.mixins') __all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin', 'TaskManagerUnifiedJobMixin', 'TaskManagerJobMixin', 'TaskManagerProjectUpdateMixin', - 'TaskManagerInventoryUpdateMixin', 'CustomVirtualEnvMixin'] + 'TaskManagerInventoryUpdateMixin', 'ExecutionEnvironmentMixin', 'CustomVirtualEnvMixin'] class ResourceMixin(models.Model): @@ -441,6 +441,22 @@ class TaskManagerInventoryUpdateMixin(TaskManagerUpdateOnLaunchMixin): abstract = True +class ExecutionEnvironmentMixin(models.Model): + class Meta: + abstract = True + + execution_environment = models.ForeignKey( + 'ExecutionEnvironment', + null=True, + blank=True, + default=None, + on_delete=models.SET_NULL, + related_name='%(class)ss', + help_text=_('The container image to be used for execution.'), + ) + pull = models.BooleanField(default=True) + + class CustomVirtualEnvMixin(models.Model): class Meta: abstract = True diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index bf2e07d255..3730fe9af1 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -61,6 +61,15 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi blank=True, related_name='%(class)s_notification_templates_for_approvals' ) + default_environment = models.ForeignKey( + 'ExecutionEnvironment', + null=True, + blank=True, + default=None, + on_delete=models.SET_NULL, + related_name='+', + help_text=_('The default execution environment for jobs run by this organization.'), + ) admin_role = ImplicitRoleField( parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 064585c6c1..f4a9e1ba45 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -39,7 +39,7 @@ from awx.main.models.base import ( from awx.main.dispatch import get_local_queuename from awx.main.dispatch.control import Control as ControlDispatcher from awx.main.registrar import activity_stream_registrar -from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin +from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin from awx.main.utils import ( camelcase_to_underscore, get_model_for_type, encrypt_dict, decrypt_field, _inventory_updates, @@ -59,7 +59,7 @@ logger_job_lifecycle = logging.getLogger('awx.analytics.job_lifecycle') # NOTE: ACTIVE_STATES moved to constants because it is used by parent modules -class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, NotificationFieldsModel): +class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEnvironmentMixin, NotificationFieldsModel): ''' Concrete base class for unified job templates. ''' @@ -527,7 +527,7 @@ class StdoutMaxBytesExceeded(Exception): class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique, - UnifiedJobTypeStringMixin, TaskManagerUnifiedJobMixin): + UnifiedJobTypeStringMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin): ''' Concrete base class for unified job run by the task engine. ''' From 9697999ddd3143dd2317c39fccfb4c80102dfe89 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 1 Jul 2020 16:12:24 -0400 Subject: [PATCH 002/157] Create the RBAC access class for execution environments --- awx/main/access.py | 51 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index 89a6c0607d..24e6bbc569 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -29,9 +29,9 @@ from awx.main.utils import ( ) from awx.main.models import ( ActivityStream, AdHocCommand, AdHocCommandEvent, Credential, CredentialType, - CredentialInputSource, CustomInventoryScript, Group, Host, Instance, InstanceGroup, - Inventory, InventorySource, InventoryUpdate, InventoryUpdateEvent, Job, JobEvent, - JobHostSummary, JobLaunchConfig, JobTemplate, Label, Notification, + CredentialInputSource, CustomInventoryScript, ExecutionEnvironment, Group, Host, Instance, + InstanceGroup, Inventory, InventorySource, InventoryUpdate, InventoryUpdateEvent, Job, + JobEvent, JobHostSummary, JobLaunchConfig, JobTemplate, Label, Notification, NotificationTemplate, Organization, Project, ProjectUpdate, ProjectUpdateEvent, Role, Schedule, SystemJob, SystemJobEvent, SystemJobTemplate, Team, UnifiedJob, UnifiedJobTemplate, WorkflowJob, @@ -1308,6 +1308,51 @@ class TeamAccess(BaseAccess): *args, **kwargs) +class ExecutionEnvironmentAccess(BaseAccess): + """ + I can see an execution environment when: + - I'm a superuser + - I'm a member of the organization + - it is a global ExecutionEnvironment + I can create/change an execution environment when: + - I'm a superuser + - I'm an admin for the organization(s) + """ + + model = ExecutionEnvironment + + def filtered_queryset(self): + return ExecutionEnvironment.objects.filter( + Q(organization__in=Organization.accessible_pk_qs(self.user, 'admin_role')) | + Q(organization__isnull=True) + ).distinct() + + @check_superuser + def can_add(self, data): + if not data: # So the browseable API will work + return Organization.accessible_objects(self.user, 'admin_role').exists() + return self.check_related('organization', Organization, data) + + @check_superuser + def can_change(self, obj, data): + if obj and obj.organization_id is None: + raise PermissionDenied + if self.user not in obj.organization.admin_role: + raise PermissionDenied + org_pk = get_pk_from_dict(data, 'organization') + if obj and obj.organization_id != org_pk: + # Prevent moving an EE to a different organization, unless a superuser or admin on both orgs. + if obj.organization_id is None or org_pk is None: + raise PermissionDenied + if self.user not in Organization.objects.get(id=org_pk).admin_role: + raise PermissionDenied + + return True + + def can_delete(self, obj): + return self.can_change(obj, None) + + class ProjectAccess(NotificationAttachMixin, BaseAccess): ''' I can see projects when: From 61cbd34586f687f4ef45d830ddfef2c962ba4755 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 1 Jul 2020 16:14:39 -0400 Subject: [PATCH 003/157] Add in the basic list and detail api views --- awx/api/serializers.py | 17 ++++++++++++++++- awx/api/urls/execution_environments.py | 14 ++++++++++++++ awx/api/urls/urls.py | 2 ++ awx/api/views/__init__.py | 12 ++++++++++++ awx/api/views/root.py | 1 + awx/main/models/execution_environments.py | 4 ++++ 6 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 awx/api/urls/execution_environments.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index d34c0d924a..cb47c99cb3 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -50,7 +50,7 @@ from awx.main.constants import ( ) from awx.main.models import ( ActivityStream, AdHocCommand, AdHocCommandEvent, Credential, CredentialInputSource, - CredentialType, CustomInventoryScript, Group, Host, Instance, + CredentialType, CustomInventoryScript, ExecutionEnvironment, Group, Host, Instance, InstanceGroup, Inventory, InventorySource, InventoryUpdate, InventoryUpdateEvent, Job, JobEvent, JobHostSummary, JobLaunchConfig, JobNotificationMixin, JobTemplate, Label, Notification, NotificationTemplate, @@ -1347,6 +1347,21 @@ class ProjectOptionsSerializer(BaseSerializer): return super(ProjectOptionsSerializer, self).validate(attrs) +class ExecutionEnvironmentSerializer(BaseSerializer): + class Meta: + model = ExecutionEnvironment + fields = ('*', '-name', 'organization', 'image', 'managed_by_tower', 'credential') + + def get_related(self, obj): + res = super(ExecutionEnvironmentSerializer, self).get_related(obj) + if obj.organization: + res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) + if obj.credential: + res['credential'] = self.reverse('api:credential_detail', + kwargs={'pk': obj.credential.pk}) + return res + + class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): status = serializers.ChoiceField(choices=Project.PROJECT_STATUS_CHOICES, read_only=True) diff --git a/awx/api/urls/execution_environments.py b/awx/api/urls/execution_environments.py new file mode 100644 index 0000000000..6e59f8fc45 --- /dev/null +++ b/awx/api/urls/execution_environments.py @@ -0,0 +1,14 @@ +from django.conf.urls import url + +from awx.api.views import ( + ExecutionEnvironmentList, + ExecutionEnvironmentDetail, +) + + +urls = [ + url(r'^$', ExecutionEnvironmentList.as_view(), name='execution_environment_list'), + url(r'^(?P[0-9]+)/$', ExecutionEnvironmentDetail.as_view(), name='execution_environment_detail'), +] + +__all__ = ['urls'] diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 636e68e4bd..2beeb47a47 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -42,6 +42,7 @@ from .user import urls as user_urls from .project import urls as project_urls from .project_update import urls as project_update_urls from .inventory import urls as inventory_urls +from .execution_environments import urls as execution_environment_urls from .team import urls as team_urls from .host import urls as host_urls from .group import urls as group_urls @@ -106,6 +107,7 @@ v2_urls = [ url(r'^schedules/', include(schedule_urls)), url(r'^organizations/', include(organization_urls)), url(r'^users/', include(user_urls)), + url(r'^execution_environments/', include(execution_environment_urls)), url(r'^projects/', include(project_urls)), url(r'^project_updates/', include(project_update_urls)), url(r'^teams/', include(team_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 43e845af0c..ff4689f78e 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -685,6 +685,18 @@ class TeamAccessList(ResourceAccessList): parent_model = models.Team +class ExecutionEnvironmentList(ListCreateAPIView): + + model = models.ExecutionEnvironment + serializer_class = serializers.ExecutionEnvironmentSerializer + + +class ExecutionEnvironmentDetail(RetrieveUpdateDestroyAPIView): + + model = models.ExecutionEnvironment + serializer_class = serializers.ExecutionEnvironmentSerializer + + class ProjectList(ListCreateAPIView): model = models.Project diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 0f5e7e6cdd..d6fc20d105 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -100,6 +100,7 @@ class ApiVersionRootView(APIView): data['dashboard'] = reverse('api:dashboard_view', request=request) data['organizations'] = reverse('api:organization_list', request=request) data['users'] = reverse('api:user_list', request=request) + data['execution_environments'] = reverse('api:execution_environment_list', request=request) data['projects'] = reverse('api:project_list', request=request) data['project_updates'] = reverse('api:project_update_list', request=request) data['teams'] = reverse('api:team_list', request=request) diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index c4d6fcb155..bdbe75eb49 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -1,6 +1,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ +from awx.api.versioning import reverse from awx.main.models.base import PrimordialModel @@ -35,3 +36,6 @@ class ExecutionEnvironment(PrimordialModel): default=None, on_delete=models.SET_NULL, ) + + def get_absolute_url(self, request=None): + return reverse('api:execution_environment_detail', kwargs={'pk': self.pk}, request=request) From 3c637cd54c15aa134de375700002d6ee0de13adf Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 2 Jul 2020 11:01:34 -0400 Subject: [PATCH 004/157] Change OrganizationSerializer to show and set default_environment --- awx/api/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index cb47c99cb3..392a25659f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -107,6 +107,7 @@ SUMMARIZABLE_FK_FIELDS = { 'insights_credential_id',), 'host': DEFAULT_SUMMARY_FIELDS, 'group': DEFAULT_SUMMARY_FIELDS, + 'default_environment': ('id', 'organization_id', 'image', 'description'), 'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',), @@ -1243,7 +1244,7 @@ class OrganizationSerializer(BaseSerializer): class Meta: model = Organization - fields = ('*', 'max_hosts', 'custom_virtualenv',) + fields = ('*', 'max_hosts', 'custom_virtualenv', 'default_environment',) def get_related(self, obj): res = super(OrganizationSerializer, self).get_related(obj) @@ -1268,6 +1269,9 @@ class OrganizationSerializer(BaseSerializer): instance_groups = self.reverse('api:organization_instance_groups_list', kwargs={'pk': obj.pk}), galaxy_credentials = self.reverse('api:organization_galaxy_credentials_list', kwargs={'pk': obj.pk}), )) + if obj.default_environment: + res['default_environment'] = self.reverse('api:execution_environment_detail', + kwargs={'pk': obj.default_environment_id}) return res def get_summary_fields(self, obj): From cb766c6a95b5d994f781e5f205efc71cb47ab6de Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 6 Jul 2020 11:00:54 -0400 Subject: [PATCH 005/157] Add execution_environment and pull to the fields for UJs and UJTs --- awx/api/serializers.py | 10 +++++++++- .../tests/unit/api/serializers/test_job_serializers.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 392a25659f..c5ea30e0f2 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -108,6 +108,7 @@ SUMMARIZABLE_FK_FIELDS = { 'host': DEFAULT_SUMMARY_FIELDS, 'group': DEFAULT_SUMMARY_FIELDS, 'default_environment': ('id', 'organization_id', 'image', 'description'), + 'execution_environment': ('id', 'organization_id', 'image', 'description'), 'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',), @@ -648,7 +649,7 @@ class UnifiedJobTemplateSerializer(BaseSerializer): class Meta: model = UnifiedJobTemplate fields = ('*', 'last_job_run', 'last_job_failed', - 'next_job_run', 'status') + 'next_job_run', 'status', 'execution_environment', 'pull') def get_related(self, obj): res = super(UnifiedJobTemplateSerializer, self).get_related(obj) @@ -658,6 +659,9 @@ class UnifiedJobTemplateSerializer(BaseSerializer): res['last_job'] = obj.last_job.get_absolute_url(request=self.context.get('request')) if obj.next_schedule: res['next_schedule'] = obj.next_schedule.get_absolute_url(request=self.context.get('request')) + if obj.execution_environment_id: + res['execution_environment'] = self.reverse('api:execution_environment_detail', + kwargs={'pk': obj.execution_environment_id}) return res def get_types(self): @@ -712,6 +716,7 @@ class UnifiedJobSerializer(BaseSerializer): class Meta: model = UnifiedJob fields = ('*', 'unified_job_template', 'launch_type', 'status', + 'execution_environment', 'pull', 'failed', 'started', 'finished', 'canceled_on', 'elapsed', 'job_args', 'job_cwd', 'job_env', 'job_explanation', 'execution_node', 'controller_node', @@ -749,6 +754,9 @@ class UnifiedJobSerializer(BaseSerializer): res['stdout'] = self.reverse('api:ad_hoc_command_stdout', kwargs={'pk': obj.pk}) if obj.workflow_job_id: res['source_workflow_job'] = self.reverse('api:workflow_job_detail', kwargs={'pk': obj.workflow_job_id}) + if obj.execution_environment_id: + res['execution_environment'] = self.reverse('api:execution_environment_detail', + kwargs={'pk': obj.execution_environment_id}) return res def get_summary_fields(self, obj): diff --git a/awx/main/tests/unit/api/serializers/test_job_serializers.py b/awx/main/tests/unit/api/serializers/test_job_serializers.py index e7b0ee7792..53cc07676d 100644 --- a/awx/main/tests/unit/api/serializers/test_job_serializers.py +++ b/awx/main/tests/unit/api/serializers/test_job_serializers.py @@ -40,7 +40,7 @@ def project_update(mocker): @pytest.fixture def job(mocker, job_template, project_update): return mocker.MagicMock(pk=5, job_template=job_template, project_update=project_update, - workflow_job_id=None) + workflow_job_id=None, execution_environment_id=None) @pytest.fixture From cc429f97411e59b1da91a2efc531bc9db0e10a34 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 6 Jul 2020 13:01:44 -0400 Subject: [PATCH 006/157] Expose an API view for all of the execution environments under an org --- awx/api/serializers.py | 5 +++-- awx/api/urls/organization.py | 2 ++ awx/api/views/__init__.py | 1 + awx/api/views/organization.py | 11 +++++++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c5ea30e0f2..fa15c45e53 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1256,7 +1256,8 @@ class OrganizationSerializer(BaseSerializer): def get_related(self, obj): res = super(OrganizationSerializer, self).get_related(obj) - res.update(dict( + res.update( + execution_environments = self.reverse('api:organization_execution_environments_list', kwargs={'pk': obj.pk}), projects = self.reverse('api:organization_projects_list', kwargs={'pk': obj.pk}), inventories = self.reverse('api:organization_inventories_list', kwargs={'pk': obj.pk}), job_templates = self.reverse('api:organization_job_templates_list', kwargs={'pk': obj.pk}), @@ -1276,7 +1277,7 @@ class OrganizationSerializer(BaseSerializer): access_list = self.reverse('api:organization_access_list', kwargs={'pk': obj.pk}), instance_groups = self.reverse('api:organization_instance_groups_list', kwargs={'pk': obj.pk}), galaxy_credentials = self.reverse('api:organization_galaxy_credentials_list', kwargs={'pk': obj.pk}), - )) + ) if obj.default_environment: res['default_environment'] = self.reverse('api:execution_environment_detail', kwargs={'pk': obj.default_environment_id}) diff --git a/awx/api/urls/organization.py b/awx/api/urls/organization.py index 12b2807905..9d8fecf4bc 100644 --- a/awx/api/urls/organization.py +++ b/awx/api/urls/organization.py @@ -9,6 +9,7 @@ from awx.api.views import ( OrganizationUsersList, OrganizationAdminsList, OrganizationInventoriesList, + OrganizationExecutionEnvironmentsList, OrganizationProjectsList, OrganizationJobTemplatesList, OrganizationWorkflowJobTemplatesList, @@ -34,6 +35,7 @@ urls = [ url(r'^(?P[0-9]+)/users/$', OrganizationUsersList.as_view(), name='organization_users_list'), url(r'^(?P[0-9]+)/admins/$', OrganizationAdminsList.as_view(), name='organization_admins_list'), url(r'^(?P[0-9]+)/inventories/$', OrganizationInventoriesList.as_view(), name='organization_inventories_list'), + url(r'^(?P[0-9]+)/execution_environments/$', OrganizationExecutionEnvironmentsList.as_view(), name='organization_execution_environments_list'), url(r'^(?P[0-9]+)/projects/$', OrganizationProjectsList.as_view(), name='organization_projects_list'), url(r'^(?P[0-9]+)/job_templates/$', OrganizationJobTemplatesList.as_view(), name='organization_job_templates_list'), url(r'^(?P[0-9]+)/workflow_job_templates/$', OrganizationWorkflowJobTemplatesList.as_view(), name='organization_workflow_job_templates_list'), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index ff4689f78e..47a0e0a3a7 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -112,6 +112,7 @@ from awx.api.views.organization import ( # noqa OrganizationInventoriesList, OrganizationUsersList, OrganizationAdminsList, + OrganizationExecutionEnvironmentsList, OrganizationProjectsList, OrganizationJobTemplatesList, OrganizationWorkflowJobTemplatesList, diff --git a/awx/api/views/organization.py b/awx/api/views/organization.py index d03dfcc86f..b0955a3a42 100644 --- a/awx/api/views/organization.py +++ b/awx/api/views/organization.py @@ -15,6 +15,7 @@ from awx.main.models import ( Inventory, Host, Project, + ExecutionEnvironment, JobTemplate, WorkflowJobTemplate, Organization, @@ -45,6 +46,7 @@ from awx.api.serializers import ( RoleSerializer, NotificationTemplateSerializer, InstanceGroupSerializer, + ExecutionEnvironmentSerializer, ProjectSerializer, JobTemplateSerializer, WorkflowJobTemplateSerializer, CredentialSerializer ) @@ -141,6 +143,15 @@ class OrganizationProjectsList(SubListCreateAPIView): parent_key = 'organization' +class OrganizationExecutionEnvironmentsList(SubListCreateAttachDetachAPIView): + + model = ExecutionEnvironment + serializer_class = ExecutionEnvironmentSerializer + parent_model = Organization + relationship = 'executionenvironments' + parent_key = 'organization' + + class OrganizationJobTemplatesList(SubListCreateAPIView): model = JobTemplate From c05e4e07ee9bb8f264a257476423eaf191779758 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 6 Jul 2020 15:44:06 -0400 Subject: [PATCH 007/157] Expose execution environments in awxkit and awx-cli --- awxkit/awxkit/api/pages/__init__.py | 1 + awxkit/awxkit/api/pages/api.py | 2 ++ .../api/pages/execution_environments.py | 33 +++++++++++++++++++ awxkit/awxkit/api/resources.py | 3 ++ 4 files changed, 39 insertions(+) create mode 100644 awxkit/awxkit/api/pages/execution_environments.py diff --git a/awxkit/awxkit/api/pages/__init__.py b/awxkit/awxkit/api/pages/__init__.py index fafe5dc08f..1d78d4ba5e 100644 --- a/awxkit/awxkit/api/pages/__init__.py +++ b/awxkit/awxkit/api/pages/__init__.py @@ -14,6 +14,7 @@ from .teams import * # NOQA from .credentials import * # NOQA from .unified_jobs import * # NOQA from .unified_job_templates import * # NOQA +from .execution_environments import * # NOQA from .projects import * # NOQA from .inventory import * # NOQA from .system_job_templates import * # NOQA diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 3209232352..4edc07857f 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -23,6 +23,7 @@ EXPORTABLE_RESOURCES = [ 'inventory_sources', 'job_templates', 'workflow_job_templates', + 'execution_environments', ] @@ -33,6 +34,7 @@ EXPORTABLE_RELATIONS = [ 'Credentials', 'Hosts', 'Groups', + 'ExecutionEnvironments', ] diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py new file mode 100644 index 0000000000..e48ef324bd --- /dev/null +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -0,0 +1,33 @@ +import logging + +from awxkit.api.mixins import HasCreate +from awxkit.api.pages import ( + Credential, + Organization, +) +from awxkit.api.resources import resources + +from . import base +from . import page + + +log = logging.getLogger(__name__) + + +class ExecutionEnvironment(HasCreate, base.Base): + + dependencies = [Organization, Credential] + NATURAL_KEY = ('organization', 'image') + + +page.register_page([resources.execution_environment, + (resources.execution_environments, 'post'), + (resources.organization_execution_environments, 'post')], ExecutionEnvironment) + + +class ExecutionEnvironments(page.PageList, ExecutionEnvironment): + pass + + +page.register_page([resources.execution_environments, + resources.organization_execution_environments], ExecutionEnvironments) diff --git a/awxkit/awxkit/api/resources.py b/awxkit/awxkit/api/resources.py index d6340cd2d7..997ada1e70 100644 --- a/awxkit/awxkit/api/resources.py +++ b/awxkit/awxkit/api/resources.py @@ -28,6 +28,8 @@ class Resources(object): _credential_types = 'credential_types/' _credentials = 'credentials/' _dashboard = 'dashboard/' + _execution_environment = r'execution_environments/\d+/' + _execution_environments = 'execution_environments/' _fact_view = r'hosts/\d+/fact_view/' _group = r'groups/\d+/' _group_access_list = r'groups/\d+/access_list/' @@ -141,6 +143,7 @@ class Resources(object): _organization_access_list = r'organizations/\d+/access_list/' _organization_admins = r'organizations/\d+/admins/' _organization_applications = r'organizations/\d+/applications/' + _organization_execution_environments = r'organizations/\d+/execution_environments/' _organization_inventories = r'organizations/\d+/inventories/' _organization_users = r'organizations/\d+/users/' _organizations = 'organizations/' From 5ec7378135f072e762f4e501b2e65258b7053c01 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 6 Jul 2020 16:23:01 -0400 Subject: [PATCH 008/157] Add a new Swagger topic --- awx/api/views/__init__.py | 2 ++ awx/api/views/organization.py | 1 + 2 files changed, 3 insertions(+) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 47a0e0a3a7..0bfa00d18c 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -690,12 +690,14 @@ class ExecutionEnvironmentList(ListCreateAPIView): model = models.ExecutionEnvironment serializer_class = serializers.ExecutionEnvironmentSerializer + swagger_topic = "Execution Environments" class ExecutionEnvironmentDetail(RetrieveUpdateDestroyAPIView): model = models.ExecutionEnvironment serializer_class = serializers.ExecutionEnvironmentSerializer + swagger_topic = "Execution Environments" class ProjectList(ListCreateAPIView): diff --git a/awx/api/views/organization.py b/awx/api/views/organization.py index b0955a3a42..b33259a8ad 100644 --- a/awx/api/views/organization.py +++ b/awx/api/views/organization.py @@ -150,6 +150,7 @@ class OrganizationExecutionEnvironmentsList(SubListCreateAttachDetachAPIView): parent_model = Organization relationship = 'executionenvironments' parent_key = 'organization' + swagger_topic = "Execution Environments" class OrganizationJobTemplatesList(SubListCreateAPIView): From f9741b619c88085b96b749fa4a08215226fbdbeb Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 8 Jul 2020 11:49:23 -0400 Subject: [PATCH 009/157] Make changes to support capture by the activity stream Including exposing a new API view for a particular EE's activity stream objects. --- awx/api/serializers.py | 3 +++ awx/api/urls/execution_environments.py | 2 ++ awx/api/views/__init__.py | 16 ++++++++++++++++ awx/main/models/__init__.py | 1 + awx/main/signals.py | 1 + 5 files changed, 23 insertions(+) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index fa15c45e53..4aff55e926 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1367,6 +1367,9 @@ class ExecutionEnvironmentSerializer(BaseSerializer): def get_related(self, obj): res = super(ExecutionEnvironmentSerializer, self).get_related(obj) + res.update( + activity_stream = self.reverse('api:execution_environment_activity_stream_list', kwargs={'pk': obj.pk}), + ) if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) if obj.credential: diff --git a/awx/api/urls/execution_environments.py b/awx/api/urls/execution_environments.py index 6e59f8fc45..e2310caec3 100644 --- a/awx/api/urls/execution_environments.py +++ b/awx/api/urls/execution_environments.py @@ -3,12 +3,14 @@ from django.conf.urls import url from awx.api.views import ( ExecutionEnvironmentList, ExecutionEnvironmentDetail, + ExecutionEnvironmentActivityStreamList, ) urls = [ url(r'^$', ExecutionEnvironmentList.as_view(), name='execution_environment_list'), url(r'^(?P[0-9]+)/$', ExecutionEnvironmentDetail.as_view(), name='execution_environment_detail'), + url(r'^(?P[0-9]+)/activity_stream/$', ExecutionEnvironmentActivityStreamList.as_view(), name='execution_environment_activity_stream_list'), ] __all__ = ['urls'] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 0bfa00d18c..4c4ec799fe 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -700,6 +700,22 @@ class ExecutionEnvironmentDetail(RetrieveUpdateDestroyAPIView): swagger_topic = "Execution Environments" +class ExecutionEnvironmentActivityStreamList(SubListAPIView): + + model = models.ActivityStream + serializer_class = serializers.ActivityStreamSerializer + parent_model = models.ExecutionEnvironment + relationship = 'activitystream_set' + search_fields = ('changes',) + + def get_queryset(self): + parent = self.get_parent_object() + self.check_parent_access(parent) + + qs = self.request.user.get_queryset(self.model) + return qs.filter(execution_environment=parent) + + class ProjectList(ListCreateAPIView): model = models.Project diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 7e3209bd6d..52cabf3774 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -222,6 +222,7 @@ activity_stream_registrar.connect(CredentialType) activity_stream_registrar.connect(Team) activity_stream_registrar.connect(Project) #activity_stream_registrar.connect(ProjectUpdate) +activity_stream_registrar.connect(ExecutionEnvironment) activity_stream_registrar.connect(JobTemplate) activity_stream_registrar.connect(Job) activity_stream_registrar.connect(AdHocCommand) diff --git a/awx/main/signals.py b/awx/main/signals.py index 0a29fa9d6c..ac7a3d2301 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -368,6 +368,7 @@ def model_serializer_mapping(): models.Credential: serializers.CredentialSerializer, models.Team: serializers.TeamSerializer, models.Project: serializers.ProjectSerializer, + models.ExecutionEnvironment: serializers.ExecutionEnvironmentSerializer, models.JobTemplate: serializers.JobTemplateWithSpecSerializer, models.Job: serializers.JobSerializer, models.AdHocCommand: serializers.AdHocCommandSerializer, From 45a0084f78ebf650803967126bd12462b3d39141 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 8 Jul 2020 14:43:14 -0400 Subject: [PATCH 010/157] Add a sublist api view for the UJTs that use a given execution environment --- awx/api/serializers.py | 3 ++- awx/api/urls/execution_environments.py | 2 ++ awx/api/views/__init__.py | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 4aff55e926..46e430a8e3 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1368,7 +1368,8 @@ class ExecutionEnvironmentSerializer(BaseSerializer): def get_related(self, obj): res = super(ExecutionEnvironmentSerializer, self).get_related(obj) res.update( - activity_stream = self.reverse('api:execution_environment_activity_stream_list', kwargs={'pk': obj.pk}), + activity_stream=self.reverse('api:execution_environment_activity_stream_list', kwargs={'pk': obj.pk}), + unified_job_templates=self.reverse('api:execution_environment_job_template_list', kwargs={'pk': obj.pk}), ) if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) diff --git a/awx/api/urls/execution_environments.py b/awx/api/urls/execution_environments.py index e2310caec3..08f852be08 100644 --- a/awx/api/urls/execution_environments.py +++ b/awx/api/urls/execution_environments.py @@ -3,6 +3,7 @@ from django.conf.urls import url from awx.api.views import ( ExecutionEnvironmentList, ExecutionEnvironmentDetail, + ExecutionEnvironmentJobTemplateList, ExecutionEnvironmentActivityStreamList, ) @@ -10,6 +11,7 @@ from awx.api.views import ( urls = [ url(r'^$', ExecutionEnvironmentList.as_view(), name='execution_environment_list'), url(r'^(?P[0-9]+)/$', ExecutionEnvironmentDetail.as_view(), name='execution_environment_detail'), + url(r'^(?P[0-9]+)/unified_job_templates/$', ExecutionEnvironmentJobTemplateList.as_view(), name='execution_environment_job_template_list'), url(r'^(?P[0-9]+)/activity_stream/$', ExecutionEnvironmentActivityStreamList.as_view(), name='execution_environment_activity_stream_list'), ] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 4c4ec799fe..7ec932c9ce 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -700,6 +700,14 @@ class ExecutionEnvironmentDetail(RetrieveUpdateDestroyAPIView): swagger_topic = "Execution Environments" +class ExecutionEnvironmentJobTemplateList(SubListAPIView): + + model = models.UnifiedJobTemplate + serializer_class = serializers.UnifiedJobTemplateSerializer + parent_model = models.ExecutionEnvironment + relationship = 'unifiedjobtemplates' + + class ExecutionEnvironmentActivityStreamList(SubListAPIView): model = models.ActivityStream From 3cbf384ad11aae7832fc0b4c41a7db66e798b3c3 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 6 Aug 2020 13:48:46 -0400 Subject: [PATCH 011/157] Run a receptor node in the dev environment --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 6 ++++++ tools/docker-compose-cluster.yml | 3 +++ .../ansible/roles/sources/templates/docker-compose.yml.j2 | 1 + tools/docker-compose/supervisor.conf | 8 ++++++++ 4 files changed, 18 insertions(+) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 7acf365c25..a03ac7d238 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -135,6 +135,11 @@ RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if buil RUN rm -rf /root/.cache && rm -rf /tmp/* +# Install Receptor +RUN cd /usr/local/bin && \ + curl -L http://nightlies.testing.ansible.com/receptor/receptor --output receptor && \ + chmod a+x receptor + # Install OpenShift CLI RUN cd /usr/local/bin && \ curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \ @@ -216,6 +221,7 @@ RUN for dir in \ /var/log/nginx \ /var/lib/postgresql \ /var/run/supervisor \ + /var/run/receptor \ /var/lib/nginx ; \ do mkdir -m 0775 -p $dir ; chmod g+rw $dir ; chgrp root $dir ; done && \ for file in \ diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 8532b6e942..7a90aa88c9 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -32,6 +32,7 @@ services: - "./redis/redis_socket_ha_1:/var/run/redis/" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" ports: + - "2222:2222" - "5899-5999:5899-5999" awx-2: user: ${CURRENT_UID} @@ -51,6 +52,7 @@ services: - "./redis/redis_socket_ha_2:/var/run/redis/" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" ports: + - "2223:2222" - "7899-7999:7899-7999" awx-3: user: ${CURRENT_UID} @@ -70,6 +72,7 @@ services: - "./redis/redis_socket_ha_3:/var/run/redis/" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" ports: + - "2224:2222" - "8899-8999:8899-8999" redis_1: user: ${CURRENT_UID} diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 72181cfb0b..0cdf4f6728 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -14,6 +14,7 @@ services: SDB_PORT: 7899 AWX_GROUP_QUEUES: tower ports: + - "2222:2222" - "8888:8888" - "8080:8080" - "8013:8013" diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index 82e8962a9b..04ddb66838 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -83,6 +83,14 @@ redirect_stderr=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 +[program:awx-receptor] +command = receptor --node id=%(ENV_HOSTNAME)s --control-service filename=/var/run/receptor/receptor.sock --tcp-listener port=2222 +autostart = true +autorestart = true +stopsignal = KILL +stopasgroup = true +killasgroup = true + [group:tower-processes] programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast,awx-rsyslogd priority=5 From 297fecba3a43bb27ceb09476106c51029a4ea8b4 Mon Sep 17 00:00:00 2001 From: Kersom <9053044+nixocio@users.noreply.github.com> Date: Wed, 19 Aug 2020 11:28:03 -0400 Subject: [PATCH 012/157] Add execution environments files (#7909) Update navigation bar and routing system to add execution environments. Also, add stub files for the remaining related work. See: https://github.com/ansible/awx/issues/7885 Also: https://github.com/ansible/awx/issues/7884 --- awx/ui_next/src/routeConfig.js | 10 +++- .../ExecutionEnvironment.jsx | 25 +++++++++ .../ExecutionEnvironmentAdd.jsx | 14 +++++ .../ExecutionEnvironmentAdd/index.js | 1 + .../ExecutionEnvironmentDetails.jsx | 14 +++++ .../ExecutionEnvironmentDetails/index.js | 1 + .../ExecutionEnvironmentEdit.jsx | 14 +++++ .../ExecutionEnvironmentEdit/index.js | 1 + .../ExecutionEnvironmentList.jsx | 14 +++++ .../ExecutionEnvironmentList/index.js | 1 + .../ExecutionEnvironments.jsx | 53 +++++++++++++++++++ .../ExecutionEnvironments.test.jsx | 25 +++++++++ .../src/screens/ExecutionEnvironment/index.js | 1 + 13 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/index.js create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/index.js create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/index.js create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/index.js create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.test.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/index.js diff --git a/awx/ui_next/src/routeConfig.js b/awx/ui_next/src/routeConfig.js index a343a7d1e0..c0aff394f0 100644 --- a/awx/ui_next/src/routeConfig.js +++ b/awx/ui_next/src/routeConfig.js @@ -2,13 +2,13 @@ import { t } from '@lingui/macro'; import ActivityStream from './screens/ActivityStream'; import Applications from './screens/Application'; -import Credentials from './screens/Credential'; import CredentialTypes from './screens/CredentialType'; +import Credentials from './screens/Credential'; import Dashboard from './screens/Dashboard'; +import ExecutionEnvironments from './screens/ExecutionEnvironment'; import Hosts from './screens/Host'; import InstanceGroups from './screens/InstanceGroup'; import Inventory from './screens/Inventory'; -import { Jobs } from './screens/Job'; import ManagementJobs from './screens/ManagementJob'; import NotificationTemplates from './screens/NotificationTemplate'; import Organizations from './screens/Organization'; @@ -19,6 +19,7 @@ import Teams from './screens/Team'; import Templates from './screens/Template'; import Users from './screens/User'; import WorkflowApprovals from './screens/WorkflowApproval'; +import { Jobs } from './screens/Job'; // Ideally, this should just be a regular object that we export, but we // need the i18n. When lingui3 arrives, we will be able to import i18n @@ -138,6 +139,11 @@ function getRouteConfig(i18n) { path: '/applications', screen: Applications, }, + { + title: i18n._(t`Execution environments`), + path: '/execution_environments', + screen: ExecutionEnvironments, + }, ], }, { diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx new file mode 100644 index 0000000000..9575a3b568 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx @@ -0,0 +1,25 @@ +import React from 'react'; +import { Route, Redirect, Switch } from 'react-router-dom'; + +import ExecutionEnvironmentDetails from './ExecutionEnvironmentDetails'; +import ExecutionEnvironmentEdit from './ExecutionEnvironmentEdit'; + +function ExecutionEnvironment() { + return ( + + + + + + + + + + ); +} + +export default ExecutionEnvironment; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx new file mode 100644 index 0000000000..e188990878 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { Card, PageSection } from '@patternfly/react-core'; + +function ExecutionEnvironmentAdd() { + return ( + + +
Add Execution Environments
+ + + ); +} + +export default ExecutionEnvironmentAdd; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/index.js b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/index.js new file mode 100644 index 0000000000..69765fcf3b --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/index.js @@ -0,0 +1 @@ +export { default } from './ExecutionEnvironmentAdd'; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx new file mode 100644 index 0000000000..f6902d1735 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { Card, PageSection } from '@patternfly/react-core'; + +function ExecutionEnvironmentDetails() { + return ( + + +
Execution environments details
+ + + ); +} + +export default ExecutionEnvironmentDetails; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/index.js b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/index.js new file mode 100644 index 0000000000..36121ea0d9 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/index.js @@ -0,0 +1 @@ +export { default } from './ExecutionEnvironmentDetails'; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx new file mode 100644 index 0000000000..91e3096ce3 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { Card, PageSection } from '@patternfly/react-core'; + +function ExecutionEnvironmentEdit() { + return ( + + +
Edit Execution environments
+ + + ); +} + +export default ExecutionEnvironmentEdit; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/index.js b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/index.js new file mode 100644 index 0000000000..6ab135ca05 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/index.js @@ -0,0 +1 @@ +export { default } from './ExecutionEnvironmentEdit'; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx new file mode 100644 index 0000000000..96d2d07e10 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { Card, PageSection } from '@patternfly/react-core'; + +function ExecutionEnvironmentList() { + return ( + + +
List Execution environments
+ + + ); +} + +export default ExecutionEnvironmentList; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/index.js b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/index.js new file mode 100644 index 0000000000..a8aa4263d7 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/index.js @@ -0,0 +1 @@ +export { default } from './ExecutionEnvironmentList'; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx new file mode 100644 index 0000000000..7db470e2f7 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx @@ -0,0 +1,53 @@ +import React, { useState, useCallback } from 'react'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Route, Switch } from 'react-router-dom'; + +import ExecutionEnvironment from './ExecutionEnvironment'; +import ExecutionEnvironmentAdd from './ExecutionEnvironmentAdd'; +import ExecutionEnvironmentList from './ExecutionEnvironmentList'; +import Breadcrumbs from '../../components/Breadcrumbs'; + +function ExecutionEnvironments({ i18n }) { + const [breadcrumbConfig, setBreadcrumbConfig] = useState({ + '/execution_environments': i18n._(t`Execution environments`), + '/execution_environments/add': i18n._(t`Create Execution environments`), + }); + + const buildBreadcrumbConfig = useCallback( + executionEnvironments => { + if (!executionEnvironments) { + return; + } + setBreadcrumbConfig({ + '/execution_environments': i18n._(t`Execution environments`), + '/execution_environments/add': i18n._(t`Create Execution environments`), + [`/execution_environments/${executionEnvironments.id}`]: `${executionEnvironments.name}`, + [`/execution_environments/${executionEnvironments.id}/edit`]: i18n._( + t`Edit details` + ), + [`/execution_environments/${executionEnvironments.id}/details`]: i18n._( + t`Details` + ), + }); + }, + [i18n] + ); + return ( + <> + + + + + + + + + + + + + + ); +} +export default withI18n()(ExecutionEnvironments); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.test.jsx new file mode 100644 index 0000000000..5ceb36ac93 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.test.jsx @@ -0,0 +1,25 @@ +import React from 'react'; + +import { mountWithContexts } from '../../../testUtils/enzymeHelpers'; + +import ExecutionEnvironments from './ExecutionEnvironments'; + +describe('', () => { + let pageWrapper; + let pageSections; + + beforeEach(() => { + pageWrapper = mountWithContexts(); + pageSections = pageWrapper.find('PageSection'); + }); + + afterEach(() => { + pageWrapper.unmount(); + }); + + test('initially renders without crashing', () => { + expect(pageWrapper.length).toBe(1); + expect(pageSections.length).toBe(1); + expect(pageSections.first().props().variant).toBe('light'); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/index.js b/awx/ui_next/src/screens/ExecutionEnvironment/index.js new file mode 100644 index 0000000000..f66a2b3cf3 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/index.js @@ -0,0 +1 @@ +export { default } from './ExecutionEnvironments'; From 06d7a61ca11242f63a7d7cb8e7c108f35bde1b32 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 31 Jul 2020 09:43:46 -0400 Subject: [PATCH 013/157] Initial EE integration --- awx/main/tasks.py | 49 ++++++++++++++++--- awx/settings/defaults.py | 2 + requirements/requirements.txt | 2 +- requirements/requirements_git.txt | 1 + .../roles/dockerfile/templates/Dockerfile.j2 | 8 +++ .../sources/templates/docker-compose.yml.j2 | 1 + tools/docker-compose/entrypoint.sh | 18 +++++-- 7 files changed, 68 insertions(+), 13 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b6ab905837..16f79ee5ef 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -887,6 +887,9 @@ class BaseTask(object): ''' return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) + def build_execution_environment_params(self, instance): + return {} + def build_private_data(self, instance, private_data_dir): ''' Return SSH private key data (only if stored in DB as ssh_key_data). @@ -1129,12 +1132,13 @@ class BaseTask(object): for hostname, hv in script_data.get('_meta', {}).get('hostvars', {}).items() } json_data = json.dumps(script_data) - handle, path = tempfile.mkstemp(dir=private_data_dir) - f = os.fdopen(handle, 'w') - f.write('#! /usr/bin/env python\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json_data) - f.close() - os.chmod(path, stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR) - return path + path = os.path.join(private_data_dir, 'inventory') + os.makedirs(path, mode=0o700) + fn = os.path.join(path, 'hosts') + with open(fn, 'w') as f: + os.chmod(fn, stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR) + f.write('#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json_data) + return fn def build_args(self, instance, private_data_dir, passwords): raise NotImplementedError @@ -1429,6 +1433,7 @@ class BaseTask(object): process_isolation_params = self.build_params_process_isolation(self.instance, private_data_dir, cwd) + execution_environment_params = self.build_execution_environment_params(self.instance) env = self.build_env(self.instance, private_data_dir, isolated, private_data_files=private_data_files) self.safe_env = build_safe_env(env) @@ -1463,7 +1468,8 @@ class BaseTask(object): 'settings': { 'job_timeout': self.get_instance_timeout(self.instance), 'suppress_ansible_output': True, - **process_isolation_params, + #**process_isolation_params, + **execution_environment_params, **resource_profiling_params, }, } @@ -2003,6 +2009,14 @@ class RunJob(BaseTask): if inventory is not None: update_inventory_computed_fields.delay(inventory.id) + def build_execution_environment_params(self, instance): + execution_environment_params = { + "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, + "process_isolation": True + } + return execution_environment_params + + @task(queue=get_local_queuename) class RunProjectUpdate(BaseTask): @@ -2349,7 +2363,7 @@ class RunProjectUpdate(BaseTask): # the project update playbook is not in a git repo, but uses a vendoring directory # to be consistent with the ansible-runner model, - # that is moved into the runner projecct folder here + # that is moved into the runner project folder here awx_playbooks = self.get_path_to('..', 'playbooks') copy_tree(awx_playbooks, os.path.join(private_data_dir, 'project')) @@ -2484,6 +2498,18 @@ class RunProjectUpdate(BaseTask): ''' return getattr(settings, 'AWX_PROOT_ENABLED', False) + def build_execution_environment_params(self, instance): + project_path = os.path.dirname(instance.get_project_path(check_if_exists=False)) + execution_environment_params = { + "process_isolation": True, + "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, + "container_volume_mounts": [ + f"{project_path}:{project_path}", + ] + + } + return execution_environment_params + @task(queue=get_local_queuename) class RunInventoryUpdate(BaseTask): @@ -2983,6 +3009,13 @@ class RunAdHocCommand(BaseTask): if isolated_manager_instance: isolated_manager_instance.cleanup() + def build_execution_environment_params(self, instance): + execution_environment_params = { + "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, + "process_isolation": True + } + return execution_environment_params + @task(queue=get_local_queuename) class RunSystemJob(BaseTask): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index e0c1db197b..849d6220bc 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -65,6 +65,8 @@ AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = 'default' AWX_CONTAINER_GROUP_DEFAULT_IMAGE = 'ansible/ansible-runner' +AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/ansible-runner:devel' + # Internationalization # https://docs.djangoproject.com/en/dev/topics/i18n/ # diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5829401c78..a6901fd3ef 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,7 +1,7 @@ adal==1.2.2 # via msrestazure aiohttp==3.6.2 # via -r /awx_devel/requirements/requirements.in aioredis==1.3.1 # via channels-redis -ansible-runner==1.4.7 # via -r /awx_devel/requirements/requirements.in +# ansible-runner==1.4.7 # via -r /awx_devel/requirements/requirements.in ansiconv==1.0.0 # via -r /awx_devel/requirements/requirements.in asciichartpy==1.5.25 # via -r /awx_devel/requirements/requirements.in asgiref==3.2.5 # via channels, channels-redis, daphne diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index 340cbfdcc7..04eac859a4 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1 +1,2 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi +git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index a03ac7d238..e056b10fb1 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -103,6 +103,7 @@ RUN dnf -y update && \ krb5-workstation \ libcgroup-tools \ nginx \ + podman \ @postgresql:12 \ python3-devel \ python3-libselinux \ @@ -216,6 +217,7 @@ RUN for dir in \ /var/lib/awx \ /var/lib/awx/rsyslog \ /var/lib/awx/rsyslog/conf.d \ + /var/lib/awx/.local/share/containers/storage \ /var/run/awx-rsyslog \ /var/log/tower \ /var/log/nginx \ @@ -225,6 +227,8 @@ RUN for dir in \ /var/lib/nginx ; \ do mkdir -m 0775 -p $dir ; chmod g+rw $dir ; chgrp root $dir ; done && \ for file in \ + /etc/subuid \ + /etc/subgid \ /etc/passwd \ /var/lib/awx/rsyslog/rsyslog.conf ; \ do touch $file ; chmod g+rw $file ; chgrp root $file ; done @@ -255,6 +259,8 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log && \ ln -sf /dev/stderr /var/log/nginx/error.log {% endif %} +RUN echo -e 'cgroup_manager = "cgroupfs"\nevents_logger = "file"' > /etc/containers/libpod.conf + ENV HOME="/var/lib/awx" ENV PATH="/usr/pgsql-10/bin:${PATH}" @@ -272,3 +278,5 @@ ENTRYPOINT ["/usr/bin/tini", "--"] CMD /usr/bin/launch_awx.sh VOLUME /var/lib/nginx {% endif %} + +VOLUME /var/lib/awx/.local/share/containers/storage diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 0cdf4f6728..754bc803f9 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -33,6 +33,7 @@ services: - "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py" - "../../docker-compose/_sources/SECRET_KEY:/etc/tower/SECRET_KEY" - "redis_socket:/var/run/redis/:rw" + - "/sys/fs/cgroup:/sys/fs/cgroup" privileged: true tty: true # A useful container that simply passes through log messages to the console diff --git a/tools/docker-compose/entrypoint.sh b/tools/docker-compose/entrypoint.sh index 8ed9bf2abd..13c858b441 100755 --- a/tools/docker-compose/entrypoint.sh +++ b/tools/docker-compose/entrypoint.sh @@ -2,13 +2,23 @@ if [ `id -u` -ge 500 ] || [ -z "${CURRENT_UID}" ]; then - cat << EOF > /tmp/passwd +cat << EOF > /etc/passwd root:x:0:0:root:/root:/bin/bash -awx:x:`id -u`:`id -g`:,,,:/tmp:/bin/bash +awx:x:`id -u`:`id -g`:,,,:/var/lib/awx:/bin/bash +EOF + +cat < /etc/subuid +awx:100000:50001 +EOF + +cat < /etc/subgid +awx:100000:50001 EOF - cat /tmp/passwd > /etc/passwd - rm /tmp/passwd fi +# Required to get rootless podman working after +# writing out the sub*id files above +podman system migrate + exec $@ From 130bf076f468a60eec53d2f65c74e322042a10fc Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 24 Aug 2020 16:02:11 -0400 Subject: [PATCH 014/157] Add Z to volume mount Set ansible-runner back to main fork due to merge --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 16f79ee5ef..cdac35737d 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2504,7 +2504,7 @@ class RunProjectUpdate(BaseTask): "process_isolation": True, "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, "container_volume_mounts": [ - f"{project_path}:{project_path}", + f"{project_path}:{project_path}:Z", ] } From a3f0158a943f063296a24368427706fcff47e880 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 24 Aug 2020 16:02:11 -0400 Subject: [PATCH 015/157] Add Z to volume mount Update to AWX execution environment use the special 2.9 container image revert setting back for merge Fix another permission error by mapping 2 folders also create folders before running --- awx/main/tasks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index cdac35737d..ed05c61828 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2344,10 +2344,14 @@ class RunProjectUpdate(BaseTask): # re-create root project folder if a natural disaster has destroyed it if not os.path.exists(settings.PROJECTS_ROOT): os.mkdir(settings.PROJECTS_ROOT) + project_path = instance.project.get_project_path(check_if_exists=False) + if not os.path.exists(project_path): + os.makedirs(project_path) # used as container mount + self.acquire_lock(instance) + self.original_branch = None if instance.scm_type == 'git' and instance.branch_override: - project_path = instance.project.get_project_path(check_if_exists=False) if os.path.exists(project_path): git_repo = git.Repo(project_path) if git_repo.head.is_detached: @@ -2499,12 +2503,14 @@ class RunProjectUpdate(BaseTask): return getattr(settings, 'AWX_PROOT_ENABLED', False) def build_execution_environment_params(self, instance): - project_path = os.path.dirname(instance.get_project_path(check_if_exists=False)) + project_path = instance.get_project_path(check_if_exists=False) + cache_path = instance.get_cache_path() execution_environment_params = { "process_isolation": True, "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, "container_volume_mounts": [ f"{project_path}:{project_path}:Z", + f"{cache_path}:{cache_path}:Z", ] } From 50433789ae505a20345c31683e02d5c7839e9b57 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 28 Aug 2020 22:16:34 -0400 Subject: [PATCH 016/157] Purge environment variables to work with ansible-runner changes Remove inventory scripts show because they no longer exist Remove reference to non-existent callback directory Remove more references to removed paths --- awx/main/tasks.py | 78 +++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 53 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ed05c61828..b773031e10 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -74,7 +74,6 @@ from awx.main.utils import (update_scm_url, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, get_awx_version) from awx.main.utils.ansible import read_ansible_config -from awx.main.utils.common import get_custom_venv_choices from awx.main.utils.external_logging import reconfigure_rsyslog from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services @@ -1066,30 +1065,18 @@ class BaseTask(object): os.chmod(path, stat.S_IRUSR) return path - def add_ansible_venv(self, venv_path, env, isolated=False): - env['VIRTUAL_ENV'] = venv_path - env['PATH'] = os.path.join(venv_path, "bin") + ":" + env['PATH'] - venv_libdir = os.path.join(venv_path, "lib") - - if not isolated and ( - not os.path.exists(venv_libdir) or - os.path.join(venv_path, '') not in get_custom_venv_choices() - ): - raise InvalidVirtualenvError(_( - 'Invalid virtual environment selected: {}'.format(venv_path) - )) - - isolated_manager.set_pythonpath(venv_libdir, env) - def add_awx_venv(self, env): env['VIRTUAL_ENV'] = settings.AWX_VENV_PATH - env['PATH'] = os.path.join(settings.AWX_VENV_PATH, "bin") + ":" + env['PATH'] + if 'PATH' in env: + env['PATH'] = os.path.join(settings.AWX_VENV_PATH, "bin") + ":" + env['PATH'] + else: + env['PATH'] = os.path.join(settings.AWX_VENV_PATH, "bin") def build_env(self, instance, private_data_dir, isolated, private_data_files=None): ''' Build environment dictionary for ansible-playbook. ''' - env = dict(os.environ.items()) + env = {} # Add ANSIBLE_* settings to the subprocess environment. for attr in dir(settings): if attr == attr.upper() and attr.startswith('ANSIBLE_'): @@ -1097,14 +1084,6 @@ class BaseTask(object): # Also set environment variables configured in AWX_TASK_ENV setting. for key, value in settings.AWX_TASK_ENV.items(): env[key] = str(value) - # Set environment variables needed for inventory and job event - # callbacks to work. - # Update PYTHONPATH to use local site-packages. - # NOTE: - # Derived class should call add_ansible_venv() or add_awx_venv() - if self.should_use_proot(instance): - env['PROOT_TMP_DIR'] = settings.AWX_PROOT_BASE_PATH - env['AWX_PRIVATE_DATA_DIR'] = private_data_dir return env def should_use_resource_profiling(self, job): @@ -1696,7 +1675,6 @@ class RunJob(BaseTask): private_data_files=private_data_files) if private_data_files is None: private_data_files = {} - self.add_ansible_venv(job.ansible_virtualenv_path, env, isolated=isolated) # Set environment variables needed for inventory and job event # callbacks to work. env['JOB_ID'] = str(job.pk) @@ -1715,7 +1693,8 @@ class RunJob(BaseTask): cp_dir = os.path.join(private_data_dir, 'cp') if not os.path.exists(cp_dir): os.mkdir(cp_dir, 0o700) - env['ANSIBLE_SSH_CONTROL_PATH_DIR'] = cp_dir + # FIXME: more elegant way to manage this path in container + env['ANSIBLE_SSH_CONTROL_PATH_DIR'] = '/runner/cp' # Set environment variables for cloud credentials. cred_files = private_data_files.get('credentials', {}) @@ -1752,7 +1731,8 @@ class RunJob(BaseTask): for path in config_values[config_setting].split(':'): if path not in paths: paths = [config_values[config_setting]] + paths - paths = [os.path.join(private_data_dir, folder)] + paths + # FIXME: again, figure out more elegant way for inside container + paths = [os.path.join('/runner', folder)] + paths env[env_key] = os.pathsep.join(paths) return env @@ -2082,7 +2062,6 @@ class RunProjectUpdate(BaseTask): env = super(RunProjectUpdate, self).build_env(project_update, private_data_dir, isolated=isolated, private_data_files=private_data_files) - self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env) env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False) env['ANSIBLE_ASK_PASS'] = str(False) env['ANSIBLE_BECOME_ASK_PASS'] = str(False) @@ -2524,18 +2503,6 @@ class RunInventoryUpdate(BaseTask): event_model = InventoryUpdateEvent event_data_key = 'inventory_update_id' - # TODO: remove once inv updates run in containers - def should_use_proot(self, inventory_update): - ''' - Return whether this task should use proot. - ''' - return getattr(settings, 'AWX_PROOT_ENABLED', False) - - # TODO: remove once inv updates run in containers - @property - def proot_show_paths(self): - return [settings.AWX_ANSIBLE_COLLECTIONS_PATHS] - def build_private_data(self, inventory_update, private_data_dir): """ Return private data needed for inventory update. @@ -2562,17 +2529,18 @@ class RunInventoryUpdate(BaseTask): are accomplished by the inventory source injectors (in this method) or custom credential type injectors (in main run method). """ - env = super(RunInventoryUpdate, self).build_env(inventory_update, + base_env = super(RunInventoryUpdate, self).build_env(inventory_update, private_data_dir, isolated, private_data_files=private_data_files) + # TODO: this is able to run by turning off isolation + # the goal is to run it a container instead + env = dict(os.environ.items()) + env.update(base_env) + if private_data_files is None: private_data_files = {} - # TODO: remove once containers replace custom venvs - self.add_ansible_venv(inventory_update.ansible_virtualenv_path, env, isolated=isolated) - - # Legacy environment variables, were used as signal to awx-manage command - # now they are provided in case some scripts may be relying on them + # Pass inventory source ID to inventory script. env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id) env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk) env.update(STANDARD_INVENTORY_UPDATE_ENV) @@ -2610,7 +2578,8 @@ class RunInventoryUpdate(BaseTask): for path in config_values[config_setting].split(':'): if path not in paths: paths = [config_values[config_setting]] + paths - paths = [os.path.join(private_data_dir, folder)] + paths + # FIXME: containers + paths = [os.path.join('/runner', folder)] + paths env[env_key] = os.pathsep.join(paths) return env @@ -2885,7 +2854,6 @@ class RunAdHocCommand(BaseTask): env = super(RunAdHocCommand, self).build_env(ad_hoc_command, private_data_dir, isolated=isolated, private_data_files=private_data_files) - self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env) # Set environment variables needed for inventory and ad hoc event # callbacks to work. env['AD_HOC_COMMAND_ID'] = str(ad_hoc_command.pk) @@ -2899,7 +2867,8 @@ class RunAdHocCommand(BaseTask): cp_dir = os.path.join(private_data_dir, 'cp') if not os.path.exists(cp_dir): os.mkdir(cp_dir, 0o700) - env['ANSIBLE_SSH_CONTROL_PATH'] = cp_dir + # FIXME: more elegant way to manage this path in container + env['ANSIBLE_SSH_CONTROL_PATH'] = '/runner/cp' return env @@ -3061,10 +3030,13 @@ class RunSystemJob(BaseTask): return path def build_env(self, instance, private_data_dir, isolated=False, private_data_files=None): - env = super(RunSystemJob, self).build_env(instance, private_data_dir, + base_env = super(RunSystemJob, self).build_env(instance, private_data_dir, isolated=isolated, private_data_files=private_data_files) - self.add_awx_venv(env) + # TODO: this is able to run by turning off isolation + # the goal is to run it a container instead + env = dict(os.environ.items()) + env.update(base_env) return env def build_cwd(self, instance, private_data_dir): From 6e2010ca402d31a2ab55c414e4dfbfa063e342c4 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 2 Sep 2020 16:43:44 -0400 Subject: [PATCH 017/157] Respect user proot show paths when using containers --- awx/main/tasks.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b773031e10..b929c231b1 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -887,7 +887,12 @@ class BaseTask(object): return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) def build_execution_environment_params(self, instance): - return {} + params = {} + if settings.AWX_PROOT_SHOW_PATHS: + params['container_volume_mounts'] = [] + for this_path in settings.AWX_PROOT_SHOW_PATHS: + params['container_volume_mounts'].append(f'{this_path}:{this_path}:Z') + return params def build_private_data(self, instance, private_data_dir): ''' @@ -1990,11 +1995,12 @@ class RunJob(BaseTask): update_inventory_computed_fields.delay(inventory.id) def build_execution_environment_params(self, instance): - execution_environment_params = { + params = super(RunJob, self).build_execution_environment_params(instance) + params.update({ "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, "process_isolation": True - } - return execution_environment_params + }) + return params @@ -2482,18 +2488,17 @@ class RunProjectUpdate(BaseTask): return getattr(settings, 'AWX_PROOT_ENABLED', False) def build_execution_environment_params(self, instance): + params = super(RunProjectUpdate, self).build_execution_environment_params(instance) project_path = instance.get_project_path(check_if_exists=False) cache_path = instance.get_cache_path() - execution_environment_params = { - "process_isolation": True, - "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, - "container_volume_mounts": [ - f"{project_path}:{project_path}:Z", - f"{cache_path}:{cache_path}:Z", - ] - - } - return execution_environment_params + params['process_isolation'] = True + params['container_image'] = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE + params.setdefault('container_volume_mounts', []) + params['container_volume_mounts'].extend([ + f"{project_path}:{project_path}:Z", + f"{cache_path}:{cache_path}:Z", + ]) + return params @task(queue=get_local_queuename) From 73418e41f30ae3d4cd26e6622dc439fb543f1d9c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 3 Sep 2020 11:02:08 -0400 Subject: [PATCH 018/157] Fix pathing issue with custom credentials also fix some minor flake8 issues --- awx/main/models/credential/__init__.py | 7 ++++++- awx/main/tasks.py | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index e8a2884083..15afd55018 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -564,7 +564,12 @@ class CredentialType(CommonModelNameNotUnique): if extra_vars: path = build_extra_vars_file(extra_vars, private_data_dir) - args.extend(['-e', '@%s' % path]) + # FIXME: develop some better means of referencing paths inside containers + container_path = os.path.join( + '/runner', + os.path.basename(path) + ) + args.extend(['-e', '@%s' % container_path]) class ManagedCredentialType(SimpleNamespace): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b929c231b1..dcd752f72c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1414,9 +1414,10 @@ class BaseTask(object): cwd = self.build_cwd(self.instance, private_data_dir) resource_profiling_params = self.build_params_resource_profiling(self.instance, private_data_dir) - process_isolation_params = self.build_params_process_isolation(self.instance, - private_data_dir, - cwd) + # TODO: Remove if fully replaced with containerized runs + # process_isolation_params = self.build_params_process_isolation(self.instance, + # private_data_dir, + # cwd) execution_environment_params = self.build_execution_environment_params(self.instance) env = self.build_env(self.instance, private_data_dir, isolated, private_data_files=private_data_files) @@ -2534,10 +2535,9 @@ class RunInventoryUpdate(BaseTask): are accomplished by the inventory source injectors (in this method) or custom credential type injectors (in main run method). """ - base_env = super(RunInventoryUpdate, self).build_env(inventory_update, - private_data_dir, - isolated, - private_data_files=private_data_files) + base_env = super(RunInventoryUpdate, self).build_env( + inventory_update, private_data_dir, isolated, + private_data_files=private_data_files) # TODO: this is able to run by turning off isolation # the goal is to run it a container instead env = dict(os.environ.items()) @@ -3035,9 +3035,9 @@ class RunSystemJob(BaseTask): return path def build_env(self, instance, private_data_dir, isolated=False, private_data_files=None): - base_env = super(RunSystemJob, self).build_env(instance, private_data_dir, - isolated=isolated, - private_data_files=private_data_files) + base_env = super(RunSystemJob, self).build_env( + instance, private_data_dir, isolated=isolated, + private_data_files=private_data_files) # TODO: this is able to run by turning off isolation # the goal is to run it a container instead env = dict(os.environ.items()) From 64f45da4d27e3ed6d22978ec2b4193cf277870d7 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 3 Sep 2020 15:48:10 -0400 Subject: [PATCH 019/157] Fix pathing issue for credential file references --- awx/main/models/credential/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index 15afd55018..ebab3bc22f 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -528,15 +528,20 @@ class CredentialType(CommonModelNameNotUnique): with open(path, 'w') as f: f.write(data) os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + # FIXME: develop some better means of referencing paths inside containers + container_path = os.path.join( + '/runner', + os.path.basename(path) + ) # determine if filename indicates single file or many if file_label.find('.') == -1: - tower_namespace.filename = path + tower_namespace.filename = container_path else: if not hasattr(tower_namespace, 'filename'): tower_namespace.filename = TowerNamespace() file_label = file_label.split('.')[1] - setattr(tower_namespace.filename, file_label, path) + setattr(tower_namespace.filename, file_label, container_path) injector_field = self._meta.get_field('injectors') for env_var, tmpl in self.injectors.get('env', {}).items(): From 9660e27246bb4edf61414bd7dcbbbea76d4f9f1d Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 4 Sep 2020 07:50:21 -0400 Subject: [PATCH 020/157] Fix project folder deletion Fix another absolute path reference in containers --- awx/main/tasks.py | 4 +++- awx/playbooks/project_update.yml | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index dcd752f72c..c81900c9c6 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1038,6 +1038,8 @@ class BaseTask(object): results_dir = os.path.join(private_data_dir, 'artifacts/playbook_profiling') if not os.path.isdir(results_dir): os.makedirs(results_dir, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + # FIXME: develop some better means of referencing paths inside containers + container_results_dir = os.path.join('/runner', 'artifacts/playbook_profiling') logger.debug('Collected the following resource profiling intervals: cpu: {} mem: {} pid: {}' .format(cpu_poll_interval, mem_poll_interval, pid_poll_interval)) @@ -1047,7 +1049,7 @@ class BaseTask(object): 'resource_profiling_cpu_poll_interval': cpu_poll_interval, 'resource_profiling_memory_poll_interval': mem_poll_interval, 'resource_profiling_pid_poll_interval': pid_poll_interval, - 'resource_profiling_results_dir': results_dir}) + 'resource_profiling_results_dir': container_results_dir}) return resource_profiling_params diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index a7b7007d56..49618909fb 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -24,9 +24,7 @@ tasks: - name: delete project directory before update - file: - path: "{{project_path|quote}}" - state: absent + command: "rm -rf {{project_path}}/*" # volume mounted, cannot delete folder itself tags: - delete From 332c802317da54e7ca3615f867f3ce00b9b02601 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 8 Sep 2020 10:30:10 -0400 Subject: [PATCH 021/157] Deal with missing HOME env var --- awx/main/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index c81900c9c6..b95015dc5c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1463,7 +1463,8 @@ class BaseTask(object): if containerized: # We don't want HOME passed through to container groups. - params['envvars'].pop('HOME') + # TODO: remove this conditional after everything is containerized + params['envvars'].pop('HOME', None) if isinstance(self.instance, AdHocCommand): params['module'] = self.build_module_name(self.instance) From 9d806ddb8202dde389f98466e555e0964a473c82 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 9 Sep 2020 09:05:23 -0400 Subject: [PATCH 022/157] Initial minimal hooking up of JT EEs to jobs --- awx/main/models/jobs.py | 2 +- awx/main/tasks.py | 37 ++++++++++++++++++------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 638954e53c..31f4784962 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -284,7 +284,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour def _get_unified_job_field_names(cls): return set(f.name for f in JobOptions._meta.fields) | set( ['name', 'description', 'organization', 'survey_passwords', 'labels', 'credentials', - 'job_slice_number', 'job_slice_count'] + 'job_slice_number', 'job_slice_count', 'execution_environment'] ) @property diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b95015dc5c..50230c0e98 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -887,7 +887,18 @@ class BaseTask(object): return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) def build_execution_environment_params(self, instance): - params = {} + if getattr(instance, 'execution_environment', None): + # TODO: process heirarchy, JT-project-org, maybe here + # or maybe in create_unified_job + logger.info('using custom image {}'.format(instance.execution_environment.image)) + image = instance.execution_environment.image + else: + logger.info('using default image') + image = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE + params = { + "container_image": image, + "process_isolation": True + } if settings.AWX_PROOT_SHOW_PATHS: params['container_volume_mounts'] = [] for this_path in settings.AWX_PROOT_SHOW_PATHS: @@ -1998,15 +2009,6 @@ class RunJob(BaseTask): if inventory is not None: update_inventory_computed_fields.delay(inventory.id) - def build_execution_environment_params(self, instance): - params = super(RunJob, self).build_execution_environment_params(instance) - params.update({ - "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, - "process_isolation": True - }) - return params - - @task(queue=get_local_queuename) class RunProjectUpdate(BaseTask): @@ -2495,8 +2497,6 @@ class RunProjectUpdate(BaseTask): params = super(RunProjectUpdate, self).build_execution_environment_params(instance) project_path = instance.get_project_path(check_if_exists=False) cache_path = instance.get_cache_path() - params['process_isolation'] = True - params['container_image'] = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE params.setdefault('container_volume_mounts', []) params['container_volume_mounts'].extend([ f"{project_path}:{project_path}:Z", @@ -2531,6 +2531,9 @@ class RunInventoryUpdate(BaseTask): injector = InventorySource.injectors[inventory_update.source]() return injector.build_private_data(inventory_update, private_data_dir) + def build_execution_environment_params(self, inventory_update): + return {} # TODO: containerize inventory updates + def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None): """Build environment dictionary for ansible-inventory. @@ -2992,13 +2995,6 @@ class RunAdHocCommand(BaseTask): if isolated_manager_instance: isolated_manager_instance.cleanup() - def build_execution_environment_params(self, instance): - execution_environment_params = { - "container_image": settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE, - "process_isolation": True - } - return execution_environment_params - @task(queue=get_local_queuename) class RunSystemJob(BaseTask): @@ -3007,6 +3003,9 @@ class RunSystemJob(BaseTask): event_model = SystemJobEvent event_data_key = 'system_job_id' + def build_execution_environment_params(self, system_job): + return {} + def build_args(self, system_job, private_data_dir, passwords): args = ['awx-manage', system_job.job_type] try: From b7209d16940594d594f3129047f9fcad9612f3df Mon Sep 17 00:00:00 2001 From: Kersom <9053044+nixocio@users.noreply.github.com> Date: Tue, 15 Sep 2020 20:25:23 -0400 Subject: [PATCH 023/157] Add list Execution Environments (#8148) See: https://github.com/ansible/awx/issues/7886 --- awx/ui_next/src/api/index.js | 3 + .../src/api/models/ExecutionEnvironments.js | 10 + .../ExecutionEnviromentList.test.jsx | 103 +++++++++ .../ExecutionEnvironmentList.jsx | 206 +++++++++++++++++- .../ExecutionEnvironmentListItem.jsx | 85 ++++++++ .../ExecutionEnvironmentListItem.test.jsx | 52 +++++ awx/ui_next/src/types.js | 10 + 7 files changed, 461 insertions(+), 8 deletions(-) create mode 100644 awx/ui_next/src/api/models/ExecutionEnvironments.js create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx diff --git a/awx/ui_next/src/api/index.js b/awx/ui_next/src/api/index.js index 3160ebd907..d048237b74 100644 --- a/awx/ui_next/src/api/index.js +++ b/awx/ui_next/src/api/index.js @@ -7,6 +7,7 @@ import CredentialInputSources from './models/CredentialInputSources'; import CredentialTypes from './models/CredentialTypes'; import Credentials from './models/Credentials'; import Dashboard from './models/Dashboard'; +import ExecutionEnvironments from './models/ExecutionEnvironments'; import Groups from './models/Groups'; import Hosts from './models/Hosts'; import InstanceGroups from './models/InstanceGroups'; @@ -50,6 +51,7 @@ const CredentialInputSourcesAPI = new CredentialInputSources(); const CredentialTypesAPI = new CredentialTypes(); const CredentialsAPI = new Credentials(); const DashboardAPI = new Dashboard(); +const ExecutionEnvironmentsAPI = new ExecutionEnvironments(); const GroupsAPI = new Groups(); const HostsAPI = new Hosts(); const InstanceGroupsAPI = new InstanceGroups(); @@ -94,6 +96,7 @@ export { CredentialTypesAPI, CredentialsAPI, DashboardAPI, + ExecutionEnvironmentsAPI, GroupsAPI, HostsAPI, InstanceGroupsAPI, diff --git a/awx/ui_next/src/api/models/ExecutionEnvironments.js b/awx/ui_next/src/api/models/ExecutionEnvironments.js new file mode 100644 index 0000000000..2df933d53a --- /dev/null +++ b/awx/ui_next/src/api/models/ExecutionEnvironments.js @@ -0,0 +1,10 @@ +import Base from '../Base'; + +class ExecutionEnvironments extends Base { + constructor(http) { + super(http); + this.baseUrl = '/api/v2/execution_environments/'; + } +} + +export default ExecutionEnvironments; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx new file mode 100644 index 0000000000..8b04233404 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx @@ -0,0 +1,103 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { + mountWithContexts, + waitForElement, +} from '../../../../testUtils/enzymeHelpers'; + +import { ExecutionEnvironmentsAPI } from '../../../api'; +import ExecutionEnvironmentList from './ExecutionEnvironmentList'; + +jest.mock('../../../api/models/ExecutionEnvironments'); + +const executionEnvironments = { + data: { + results: [ + { + id: 1, + image: 'https://registry.com/r/image/manifest', + organization: null, + credential: null, + }, + { + id: 2, + image: 'https://registry.com/r/image2/manifest', + organization: null, + credential: null, + }, + ], + count: 2, + }, +}; + +const options = { data: { actions: { POST: true } } }; + +describe('', () => { + let wrapper; + + test('should mount successfully', async () => { + await act(async () => { + wrapper = mountWithContexts(); + }); + await waitForElement( + wrapper, + 'ExecutionEnvironmentList', + el => el.length > 0 + ); + }); + + test('should have data fetched and render 2 rows', async () => { + ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); + + await act(async () => { + wrapper = mountWithContexts(); + }); + await waitForElement( + wrapper, + 'ExecutionEnvironmentList', + el => el.length > 0 + ); + + expect(wrapper.find('ExecutionEnvironmentListItem').length).toBe(2); + expect(ExecutionEnvironmentsAPI.read).toBeCalled(); + expect(ExecutionEnvironmentsAPI.readOptions).toBeCalled(); + }); + + test('should thrown content error', async () => { + ExecutionEnvironmentsAPI.read.mockRejectedValue( + new Error({ + response: { + config: { + method: 'GET', + url: '/api/v2/execution_environments', + }, + data: 'An error occurred', + }, + }) + ); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); + await act(async () => { + wrapper = mountWithContexts(); + }); + await waitForElement( + wrapper, + 'ExecutionEnvironmentList', + el => el.length > 0 + ); + expect(wrapper.find('ContentError').length).toBe(1); + }); + + test('should not render add button', async () => { + ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue({ + data: { actions: { POST: false } }, + }); + await act(async () => { + wrapper = mountWithContexts(); + }); + waitForElement(wrapper, 'ExecutionEnvironmentList', el => el.length > 0); + expect(wrapper.find('ToolbarAddButton').length).toBe(0); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx index 96d2d07e10..452fc4fbdb 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx @@ -1,14 +1,204 @@ -import React from 'react'; +import React, { useEffect, useCallback } from 'react'; +import { useLocation, useRouteMatch } from 'react-router-dom'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; import { Card, PageSection } from '@patternfly/react-core'; -function ExecutionEnvironmentList() { +import { ExecutionEnvironmentsAPI } from '../../../api'; +import { getQSConfig, parseQueryString } from '../../../util/qs'; +import useRequest, { useDeleteItems } from '../../../util/useRequest'; +import useSelected from '../../../util/useSelected'; +import PaginatedDataList, { + ToolbarDeleteButton, + ToolbarAddButton, +} from '../../../components/PaginatedDataList'; +import ErrorDetail from '../../../components/ErrorDetail'; +import AlertModal from '../../../components/AlertModal'; +import DatalistToolbar from '../../../components/DataListToolbar'; + +import ExecutionEnvironmentsListItem from './ExecutionEnvironmentListItem'; + +const QS_CONFIG = getQSConfig('execution_environments', { + page: 1, + page_size: 20, + managed_by_tower: false, + order_by: 'image', +}); + +function ExecutionEnvironmentList({ i18n }) { + const location = useLocation(); + const match = useRouteMatch(); + + const { + error: contentError, + isLoading, + request: fetchExecutionEnvironments, + result: { + executionEnvironments, + executionEnvironmentsCount, + actions, + relatedSearchableKeys, + searchableKeys, + }, + } = useRequest( + useCallback(async () => { + const params = parseQueryString(QS_CONFIG, location.search); + + const [response, responseActions] = await Promise.all([ + ExecutionEnvironmentsAPI.read(params), + ExecutionEnvironmentsAPI.readOptions(), + ]); + + return { + executionEnvironments: response.data.results, + executionEnvironmentsCount: response.data.count, + actions: responseActions.data.actions, + relatedSearchableKeys: ( + responseActions?.data?.related_search_fields || [] + ).map(val => val.slice(0, -8)), + searchableKeys: Object.keys( + responseActions.data.actions?.GET || {} + ).filter(key => responseActions.data.actions?.GET[key].filterable), + }; + }, [location]), + { + executionEnvironments: [], + executionEnvironmentsCount: 0, + actions: {}, + relatedSearchableKeys: [], + searchableKeys: [], + } + ); + + useEffect(() => { + fetchExecutionEnvironments(); + }, [fetchExecutionEnvironments]); + + const { selected, isAllSelected, handleSelect, setSelected } = useSelected( + executionEnvironments + ); + + const { + isLoading: deleteLoading, + deletionError, + deleteItems: deleteExecutionEnvironments, + clearDeletionError, + } = useDeleteItems( + useCallback(async () => { + await Promise.all( + selected.map(({ id }) => ExecutionEnvironmentsAPI.destroy(id)) + ); + }, [selected]), + { + qsConfig: QS_CONFIG, + allItemsSelected: isAllSelected, + fetchItems: fetchExecutionEnvironments, + } + ); + + const handleDelete = async () => { + await deleteExecutionEnvironments(); + setSelected([]); + }; + + const canAdd = actions && actions.POST; + return ( - - -
List Execution environments
- - + <> + + + ( + + setSelected(isSelected ? [...executionEnvironments] : []) + } + qsConfig={QS_CONFIG} + additionalControls={[ + ...(canAdd + ? [ + , + ] + : []), + , + ]} + /> + )} + renderItem={executionEnvironment => ( + handleSelect(executionEnvironment)} + isSelected={selected.some( + row => row.id === executionEnvironment.id + )} + /> + )} + emptyStateControls={ + canAdd && ( + + ) + } + /> + + + + {i18n._(t`Failed to delete one or more execution environments`)} + + + ); } -export default ExecutionEnvironmentList; +export default withI18n()(ExecutionEnvironmentList); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx new file mode 100644 index 0000000000..7789facfbb --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx @@ -0,0 +1,85 @@ +import React from 'react'; +import { string, bool, func } from 'prop-types'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Link } from 'react-router-dom'; +import { + Button, + DataListAction, + DataListCheck, + DataListItem, + DataListItemRow, + DataListItemCells, + Tooltip, +} from '@patternfly/react-core'; +import { PencilAltIcon } from '@patternfly/react-icons'; + +import DataListCell from '../../../components/DataListCell'; +import { ExecutionEnvironment } from '../../../types'; + +function ExecutionEnvironmentListItem({ + executionEnvironment, + detailUrl, + isSelected, + onSelect, + i18n, +}) { + const labelId = `check-action-${executionEnvironment.id}`; + + return ( + + + + + + {executionEnvironment.image} + + , + ]} + /> + + + + + + + + ); +} + +ExecutionEnvironmentListItem.prototype = { + executionEnvironment: ExecutionEnvironment.isRequired, + detailUrl: string.isRequired, + isSelected: bool.isRequired, + onSelect: func.isRequired, +}; + +export default withI18n()(ExecutionEnvironmentListItem); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx new file mode 100644 index 0000000000..4f51a51672 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx @@ -0,0 +1,52 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; + +import ExecutionEnvironmentListItem from './ExecutionEnvironmentListItem'; + +describe('', () => { + let wrapper; + const executionEnvironment = { + id: 1, + image: 'https://registry.com/r/image/manifest', + organization: null, + credential: null, + }; + + test('should mount successfully', async () => { + await act(async () => { + wrapper = mountWithContexts( + {}} + /> + ); + }); + expect(wrapper.find('ExecutionEnvironmentListItem').length).toBe(1); + }); + + test('should render the proper data', async () => { + await act(async () => { + wrapper = mountWithContexts( + {}} + /> + ); + }); + expect( + wrapper + .find('DataListCell[aria-label="execution environment image"]') + .text() + ).toBe(executionEnvironment.image); + expect(wrapper.find('PencilAltIcon').length).toBe(1); + expect( + wrapper.find('input#select-execution-environment-1').prop('checked') + ).toBe(false); + }); +}); diff --git a/awx/ui_next/src/types.js b/awx/ui_next/src/types.js index 760ecf4ed2..53c7ed3b3d 100644 --- a/awx/ui_next/src/types.js +++ b/awx/ui_next/src/types.js @@ -407,3 +407,13 @@ export const WorkflowApproval = shape({ approval_expiration: string, timed_out: bool, }); + +export const ExecutionEnvironment = shape({ + id: number.isRequired, + organization: number, + credential: number, + image: string.isRequired, + url: string, + summary_fields: shape({}), + description: string, +}); From 9530c6ca50b9e9afd15b39e4efc22a969e03d00d Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 16 Sep 2020 11:34:24 -0400 Subject: [PATCH 024/157] Changes to get execution environments factories working (#8126) --- .../api/pages/execution_environments.py | 26 ++++++++++++++++++- awxkit/awxkit/api/pages/job_templates.py | 21 +++++++-------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py index e48ef324bd..87225d1052 100644 --- a/awxkit/awxkit/api/pages/execution_environments.py +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -1,11 +1,12 @@ import logging -from awxkit.api.mixins import HasCreate +from awxkit.api.mixins import DSAdapter, HasCreate from awxkit.api.pages import ( Credential, Organization, ) from awxkit.api.resources import resources +from awxkit.utils import random_title, PseudoNamespace from . import base from . import page @@ -19,6 +20,29 @@ class ExecutionEnvironment(HasCreate, base.Base): dependencies = [Organization, Credential] NATURAL_KEY = ('organization', 'image') + # fields are image, organization, managed_by_tower, credential + def create(self, image='quay.io/ansible/ansible-runner:devel', credential=None, **kwargs): + # we do not want to make a credential by default + payload = self.create_payload(image=image, credential=credential, **kwargs) + ret = self.update_identity(ExecutionEnvironments(self.connection).post(payload)) + return ret + + def create_payload(self, organization=Organization, **kwargs): + self.create_and_update_dependencies(organization) + payload = self.payload(organization=self.ds.organization, **kwargs) + payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) + return payload + + def payload(self, image=None, organization=None, credential=None, **kwargs): + payload = PseudoNamespace( + image=image or random_title(10), + organization=organization.id if organization else None, + credential=credential.id if credential else None, + **kwargs + ) + + return payload + page.register_page([resources.execution_environment, (resources.execution_environments, 'post'), diff --git a/awxkit/awxkit/api/pages/job_templates.py b/awxkit/awxkit/api/pages/job_templates.py index cd45fc0c87..9f008078a9 100644 --- a/awxkit/awxkit/api/pages/job_templates.py +++ b/awxkit/awxkit/api/pages/job_templates.py @@ -101,18 +101,17 @@ class JobTemplate( if kwargs.get('project'): payload.update(project=kwargs.get('project').id, playbook=playbook) - if kwargs.get('inventory'): - payload.update(inventory=kwargs.get('inventory').id) - if kwargs.get('credential'): - payload.update(credential=kwargs.get('credential').id) - if kwargs.get('webhook_credential'): - webhook_cred = kwargs.get('webhook_credential') - if isinstance(webhook_cred, int): - payload.update(webhook_credential=int(webhook_cred)) - elif hasattr(webhook_cred, 'id'): - payload.update(webhook_credential=webhook_cred.id) + + for fk_field in ('inventory', 'credential', 'webhook_credential', 'execution_environment'): + rel_obj = kwargs.get(fk_field) + if rel_obj is None: + continue + elif isinstance(rel_obj, int): + payload.update(**{fk_field: int(rel_obj)}) + elif hasattr(rel_obj, 'id'): + payload.update(**{fk_field: rel_obj.id}) else: - raise AttributeError("Webhook credential must either be integer of pkid or Credential object") + raise AttributeError(f'Related field {fk_field} must be either integer of pkid or object') return payload From 684b9bd47aa56111466b4d11e682bc1d470c5a78 Mon Sep 17 00:00:00 2001 From: Kersom <9053044+nixocio@users.noreply.github.com> Date: Fri, 18 Sep 2020 08:40:29 -0400 Subject: [PATCH 025/157] Add feature to Add/Edit Execution Environments (#8165) * Add feature to Add/Edit Execution Environments Add feature to Add/Edit Execution Environments. Also, add key for `ExecutionEnvironmentsList`. See: https://github.com/ansible/awx/issues/7887 * Update registry credential label --- awx/ui_next/src/routeConfig.js | 2 +- .../ExecutionEnvironment.jsx | 133 +++++++++++++++--- .../ExecutionEnvironmentAdd.jsx | 33 ++++- .../ExecutionEnvironmentAdd.test.jsx | 80 +++++++++++ .../ExecutionEnvironmentEdit.jsx | 42 ++++-- .../ExecutionEnvironmentEdit.test.jsx | 103 ++++++++++++++ .../ExecutionEnviromentList.test.jsx | 2 + .../ExecutionEnvironmentList.jsx | 1 + .../ExecutionEnvironments.jsx | 2 +- .../shared/ExecutionEnvironmentForm.jsx | 86 +++++++++++ .../shared/ExecutionEnvironmentForm.test.jsx | 115 +++++++++++++++ 11 files changed, 570 insertions(+), 29 deletions(-) create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx diff --git a/awx/ui_next/src/routeConfig.js b/awx/ui_next/src/routeConfig.js index c0aff394f0..507bc4e6d7 100644 --- a/awx/ui_next/src/routeConfig.js +++ b/awx/ui_next/src/routeConfig.js @@ -140,7 +140,7 @@ function getRouteConfig(i18n) { screen: Applications, }, { - title: i18n._(t`Execution environments`), + title: i18n._(t`Execution Environments`), path: '/execution_environments', screen: ExecutionEnvironments, }, diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx index 9575a3b568..78ead63048 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx @@ -1,25 +1,124 @@ -import React from 'react'; -import { Route, Redirect, Switch } from 'react-router-dom'; +import React, { useEffect, useCallback } from 'react'; +import { + Link, + Redirect, + Route, + Switch, + useLocation, + useParams, +} from 'react-router-dom'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Card, PageSection } from '@patternfly/react-core'; +import { CaretLeftIcon } from '@patternfly/react-icons'; + +import useRequest from '../../util/useRequest'; +import { ExecutionEnvironmentsAPI } from '../../api'; +import RoutedTabs from '../../components/RoutedTabs'; +import ContentError from '../../components/ContentError'; +import ContentLoading from '../../components/ContentLoading'; import ExecutionEnvironmentDetails from './ExecutionEnvironmentDetails'; import ExecutionEnvironmentEdit from './ExecutionEnvironmentEdit'; -function ExecutionEnvironment() { +function ExecutionEnvironment({ i18n, setBreadcrumb }) { + const { id } = useParams(); + const { pathname } = useLocation(); + + const { + isLoading, + error: contentError, + request: fetchExecutionEnvironments, + result: executionEnvironment, + } = useRequest( + useCallback(async () => { + const { data } = await ExecutionEnvironmentsAPI.readDetail(id); + return data; + }, [id]), + null + ); + + useEffect(() => { + fetchExecutionEnvironments(); + }, [fetchExecutionEnvironments, pathname]); + + useEffect(() => { + if (executionEnvironment) { + setBreadcrumb(executionEnvironment); + } + }, [executionEnvironment, setBreadcrumb]); + + const tabsArray = [ + { + name: ( + <> + + {i18n._(t`Back to execution environments`)} + + ), + link: '/execution_environments', + id: 99, + }, + { + name: i18n._(t`Details`), + link: `/execution_environments/${id}/details`, + id: 0, + }, + ]; + + if (!isLoading && contentError) { + return ( + + + + {contentError.response?.status === 404 && ( + + {i18n._(t`Execution environment not found.`)}{' '} + + {i18n._(t`View all execution environments`)} + + + )} + + + + ); + } + + let cardHeader = ; + if (pathname.endsWith('edit')) { + cardHeader = null; + } + return ( - - - - - - - - - + + + {cardHeader} + {isLoading && } + {!isLoading && executionEnvironment && ( + + + {executionEnvironment && ( + <> + + + + + + + + )} + + )} + + ); } -export default ExecutionEnvironment; +export default withI18n()(ExecutionEnvironment); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx index e188990878..5f162f352e 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.jsx @@ -1,11 +1,40 @@ -import React from 'react'; +import React, { useState } from 'react'; import { Card, PageSection } from '@patternfly/react-core'; +import { useHistory } from 'react-router-dom'; + +import ExecutionEnvironmentForm from '../shared/ExecutionEnvironmentForm'; +import { CardBody } from '../../../components/Card'; +import { ExecutionEnvironmentsAPI } from '../../../api'; function ExecutionEnvironmentAdd() { + const history = useHistory(); + const [submitError, setSubmitError] = useState(null); + + const handleSubmit = async values => { + try { + const { data: response } = await ExecutionEnvironmentsAPI.create({ + ...values, + credential: values?.credential?.id, + }); + history.push(`/execution_environments/${response.id}/details`); + } catch (error) { + setSubmitError(error); + } + }; + + const handleCancel = () => { + history.push(`/execution_environments`); + }; return ( -
Add Execution Environments
+ + + ); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx new file mode 100644 index 0000000000..781501b19e --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx @@ -0,0 +1,80 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { createMemoryHistory } from 'history'; + +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; +import { ExecutionEnvironmentsAPI } from '../../../api'; +import ExecutionEnvironmentAdd from './ExecutionEnvironmentAdd'; + +jest.mock('../../../api'); + +const executionEnvironmentData = { + credential: 4, + description: 'A simple EE', + image: 'https://registry.com/image/container', +}; + +ExecutionEnvironmentsAPI.create.mockResolvedValue({ + data: { + id: 42, + }, +}); + +describe('', () => { + let wrapper; + let history; + + beforeEach(async () => { + history = createMemoryHistory({ + initialEntries: ['/execution_environments'], + }); + await act(async () => { + wrapper = mountWithContexts(, { + context: { router: { history } }, + }); + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + wrapper.unmount(); + }); + + test('handleSubmit should call the api and redirect to details page', async () => { + await act(async () => { + wrapper.find('ExecutionEnvironmentForm').prop('onSubmit')({ + executionEnvironmentData, + }); + }); + wrapper.update(); + expect(ExecutionEnvironmentsAPI.create).toHaveBeenCalledWith({ + executionEnvironmentData, + }); + expect(history.location.pathname).toBe( + '/execution_environments/42/details' + ); + }); + + test('handleCancel should return the user back to the execution environments list', async () => { + wrapper.find('Button[aria-label="Cancel"]').simulate('click'); + expect(history.location.pathname).toEqual('/execution_environments'); + }); + + test('failed form submission should show an error message', async () => { + const error = { + response: { + data: { detail: 'An error occurred' }, + }, + }; + ExecutionEnvironmentsAPI.create.mockImplementationOnce(() => + Promise.reject(error) + ); + await act(async () => { + wrapper.find('ExecutionEnvironmentForm').invoke('onSubmit')( + executionEnvironmentData + ); + }); + wrapper.update(); + expect(wrapper.find('FormSubmitError').length).toBe(1); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx index 91e3096ce3..6d8cbc9520 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx @@ -1,13 +1,39 @@ -import React from 'react'; -import { Card, PageSection } from '@patternfly/react-core'; +import React, { useState } from 'react'; +import { useHistory } from 'react-router-dom'; -function ExecutionEnvironmentEdit() { +import { CardBody } from '../../../components/Card'; +import { ExecutionEnvironmentsAPI } from '../../../api'; +import ExecutionEnvironmentForm from '../shared/ExecutionEnvironmentForm'; + +function ExecutionEnvironmentEdit({ executionEnvironment }) { + const history = useHistory(); + const [submitError, setSubmitError] = useState(null); + const detailsUrl = `/execution_environments/${executionEnvironment.id}/details`; + + const handleSubmit = async values => { + try { + await ExecutionEnvironmentsAPI.update(executionEnvironment.id, { + ...values, + credential: values.credential ? values.credential.id : null, + }); + history.push(detailsUrl); + } catch (error) { + setSubmitError(error); + } + }; + + const handleCancel = () => { + history.push(detailsUrl); + }; return ( - - -
Edit Execution environments
- - + + + ); } diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx new file mode 100644 index 0000000000..2d4f916aba --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx @@ -0,0 +1,103 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { createMemoryHistory } from 'history'; + +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; +import { ExecutionEnvironmentsAPI } from '../../../api'; + +import ExecutionEnvironmentEdit from './ExecutionEnvironmentEdit'; + +jest.mock('../../../api'); + +const executionEnvironmentData = { + id: 42, + credential: { id: 4 }, + description: 'A simple EE', + image: 'https://registry.com/image/container', +}; + +const updateExecutionEnvironmentData = { + image: 'https://registry.com/image/container2', + description: 'Updated new description', +}; + +describe('', () => { + let wrapper; + let history; + + beforeAll(async () => { + history = createMemoryHistory(); + await act(async () => { + wrapper = mountWithContexts( + , + { + context: { router: { history } }, + } + ); + }); + }); + + afterAll(() => { + jest.clearAllMocks(); + wrapper.unmount(); + }); + + test('handleSubmit should call the api and redirect to details page', async () => { + await act(async () => { + wrapper.find('ExecutionEnvironmentForm').invoke('onSubmit')( + updateExecutionEnvironmentData + ); + wrapper.update(); + expect(ExecutionEnvironmentsAPI.update).toHaveBeenCalledWith(42, { + ...updateExecutionEnvironmentData, + credential: null, + }); + }); + + expect(history.location.pathname).toEqual( + '/execution_environments/42/details' + ); + }); + + test('should navigate to execution environments details when cancel is clicked', async () => { + await act(async () => { + wrapper.find('button[aria-label="Cancel"]').prop('onClick')(); + }); + expect(history.location.pathname).toEqual( + '/execution_environments/42/details' + ); + }); + + test('should navigate to execution environments detail after successful submission', async () => { + await act(async () => { + wrapper.find('ExecutionEnvironmentForm').invoke('onSubmit')({ + updateExecutionEnvironmentData, + }); + }); + wrapper.update(); + expect(wrapper.find('FormSubmitError').length).toBe(0); + expect(history.location.pathname).toEqual( + '/execution_environments/42/details' + ); + }); + + test('failed form submission should show an error message', async () => { + const error = { + response: { + data: { detail: 'An error occurred' }, + }, + }; + ExecutionEnvironmentsAPI.update.mockImplementationOnce(() => + Promise.reject(error) + ); + await act(async () => { + wrapper.find('ExecutionEnvironmentForm').invoke('onSubmit')( + updateExecutionEnvironmentData + ); + }); + wrapper.update(); + expect(wrapper.find('FormSubmitError').length).toBe(1); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx index 8b04233404..4371d4c727 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx @@ -19,12 +19,14 @@ const executionEnvironments = { image: 'https://registry.com/r/image/manifest', organization: null, credential: null, + url: '/api/v2/execution_environments/1/', }, { id: 2, image: 'https://registry.com/r/image2/manifest', organization: null, credential: null, + url: '/api/v2/execution_environments/2/', }, ], count: 2, diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx index 452fc4fbdb..02ce49ee9f 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx @@ -171,6 +171,7 @@ function ExecutionEnvironmentList({ i18n }) { )} renderItem={executionEnvironment => ( handleSelect(executionEnvironment)} diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx index 7db470e2f7..7db0baaedc 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx @@ -22,7 +22,7 @@ function ExecutionEnvironments({ i18n }) { setBreadcrumbConfig({ '/execution_environments': i18n._(t`Execution environments`), '/execution_environments/add': i18n._(t`Create Execution environments`), - [`/execution_environments/${executionEnvironments.id}`]: `${executionEnvironments.name}`, + [`/execution_environments/${executionEnvironments.id}`]: `${executionEnvironments.image}`, [`/execution_environments/${executionEnvironments.id}/edit`]: i18n._( t`Edit details` ), diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx new file mode 100644 index 0000000000..60988ab278 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx @@ -0,0 +1,86 @@ +import React from 'react'; +import { func, shape } from 'prop-types'; +import { Formik, useField } from 'formik'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; + +import { Form } from '@patternfly/react-core'; +import FormField, { FormSubmitError } from '../../../components/FormField'; +import FormActionGroup from '../../../components/FormActionGroup'; +import CredentialLookup from '../../../components/Lookup/CredentialLookup'; +import { url } from '../../../util/validators'; +import { FormColumnLayout } from '../../../components/FormLayout'; + +function ExecutionEnvironmentFormFields({ i18n }) { + const [credentialField, , credentialHelpers] = useField('credential'); + return ( + <> + + + credentialHelpers.setValue(value)} + value={credentialField.value || null} + /> + + ); +} + +function ExecutionEnvironmentForm({ + executionEnvironment = {}, + onSubmit, + onCancel, + submitError, + ...rest +}) { + const initialValues = { + image: executionEnvironment.image || '', + description: executionEnvironment.description || '', + credential: executionEnvironment?.summary_fields?.credential || null, + }; + return ( + onSubmit(values)}> + {formik => ( +
+ + + {submitError && } + + +
+ )} + + ); +} + +ExecutionEnvironmentForm.propTypes = { + executionEnvironment: shape({}), + onCancel: func.isRequired, + onSubmit: func.isRequired, + submitError: shape({}), +}; + +ExecutionEnvironmentForm.defaultProps = { + executionEnvironment: {}, + submitError: null, +}; + +export default withI18n()(ExecutionEnvironmentForm); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx new file mode 100644 index 0000000000..717b264354 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx @@ -0,0 +1,115 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; + +import ExecutionEnvironmentForm from './ExecutionEnvironmentForm'; + +jest.mock('../../../api'); + +const executionEnvironment = { + id: 16, + type: 'execution_environment', + url: '/api/v2/execution_environments/16/', + related: { + created_by: '/api/v2/users/1/', + modified_by: '/api/v2/users/1/', + activity_stream: '/api/v2/execution_environments/16/activity_stream/', + unified_job_templates: + '/api/v2/execution_environments/16/unified_job_templates/', + credential: '/api/v2/credentials/4/', + }, + summary_fields: { + credential: { + id: 4, + name: 'Container Registry', + }, + }, + created: '2020-09-17T16:06:57.346128Z', + modified: '2020-09-17T16:06:57.346147Z', + description: 'A simple EE', + organization: null, + image: 'https://registry.com/image/container', + managed_by_tower: false, + credential: 4, +}; + +describe('', () => { + let wrapper; + let onCancel; + let onSubmit; + + beforeEach(async () => { + onCancel = jest.fn(); + onSubmit = jest.fn(); + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + wrapper.unmount(); + }); + + test('Initially renders successfully', () => { + expect(wrapper.length).toBe(1); + }); + + test('should display form fields properly', () => { + expect(wrapper.find('FormGroup[label="Image"]').length).toBe(1); + expect(wrapper.find('FormGroup[label="Description"]').length).toBe(1); + expect(wrapper.find('CredentialLookup').length).toBe(1); + }); + + test('should call onSubmit when form submitted', async () => { + expect(onSubmit).not.toHaveBeenCalled(); + await act(async () => { + wrapper.find('button[aria-label="Save"]').simulate('click'); + }); + expect(onSubmit).toHaveBeenCalledTimes(1); + }); + + test('should update form values', () => { + act(() => { + wrapper.find('input#execution-environment-image').simulate('change', { + target: { + value: 'https://registry.com/image/container2', + name: 'image', + }, + }); + wrapper + .find('input#execution-environment-description') + .simulate('change', { + target: { value: 'New description', name: 'description' }, + }); + wrapper.find('CredentialLookup').invoke('onBlur')(); + wrapper.find('CredentialLookup').invoke('onChange')({ + id: 99, + name: 'credential', + }); + }); + wrapper.update(); + expect( + wrapper.find('input#execution-environment-image').prop('value') + ).toEqual('https://registry.com/image/container2'); + expect( + wrapper.find('input#execution-environment-description').prop('value') + ).toEqual('New description'); + expect(wrapper.find('CredentialLookup').prop('value')).toEqual({ + id: 99, + name: 'credential', + }); + }); + + test('should call handleCancel when Cancel button is clicked', async () => { + expect(onCancel).not.toHaveBeenCalled(); + wrapper.find('button[aria-label="Cancel"]').invoke('onClick')(); + expect(onCancel).toBeCalled(); + }); +}); From 54d0f173dce26c03fb4d3a783b02dfe9b214337b Mon Sep 17 00:00:00 2001 From: Kersom <9053044+nixocio@users.noreply.github.com> Date: Fri, 18 Sep 2020 09:41:09 -0400 Subject: [PATCH 026/157] Add details page for Execution Environments (#8172) * Add feature to Add/Edit Execution Environments Add feature to Add/Edit Execution Environments. Also, add key for `ExecutionEnvironmentsList`. See: https://github.com/ansible/awx/issues/7887 * Add details page for execution environments Add details page for execution environments See: https://github.com/ansible/awx/issues/8171 --- .../ExecutionEnvironment.jsx | 4 +- .../ExecutionEnvironmentDetails.jsx | 100 ++++++++++++++++-- .../ExecutionEnvironmentDetails.test.jsx | 98 +++++++++++++++++ .../shared/ExecutionEnvironmentForm.jsx | 6 +- 4 files changed, 195 insertions(+), 13 deletions(-) create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx index 78ead63048..55a3228e13 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx @@ -110,7 +110,9 @@ function ExecutionEnvironment({ i18n, setBreadcrumb }) { /> - + )} diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index f6902d1735..abb6cf5ddc 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -1,14 +1,96 @@ -import React from 'react'; -import { Card, PageSection } from '@patternfly/react-core'; +import React, { useCallback } from 'react'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Link, useHistory } from 'react-router-dom'; +import { Button, Label } from '@patternfly/react-core'; + +import AlertModal from '../../../components/AlertModal'; +import { CardBody, CardActionsRow } from '../../../components/Card'; +import DeleteButton from '../../../components/DeleteButton'; +import { + Detail, + DetailList, + UserDateDetail, +} from '../../../components/DetailList'; +import useRequest, { useDismissableError } from '../../../util/useRequest'; +import { ExecutionEnvironmentsAPI } from '../../../api'; + +function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { + const history = useHistory(); + const { id, image, description } = executionEnvironment; + + const { + request: deleteExecutionEnvironment, + isLoading, + error: deleteError, + } = useRequest( + useCallback(async () => { + await ExecutionEnvironmentsAPI.destroy(id); + history.push(`/execution_environments`); + }, [id, history]) + ); + + const { error, dismissError } = useDismissableError(deleteError); -function ExecutionEnvironmentDetails() { return ( - - -
Execution environments details
- - + + + + + {executionEnvironment.summary_fields.credential && ( + + {executionEnvironment.summary_fields.credential.name} + + } + dataCy="execution-environment-credential" + /> + )} + + + + + + + {i18n._(t`Delete`)} + + + + {error && ( + + )} + ); } -export default ExecutionEnvironmentDetails; +export default withI18n()(ExecutionEnvironmentDetails); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx new file mode 100644 index 0000000000..6e264964d4 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx @@ -0,0 +1,98 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { createMemoryHistory } from 'history'; + +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; +import { ExecutionEnvironmentsAPI } from '../../../api'; + +import ExecutionEnvironmentDetails from './ExecutionEnvironmentDetails'; + +jest.mock('../../../api'); + +const executionEnvironment = { + id: 17, + type: 'execution_environment', + url: '/api/v2/execution_environments/17/', + related: { + created_by: '/api/v2/users/1/', + modified_by: '/api/v2/users/1/', + activity_stream: '/api/v2/execution_environments/17/activity_stream/', + unified_job_templates: + '/api/v2/execution_environments/17/unified_job_templates/', + credential: '/api/v2/credentials/4/', + }, + summary_fields: { + credential: { + id: 4, + name: 'Container Registry', + }, + created_by: { + id: 1, + username: 'admin', + first_name: '', + last_name: '', + }, + modified_by: { + id: 1, + username: 'admin', + first_name: '', + last_name: '', + }, + }, + created: '2020-09-17T20:14:15.408782Z', + modified: '2020-09-17T20:14:15.408802Z', + description: 'Foo', + organization: null, + image: 'https://localhost:90/12345/ma', + managed_by_tower: false, + credential: 4, +}; + +describe('', () => { + let wrapper; + test('should render details properly', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + wrapper.update(); + + expect(wrapper.find('Detail[label="Image"]').prop('value')).toEqual( + executionEnvironment.image + ); + expect(wrapper.find('Detail[label="Description"]').prop('value')).toEqual( + 'Foo' + ); + expect( + wrapper.find('Detail[label="Credential"]').prop('value').props.children + ).toEqual(executionEnvironment.summary_fields.credential.name); + const dates = wrapper.find('UserDateDetail'); + expect(dates).toHaveLength(2); + expect(dates.at(0).prop('date')).toEqual(executionEnvironment.created); + expect(dates.at(1).prop('date')).toEqual(executionEnvironment.modified); + }); + + test('expected api call is made for delete', async () => { + const history = createMemoryHistory({ + initialEntries: ['/execution_environments/42/details'], + }); + await act(async () => { + wrapper = mountWithContexts( + , + { + context: { router: { history } }, + } + ); + }); + await act(async () => { + wrapper.find('DeleteButton').invoke('onConfirm')(); + }); + expect(ExecutionEnvironmentsAPI.destroy).toHaveBeenCalledTimes(1); + expect(history.location.pathname).toBe('/execution_environments'); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx index 60988ab278..a5170ae3bb 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx @@ -17,7 +17,7 @@ function ExecutionEnvironmentFormFields({ i18n }) { <> credentialHelpers.setValue(value)} - value={credentialField.value || null} + value={credentialField.value} /> ); From 3d233faed842005cdaa15270e93d86924e049db7 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 22 Sep 2020 16:40:32 -0400 Subject: [PATCH 027/157] Expose the user capabilities dict for EEs (#8208) --- awx/api/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 46e430a8e3..435f2ad444 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1361,6 +1361,8 @@ class ProjectOptionsSerializer(BaseSerializer): class ExecutionEnvironmentSerializer(BaseSerializer): + show_capabilities = ['edit', 'delete'] + class Meta: model = ExecutionEnvironment fields = ('*', '-name', 'organization', 'image', 'managed_by_tower', 'credential') From 6e6cd51b4de4986cb28a7a88b9b286006fb64ed9 Mon Sep 17 00:00:00 2001 From: Kersom <9053044+nixocio@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:44:08 -0400 Subject: [PATCH 028/157] Update usage of summary_fields for execution environments (#8217) Update usage of summary_fields for execution environments. Also, update unit-tests to cover this change. See: https://github.com/ansible/awx/issues/8216 --- .../PaginatedDataList/ToolbarDeleteButton.jsx | 25 ++++-- .../ExecutionEnviromentList.test.jsx | 77 +++++++++++++++++++ .../shared/ExecutionEnvironmentForm.test.jsx | 2 +- 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/awx/ui_next/src/components/PaginatedDataList/ToolbarDeleteButton.jsx b/awx/ui_next/src/components/PaginatedDataList/ToolbarDeleteButton.jsx index f9afa31179..4874ab9f64 100644 --- a/awx/ui_next/src/components/PaginatedDataList/ToolbarDeleteButton.jsx +++ b/awx/ui_next/src/components/PaginatedDataList/ToolbarDeleteButton.jsx @@ -20,11 +20,11 @@ const WarningMessage = styled(Alert)` margin-top: 10px; `; -const requireNameOrUsername = props => { - const { name, username } = props; - if (!name && !username) { +const requiredField = props => { + const { name, username, image } = props; + if (!name && !username && !image) { return new Error( - `One of 'name' or 'username' is required by ItemToDelete component.` + `One of 'name', 'username' or 'image' is required by ItemToDelete component.` ); } if (name) { @@ -47,13 +47,24 @@ const requireNameOrUsername = props => { 'ItemToDelete' ); } + if (image) { + checkPropTypes( + { + image: string, + }, + { image: props.image }, + 'prop', + 'ItemToDelete' + ); + } return null; }; const ItemToDelete = shape({ id: number.isRequired, - name: requireNameOrUsername, - username: requireNameOrUsername, + name: requiredField, + username: requiredField, + image: requiredField, summary_fields: shape({ user_capabilities: shape({ delete: bool.isRequired, @@ -171,7 +182,7 @@ function ToolbarDeleteButton({
{i18n._(t`This action will delete the following:`)}
{itemsToDelete.map(item => ( - {item.name || item.username} + {item.name || item.username || item.image}
))} diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx index 4371d4c727..475dd1a8b5 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx @@ -20,6 +20,7 @@ const executionEnvironments = { organization: null, credential: null, url: '/api/v2/execution_environments/1/', + summary_fields: { user_capabilities: { edit: true, delete: true } }, }, { id: 2, @@ -27,6 +28,7 @@ const executionEnvironments = { organization: null, credential: null, url: '/api/v2/execution_environments/2/', + summary_fields: { user_capabilities: { edit: false, delete: true } }, }, ], count: 2, @@ -67,6 +69,81 @@ describe('', () => { expect(ExecutionEnvironmentsAPI.readOptions).toBeCalled(); }); + test('should delete item successfully', async () => { + ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); + + await act(async () => { + wrapper = mountWithContexts(); + }); + await waitForElement( + wrapper, + 'ExecutionEnvironmentList', + el => el.length > 0 + ); + + wrapper + .find('input#select-execution-environment-1') + .simulate('change', executionEnvironments.data.results[0]); + wrapper.update(); + + expect( + wrapper.find('input#select-execution-environment-1').prop('checked') + ).toBe(true); + + await act(async () => { + wrapper.find('Button[aria-label="Delete"]').prop('onClick')(); + }); + wrapper.update(); + + await act(async () => { + wrapper.find('Button[aria-label="confirm delete"]').prop('onClick')(); + }); + + expect(ExecutionEnvironmentsAPI.destroy).toBeCalledWith( + executionEnvironments.data.results[0].id + ); + }); + + test('should render deletion error modal', async () => { + ExecutionEnvironmentsAPI.destroy.mockRejectedValue( + new Error({ + response: { + config: { + method: 'DELETE', + url: '/api/v2/execution_environments', + }, + data: 'An error occurred', + }, + }) + ); + ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); + await act(async () => { + wrapper = mountWithContexts(); + }); + waitForElement(wrapper, 'ExecutionEnvironmentList', el => el.length > 0); + + wrapper + .find('input#select-execution-environment-1') + .simulate('change', 'a'); + wrapper.update(); + expect( + wrapper.find('input#select-execution-environment-1').prop('checked') + ).toBe(true); + + await act(async () => + wrapper.find('Button[aria-label="Delete"]').prop('onClick')() + ); + wrapper.update(); + + await act(async () => + wrapper.find('Button[aria-label="confirm delete"]').prop('onClick')() + ); + wrapper.update(); + expect(wrapper.find('ErrorDetail').length).toBe(1); + }); + test('should thrown content error', async () => { ExecutionEnvironmentsAPI.read.mockRejectedValue( new Error({ diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx index 717b264354..98164b6964 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx @@ -62,7 +62,7 @@ describe('', () => { }); test('should display form fields properly', () => { - expect(wrapper.find('FormGroup[label="Image"]').length).toBe(1); + expect(wrapper.find('FormGroup[label="Image name"]').length).toBe(1); expect(wrapper.find('FormGroup[label="Description"]').length).toBe(1); expect(wrapper.find('CredentialLookup').length).toBe(1); }); From 7c6975baec64acf89728b02a2a3e7e0b9fc25382 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 29 Sep 2020 23:17:42 -0400 Subject: [PATCH 029/157] Collections volume permission fix, and container group fix Use same image for both types of container isolation Inventory move fix related to container groups --- awx/main/tasks.py | 15 ++++++++++----- awx/playbooks/project_update.yml | 3 +++ awx/settings/defaults.py | 6 +++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 50230c0e98..1c5b04fff9 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1467,7 +1467,6 @@ class BaseTask(object): 'job_timeout': self.get_instance_timeout(self.instance), 'suppress_ansible_output': True, #**process_isolation_params, - **execution_environment_params, **resource_profiling_params, }, } @@ -1476,6 +1475,10 @@ class BaseTask(object): # We don't want HOME passed through to container groups. # TODO: remove this conditional after everything is containerized params['envvars'].pop('HOME', None) + else: + # TODO: container group jobs will not work with container isolation settings + # but both will run with same settings when worker_in and worker_out are added + params['settings'].update(execution_environment_params) if isinstance(self.instance, AdHocCommand): params['module'] = self.build_module_name(self.instance) @@ -1503,10 +1506,12 @@ class BaseTask(object): module_args = ansible_runner.utils.args2cmdline( params.get('module_args'), ) - shutil.move( - params.pop('inventory'), - os.path.join(private_data_dir, 'inventory') - ) + # TODO on merge: delete if https://github.com/ansible/awx/pull/8185 is merged + if not os.path.exists(os.path.join(private_data_dir, 'inventory')): + shutil.move( + params.pop('inventory'), + os.path.join(private_data_dir, 'inventory') + ) ansible_runner.utils.dump_artifacts(params) isolated_manager_instance = isolated_manager.IsolatedManager( diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 49618909fb..664f189a28 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -204,6 +204,9 @@ ANSIBLE_FORCE_COLOR: false ANSIBLE_COLLECTIONS_PATHS: "{{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_collections" GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no" + # Put the local tmp directory in same volume as collection destination + # otherwise, files cannot be moved accross volumes and will cause error + ANSIBLE_LOCAL_TEMP: "{{projects_root}}/.__awx_cache/{{local_path}}/stage/tmp" when: - "ansible_version.full is version_compare('2.9', '>=')" diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 849d6220bc..0a2a7043d6 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -59,13 +59,13 @@ DATABASES = { } } +AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/ansible-runner:devel' + AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = 'default' -AWX_CONTAINER_GROUP_DEFAULT_IMAGE = 'ansible/ansible-runner' - -AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/ansible-runner:devel' +AWX_CONTAINER_GROUP_DEFAULT_IMAGE = AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE # Internationalization # https://docs.djangoproject.com/en/dev/topics/i18n/ From 87b13ead12d0da31985d70da2b7929ecfffecffb Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 13:40:51 -0400 Subject: [PATCH 030/157] REVERT ME: Install ansible/devel for now --- .../roles/dockerfile/templates/Dockerfile.j2 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index e056b10fb1..6fa2c08cd4 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -6,10 +6,13 @@ # Locations - set globally to be used across stages ARG COLLECTION_BASE="/var/lib/awx/vendor/awx_ansible_collections" +ARG ANSIBLE_BRANCH=devel # Build container FROM centos:8 as builder +ARG ANSIBLE_BRANCH + ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 @@ -22,7 +25,7 @@ RUN dnf -y update && \ dnf -y install epel-release 'dnf-command(config-manager)' && \ dnf module -y enable 'postgresql:12' && \ dnf config-manager --set-enabled powertools && \ - dnf -y install ansible \ + dnf -y install \ gcc \ gcc-c++ \ git-core \ @@ -46,7 +49,8 @@ RUN dnf -y update && \ xmlsec1-devel \ xmlsec1-openssl-devel -RUN python3 -m ensurepip && pip3 install "virtualenv < 20" +RUN python3 -m ensurepip && pip3 install "virtualenv < 20" && \ + pip3 install --no-cache-dir https://github.com/ansible/ansible/archive/${ANSIBLE_BRANCH}.tar.gz # Install & build requirements ADD Makefile /tmp/Makefile @@ -81,6 +85,7 @@ RUN make sdist && \ FROM centos:8 ARG COLLECTION_BASE +ARG ANSIBLE_BRANCH ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en @@ -95,7 +100,6 @@ RUN dnf -y update && \ dnf module -y enable 'postgresql:12' && \ dnf config-manager --set-enabled powertools && \ dnf -y install acl \ - ansible \ bubblewrap \ git-core \ git-lfs \ @@ -132,7 +136,8 @@ RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master RUN curl -L -o /usr/bin/tini https://github.com/krallin/tini/releases/download/v0.19.0/tini-{{ tini_architecture | default('amd64') }} && \ chmod +x /usr/bin/tini -RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if build_dev|bool %}flake8{% endif %} +RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if build_dev|bool %}flake8{% endif %} && \ + pip3 install --no-cache-dir https://github.com/ansible/ansible/archive/${ANSIBLE_BRANCH}.tar.gz RUN rm -rf /root/.cache && rm -rf /tmp/* From efb25b7b9e75040380d954bb5faa1ce69066efcf Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 13:46:16 -0400 Subject: [PATCH 031/157] Use WIP version of collections_requirements.yml Pulled from ansible-builder/test/data/awx --- requirements/collections_requirements.yml | 25 +++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 2a8f46fa24..f750aafce4 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -1,12 +1,21 @@ --- +# from https://github.com/ansible/awx/blob/devel/ +# requirements/collections_requirements.yml collections: - name: awx.awx - - name: azure.azcollection - - name: amazon.aws - - name: theforeman.foreman - - name: google.cloud - - name: openstack.cloud - - name: community.vmware + - name: azure.azcollection # PR 220 is in 1.1.0 + - name: amazon.aws # PR 125 is in 1.1.0 + - name: theforeman.foreman # has requirements.txt (which -r to another file) + - name: google.cloud # has requirements.txt, mainly for google-auth + # forked from opendev.org + - name: https://github.com/AlanCoding/ansible-collections-openstack.git + version: ee_req_install + type: git + - name: community.vmware # has requirements.txt, but may add pyvcloud - name: ovirt.ovirt - - name: community.kubernetes # required for isolated management playbooks - - name: ansible.posix # required for isolated management playbooks + - name: https://github.com/ansible-collections/community.kubernetes.git + type: git + # adds openshift python lib + # needs kubectl for yum / dnf / apt-get + # needs to install snap, then use snap to install helm + - name: ansible.posix From 46f5cb6b7af8e9f1b0fb84f0de407aa33d7f74ba Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 13:47:19 -0400 Subject: [PATCH 032/157] Install receptorctl in awx venv --- requirements/requirements_git.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index 04eac859a4..74f1fb4a9e 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,2 +1,3 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner +git+https://github.com/project-receptor/receptor.git@#egg=receptorctl&subdirectory=receptorctl From 490f719fd922f0deddcc50b9d77bb896035d19c7 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 13:49:05 -0400 Subject: [PATCH 033/157] Add new ee container --- .../roles/sources/templates/docker-compose.yml.j2 | 14 ++++++++++++-- tools/docker-compose/receptor.cfg | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tools/docker-compose/receptor.cfg diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 754bc803f9..ea330d66f8 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -33,6 +33,7 @@ services: - "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py" - "../../docker-compose/_sources/SECRET_KEY:/etc/tower/SECRET_KEY" - "redis_socket:/var/run/redis/:rw" + - "receptor:/var/run/receptor/" - "/sys/fs/cgroup:/sys/fs/cgroup" privileged: true tty: true @@ -42,8 +43,16 @@ services: # build: # context: ./docker-compose # dockerfile: Dockerfile-logstash - - # Postgres Database Container + ee: + image: quay.io/awx/ee + user: ${CURRENT_UID} + volumes: + - "./docker-compose/receptor.cfg:/receptor.cfg" + - "receptor:/var/run/receptor/" + command: + - receptor + - --config + - /receptor.cfg postgres: image: postgres:12 container_name: tools_postgres_1 @@ -66,3 +75,4 @@ services: volumes: awx_db: redis_socket: + receptor: diff --git a/tools/docker-compose/receptor.cfg b/tools/docker-compose/receptor.cfg new file mode 100644 index 0000000000..137d15cdf6 --- /dev/null +++ b/tools/docker-compose/receptor.cfg @@ -0,0 +1,15 @@ +--- +- log-level: debug + +- control-service: + service: control + filename: /var/run/receptor/receptor.sock + +- tcp-listener: + port: 2222 + +- work-command: + worktype: worker + command: ansible-runner + params: worker + allowruntimeparams: true From 82a641e173e3e6e2db8e9c32eb45483848d6b0d1 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 13:49:30 -0400 Subject: [PATCH 034/157] Add AWX EE definition --- execution-environment.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 execution-environment.yml diff --git a/execution-environment.yml b/execution-environment.yml new file mode 100644 index 0000000000..79efbcbdb8 --- /dev/null +++ b/execution-environment.yml @@ -0,0 +1,10 @@ +--- +version: 1 +dependencies: + galaxy: requirements/collections_requirements.yml +additional_build_steps: + prepend: + - RUN pip3 install --upgrade pip setuptools + append: + - COPY --from=quay.io/shanemcd/receptor /usr/bin/receptor /usr/bin/receptor + - RUN mkdir -m 0770 /var/run/receptor From fd9373a9ec1f4cfff59b5bcd0fc661e11e897734 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 16:46:31 -0400 Subject: [PATCH 035/157] Use official receptor image --- execution-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution-environment.yml b/execution-environment.yml index 79efbcbdb8..ad1005cc2f 100644 --- a/execution-environment.yml +++ b/execution-environment.yml @@ -6,5 +6,5 @@ additional_build_steps: prepend: - RUN pip3 install --upgrade pip setuptools append: - - COPY --from=quay.io/shanemcd/receptor /usr/bin/receptor /usr/bin/receptor + - COPY --from=quay.io/project-receptor/receptor /usr/bin/receptor /usr/bin/receptor - RUN mkdir -m 0770 /var/run/receptor From 5c2b2dea0c280e9a9dc1843f413f3b3c4fa32e65 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 8 Oct 2020 18:33:43 -0400 Subject: [PATCH 036/157] REVERT ME: Install community.general in image This is needed for the wait_fors in the launch scripts to work --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 6fa2c08cd4..ecdcb32733 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -139,6 +139,9 @@ RUN curl -L -o /usr/bin/tini https://github.com/krallin/tini/releases/download/v RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if build_dev|bool %}flake8{% endif %} && \ pip3 install --no-cache-dir https://github.com/ansible/ansible/archive/${ANSIBLE_BRANCH}.tar.gz +# TODO: Remove this once launch script removes need for postres modules +RUN ansible-galaxy collection install --collections-path /usr/share/ansible/collections community.general + RUN rm -rf /root/.cache && rm -rf /tmp/* # Install Receptor From f554f45288a23acc25529c8542e4312ac55c4719 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 9 Oct 2020 19:16:24 -0400 Subject: [PATCH 037/157] Add license for receptor --- docs/licenses/receptor.txt | 168 +++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 docs/licenses/receptor.txt diff --git a/docs/licenses/receptor.txt b/docs/licenses/receptor.txt new file mode 100644 index 0000000000..bb0a7c7983 --- /dev/null +++ b/docs/licenses/receptor.txt @@ -0,0 +1,168 @@ +Apache License +============== + +_Version 2.0, January 2004_ +_<>_ + +### Terms and Conditions for use, reproduction, and distribution + +#### 1. Definitions + +“License” shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +“Licensor” shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +“Legal Entity” shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, “control” means **(i)** the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or **(ii)** ownership of fifty percent (50%) or more of the +outstanding shares, or **(iii)** beneficial ownership of such entity. + +“You” (or “Your”) shall mean an individual or Legal Entity exercising +permissions granted by this License. + +“Source” form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +“Object” form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +“Work” shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +“Derivative Works” shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +“Contribution” shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +“submitted” means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as “Not a Contribution.” + +“Contributor” shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +#### 2. Grant of Copyright License + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +#### 3. Grant of Patent License + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +#### 4. Redistribution + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +* **(a)** You must give any other recipients of the Work or Derivative Works a copy of +this License; and +* **(b)** You must cause any modified files to carry prominent notices stating that You +changed the files; and +* **(c)** You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +* **(d)** If the Work includes a “NOTICE” text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. + +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +#### 5. Submission of Contributions + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +#### 6. Trademarks + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +#### 7. Disclaimer of Warranty + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an “AS IS” BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +#### 8. Limitation of Liability + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +#### 9. Accepting Warranty or Additional Liability + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. From 966bb6fc74979c75d3c90c3978d6b8c93e5545af Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 9 Oct 2020 19:17:55 -0400 Subject: [PATCH 038/157] Back to green --- awx/main/models/credential/injectors.py | 4 +- awx/main/models/inventory.py | 2 +- awx/main/tasks.py | 3 + .../test_inventory_source_injectors.py | 5 +- awx/main/tests/unit/test_tasks.py | 65 +++++++------------ 5 files changed, 34 insertions(+), 45 deletions(-) diff --git a/awx/main/models/credential/injectors.py b/awx/main/models/credential/injectors.py index 90615f2d66..4d7ef26054 100644 --- a/awx/main/models/credential/injectors.py +++ b/awx/main/models/credential/injectors.py @@ -35,8 +35,8 @@ def gce(cred, env, private_data_dir): json.dump(json_cred, f, indent=2) f.close() os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - env['GCE_CREDENTIALS_FILE_PATH'] = path - env['GCP_SERVICE_ACCOUNT_FILE'] = path + env['GCE_CREDENTIALS_FILE_PATH'] = os.path.join('/runner', os.path.basename(path)) + env['GCP_SERVICE_ACCOUNT_FILE'] = os.path.join('/runner', os.path.basename(path)) # Handle env variables for new module types. # This includes gcp_compute inventory plugin and diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 5305e6e532..94ddcc4e90 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1544,7 +1544,7 @@ class openstack(PluginFileInjector): env = super(openstack, self).get_plugin_env(inventory_update, private_data_dir, private_data_files) credential = inventory_update.get_cloud_credential() cred_data = private_data_files['credentials'] - env['OS_CLIENT_CONFIG_FILE'] = cred_data[credential] + env['OS_CLIENT_CONFIG_FILE'] = os.path.join('/runner', os.path.basename(cred_data[credential])) return env diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 1c5b04fff9..bd4c5bd81c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1102,6 +1102,9 @@ class BaseTask(object): # Also set environment variables configured in AWX_TASK_ENV setting. for key, value in settings.AWX_TASK_ENV.items(): env[key] = str(value) + + env['AWX_PRIVATE_DATA_DIR'] = private_data_dir + return env def should_use_resource_profiling(self, job): diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index fc28c92294..84660c79e3 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -110,7 +110,8 @@ def read_content(private_data_dir, raw_env, inventory_update): continue # Ansible runner abs_file_path = os.path.join(private_data_dir, filename) file_aliases[abs_file_path] = filename - if abs_file_path in inverse_env: + runner_path = os.path.join('/runner', os.path.basename(abs_file_path)) + if runner_path in inverse_env: referenced_paths.add(abs_file_path) alias = 'file_reference' for i in range(10): @@ -121,7 +122,7 @@ def read_content(private_data_dir, raw_env, inventory_update): raise RuntimeError('Test not able to cope with >10 references by env vars. ' 'Something probably went very wrong.') file_aliases[abs_file_path] = alias - for env_key in inverse_env[abs_file_path]: + for env_key in inverse_env[runner_path]: env[env_key] = '{{{{ {} }}}}'.format(alias) try: with open(abs_file_path, 'r') as f: diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 053745cc64..6b6811c86f 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -347,11 +347,12 @@ def pytest_generate_tests(metafunc): ) -def parse_extra_vars(args): +def parse_extra_vars(args, private_data_dir): extra_vars = {} for chunk in args: - if chunk.startswith('@/tmp/'): - with open(chunk.strip('@'), 'r') as f: + if chunk.startswith('@/runner/'): + local_path = os.path.join(private_data_dir, os.path.basename(chunk.strip('@'))) + with open(local_path, 'r') as f: extra_vars.update(yaml.load(f, Loader=SafeLoader)) return extra_vars @@ -597,7 +598,7 @@ class TestGenericRun(): assert resource_profiling_params['resource_profiling_cpu_poll_interval'] == '0.25' assert resource_profiling_params['resource_profiling_memory_poll_interval'] == '0.25' assert resource_profiling_params['resource_profiling_pid_poll_interval'] == '0.25' - assert resource_profiling_params['resource_profiling_results_dir'] == '/fake_private_data_dir/artifacts/playbook_profiling' + assert resource_profiling_params['resource_profiling_results_dir'] == '/runner/artifacts/playbook_profiling' @pytest.mark.parametrize("scenario, profiling_enabled", [ @@ -656,30 +657,6 @@ class TestGenericRun(): env = task.build_env(job, private_data_dir) assert env['FOO'] == 'BAR' - def test_valid_custom_virtualenv(self, patch_Job, private_data_dir): - job = Job(project=Project(), inventory=Inventory()) - - with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as tempdir: - job.project.custom_virtualenv = tempdir - os.makedirs(os.path.join(tempdir, 'lib')) - os.makedirs(os.path.join(tempdir, 'bin', 'activate')) - - task = tasks.RunJob() - env = task.build_env(job, private_data_dir) - - assert env['PATH'].startswith(os.path.join(tempdir, 'bin')) - assert env['VIRTUAL_ENV'] == tempdir - - def test_invalid_custom_virtualenv(self, patch_Job, private_data_dir): - job = Job(project=Project(), inventory=Inventory()) - job.project.custom_virtualenv = '/var/lib/awx/venv/missing' - task = tasks.RunJob() - - with pytest.raises(tasks.InvalidVirtualenvError) as e: - task.build_env(job, private_data_dir) - - assert 'Invalid virtual environment selected: /var/lib/awx/venv/missing' == str(e.value) - class TestAdhocRun(TestJobExecution): @@ -1203,7 +1180,9 @@ class TestJobCredentials(TestJobExecution): credential.credential_type.inject_credential( credential, env, safe_env, [], private_data_dir ) - json_data = json.load(open(env['GCE_CREDENTIALS_FILE_PATH'], 'rb')) + runner_path = env['GCE_CREDENTIALS_FILE_PATH'] + local_path = os.path.join(private_data_dir, os.path.basename(runner_path)) + json_data = json.load(open(local_path, 'rb')) assert json_data['type'] == 'service_account' assert json_data['private_key'] == self.EXAMPLE_PRIVATE_KEY assert json_data['client_email'] == 'bob' @@ -1344,7 +1323,7 @@ class TestJobCredentials(TestJobExecution): ) config = configparser.ConfigParser() - config.read(env['OVIRT_INI_PATH']) + config.read(os.path.join(private_data_dir, os.path.basename(env['OVIRT_INI_PATH']))) assert config.get('ovirt', 'ovirt_url') == 'some-ovirt-host.example.org' assert config.get('ovirt', 'ovirt_username') == 'bob' assert config.get('ovirt', 'ovirt_password') == 'some-pass' @@ -1577,7 +1556,7 @@ class TestJobCredentials(TestJobExecution): credential.credential_type.inject_credential( credential, {}, {}, args, private_data_dir ) - extra_vars = parse_extra_vars(args) + extra_vars = parse_extra_vars(args, private_data_dir) assert extra_vars["api_token"] == "ABC123" assert hasattr(extra_vars["api_token"], '__UNSAFE__') @@ -1612,7 +1591,7 @@ class TestJobCredentials(TestJobExecution): credential.credential_type.inject_credential( credential, {}, {}, args, private_data_dir ) - extra_vars = parse_extra_vars(args) + extra_vars = parse_extra_vars(args, private_data_dir) assert extra_vars["turbo_button"] == "True" return ['successful', 0] @@ -1647,7 +1626,7 @@ class TestJobCredentials(TestJobExecution): credential.credential_type.inject_credential( credential, {}, {}, args, private_data_dir ) - extra_vars = parse_extra_vars(args) + extra_vars = parse_extra_vars(args, private_data_dir) assert extra_vars["turbo_button"] == "FAST!" @@ -1687,7 +1666,7 @@ class TestJobCredentials(TestJobExecution): credential, {}, {}, args, private_data_dir ) - extra_vars = parse_extra_vars(args) + extra_vars = parse_extra_vars(args, private_data_dir) assert extra_vars["password"] == "SUPER-SECRET-123" def test_custom_environment_injectors_with_file(self, private_data_dir): @@ -1722,7 +1701,8 @@ class TestJobCredentials(TestJobExecution): credential, env, {}, [], private_data_dir ) - assert open(env['MY_CLOUD_INI_FILE'], 'r').read() == '[mycloud]\nABC123' + path = os.path.join(private_data_dir, os.path.basename(env['MY_CLOUD_INI_FILE'])) + assert open(path, 'r').read() == '[mycloud]\nABC123' def test_custom_environment_injectors_with_unicode_content(self, private_data_dir): value = 'Iñtërnâtiônàlizætiøn' @@ -1746,7 +1726,8 @@ class TestJobCredentials(TestJobExecution): credential, env, {}, [], private_data_dir ) - assert open(env['MY_CLOUD_INI_FILE'], 'r').read() == value + path = os.path.join(private_data_dir, os.path.basename(env['MY_CLOUD_INI_FILE'])) + assert open(path, 'r').read() == value def test_custom_environment_injectors_with_files(self, private_data_dir): some_cloud = CredentialType( @@ -1786,8 +1767,10 @@ class TestJobCredentials(TestJobExecution): credential, env, {}, [], private_data_dir ) - assert open(env['MY_CERT_INI_FILE'], 'r').read() == '[mycert]\nCERT123' - assert open(env['MY_KEY_INI_FILE'], 'r').read() == '[mykey]\nKEY123' + cert_path = os.path.join(private_data_dir, os.path.basename(env['MY_CERT_INI_FILE'])) + key_path = os.path.join(private_data_dir, os.path.basename(env['MY_KEY_INI_FILE'])) + assert open(cert_path, 'r').read() == '[mycert]\nCERT123' + assert open(key_path, 'r').read() == '[mykey]\nKEY123' def test_multi_cloud(self, private_data_dir): gce = CredentialType.defaults['gce']() @@ -1826,7 +1809,8 @@ class TestJobCredentials(TestJobExecution): assert env['AZURE_AD_USER'] == 'bob' assert env['AZURE_PASSWORD'] == 'secret' - json_data = json.load(open(env['GCE_CREDENTIALS_FILE_PATH'], 'rb')) + path = os.path.join(private_data_dir, os.path.basename(env['GCE_CREDENTIALS_FILE_PATH'])) + json_data = json.load(open(path, 'rb')) assert json_data['type'] == 'service_account' assert json_data['private_key'] == self.EXAMPLE_PRIVATE_KEY assert json_data['client_email'] == 'bob' @@ -2307,7 +2291,8 @@ class TestInventoryUpdateCredentials(TestJobExecution): private_data_files = task.build_private_data_files(inventory_update, private_data_dir) env = task.build_env(inventory_update, private_data_dir, False, private_data_files) - shade_config = open(env['OS_CLIENT_CONFIG_FILE'], 'r').read() + path = os.path.join(private_data_dir, os.path.basename(env['OS_CLIENT_CONFIG_FILE'])) + shade_config = open(path, 'r').read() assert '\n'.join([ 'clouds:', ' devstack:', From 1f4a45a6987c59892af00f4b9d47244431ac6ae7 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 10 Oct 2020 11:01:40 -0400 Subject: [PATCH 039/157] Remove "pull" field from EE mixin I think this should go on the EE definition itself --- awx/api/serializers.py | 4 ++-- awx/main/migrations/0124_execution_environments.py | 10 ---------- awx/main/models/mixins.py | 1 - 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 435f2ad444..dcf403c47e 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -649,7 +649,7 @@ class UnifiedJobTemplateSerializer(BaseSerializer): class Meta: model = UnifiedJobTemplate fields = ('*', 'last_job_run', 'last_job_failed', - 'next_job_run', 'status', 'execution_environment', 'pull') + 'next_job_run', 'status', 'execution_environment') def get_related(self, obj): res = super(UnifiedJobTemplateSerializer, self).get_related(obj) @@ -716,7 +716,7 @@ class UnifiedJobSerializer(BaseSerializer): class Meta: model = UnifiedJob fields = ('*', 'unified_job_template', 'launch_type', 'status', - 'execution_environment', 'pull', + 'execution_environment', 'failed', 'started', 'finished', 'canceled_on', 'elapsed', 'job_args', 'job_cwd', 'job_env', 'job_explanation', 'execution_node', 'controller_node', diff --git a/awx/main/migrations/0124_execution_environments.py b/awx/main/migrations/0124_execution_environments.py index 982b1b21e8..18aad9a174 100644 --- a/awx/main/migrations/0124_execution_environments.py +++ b/awx/main/migrations/0124_execution_environments.py @@ -16,16 +16,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='unifiedjob', - name='pull', - field=models.BooleanField(default=True), - ), - migrations.AddField( - model_name='unifiedjobtemplate', - name='pull', - field=models.BooleanField(default=True), - ), migrations.CreateModel( name='ExecutionEnvironment', fields=[ diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 54c8a000a4..1cd1366a92 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -454,7 +454,6 @@ class ExecutionEnvironmentMixin(models.Model): related_name='%(class)ss', help_text=_('The container image to be used for execution.'), ) - pull = models.BooleanField(default=True) class CustomVirtualEnvMixin(models.Model): From ee1d322336e233e4f320444166a560c044eef1f6 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 10 Oct 2020 11:02:16 -0400 Subject: [PATCH 040/157] WIP: Module for EEs --- .../plugins/modules/tower_ad_hoc_command.py | 5 ++ .../modules/tower_execution_environment.py | 86 +++++++++++++++++++ .../plugins/modules/tower_inventory_source.py | 4 + .../plugins/modules/tower_job_template.py | 4 + .../plugins/modules/tower_organization.py | 4 + .../plugins/modules/tower_project.py | 4 + .../modules/tower_workflow_job_template.py | 4 + awx_collection/test/awx/test_completeness.py | 1 + 8 files changed, 112 insertions(+) create mode 100644 awx_collection/plugins/modules/tower_execution_environment.py diff --git a/awx_collection/plugins/modules/tower_ad_hoc_command.py b/awx_collection/plugins/modules/tower_ad_hoc_command.py index 00f16d9f13..2d099b2b1d 100644 --- a/awx_collection/plugins/modules/tower_ad_hoc_command.py +++ b/awx_collection/plugins/modules/tower_ad_hoc_command.py @@ -28,6 +28,11 @@ options: - Job_type to use for the ad hoc command. type: str choices: [ 'run', 'check' ] + execution_environment: + description: + - Execution Environment to use for the ad hoc command. + required: False + type: str inventory: description: - Inventory to use for the ad hoc command. diff --git a/awx_collection/plugins/modules/tower_execution_environment.py b/awx_collection/plugins/modules/tower_execution_environment.py new file mode 100644 index 0000000000..978d23298c --- /dev/null +++ b/awx_collection/plugins/modules/tower_execution_environment.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# coding: utf-8 -*- + +# (c) 2020, Shane McDonald +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + + +DOCUMENTATION = ''' +--- +module: tower_execution_environment +author: "Shane McDonald" +short_description: create, update, or destroy Execution Environments in Ansible Tower. +description: + - Create, update, or destroy Execution Environments in Ansible Tower. See + U(https://www.ansible.com/tower) for an overview. +options: + image: + description: + - The fully qualified name of the container image + required: True + type: str + state: + description: + - Desired state of the resource. + choices: ["present", "absent"] + default: "present" + type: str + credential: + description: + - Name of the credential to use for the job template. + - Deprecated, use 'credentials'. + type: str + description: + description: + - Description to use for the job template. + type: str + organization: + description: + - TODO + type: str +extends_documentation_fragment: awx.awx.auth +''' + + +EXAMPLES = ''' +- name: Add EE to Tower + tower_execution_environment: + image: quay.io/awx/ee +''' + + +from ..module_utils.tower_api import TowerAPIModule +import json + + +def main(): + # Any additional arguments that are not fields of the item can be added here + argument_spec = dict( + image=dict(required=True), + ) + + # Create a module for ourselves + module = TowerAPIModule(argument_spec=argument_spec) + + # Extract our parameters + image = module.params.get('image') + state = module.params.get('state') + + existing_item = module.get_one('execution_environments', name_or_id=image) + + if state == 'absent': + module.delete_if_needed(image) + + module.create_or_update_if_needed(existing_item, image, endpoint='execution_environments', item_type='execution_environment') + + +if __name__ == '__main__': + main() diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 5945d411d8..ceb0e8b5a6 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -73,6 +73,10 @@ options: description: - Credential to use for the source. type: str + execution_environment: + description: + - Execution Environment to use for the source. + type: str overwrite: description: - Delete child groups and hosts not found in source. diff --git a/awx_collection/plugins/modules/tower_job_template.py b/awx_collection/plugins/modules/tower_job_template.py index 787c145a20..b2b1530d6f 100644 --- a/awx_collection/plugins/modules/tower_job_template.py +++ b/awx_collection/plugins/modules/tower_job_template.py @@ -60,6 +60,10 @@ options: description: - Path to the playbook to use for the job template within the project provided. type: str + execution_environment: + description: + - Execution Environment to use for the JT. + type: str credential: description: - Name of the credential to use for the job template. diff --git a/awx_collection/plugins/modules/tower_organization.py b/awx_collection/plugins/modules/tower_organization.py index 0402056bbf..7d88d2a421 100644 --- a/awx_collection/plugins/modules/tower_organization.py +++ b/awx_collection/plugins/modules/tower_organization.py @@ -36,6 +36,10 @@ options: - Local absolute file path containing a custom Python virtualenv to use. type: str default: '' + default_environment: + description: + - Default Execution Environment to use for the Organization. + type: str max_hosts: description: - The max hosts allowed in this organizations diff --git a/awx_collection/plugins/modules/tower_project.py b/awx_collection/plugins/modules/tower_project.py index 76cef63f10..f6ab7d144c 100644 --- a/awx_collection/plugins/modules/tower_project.py +++ b/awx_collection/plugins/modules/tower_project.py @@ -31,6 +31,10 @@ options: description: - Description to use for the project. type: str + execution_environment: + description: + - Execution Environment to use for the project. + type: str scm_type: description: - Type of SCM resource. diff --git a/awx_collection/plugins/modules/tower_workflow_job_template.py b/awx_collection/plugins/modules/tower_workflow_job_template.py index 7836b42cc4..54b6695b03 100644 --- a/awx_collection/plugins/modules/tower_workflow_job_template.py +++ b/awx_collection/plugins/modules/tower_workflow_job_template.py @@ -40,6 +40,10 @@ options: description: - Variables which will be made available to jobs ran inside the workflow. type: dict + execution_environment: + description: + - Execution Environment to use for the WFJT. + type: str organization: description: - Organization the workflow job template exists in. diff --git a/awx_collection/test/awx/test_completeness.py b/awx_collection/test/awx/test_completeness.py index 467ee9357b..7ff6e22a31 100644 --- a/awx_collection/test/awx/test_completeness.py +++ b/awx_collection/test/awx/test_completeness.py @@ -138,6 +138,7 @@ def determine_state(module_id, endpoint, module, parameter, api_option, module_o if not api_option and module_option and module_option.get('type', 'str') == 'list': return "OK, Field appears to be relation" # TODO, at some point try and check the object model to confirm its actually a relation + return cause_error('Failed, option mismatch') # We made it through all of the checks so we are ok From ecaa66c13be4b0f2ad865fabd485798a4f5560cc Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 10 Oct 2020 11:09:40 -0400 Subject: [PATCH 041/157] Fix linter --- awx/main/tests/unit/test_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 6b6811c86f..2fff642eef 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -6,7 +6,6 @@ import os import shutil import tempfile -from backports.tempfile import TemporaryDirectory import fcntl from unittest import mock import pytest From 9786dc08d3f1bde17b5ebdb633ea96771e8e259a Mon Sep 17 00:00:00 2001 From: nixocio Date: Fri, 2 Oct 2020 17:27:12 -0400 Subject: [PATCH 042/157] Add organization as part of creating/editing an execution environments Add organization as part of creating/editing an execution environments If one is a `system admin` the Organization is an optional field. Not providing an Organization makes the execution environment globally available. If one is a `org admin` the Organization is a required field. See: https://github.com/ansible/awx/issues/7887 --- .../components/Lookup/OrganizationLookup.jsx | 2 + .../ExecutionEnvironmentAdd.jsx | 23 ++++--- .../ExecutionEnvironmentAdd.test.jsx | 7 +- .../ExecutionEnvironmentEdit.jsx | 19 +++-- .../ExecutionEnvironmentEdit.test.jsx | 7 ++ .../shared/ExecutionEnvironmentForm.jsx | 69 +++++++++++++++---- .../shared/ExecutionEnvironmentForm.test.jsx | 21 +++++- 7 files changed, 118 insertions(+), 30 deletions(-) diff --git a/awx/ui_next/src/components/Lookup/OrganizationLookup.jsx b/awx/ui_next/src/components/Lookup/OrganizationLookup.jsx index 3fb443426e..8252c9035c 100644 --- a/awx/ui_next/src/components/Lookup/OrganizationLookup.jsx +++ b/awx/ui_next/src/components/Lookup/OrganizationLookup.jsx @@ -30,6 +30,7 @@ function OrganizationLookup({ history, autoPopulate, isDisabled, + helperText, }) { const autoPopulateLookup = useAutoPopulateLookup(onChange); @@ -79,6 +80,7 @@ function OrganizationLookup({ isRequired={required} validated={isValid ? 'default' : 'error'} label={i18n._(t`Organization`)} + helperText={helperText} > - + + {({ me }) => ( + + )} + diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx index 781501b19e..5396746223 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx @@ -8,6 +8,11 @@ import ExecutionEnvironmentAdd from './ExecutionEnvironmentAdd'; jest.mock('../../../api'); +const mockMe = { + is_superuser: true, + is_system_auditor: false, +}; + const executionEnvironmentData = { credential: 4, description: 'A simple EE', @@ -29,7 +34,7 @@ describe('', () => { initialEntries: ['/execution_environments'], }); await act(async () => { - wrapper = mountWithContexts(, { + wrapper = mountWithContexts(, { context: { router: { history } }, }); }); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx index 6d8cbc9520..ea4943b2da 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.jsx @@ -4,6 +4,7 @@ import { useHistory } from 'react-router-dom'; import { CardBody } from '../../../components/Card'; import { ExecutionEnvironmentsAPI } from '../../../api'; import ExecutionEnvironmentForm from '../shared/ExecutionEnvironmentForm'; +import { Config } from '../../../contexts/Config'; function ExecutionEnvironmentEdit({ executionEnvironment }) { const history = useHistory(); @@ -15,6 +16,7 @@ function ExecutionEnvironmentEdit({ executionEnvironment }) { await ExecutionEnvironmentsAPI.update(executionEnvironment.id, { ...values, credential: values.credential ? values.credential.id : null, + organization: values.organization ? values.organization.id : null, }); history.push(detailsUrl); } catch (error) { @@ -27,12 +29,17 @@ function ExecutionEnvironmentEdit({ executionEnvironment }) { }; return ( - + + {({ me }) => ( + + )} + ); } diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx index 2d4f916aba..94eff7616f 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentEdit/ExecutionEnvironmentEdit.test.jsx @@ -9,6 +9,11 @@ import ExecutionEnvironmentEdit from './ExecutionEnvironmentEdit'; jest.mock('../../../api'); +const mockMe = { + is_superuser: true, + is_system_auditor: false, +}; + const executionEnvironmentData = { id: 42, credential: { id: 4 }, @@ -31,6 +36,7 @@ describe('', () => { wrapper = mountWithContexts( , { context: { router: { history } }, @@ -53,6 +59,7 @@ describe('', () => { expect(ExecutionEnvironmentsAPI.update).toHaveBeenCalledWith(42, { ...updateExecutionEnvironmentData, credential: null, + organization: null, }); }); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx index a5170ae3bb..5d7a16d217 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx @@ -1,18 +1,42 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { func, shape } from 'prop-types'; -import { Formik, useField } from 'formik'; +import { Formik, useField, useFormikContext } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; - import { Form } from '@patternfly/react-core'; -import FormField, { FormSubmitError } from '../../../components/FormField'; -import FormActionGroup from '../../../components/FormActionGroup'; -import CredentialLookup from '../../../components/Lookup/CredentialLookup'; -import { url } from '../../../util/validators'; -import { FormColumnLayout } from '../../../components/FormLayout'; -function ExecutionEnvironmentFormFields({ i18n }) { - const [credentialField, , credentialHelpers] = useField('credential'); +import CredentialLookup from '../../../components/Lookup/CredentialLookup'; +import FormActionGroup from '../../../components/FormActionGroup'; +import FormField, { FormSubmitError } from '../../../components/FormField'; +import { FormColumnLayout } from '../../../components/FormLayout'; +import { OrganizationLookup } from '../../../components/Lookup'; +import { required, url } from '../../../util/validators'; + +function ExecutionEnvironmentFormFields({ i18n, me, executionEnvironment }) { + const [credentialField] = useField('credential'); + const [organizationField, organizationMeta, organizationHelpers] = useField({ + name: 'organization', + validate: + !me?.is_superuser && + required(i18n._(t`Select a value for this field`), i18n), + }); + + const { setFieldValue } = useFormikContext(); + + const onCredentialChange = useCallback( + value => { + setFieldValue('credential', value); + }, + [setFieldValue] + ); + + const onOrganizationChange = useCallback( + value => { + setFieldValue('organization', value); + }, + [setFieldValue] + ); + return ( <> + organizationHelpers.setTouched()} + onChange={onOrganizationChange} + value={organizationField.value} + required={!me.is_superuser} + helperText={ + me?.is_superuser + ? i18n._( + t`Leave this field blank to make the execution environment globally available.` + ) + : null + } + autoPopulate={!me?.is_superuser ? !executionEnvironment?.id : null} + /> + credentialHelpers.setValue(value)} + onChange={onCredentialChange} value={credentialField.value} /> @@ -46,19 +87,21 @@ function ExecutionEnvironmentForm({ onSubmit, onCancel, submitError, + me, ...rest }) { const initialValues = { image: executionEnvironment.image || '', description: executionEnvironment.description || '', - credential: executionEnvironment?.summary_fields?.credential || null, + credential: executionEnvironment.summary_fields?.credential || null, + organization: executionEnvironment.summary_fields?.organization || null, }; return ( onSubmit(values)}> {formik => (
- + {submitError && } ', () => { onCancel={onCancel} onSubmit={onSubmit} executionEnvironment={executionEnvironment} + me={mockMe} /> ); }); @@ -75,8 +81,8 @@ describe('', () => { expect(onSubmit).toHaveBeenCalledTimes(1); }); - test('should update form values', () => { - act(() => { + test('should update form values', async () => { + await act(async () => { wrapper.find('input#execution-environment-image').simulate('change', { target: { value: 'https://registry.com/image/container2', @@ -93,8 +99,19 @@ describe('', () => { id: 99, name: 'credential', }); + + wrapper.find('OrganizationLookup').invoke('onBlur')(); + wrapper.find('OrganizationLookup').invoke('onChange')({ + id: 3, + name: 'organization', + }); }); + wrapper.update(); + expect(wrapper.find('OrganizationLookup').prop('value')).toEqual({ + id: 3, + name: 'organization', + }); expect( wrapper.find('input#execution-environment-image').prop('value') ).toEqual('https://registry.com/image/container2'); From 6ff1424e8c924709b40ee30612eb8e0403164355 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 30 Nov 2020 18:53:15 -0500 Subject: [PATCH 043/157] Fix tests after rebasing in inventory update refactor --- awx/main/tests/functional/test_inventory_source_injectors.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 84660c79e3..bb26b7c029 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -215,9 +215,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential f"'{inventory_filename}' file not found in inventory update runtime files {content.keys()}" env.pop('ANSIBLE_COLLECTIONS_PATHS', None) # collection paths not relevant to this test - env.pop('PYTHONPATH') - env.pop('VIRTUAL_ENV') - env.pop('PROOT_TMP_DIR') base_dir = os.path.join(DATA, 'plugins') if not os.path.exists(base_dir): os.mkdir(base_dir) From 14a8e3da5eeee42084890d59f7d6d23f0176a586 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 3 Dec 2020 11:52:25 -0500 Subject: [PATCH 044/157] WIP: containerized inventory updates. Thanks ALAN!! --- awx/main/models/inventory.py | 1 - awx/main/tasks.py | 9 ++++----- awx/settings/defaults.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 94ddcc4e90..28f4b09948 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1441,7 +1441,6 @@ class PluginFileInjector(object): def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) - env['ANSIBLE_COLLECTIONS_PATHS'] = settings.AWX_ANSIBLE_COLLECTIONS_PATHS return env def build_private_data(self, inventory_update, private_data_dir): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index bd4c5bd81c..0e793498a3 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2539,9 +2539,6 @@ class RunInventoryUpdate(BaseTask): injector = InventorySource.injectors[inventory_update.source]() return injector.build_private_data(inventory_update, private_data_dir) - def build_execution_environment_params(self, inventory_update): - return {} # TODO: containerize inventory updates - def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None): """Build environment dictionary for ansible-inventory. @@ -2631,7 +2628,7 @@ class RunInventoryUpdate(BaseTask): args.append(source_location) args.append('--output') - args.append(os.path.join(private_data_dir, 'artifacts', 'output.json')) + args.append(os.path.join('/runner', 'artifacts', 'output.json')) if os.path.isdir(source_location): playbook_dir = source_location @@ -2667,8 +2664,10 @@ class RunInventoryUpdate(BaseTask): with open(inventory_path, 'w') as f: f.write(content) os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + + inventory_path = os.path.join('/runner', injector.filename) elif src == 'scm': - inventory_path = os.path.join(private_data_dir, 'project', inventory_update.source_path) + inventory_path = os.path.join('/runner', 'project', inventory_update.source_path) elif src == 'custom': handle, inventory_path = tempfile.mkstemp(dir=private_data_dir) f = os.fdopen(handle, 'w') diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 0a2a7043d6..b6631165a1 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -59,7 +59,7 @@ DATABASES = { } } -AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/ansible-runner:devel' +AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/shanemcd/ee' AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 From 69dcbe08656a046d91ffdb514dde991326408326 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 4 Dec 2020 11:26:56 -0500 Subject: [PATCH 045/157] More inventory update containerization fixes --- awx/main/tasks.py | 30 ++++++++++++++++-------------- awx/main/tests/unit/test_tasks.py | 3 ++- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0e793498a3..433c7fb2f8 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2546,13 +2546,9 @@ class RunInventoryUpdate(BaseTask): are accomplished by the inventory source injectors (in this method) or custom credential type injectors (in main run method). """ - base_env = super(RunInventoryUpdate, self).build_env( + env = super(RunInventoryUpdate, self).build_env( inventory_update, private_data_dir, isolated, private_data_files=private_data_files) - # TODO: this is able to run by turning off isolation - # the goal is to run it a container instead - env = dict(os.environ.items()) - env.update(base_env) if private_data_files is None: private_data_files = {} @@ -2623,17 +2619,20 @@ class RunInventoryUpdate(BaseTask): args = ['ansible-inventory', '--list', '--export'] # Add arguments for the source inventory file/script/thing - source_location = self.pseudo_build_inventory(inventory_update, private_data_dir) + rel_path = self.pseudo_build_inventory(inventory_update, private_data_dir) + container_location = os.path.join('/runner', rel_path) # TODO: make container paths elegant + source_location = os.path.join(private_data_dir, rel_path) + args.append('-i') - args.append(source_location) + args.append(container_location) args.append('--output') args.append(os.path.join('/runner', 'artifacts', 'output.json')) if os.path.isdir(source_location): - playbook_dir = source_location + playbook_dir = container_location else: - playbook_dir = os.path.dirname(source_location) + playbook_dir = os.path.dirname(container_location) args.extend(['--playbook-dir', playbook_dir]) if inventory_update.verbosity: @@ -2665,9 +2664,9 @@ class RunInventoryUpdate(BaseTask): f.write(content) os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - inventory_path = os.path.join('/runner', injector.filename) + rel_path = injector.filename elif src == 'scm': - inventory_path = os.path.join('/runner', 'project', inventory_update.source_path) + rel_path = os.path.join('project', inventory_update.source_path) elif src == 'custom': handle, inventory_path = tempfile.mkstemp(dir=private_data_dir) f = os.fdopen(handle, 'w') @@ -2676,7 +2675,9 @@ class RunInventoryUpdate(BaseTask): f.write(inventory_update.source_script.script) f.close() os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - return inventory_path + + rel_path = os.path.split(inventory_path)[-1] + return rel_path def build_cwd(self, inventory_update, private_data_dir): ''' @@ -2685,9 +2686,10 @@ class RunInventoryUpdate(BaseTask): - SCM, where source needs to live in the project folder ''' src = inventory_update.source + container_dir = '/runner' # TODO: make container paths elegant if src == 'scm' and inventory_update.source_project_update: - return os.path.join(private_data_dir, 'project') - return private_data_dir + return os.path.join(container_dir, 'project') + return container_dir def build_playbook_path_relative_to_cwd(self, inventory_update, private_data_dir): return None diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 2fff642eef..d6e70fe3fd 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -2090,7 +2090,8 @@ class TestInventoryUpdateCredentials(TestJobExecution): assert '-i' in ' '.join(args) script = args[args.index('-i') + 1] - with open(script, 'r') as f: + host_script = script.replace('/runner', private_data_dir) + with open(host_script, 'r') as f: assert f.read() == inventory_update.source_script.script assert env['FOO'] == 'BAR' if with_credential: From b716e2b099375a4e0df210f46e37a4fbc6032bf5 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 7 Dec 2020 15:11:29 -0500 Subject: [PATCH 046/157] Make insights integration tests pass again --- awx/main/tasks.py | 13 +++++++++++++ awx/settings/defaults.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 433c7fb2f8..80ae17d474 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1893,6 +1893,19 @@ class RunJob(BaseTask): return False return getattr(settings, 'AWX_PROOT_ENABLED', False) + def build_execution_environment_params(self, instance): + params = super(RunJob, self).build_execution_environment_params(instance) + # If this has an insights agent and it is not already mounted then show it + insights_dir = os.path.dirname(settings.INSIGHTS_SYSTEM_ID_FILE) + if instance.use_fact_cache and os.path.exists(insights_dir): + logger.info('not parent of others') + params.setdefault('container_volume_mounts', []) + params['container_volume_mounts'].extend([ + f"{insights_dir}:{insights_dir}:Z", + ]) + + return params + def pre_run_hook(self, job, private_data_dir): super(RunJob, self).pre_run_hook(job, private_data_dir) if job.inventory is None: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b6631165a1..058d6aeaee 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -785,6 +785,8 @@ TOWER_URL_BASE = "https://towerhost" INSIGHTS_URL_BASE = "https://example.org" INSIGHTS_AGENT_MIME = 'application/example' +# See https://github.com/ansible/awx-facts-playbooks +INSIGHTS_SYSTEM_ID_FILE='/etc/redhat-access-insights/machine-id' TOWER_SETTINGS_MANIFEST = {} From 54681eb0555d77cfbb434195e5a16268db7d8e44 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 8 Dec 2020 20:52:35 -0500 Subject: [PATCH 047/157] Add utility method to get controller private_data_dir --- awxkit/awxkit/api/pages/unified_jobs.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/awxkit/awxkit/api/pages/unified_jobs.py b/awxkit/awxkit/api/pages/unified_jobs.py index 8e07b71de9..20c6175ed3 100644 --- a/awxkit/awxkit/api/pages/unified_jobs.py +++ b/awxkit/awxkit/api/pages/unified_jobs.py @@ -135,6 +135,28 @@ class UnifiedJob(HasStatus, base.Base): raise return args + @property + def controller_dir(self): + """Returns the path to the private_data_dir on the controller node for the job + This can be used if trying to shell in and inspect the files used by the job + Cannot use job_cwd, because that is path inside EE container + """ + self.get() + job_args = self.job_args + expected_prefix = '/tmp/awx_{}'.format(self.id) + for arg1, arg2 in zip(job_args[:-1], job_args[1:]): + if arg1 == '-v': + if ':' in arg2: + host_loc = arg2.split(':')[0] + if host_loc.startswith(expected_prefix): + return host_loc + raise RuntimeError( + 'Could not find a controller private_data_dir for this job. ' + 'Searched for volume mount to {} inside of args {}'.format( + expected_prefix, job_args + ) + ) + class UnifiedJobs(page.PageList, UnifiedJob): From 2302496724e1c061dd0ed59babd2148d8b98140b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 9 Dec 2020 09:31:59 -0500 Subject: [PATCH 048/157] Add back in the subversion requirement --- execution-environment.yml | 1 + requirements/bindep_requirements.txt | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 requirements/bindep_requirements.txt diff --git a/execution-environment.yml b/execution-environment.yml index ad1005cc2f..4e4d10cb36 100644 --- a/execution-environment.yml +++ b/execution-environment.yml @@ -2,6 +2,7 @@ version: 1 dependencies: galaxy: requirements/collections_requirements.yml + system: requirements/bindep_requirements.txt additional_build_steps: prepend: - RUN pip3 install --upgrade pip setuptools diff --git a/requirements/bindep_requirements.txt b/requirements/bindep_requirements.txt new file mode 100644 index 0000000000..fe8e3c519b --- /dev/null +++ b/requirements/bindep_requirements.txt @@ -0,0 +1,2 @@ +subversion [platform:rpm] +subversion [platform:dpkg] From 7a433f4e8f41b19039e2a55b75e36a0ed6fdc19e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 9 Dec 2020 16:36:48 -0500 Subject: [PATCH 049/157] Change the shebang back to just python --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 80ae17d474..e7a6aa711a 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1137,7 +1137,7 @@ class BaseTask(object): fn = os.path.join(path, 'hosts') with open(fn, 'w') as f: os.chmod(fn, stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR) - f.write('#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json_data) + f.write('#! /usr/bin/env python\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json_data) return fn def build_args(self, instance, private_data_dir, passwords): From c0faa39b537a60343b2f4d0adb1683df29a9116c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 10 Dec 2020 09:34:40 -0500 Subject: [PATCH 050/157] Remove files moved to the ansible/awx-ee repo These have been moved to: https://github.com/ansible/awx-ee that will be the home for the processes needed to build this execution environment. --- execution-environment.yml | 11 ----------- requirements/bindep_requirements.txt | 2 -- 2 files changed, 13 deletions(-) delete mode 100644 execution-environment.yml delete mode 100644 requirements/bindep_requirements.txt diff --git a/execution-environment.yml b/execution-environment.yml deleted file mode 100644 index 4e4d10cb36..0000000000 --- a/execution-environment.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -version: 1 -dependencies: - galaxy: requirements/collections_requirements.yml - system: requirements/bindep_requirements.txt -additional_build_steps: - prepend: - - RUN pip3 install --upgrade pip setuptools - append: - - COPY --from=quay.io/project-receptor/receptor /usr/bin/receptor /usr/bin/receptor - - RUN mkdir -m 0770 /var/run/receptor diff --git a/requirements/bindep_requirements.txt b/requirements/bindep_requirements.txt deleted file mode 100644 index fe8e3c519b..0000000000 --- a/requirements/bindep_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -subversion [platform:rpm] -subversion [platform:dpkg] From c1133b3f6d6d5427b7b99b2f615f9ad8489b4151 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 17 Nov 2020 14:46:42 -0500 Subject: [PATCH 051/157] Add in more model changes around execution environments - a new unique name field to EE - a new configure-Tower-in-Tower setting DEFAULT_EXECUTION_ENVIRONMENT - an Org-level execution_environment_admin_role - a default_environment field on Project - a new Container Registry credential type - order EEs by reverse of the created timestamp - a method to resolve which EE to use on jobs --- awx/api/serializers.py | 11 +++-- awx/conf/fields.py | 1 + awx/main/access.py | 2 + awx/main/conf.py | 13 ++++++ .../0125_more_ee_modeling_changes.py | 41 +++++++++++++++++++ awx/main/models/credential/__init__.py | 32 ++++++++++++++- awx/main/models/execution_environments.py | 7 ++-- awx/main/models/organization.py | 3 ++ awx/main/models/projects.py | 9 ++++ awx/main/models/rbac.py | 2 + awx/main/models/unified_jobs.py | 20 +++++++++ awx/main/tests/functional/test_credential.py | 1 + awx/settings/defaults.py | 1 + .../api/pages/execution_environments.py | 2 +- 14 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 awx/main/migrations/0125_more_ee_modeling_changes.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index dcf403c47e..68e507a6d2 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -107,8 +107,8 @@ SUMMARIZABLE_FK_FIELDS = { 'insights_credential_id',), 'host': DEFAULT_SUMMARY_FIELDS, 'group': DEFAULT_SUMMARY_FIELDS, - 'default_environment': ('id', 'organization_id', 'image', 'description'), - 'execution_environment': ('id', 'organization_id', 'image', 'description'), + 'default_environment': DEFAULT_SUMMARY_FIELDS + ('image',), + 'execution_environment': DEFAULT_SUMMARY_FIELDS + ('image',), 'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',), @@ -1365,7 +1365,7 @@ class ExecutionEnvironmentSerializer(BaseSerializer): class Meta: model = ExecutionEnvironment - fields = ('*', '-name', 'organization', 'image', 'managed_by_tower', 'credential') + fields = ('*', 'organization', 'image', 'managed_by_tower', 'credential') def get_related(self, obj): res = super(ExecutionEnvironmentSerializer, self).get_related(obj) @@ -1395,7 +1395,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): class Meta: model = Project fields = ('*', 'organization', 'scm_update_on_launch', - 'scm_update_cache_timeout', 'allow_override', 'custom_virtualenv',) + \ + 'scm_update_cache_timeout', 'allow_override', 'custom_virtualenv', 'default_environment') + \ ('last_update_failed', 'last_updated') # Backwards compatibility def get_related(self, obj): @@ -1420,6 +1420,9 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) + if obj.default_environment: + res['default_environment'] = self.reverse('api:execution_environment_detail', + kwargs={'pk': obj.default_environment_id}) # Backwards compatibility. if obj.current_update: res['current_update'] = self.reverse('api:project_update_detail', diff --git a/awx/conf/fields.py b/awx/conf/fields.py index 7c9a94969d..90f495e293 100644 --- a/awx/conf/fields.py +++ b/awx/conf/fields.py @@ -14,6 +14,7 @@ from rest_framework.fields import ( # noqa BooleanField, CharField, ChoiceField, DictField, DateTimeField, EmailField, IntegerField, ListField, NullBooleanField ) +from rest_framework.serializers import PrimaryKeyRelatedField logger = logging.getLogger('awx.conf.fields') diff --git a/awx/main/access.py b/awx/main/access.py index 24e6bbc569..f1edc58006 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1320,6 +1320,8 @@ class ExecutionEnvironmentAccess(BaseAccess): """ model = ExecutionEnvironment + select_related = ('organization',) + prefetch_related = ('organization__admin_role',) def filtered_queryset(self): return ExecutionEnvironment.objects.filter( diff --git a/awx/main/conf.py b/awx/main/conf.py index 6bf86db214..f46371e22b 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -10,6 +10,7 @@ from rest_framework.fields import FloatField # Tower from awx.conf import fields, register, register_validate +from awx.main.models import ExecutionEnvironment logger = logging.getLogger('awx.main.conf') @@ -176,6 +177,18 @@ register( read_only=True, ) +register( + 'DEFAULT_EXECUTION_ENVIRONMENT', + field_class=fields.PrimaryKeyRelatedField, + allow_null=True, + default=None, + queryset=ExecutionEnvironment.objects.all(), + label=_('Global default execution environment'), + help_text=_('.'), + category=_('System'), + category_slug='system', +) + register( 'CUSTOM_VENV_PATHS', field_class=fields.StringListPathField, diff --git a/awx/main/migrations/0125_more_ee_modeling_changes.py b/awx/main/migrations/0125_more_ee_modeling_changes.py new file mode 100644 index 0000000000..3d5a076d8d --- /dev/null +++ b/awx/main/migrations/0125_more_ee_modeling_changes.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.16 on 2020-11-19 16:20 +import uuid + +import awx.main.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0124_execution_environments'), + ] + + operations = [ + migrations.AlterModelOptions( + name='executionenvironment', + options={'ordering': ('-created',)}, + ), + migrations.AddField( + model_name='executionenvironment', + name='name', + field=models.CharField(default=uuid.uuid4, max_length=512, unique=True), + preserve_default=False, + ), + migrations.AddField( + model_name='organization', + name='execution_environment_admin_role', + field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role='admin_role', related_name='+', to='main.Role'), + preserve_default='True', + ), + migrations.AddField( + model_name='project', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run using this project.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + migrations.AlterUniqueTogether( + name='executionenvironment', + unique_together=set(), + ), + ] diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index ebab3bc22f..00af665969 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -1133,7 +1133,6 @@ ManagedCredentialType( }, ) - ManagedCredentialType( namespace='kubernetes_bearer_token', kind='kubernetes', @@ -1165,6 +1164,37 @@ ManagedCredentialType( } ) +ManagedCredentialType( + namespace='registry', + kind='registry', + name=ugettext_noop('Container Registry'), + inputs={ + 'fields': [{ + 'id': 'host', + 'label': ugettext_noop('Authentication URL'), + 'type': 'string', + 'help_text': ugettext_noop('Authentication endpoint for the container registry.'), + }, { + 'id': 'username', + 'label': ugettext_noop('Username'), + 'type': 'string', + }, { + 'id': 'password', + 'label': ugettext_noop('Password'), + 'type': 'string', + 'secret': True, + }, { + 'id': 'token', + 'label': ugettext_noop('Access Token'), + 'type': 'string', + 'secret': True, + 'help_text': ugettext_noop('A token to use to authenticate with. ' + 'This should not be set if username/password are being used.'), + }], + 'required': ['host'], + } +) + ManagedCredentialType( namespace='galaxy_api_token', diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index bdbe75eb49..51c7c251ea 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -2,16 +2,15 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from awx.api.versioning import reverse -from awx.main.models.base import PrimordialModel +from awx.main.models.base import CommonModel __all__ = ['ExecutionEnvironment'] -class ExecutionEnvironment(PrimordialModel): +class ExecutionEnvironment(CommonModel): class Meta: - unique_together = ('organization', 'image') - ordering = (models.F('organization_id').asc(nulls_first=True), 'image') + ordering = ('-created',) organization = models.ForeignKey( 'Organization', diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index 3730fe9af1..bdf1e38d7d 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -95,6 +95,9 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi job_template_admin_role = ImplicitRoleField( parent_role='admin_role', ) + execution_environment_admin_role = ImplicitRoleField( + parent_role='admin_role', + ) auditor_role = ImplicitRoleField( parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR, ) diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 65fb8304ce..ec14a2ef76 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -259,6 +259,15 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn app_label = 'main' ordering = ('id',) + default_environment = models.ForeignKey( + 'ExecutionEnvironment', + null=True, + blank=True, + default=None, + on_delete=models.SET_NULL, + related_name='+', + help_text=_('The default execution environment for jobs run using this project.'), + ) scm_update_on_launch = models.BooleanField( default=False, help_text=_('Update the project when a job is launched that uses the project.'), diff --git a/awx/main/models/rbac.py b/awx/main/models/rbac.py index 67d21e873d..fe8d622ac6 100644 --- a/awx/main/models/rbac.py +++ b/awx/main/models/rbac.py @@ -40,6 +40,7 @@ role_names = { 'inventory_admin_role': _('Inventory Admin'), 'credential_admin_role': _('Credential Admin'), 'job_template_admin_role': _('Job Template Admin'), + 'execution_environment_admin_role': _('Execution Environment Admin'), 'workflow_admin_role': _('Workflow Admin'), 'notification_admin_role': _('Notification Admin'), 'auditor_role': _('Auditor'), @@ -60,6 +61,7 @@ role_descriptions = { 'inventory_admin_role': _('Can manage all inventories of the %s'), 'credential_admin_role': _('Can manage all credentials of the %s'), 'job_template_admin_role': _('Can manage all job templates of the %s'), + 'execution_environment_admin_role': _('Can manage all execution environments of the %s'), 'workflow_admin_role': _('Can manage all workflows of the %s'), 'notification_admin_role': _('Can manage all notifications of the %s'), 'auditor_role': _('Can view all aspects of the %s'), diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index f4a9e1ba45..53406be172 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -40,6 +40,7 @@ from awx.main.dispatch import get_local_queuename from awx.main.dispatch.control import Control as ControlDispatcher from awx.main.registrar import activity_stream_registrar from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin +from awx.main.models.execution_environments import ExecutionEnvironment from awx.main.utils import ( camelcase_to_underscore, get_model_for_type, encrypt_dict, decrypt_field, _inventory_updates, @@ -338,6 +339,23 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn from awx.main.models.notifications import NotificationTemplate return NotificationTemplate.objects.none() + def resolve_execution_environment(self): + """ + Return the execution environment that should be used when creating a new job. + """ + if self.execution_environment is not None: + return self.execution_environment + if getattr(self, 'project_id', None) and self.project.default_environment is not None: + return self.project.default_environment + if getattr(self, 'organization', None) and self.organization.default_environment is not None: + return self.organization.default_environment + if getattr(self, 'inventory', None) and self.inventory.organization is not None: + if self.inventory.organization.default_environment is not None: + return self.inventory.organization.default_environment + if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: + return settings.DEFAULT_EXECUTION_ENVIRONMENT + return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() + def create_unified_job(self, **kwargs): ''' Create a new unified job based on this unified job template. @@ -376,6 +394,8 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn for fd, val in eager_fields.items(): setattr(unified_job, fd, val) + unified_job.execution_environment = self.resolve_execution_environment() + # NOTE: slice workflow jobs _get_parent_field_name method # is not correct until this is set if not parent_field_name: diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index 27f67b96f4..4f87c249be 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -90,6 +90,7 @@ def test_default_cred_types(): 'kubernetes_bearer_token', 'net', 'openstack', + 'registry', 'rhv', 'satellite6', 'scm', diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 058d6aeaee..52a044afa0 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -175,6 +175,7 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST'] PROXY_IP_ALLOWED_LIST = [] CUSTOM_VENV_PATHS = [] +DEFAULT_EXECUTION_ENVIRONMENT = None # Note: This setting may be overridden by database settings. STDOUT_MAX_BYTES_DISPLAY = 1048576 diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py index 87225d1052..c3bcecb4bf 100644 --- a/awxkit/awxkit/api/pages/execution_environments.py +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -18,7 +18,7 @@ log = logging.getLogger(__name__) class ExecutionEnvironment(HasCreate, base.Base): dependencies = [Organization, Credential] - NATURAL_KEY = ('organization', 'image') + NATURAL_KEY = ('name',) # fields are image, organization, managed_by_tower, credential def create(self, image='quay.io/ansible/ansible-runner:devel', credential=None, **kwargs): From 6d935f740c5a69d74d99b8fcf35ec1f70231ad3a Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 19 Nov 2020 14:06:32 -0500 Subject: [PATCH 052/157] Fill in the new execution environment collection module as well as changes to other ones that need to be able to attach EEs. --- awx/conf/fields.py | 2 +- awx/main/models/unified_jobs.py | 2 +- .../modules/tower_execution_environment.py | 66 ++++++++++++++----- .../plugins/modules/tower_inventory_source.py | 4 ++ .../plugins/modules/tower_job_template.py | 15 +++-- .../plugins/modules/tower_organization.py | 6 +- .../plugins/modules/tower_project.py | 20 ++++-- .../modules/tower_workflow_job_template.py | 5 ++ 8 files changed, 91 insertions(+), 29 deletions(-) diff --git a/awx/conf/fields.py b/awx/conf/fields.py index 90f495e293..e28a44aa32 100644 --- a/awx/conf/fields.py +++ b/awx/conf/fields.py @@ -14,7 +14,7 @@ from rest_framework.fields import ( # noqa BooleanField, CharField, ChoiceField, DictField, DateTimeField, EmailField, IntegerField, ListField, NullBooleanField ) -from rest_framework.serializers import PrimaryKeyRelatedField +from rest_framework.serializers import PrimaryKeyRelatedField # noqa logger = logging.getLogger('awx.conf.fields') diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 53406be172..aae9d59a8f 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -353,7 +353,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn if self.inventory.organization.default_environment is not None: return self.inventory.organization.default_environment if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: - return settings.DEFAULT_EXECUTION_ENVIRONMENT + return settings.DEFAULT_EXECUTION_ENVIRONMENT return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() def create_unified_job(self, **kwargs): diff --git a/awx_collection/plugins/modules/tower_execution_environment.py b/awx_collection/plugins/modules/tower_execution_environment.py index 978d23298c..486a24d949 100644 --- a/awx_collection/plugins/modules/tower_execution_environment.py +++ b/awx_collection/plugins/modules/tower_execution_environment.py @@ -22,30 +22,34 @@ description: - Create, update, or destroy Execution Environments in Ansible Tower. See U(https://www.ansible.com/tower) for an overview. options: + name: + description: + - Name to use for the execution environment. + required: True + type: str image: description: - - The fully qualified name of the container image + - The fully qualified url of the container image. required: True type: str + description: + description: + - Description to use for the execution environment. + type: str + organization: + description: + - The organization the execution environment belongs to. + type: str + credential: + description: + - Name of the credential to use for the execution environment. + type: str state: description: - Desired state of the resource. choices: ["present", "absent"] default: "present" type: str - credential: - description: - - Name of the credential to use for the job template. - - Deprecated, use 'credentials'. - type: str - description: - description: - - Description to use for the job template. - type: str - organization: - description: - - TODO - type: str extends_documentation_fragment: awx.awx.auth ''' @@ -53,6 +57,7 @@ extends_documentation_fragment: awx.awx.auth EXAMPLES = ''' - name: Add EE to Tower tower_execution_environment: + name: "My EE" image: quay.io/awx/ee ''' @@ -64,22 +69,49 @@ import json def main(): # Any additional arguments that are not fields of the item can be added here argument_spec = dict( + name=dict(required=True), image=dict(required=True), + description=dict(default=''), + organization=dict(), + credential=dict(default=''), + state=dict(choices=['present', 'absent'], default='present'), ) # Create a module for ourselves module = TowerAPIModule(argument_spec=argument_spec) # Extract our parameters + name = module.params.get('name') image = module.params.get('image') + description = module.params.get('description') state = module.params.get('state') - existing_item = module.get_one('execution_environments', name_or_id=image) + existing_item = module.get_one('execution_environments', name_or_id=name) if state == 'absent': - module.delete_if_needed(image) + module.delete_if_needed(existing_item) - module.create_or_update_if_needed(existing_item, image, endpoint='execution_environments', item_type='execution_environment') + new_fields = { + 'name': name, + 'image': image, + } + if description: + new_fields['description'] = description + + # Attempt to look up the related items the user specified (these will fail the module if not found) + organization = module.params.get('organization') + if organization: + new_fields['organization'] = module.resolve_name_to_id('organizations', organization) + + credential = module.params.get('credential') + if credential: + new_fields['credential'] = module.resolve_name_to_id('credentials', credential) + + module.create_or_update_if_needed( + existing_item, new_fields, + endpoint='execution_environments', + item_type='execution_environment' + ) if __name__ == '__main__': diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index ceb0e8b5a6..9edf467617 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -177,6 +177,7 @@ def main(): enabled_value=dict(), host_filter=dict(), credential=dict(), + execution_environment=dict(), organization=dict(), overwrite=dict(type='bool'), overwrite_vars=dict(type='bool'), @@ -203,6 +204,7 @@ def main(): organization = module.params.get('organization') source_script = module.params.get('source_script') credential = module.params.get('credential') + ee = module.params.get('execution_environment') source_project = module.params.get('source_project') state = module.params.get('state') @@ -254,6 +256,8 @@ def main(): # Attempt to look up the related items the user specified (these will fail the module if not found) if credential is not None: inventory_source_fields['credential'] = module.resolve_name_to_id('credentials', credential) + if ee is not None: + inventory_source_fields['execution_environment'] = module.resolve_name_to_id('execution_environments', ee) if source_project is not None: inventory_source_fields['source_project'] = module.resolve_name_to_id('projects', source_project) if source_script is not None: diff --git a/awx_collection/plugins/modules/tower_job_template.py b/awx_collection/plugins/modules/tower_job_template.py index b2b1530d6f..131d05f924 100644 --- a/awx_collection/plugins/modules/tower_job_template.py +++ b/awx_collection/plugins/modules/tower_job_template.py @@ -60,10 +60,6 @@ options: description: - Path to the playbook to use for the job template within the project provided. type: str - execution_environment: - description: - - Execution Environment to use for the JT. - type: str credential: description: - Name of the credential to use for the job template. @@ -79,6 +75,10 @@ options: - Name of the vault credential to use for the job template. - Deprecated, use 'credentials'. type: str + execution_environment: + description: + - Execution Environment to use for the JT. + type: str forks: description: - The number of parallel or simultaneous processes to use while executing the playbook. @@ -354,6 +354,7 @@ def main(): vault_credential=dict(), custom_virtualenv=dict(), credentials=dict(type='list', elements='str'), + execution_environment=dict(), forks=dict(type='int'), limit=dict(), verbosity=dict(type='int', choices=[0, 1, 2, 3, 4], default=0), @@ -420,7 +421,11 @@ def main(): organization = module.params.get('organization') if organization: organization_id = module.resolve_name_to_id('organizations', organization) - search_fields['organization'] = new_fields['organization'] = organization_id + search_fields['organization'] = new_fields['organization'] = organization_id + + ee = module.params.get('execution_environment') + if ee: + new_fields['execution_environment'] = module.resolve_name_to_id('execution_environments', ee) # Attempt to look up an existing item based on the provided data existing_item = module.get_one('job_templates', name_or_id=name, **{'data': search_fields}) diff --git a/awx_collection/plugins/modules/tower_organization.py b/awx_collection/plugins/modules/tower_organization.py index 7d88d2a421..bcf6060ea6 100644 --- a/awx_collection/plugins/modules/tower_organization.py +++ b/awx_collection/plugins/modules/tower_organization.py @@ -38,7 +38,7 @@ options: default: '' default_environment: description: - - Default Execution Environment to use for the Organization. + - Default Execution Environment to use for jobs owned by the Organization. type: str max_hosts: description: @@ -114,6 +114,7 @@ def main(): name=dict(required=True), description=dict(), custom_virtualenv=dict(), + default_environment=dict(), max_hosts=dict(type='int', default="0"), notification_templates_started=dict(type="list", elements='str'), notification_templates_success=dict(type="list", elements='str'), @@ -130,6 +131,7 @@ def main(): name = module.params.get('name') description = module.params.get('description') custom_virtualenv = module.params.get('custom_virtualenv') + default_ee = module.params.get('default_environment') max_hosts = module.params.get('max_hosts') # instance_group_names = module.params.get('instance_groups') state = module.params.get('state') @@ -179,6 +181,8 @@ def main(): org_fields['description'] = description if custom_virtualenv is not None: org_fields['custom_virtualenv'] = custom_virtualenv + if default_ee is not None: + org_fields['default_environment'] = module.resolve_name_to_id('execution_environments', default_ee) if max_hosts is not None: org_fields['max_hosts'] = max_hosts diff --git a/awx_collection/plugins/modules/tower_project.py b/awx_collection/plugins/modules/tower_project.py index f6ab7d144c..1a8248c1fa 100644 --- a/awx_collection/plugins/modules/tower_project.py +++ b/awx_collection/plugins/modules/tower_project.py @@ -31,10 +31,6 @@ options: description: - Description to use for the project. type: str - execution_environment: - description: - - Execution Environment to use for the project. - type: str scm_type: description: - Type of SCM resource. @@ -106,6 +102,14 @@ options: - Local absolute file path containing a custom Python virtualenv to use type: str default: '' + default_environment: + description: + - Default Execution Environment to use for jobs relating to the project. + type: str + execution_environment: + description: + - Execution Environment to use for project updates. + type: str organization: description: - Name of organization for project. @@ -243,6 +247,8 @@ def main(): allow_override=dict(type='bool', aliases=['scm_allow_override']), timeout=dict(type='int', default=0, aliases=['job_timeout']), custom_virtualenv=dict(), + default_environment=dict(), + execution_environment=dict(), organization=dict(), notification_templates_started=dict(type="list", elements='str'), notification_templates_success=dict(type="list", elements='str'), @@ -274,6 +280,8 @@ def main(): allow_override = module.params.get('allow_override') timeout = module.params.get('timeout') custom_virtualenv = module.params.get('custom_virtualenv') + default_ee = module.params.get('default_environment') + ee = module.params.get('execution_environment') organization = module.params.get('organization') state = module.params.get('state') wait = module.params.get('wait') @@ -337,6 +345,10 @@ def main(): project_fields['description'] = description if credential is not None: project_fields['credential'] = credential + if default_ee is not None: + project_fields['default_environment'] = module.resolve_name_to_id('execution_environments', default_ee) + if ee is not None: + project_fields['execution_environment'] = module.resolve_name_to_id('execution_environments', ee) if allow_override is not None: project_fields['allow_override'] = allow_override if scm_type == '': diff --git a/awx_collection/plugins/modules/tower_workflow_job_template.py b/awx_collection/plugins/modules/tower_workflow_job_template.py index 54b6695b03..48759e1cc4 100644 --- a/awx_collection/plugins/modules/tower_workflow_job_template.py +++ b/awx_collection/plugins/modules/tower_workflow_job_template.py @@ -175,6 +175,7 @@ def main(): description=dict(), extra_vars=dict(type='dict'), organization=dict(), + execution_environment=dict(), survey_spec=dict(type='dict', aliases=['survey']), survey_enabled=dict(type='bool'), allow_simultaneous=dict(type='bool'), @@ -212,6 +213,10 @@ def main(): organization_id = module.resolve_name_to_id('organizations', organization) search_fields['organization'] = new_fields['organization'] = organization_id + ee = module.params.get('execution_environment') + if ee: + new_fields['execution_environment'] = module.resolve_name_to_id('execution_environments', ee) + # Attempt to look up an existing item based on the provided data existing_item = module.get_one('workflow_job_templates', name_or_id=name, **{'data': search_fields}) From 8562c378c036eb1cd671d49fc8c23613bdba4da3 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 20 Nov 2020 10:43:53 -0500 Subject: [PATCH 053/157] Make use of the EE resolver code when launching jobs --- awx/main/tasks.py | 9 +-------- awx/settings/defaults.py | 3 +++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index e7a6aa711a..ea3aa94b82 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -887,14 +887,7 @@ class BaseTask(object): return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) def build_execution_environment_params(self, instance): - if getattr(instance, 'execution_environment', None): - # TODO: process heirarchy, JT-project-org, maybe here - # or maybe in create_unified_job - logger.info('using custom image {}'.format(instance.execution_environment.image)) - image = instance.execution_environment.image - else: - logger.info('using default image') - image = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE + image = instance.execution_environment.image params = { "container_image": image, "process_isolation": True diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 52a044afa0..7812b3af31 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -59,12 +59,15 @@ DATABASES = { } } +# TODO: remove this setting in favor of a default execution environment AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/shanemcd/ee' AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = 'default' + +# TODO: remove this setting in favor of a default execution environment AWX_CONTAINER_GROUP_DEFAULT_IMAGE = AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE # Internationalization From 4993a9e6ec755f0e71537b86badcc5143e48d1ea Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 20 Nov 2020 14:08:11 -0500 Subject: [PATCH 054/157] Move the resolve_execution_environment method to the mixin class so that it can be used with AdHocCommands as well. --- awx/main/models/ad_hoc_commands.py | 4 ++-- awx/main/models/mixins.py | 19 +++++++++++++++++++ awx/main/models/unified_jobs.py | 18 ------------------ awx/main/tasks.py | 3 +++ 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 9787f01423..536ac8a912 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -198,8 +198,8 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin): def copy(self): data = {} for field in ('job_type', 'inventory_id', 'limit', 'credential_id', - 'module_name', 'module_args', 'forks', 'verbosity', - 'extra_vars', 'become_enabled', 'diff_mode'): + 'execution_environment_id', 'module_name', 'module_args', + 'forks', 'verbosity', 'extra_vars', 'become_enabled', 'diff_mode'): data[field] = getattr(self, field) return AdHocCommand.objects.create(**data) diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 1cd1366a92..459eadabcf 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -455,6 +455,25 @@ class ExecutionEnvironmentMixin(models.Model): help_text=_('The container image to be used for execution.'), ) + def resolve_execution_environment(self): + """ + Return the execution environment that should be used when creating a new job. + """ + from awx.main.models.execution_environments import ExecutionEnvironment + + if self.execution_environment is not None: + return self.execution_environment + if getattr(self, 'project_id', None) and self.project.default_environment is not None: + return self.project.default_environment + if getattr(self, 'organization', None) and self.organization.default_environment is not None: + return self.organization.default_environment + if getattr(self, 'inventory', None) and self.inventory.organization is not None: + if self.inventory.organization.default_environment is not None: + return self.inventory.organization.default_environment + if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: + return settings.DEFAULT_EXECUTION_ENVIRONMENT + return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() + class CustomVirtualEnvMixin(models.Model): class Meta: diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index aae9d59a8f..7970de1b22 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -40,7 +40,6 @@ from awx.main.dispatch import get_local_queuename from awx.main.dispatch.control import Control as ControlDispatcher from awx.main.registrar import activity_stream_registrar from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin -from awx.main.models.execution_environments import ExecutionEnvironment from awx.main.utils import ( camelcase_to_underscore, get_model_for_type, encrypt_dict, decrypt_field, _inventory_updates, @@ -339,23 +338,6 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn from awx.main.models.notifications import NotificationTemplate return NotificationTemplate.objects.none() - def resolve_execution_environment(self): - """ - Return the execution environment that should be used when creating a new job. - """ - if self.execution_environment is not None: - return self.execution_environment - if getattr(self, 'project_id', None) and self.project.default_environment is not None: - return self.project.default_environment - if getattr(self, 'organization', None) and self.organization.default_environment is not None: - return self.organization.default_environment - if getattr(self, 'inventory', None) and self.inventory.organization is not None: - if self.inventory.organization.default_environment is not None: - return self.inventory.organization.default_environment - if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: - return settings.DEFAULT_EXECUTION_ENVIRONMENT - return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() - def create_unified_job(self, **kwargs): ''' Create a new unified job based on this unified job template. diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ea3aa94b82..8f09f67f56 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -887,6 +887,9 @@ class BaseTask(object): return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) def build_execution_environment_params(self, instance): + if instance.execution_environment_id is None: + self.update_model(instance.pk, execution_environment=instance.resolve_execution_environment()) + image = instance.execution_environment.image params = { "container_image": image, From e7bf81883be4d1eb16d4becdb172a5aa546a6c8c Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 8 Dec 2020 09:42:02 -0500 Subject: [PATCH 055/157] Populate the EE name field in awxkit --- awxkit/awxkit/api/pages/execution_environments.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py index c3bcecb4bf..a01aa91011 100644 --- a/awxkit/awxkit/api/pages/execution_environments.py +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -20,21 +20,22 @@ class ExecutionEnvironment(HasCreate, base.Base): dependencies = [Organization, Credential] NATURAL_KEY = ('name',) - # fields are image, organization, managed_by_tower, credential - def create(self, image='quay.io/ansible/ansible-runner:devel', credential=None, **kwargs): + # fields are name, image, organization, managed_by_tower, credential + def create(self, name='', image='quay.io/ansible/ansible-runner:devel', credential=None, **kwargs): # we do not want to make a credential by default - payload = self.create_payload(image=image, credential=credential, **kwargs) + payload = self.create_payload(name=name, image=image, credential=credential, **kwargs) ret = self.update_identity(ExecutionEnvironments(self.connection).post(payload)) return ret - def create_payload(self, organization=Organization, **kwargs): + def create_payload(self, name='', organization=Organization, **kwargs): self.create_and_update_dependencies(organization) - payload = self.payload(organization=self.ds.organization, **kwargs) + payload = self.payload(name=name, organization=self.ds.organization, **kwargs) payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) return payload - def payload(self, image=None, organization=None, credential=None, **kwargs): + def payload(self, name='', image=None, organization=None, credential=None, **kwargs): payload = PseudoNamespace( + name=name or "EE - {}".format(random_title()), image=image or random_title(10), organization=organization.id if organization else None, credential=credential.id if credential else None, From 5f1da2b9233edcc589c243f961d42bb3a7ecf3fe Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 8 Dec 2020 09:59:45 -0500 Subject: [PATCH 056/157] Adjust ExecutionEnvironmentAccess to account for the new EE admin role --- awx/main/access.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index f1edc58006..a8a110e9c3 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1312,7 +1312,7 @@ class ExecutionEnvironmentAccess(BaseAccess): """ I can see an execution environment when: - I'm a superuser - - I'm a member of the organization + - I'm a member of the same organization - it is a global ExecutionEnvironment I can create/change an execution environment when: - I'm a superuser @@ -1321,32 +1321,32 @@ class ExecutionEnvironmentAccess(BaseAccess): model = ExecutionEnvironment select_related = ('organization',) - prefetch_related = ('organization__admin_role',) + prefetch_related = ('organization__admin_role', 'organization__execution_environment_admin_role') def filtered_queryset(self): return ExecutionEnvironment.objects.filter( - Q(organization__in=Organization.accessible_pk_qs(self.user, 'admin_role')) | + Q(organization__in=Organization.accessible_pk_qs(self.user, 'execution_environment_admin_role')) | Q(organization__isnull=True) ).distinct() @check_superuser def can_add(self, data): if not data: # So the browseable API will work - return Organization.accessible_objects(self.user, 'admin_role').exists() + return Organization.accessible_objects(self.user, 'execution_environment_admin_role').exists() return self.check_related('organization', Organization, data) @check_superuser def can_change(self, obj, data): if obj and obj.organization_id is None: raise PermissionDenied - if self.user not in obj.organization.admin_role: + if self.user not in obj.organization.execution_environment_admin_role: raise PermissionDenied org_pk = get_pk_from_dict(data, 'organization') if obj and obj.organization_id != org_pk: # Prevent moving an EE to a different organization, unless a superuser or admin on both orgs. if obj.organization_id is None or org_pk is None: raise PermissionDenied - if self.user not in Organization.objects.get(id=org_pk).admin_role: + if self.user not in Organization.objects.get(id=org_pk).execution_environment_admin_role: raise PermissionDenied return True From 4a0fc3e1af11576b6e1eeed35a7b62737eb46c9f Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 9 Dec 2020 10:28:12 -0500 Subject: [PATCH 057/157] Ensure that a fallback EE is available to be found for the failing tests. --- awx/main/tests/functional/test_inventory_source_injectors.py | 4 +++- awx/main/tests/unit/test_tasks.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index bb26b7c029..bf1a4002c5 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -6,7 +6,7 @@ import re from collections import namedtuple from awx.main.tasks import RunInventoryUpdate -from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob +from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob, ExecutionEnvironment from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV from awx.main.tests import data @@ -183,6 +183,8 @@ def create_reference_data(source_dir, env, content): @pytest.mark.django_db @pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS) def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory): + ExecutionEnvironment.objects.create(name='test EE', managed_by_tower=True) + injector = InventorySource.injectors[this_kind] if injector.plugin_name is None: pytest.skip('Use of inventory plugin is not enabled for this source') diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index d6e70fe3fd..94d622deac 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -18,6 +18,7 @@ from awx.main.models import ( AdHocCommand, Credential, CredentialType, + ExecutionEnvironment, Inventory, InventorySource, InventoryUpdate, @@ -657,9 +658,12 @@ class TestGenericRun(): assert env['FOO'] == 'BAR' +@pytest.mark.django_db class TestAdhocRun(TestJobExecution): def test_options_jinja_usage(self, adhoc_job, adhoc_update_model_wrapper): + ExecutionEnvironment.objects.create(name='test EE', managed_by_tower=True) + adhoc_job.module_args = '{{ ansible_ssh_pass }}' adhoc_job.websocket_emit_status = mock.Mock() adhoc_job.send_notification_templates = mock.Mock() From fde7a1e3e5680af91c3373e51659a765d73115e1 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Dec 2020 09:27:45 -0500 Subject: [PATCH 058/157] Ensure that the updated job instance is used when attaching an EE. --- awx/main/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 8f09f67f56..d0a78064af 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -888,7 +888,8 @@ class BaseTask(object): def build_execution_environment_params(self, instance): if instance.execution_environment_id is None: - self.update_model(instance.pk, execution_environment=instance.resolve_execution_environment()) + self.instance = instance = self.update_model( + instance.pk, execution_environment=instance.resolve_execution_environment()) image = instance.execution_environment.image params = { From 44ad6bfdce4dd7c2466969ce4654cf922477b698 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Dec 2020 11:12:24 -0500 Subject: [PATCH 059/157] Insert a default EE into the development environment --- tools/docker-compose/bootstrap_development.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index 1d3e399bf7..46fb917b08 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -26,3 +26,8 @@ make init mkdir -p /awx_devel/awx/public/static mkdir -p /awx_devel/awx/ui/static mkdir -p /awx_devel/awx/ui_next/build/static + +echo "ee, created = ExecutionEnvironment.objects.get_or_create(name='Default EE', \ + defaults={'image': 'quay.io/awx/ee', \ + 'managed_by_tower': True}); \ + print('Already exists' if not created else 'Created')" | awx-manage shell_plus --quiet-load From c74d60f3f3b9488769e99bc22674eb8b6715ab4e Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Dec 2020 11:42:52 -0500 Subject: [PATCH 060/157] Make sure that the new credential type is in the choices list --- awx/main/migrations/0125_more_ee_modeling_changes.py | 5 +++++ awx/main/models/credential/__init__.py | 1 + 2 files changed, 6 insertions(+) diff --git a/awx/main/migrations/0125_more_ee_modeling_changes.py b/awx/main/migrations/0125_more_ee_modeling_changes.py index 3d5a076d8d..be999cbb79 100644 --- a/awx/main/migrations/0125_more_ee_modeling_changes.py +++ b/awx/main/migrations/0125_more_ee_modeling_changes.py @@ -34,6 +34,11 @@ class Migration(migrations.Migration): name='default_environment', field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run using this project.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), ), + migrations.AlterField( + model_name='credentialtype', + name='kind', + field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('registry', 'Container Registry'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External'), ('kubernetes', 'Kubernetes'), ('galaxy', 'Galaxy/Automation Hub')], max_length=32), + ), migrations.AlterUniqueTogether( name='executionenvironment', unique_together=set(), diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index 00af665969..7cdd9898d3 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -331,6 +331,7 @@ class CredentialType(CommonModelNameNotUnique): ('net', _('Network')), ('scm', _('Source Control')), ('cloud', _('Cloud')), + ('registry', _('Container Registry')), ('token', _('Personal Access Token')), ('insights', _('Insights')), ('external', _('External')), From 12b8349e88ea337da4da0abf359fbb413af1fbad Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 10 Dec 2020 13:24:58 -0500 Subject: [PATCH 061/157] Show EE images that are managed by tower in UI --- .../ExecutionEnvironmentList/ExecutionEnvironmentList.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx index 02ce49ee9f..749fe8893e 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx @@ -21,7 +21,6 @@ import ExecutionEnvironmentsListItem from './ExecutionEnvironmentListItem'; const QS_CONFIG = getQSConfig('execution_environments', { page: 1, page_size: 20, - managed_by_tower: false, order_by: 'image', }); From 9964ba7c9a10acbd2596c13846928944e37dfe06 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 11 Dec 2020 10:09:27 -0500 Subject: [PATCH 062/157] Improve the behavior of EE resolution for ad hoc commands - call resolve_execution_environment during AdHocCommand.save() - wrap the fallback call of the resolver in tasks.py in disable_activity_stream() --- awx/main/models/ad_hoc_commands.py | 3 +++ awx/main/tasks.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 536ac8a912..54269fff1e 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -209,6 +209,9 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin): self.name = Truncator(u': '.join(filter(None, (self.module_name, self.module_args)))).chars(512) if 'name' not in update_fields: update_fields.append('name') + if not self.execution_environment_id: + self.execution_environment = self.resolve_execution_environment() + update_fields.append('execution_environment') super(AdHocCommand, self).save(*args, **kwargs) @property diff --git a/awx/main/tasks.py b/awx/main/tasks.py index d0a78064af..cf081a9d17 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -888,8 +888,11 @@ class BaseTask(object): def build_execution_environment_params(self, instance): if instance.execution_environment_id is None: - self.instance = instance = self.update_model( - instance.pk, execution_environment=instance.resolve_execution_environment()) + from awx.main.signals import disable_activity_stream + + with disable_activity_stream(): + self.instance = instance = self.update_model( + instance.pk, execution_environment=instance.resolve_execution_environment()) image = instance.execution_environment.image params = { From b95347822599f68ce8e121ef3323611d60cc530a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 11 Dec 2020 21:44:51 -0500 Subject: [PATCH 063/157] Change the default EE location --- awx_collection/plugins/modules/tower_execution_environment.py | 2 +- .../ansible/roles/sources/templates/docker-compose.yml.j2 | 2 +- tools/docker-compose/bootstrap_development.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/awx_collection/plugins/modules/tower_execution_environment.py b/awx_collection/plugins/modules/tower_execution_environment.py index 486a24d949..280408f72a 100644 --- a/awx_collection/plugins/modules/tower_execution_environment.py +++ b/awx_collection/plugins/modules/tower_execution_environment.py @@ -58,7 +58,7 @@ EXAMPLES = ''' - name: Add EE to Tower tower_execution_environment: name: "My EE" - image: quay.io/awx/ee + image: quay.io/ansible/awx-ee ''' diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index ea330d66f8..eae187cea6 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -44,7 +44,7 @@ services: # context: ./docker-compose # dockerfile: Dockerfile-logstash ee: - image: quay.io/awx/ee + image: quay.io/ansible/awx-ee user: ${CURRENT_UID} volumes: - "./docker-compose/receptor.cfg:/receptor.cfg" diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index 46fb917b08..3cc937c9c6 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -28,6 +28,6 @@ mkdir -p /awx_devel/awx/ui/static mkdir -p /awx_devel/awx/ui_next/build/static echo "ee, created = ExecutionEnvironment.objects.get_or_create(name='Default EE', \ - defaults={'image': 'quay.io/awx/ee', \ + defaults={'image': 'quay.io/ansible/awx-ee', \ 'managed_by_tower': True}); \ print('Already exists' if not created else 'Created')" | awx-manage shell_plus --quiet-load From ba146343184a1f20638d05ae3f1e803159f04ce5 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 11 Dec 2020 21:50:10 -0500 Subject: [PATCH 064/157] Fix collection pep8 failure --- awx_collection/plugins/modules/tower_job_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx_collection/plugins/modules/tower_job_template.py b/awx_collection/plugins/modules/tower_job_template.py index 131d05f924..1ed750b86e 100644 --- a/awx_collection/plugins/modules/tower_job_template.py +++ b/awx_collection/plugins/modules/tower_job_template.py @@ -421,7 +421,7 @@ def main(): organization = module.params.get('organization') if organization: organization_id = module.resolve_name_to_id('organizations', organization) - search_fields['organization'] = new_fields['organization'] = organization_id + search_fields['organization'] = new_fields['organization'] = organization_id ee = module.params.get('execution_environment') if ee: From 0dfb183cb6f72caa3a4558138d916c93818a901a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 11 Dec 2020 21:58:35 -0500 Subject: [PATCH 065/157] Fix another credential path-in-container bug --- awx/main/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index cf081a9d17..8182bd64cf 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1728,7 +1728,10 @@ class RunJob(BaseTask): cred_files = private_data_files.get('credentials', {}) for cloud_cred in job.cloud_credentials: if cloud_cred and cloud_cred.credential_type.namespace == 'openstack': - env['OS_CLIENT_CONFIG_FILE'] = cred_files.get(cloud_cred, '') + env['OS_CLIENT_CONFIG_FILE'] = os.path.join( + '/runner', + os.path.basename(cred_files.get(cloud_cred, '')) + ) for network_cred in job.network_credentials: env['ANSIBLE_NET_USERNAME'] = network_cred.get_input('username', default='') From 015fc29c1c0179d047f0653c0da0c2d61f06db69 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Sat, 12 Dec 2020 22:48:21 -0500 Subject: [PATCH 066/157] Fix another svn issue due to pre-existing folder --- awx/playbooks/project_update.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 664f189a28..e00bed4249 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -55,6 +55,8 @@ force: "{{scm_clean}}" username: "{{scm_username|default(omit)}}" password: "{{scm_password|default(omit)}}" + # must be in_place because folder pre-existing, because it is mounted + in_place: true environment: LC_ALL: 'en_US.UTF-8' register: svn_result From 49bdadcdbff662ff8e0a568cf903f40e695b0be2 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Sun, 13 Dec 2020 09:20:48 -0500 Subject: [PATCH 067/157] Fix yet another host vs container path bug --- awx/main/models/credential/injectors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/models/credential/injectors.py b/awx/main/models/credential/injectors.py index 4d7ef26054..75a08482cc 100644 --- a/awx/main/models/credential/injectors.py +++ b/awx/main/models/credential/injectors.py @@ -105,7 +105,8 @@ def openstack(cred, env, private_data_dir): yaml.safe_dump(openstack_data, f, default_flow_style=False, allow_unicode=True) f.close() os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - env['OS_CLIENT_CONFIG_FILE'] = path + # TODO: constant for container base path + env['OS_CLIENT_CONFIG_FILE'] = os.path.join('/runner', os.path.basename(path)) def kubernetes_bearer_token(cred, env, private_data_dir): From 10e68c6fb332873cd19fe6a78694250ce1c2525b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 14 Dec 2020 10:08:07 -0500 Subject: [PATCH 068/157] Fix unit test fallout --- awx/main/tests/unit/test_tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 94d622deac..8586ad11c7 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1288,7 +1288,11 @@ class TestJobCredentials(TestJobExecution): credential, env, {}, [], private_data_dir ) - shade_config = open(env['OS_CLIENT_CONFIG_FILE'], 'r').read() + # convert container path to host machine path + config_loc = os.path.join( + private_data_dir, os.path.basename(env['OS_CLIENT_CONFIG_FILE']) + ) + shade_config = open(config_loc, 'r').read() assert shade_config == '\n'.join([ 'clouds:', ' devstack:', From eb5bf599e3b9d6debeea8d1b1485595ba60f6a64 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 15 Dec 2020 13:29:03 -0500 Subject: [PATCH 069/157] Fix raw archive project updates Several squashed commits Fix git bug introduced by setting remote tmp in project path change shebang back to py3 again Revert shebang change --- awx/main/tasks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 8182bd64cf..9ffcad4393 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2231,6 +2231,14 @@ class RunProjectUpdate(BaseTask): elif project_update.project.allow_override: # If branch is override-able, do extra fetch for all branches extra_vars['scm_refspec'] = 'refs/heads/*:refs/remotes/origin/*' + + if project_update.scm_type == 'archive': + # for raw archive, prevent error moving files between volumes + extra_vars['ansible_remote_tmp'] = os.path.join( + project_update.get_project_path(check_if_exists=False), + '.ansible_awx', 'tmp' + ) + self._write_extra_vars_file(private_data_dir, extra_vars) def build_cwd(self, project_update, private_data_dir): From 90b9c7861c9797690a0cd0f14d7b7868f36cd2f4 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 18 Dec 2020 10:01:00 -0500 Subject: [PATCH 070/157] Allow jobs to run in the base ansible-runner image (#8949) --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 9ffcad4393..3dd9255cfb 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1137,7 +1137,7 @@ class BaseTask(object): fn = os.path.join(path, 'hosts') with open(fn, 'w') as f: os.chmod(fn, stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR) - f.write('#! /usr/bin/env python\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json_data) + f.write('#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json_data) return fn def build_args(self, instance, private_data_dir, passwords): From 0c497fa6827f92884dabf5fb551fb5fe9d8a2cf6 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 4 Jan 2021 11:47:55 -0500 Subject: [PATCH 071/157] Get podman-in-docker working under cgroups v2 --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index ecdcb32733..838b0ef7c6 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -107,7 +107,6 @@ RUN dnf -y update && \ krb5-workstation \ libcgroup-tools \ nginx \ - podman \ @postgresql:12 \ python3-devel \ python3-libselinux \ @@ -157,6 +156,7 @@ RUN cd /usr/local/bin && \ {% if (build_dev|bool) or (kube_dev|bool) %} # Install development/test requirements RUN dnf -y install \ + crun \ gdb \ gtk3 \ gettext \ @@ -170,6 +170,7 @@ RUN dnf -y install \ nss \ make \ patch \ + podman \ socat \ tmux \ wget \ From acee22435b333ab7fd4ef2bbc9a115f25c500dd2 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 25 Jan 2021 17:39:23 -0500 Subject: [PATCH 072/157] Update ExecutionEnvironments.jsx with breadcrumb replacement --- .../screens/ExecutionEnvironment/ExecutionEnvironments.jsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx index 7db0baaedc..d95c15959f 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx @@ -6,7 +6,7 @@ import { Route, Switch } from 'react-router-dom'; import ExecutionEnvironment from './ExecutionEnvironment'; import ExecutionEnvironmentAdd from './ExecutionEnvironmentAdd'; import ExecutionEnvironmentList from './ExecutionEnvironmentList'; -import Breadcrumbs from '../../components/Breadcrumbs'; +import ScreenHeader from '../../components/ScreenHeader/ScreenHeader'; function ExecutionEnvironments({ i18n }) { const [breadcrumbConfig, setBreadcrumbConfig] = useState({ @@ -35,7 +35,10 @@ function ExecutionEnvironments({ i18n }) { ); return ( <> - + From 521d3d5edbc52d54cce8c22e9443574a5a727197 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 31 Jul 2020 09:43:46 -0400 Subject: [PATCH 073/157] Initial EE integration --- awx/main/tasks.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 3dd9255cfb..d51f3cb1a7 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1430,10 +1430,6 @@ class BaseTask(object): cwd = self.build_cwd(self.instance, private_data_dir) resource_profiling_params = self.build_params_resource_profiling(self.instance, private_data_dir) - # TODO: Remove if fully replaced with containerized runs - # process_isolation_params = self.build_params_process_isolation(self.instance, - # private_data_dir, - # cwd) execution_environment_params = self.build_execution_environment_params(self.instance) env = self.build_env(self.instance, private_data_dir, isolated, private_data_files=private_data_files) @@ -1469,7 +1465,6 @@ class BaseTask(object): 'settings': { 'job_timeout': self.get_instance_timeout(self.instance), 'suppress_ansible_output': True, - #**process_isolation_params, **resource_profiling_params, }, } From f1df4c54f835a115a3c1f9d1b9e6a3573a3be83a Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 12 Nov 2020 16:34:18 -0500 Subject: [PATCH 074/157] Begin integrating receptor --- awx/main/tasks.py | 122 ++++++++++++------ .../test_inventory_source_injectors.py | 4 + requirements/requirements_dev.txt | 1 + .../roles/dockerfile/templates/Dockerfile.j2 | 6 +- .../sources/templates/docker-compose.yml.j2 | 12 +- .../{receptor.cfg => receptor.conf} | 10 +- tools/docker-compose/supervisor.conf | 2 +- 7 files changed, 99 insertions(+), 58 deletions(-) rename tools/docker-compose/{receptor.cfg => receptor.conf} (59%) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index d51f3cb1a7..7a408265d9 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -23,6 +23,9 @@ import fcntl from pathlib import Path from uuid import uuid4 import urllib.parse as urlparse +import socket +import threading +import concurrent.futures # Django from django.conf import settings @@ -49,6 +52,9 @@ from gitdb.exc import BadName as BadGitName # Runner import ansible_runner +# Receptor +from receptorctl.socket_interface import ReceptorControl + # AWX from awx import __version__ as awx_application_version from awx.main.constants import PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV @@ -1453,15 +1459,10 @@ class BaseTask(object): params = { 'ident': self.instance.id, 'private_data_dir': private_data_dir, - 'project_dir': cwd, 'playbook': self.build_playbook_path_relative_to_cwd(self.instance, private_data_dir), 'inventory': self.build_inventory(self.instance, private_data_dir), 'passwords': expect_passwords, 'envvars': env, - 'event_handler': self.event_handler, - 'cancel_callback': self.cancel_callback, - 'finished_callback': self.finished_callback, - 'status_handler': self.status_handler, 'settings': { 'job_timeout': self.get_instance_timeout(self.instance), 'suppress_ansible_output': True, @@ -1473,10 +1474,7 @@ class BaseTask(object): # We don't want HOME passed through to container groups. # TODO: remove this conditional after everything is containerized params['envvars'].pop('HOME', None) - else: - # TODO: container group jobs will not work with container isolation settings - # but both will run with same settings when worker_in and worker_out are added - params['settings'].update(execution_environment_params) + if isinstance(self.instance, AdHocCommand): params['module'] = self.build_module_name(self.instance) @@ -1497,39 +1495,85 @@ class BaseTask(object): del params[v] self.dispatcher = CallbackQueueDispatcher() - if self.instance.is_isolated() or containerized: - module_args = None - if 'module_args' in params: - # if it's adhoc, copy the module args - module_args = ansible_runner.utils.args2cmdline( - params.get('module_args'), - ) - # TODO on merge: delete if https://github.com/ansible/awx/pull/8185 is merged - if not os.path.exists(os.path.join(private_data_dir, 'inventory')): - shutil.move( - params.pop('inventory'), - os.path.join(private_data_dir, 'inventory') - ) - ansible_runner.utils.dump_artifacts(params) - isolated_manager_instance = isolated_manager.IsolatedManager( - self.event_handler, - canceled_callback=lambda: self.update_model(self.instance.pk).cancel_flag, - check_callback=self.check_handler, - pod_manager=pod_manager - ) - status, rc = isolated_manager_instance.run(self.instance, - private_data_dir, - params.get('playbook'), - params.get('module'), - module_args, - ident=str(self.instance.pk)) - self.finished_callback(None) + if not isinstance(self.instance, ProjectUpdate): + worktype='worker' + # TODO: container group jobs will not work with container isolation settings + # but both will run with same settings when worker_in and worker_out are added + params['settings'].update(execution_environment_params) else: - res = ansible_runner.interface.run(**params) - status = res.status - rc = res.rc + worktype='worker' + params['settings'].update(execution_environment_params) + + # Create a socketpair. Where the left side will be used for writing our payload + # (private data dir, kwargs). The right side will be passed to Receptor for + # reading. + sockin, sockout = socket.socketpair() + + # Spawned in a thread so Receptor can start reading before we finish writing, we + # write our payload to the left side of our socketpair. + def transmit(_socket): + ansible_runner.interface.run(streamer='transmit', + _output=_socket.makefile('wb'), + **params) + + # Socket must be shutdown here, or the reader will hang forever. + _socket.shutdown(socket.SHUT_WR) + + threading.Thread(target=transmit, args=[sockin]).start() + + self.instance.log_lifecycle("running_playbook") + # We establish a connection to the Receptor socket and submit our work, passing + # in the right side of our socketpair for reading. + receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') + result = receptor_ctl.submit_work(worktype=worktype, + payload=sockout.makefile('rb')) + sockin.close() + sockout.close() + + resultsock, resultfile = receptor_ctl.get_work_results(result['unitid'], + return_socket=True, + return_sockfile=True) + + def processor(): + return ansible_runner.interface.run(streamer='process', + quiet=True, + _input=resultfile, + event_handler=self.event_handler, + finished_callback=self.finished_callback, + status_handler=self.status_handler) + + def cancel_watcher(processor_future): + while True: + if processor_future.done(): + return + + if self.cancel_callback(): + result = namedtuple('result', ['status', 'rc']) + return result('canceled', 1) + time.sleep(1) + + # Both "processor" and "cancel_watcher" are spawned in separate threads. + # We wait for the first one to return. If cancel_watcher returns first, + # we yank the socket out from underneath the processor, which will cause it + # to exit. A reference to the processor_future is passed into the cancel_watcher_future, + # Which exits if the job has finished normally. The context manager ensures we do not + # leave any threads laying around. + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + processor_future = executor.submit(processor) + cancel_watcher_future = executor.submit(cancel_watcher, processor_future) + futures = [processor_future, cancel_watcher_future] + first_future = concurrent.futures.wait(futures, + return_when=concurrent.futures.FIRST_COMPLETED) + + res = list(first_future.done)[0].result() + if res.status == 'canceled': + resultsock.shutdown(socket.SHUT_RDWR) + resultfile.close() + + status = res.status + rc = res.rc if status == 'timeout': self.instance.job_explanation = "Job terminated due to timeout" diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index bf1a4002c5..c4f7e6a17d 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -206,6 +206,10 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential It will make assertions that the contents are correct If MAKE_INVENTORY_REFERENCE_FILES is set, it will produce reference files """ + if _kw.get('streamer') != 'transmit': + Res = namedtuple('Result', ['status', 'rc']) + return Res('successful', 0) + private_data_dir = envvars.pop('AWX_PRIVATE_DATA_DIR') assert envvars.pop('ANSIBLE_INVENTORY_ENABLED') == 'auto' set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0']) diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index fe51fff164..4788e153a2 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -20,5 +20,6 @@ matplotlib backports.tempfile # support in unit tests for py32+ tempfile.TemporaryDirectory mockldap sdb +remote-pdb gprof2dot atomicwrites==1.4.0 diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 838b0ef7c6..0d48f2ea56 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -143,11 +143,6 @@ RUN ansible-galaxy collection install --collections-path /usr/share/ansible/coll RUN rm -rf /root/.cache && rm -rf /tmp/* -# Install Receptor -RUN cd /usr/local/bin && \ - curl -L http://nightlies.testing.ansible.com/receptor/receptor --output receptor && \ - chmod a+x receptor - # Install OpenShift CLI RUN cd /usr/local/bin && \ curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \ @@ -190,6 +185,7 @@ COPY --from=builder /var/lib/awx /var/lib/awx RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage {%if build_dev|bool %} +COPY --from=quay.io/shanemcd/receptor /usr/bin/receptor /usr/bin/receptor RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr \ -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \ openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \ diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index eae187cea6..db81b4ba39 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -35,6 +35,8 @@ services: - "redis_socket:/var/run/redis/:rw" - "receptor:/var/run/receptor/" - "/sys/fs/cgroup:/sys/fs/cgroup" + - "./docker-compose/receptor.conf:/etc/receptor/receptor.conf" + - "~/.kube/config:/var/lib/awx/.kube/config" privileged: true tty: true # A useful container that simply passes through log messages to the console @@ -43,16 +45,6 @@ services: # build: # context: ./docker-compose # dockerfile: Dockerfile-logstash - ee: - image: quay.io/ansible/awx-ee - user: ${CURRENT_UID} - volumes: - - "./docker-compose/receptor.cfg:/receptor.cfg" - - "receptor:/var/run/receptor/" - command: - - receptor - - --config - - /receptor.cfg postgres: image: postgres:12 container_name: tools_postgres_1 diff --git a/tools/docker-compose/receptor.cfg b/tools/docker-compose/receptor.conf similarity index 59% rename from tools/docker-compose/receptor.cfg rename to tools/docker-compose/receptor.conf index 137d15cdf6..7df861f6ca 100644 --- a/tools/docker-compose/receptor.cfg +++ b/tools/docker-compose/receptor.conf @@ -5,11 +5,15 @@ service: control filename: /var/run/receptor/receptor.sock -- tcp-listener: - port: 2222 +- local-only: - work-command: worktype: worker command: ansible-runner params: worker - allowruntimeparams: true + +- work-kubernetes: + worktype: ocp + namespace: receptor + image: quay.io/shanemcd/ee + authmethod: kubeconfig diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index 04ddb66838..1a71b8018e 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -84,7 +84,7 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 [program:awx-receptor] -command = receptor --node id=%(ENV_HOSTNAME)s --control-service filename=/var/run/receptor/receptor.sock --tcp-listener port=2222 +command = receptor --config /etc/receptor/receptor.conf autostart = true autorestart = true stopsignal = KILL From 81f6d36a3a36615a84544de8dba11cd10358eaed Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 17 Nov 2020 13:40:45 -0500 Subject: [PATCH 075/157] Set SDB_NOTIFY_HOST for all processes --- awx/settings/development.py | 9 --------- tools/docker-compose/entrypoint.sh | 6 ++++++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/awx/settings/development.py b/awx/settings/development.py index 6181d16ec6..d181ca10fc 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -177,15 +177,6 @@ CELERYBEAT_SCHEDULE.update({ # noqa CLUSTER_HOST_ID = socket.gethostname() - -if 'Docker Desktop' in os.getenv('OS', ''): - os.environ['SDB_NOTIFY_HOST'] = 'docker.for.mac.host.internal' -else: - try: - os.environ['SDB_NOTIFY_HOST'] = os.popen('ip route').read().split(' ')[2] - except Exception: - pass - AWX_CALLBACK_PROFILE = True if 'sqlite3' not in DATABASES['default']['ENGINE']: # noqa diff --git a/tools/docker-compose/entrypoint.sh b/tools/docker-compose/entrypoint.sh index 13c858b441..c155f022e4 100755 --- a/tools/docker-compose/entrypoint.sh +++ b/tools/docker-compose/entrypoint.sh @@ -21,4 +21,10 @@ fi # writing out the sub*id files above podman system migrate +if [[ "$OS" == *"Docker Desktop"* ]]; then + export SDB_NOTIFY_HOST='docker.for.mac.host.internal' +else + export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}') +fi + exec $@ From 0184a7c267d9de3acb320f2ddb72e7772ea2dafd Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 18 Nov 2020 16:33:35 -0500 Subject: [PATCH 076/157] Create receptor mesh in cluster development environment --- tools/docker-compose-cluster.yml | 16 +++++++++++++ .../awx-1-receptor.conf | 23 +++++++++++++++++++ .../awx-2-receptor.conf | 23 +++++++++++++++++++ .../awx-3-receptor.conf | 23 +++++++++++++++++++ tools/docker-compose/supervisor.conf | 3 +++ 5 files changed, 88 insertions(+) create mode 100644 tools/docker-compose-cluster/awx-1-receptor.conf create mode 100644 tools/docker-compose-cluster/awx-2-receptor.conf create mode 100644 tools/docker-compose-cluster/awx-3-receptor.conf diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 7a90aa88c9..6065069125 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -14,6 +14,7 @@ services: - "8013:8013" - "8043:8043" - "1936:1936" + awx-1: user: ${CURRENT_UID} container_name: tools_awx_1_1 @@ -31,9 +32,11 @@ services: - "../:/awx_devel" - "./redis/redis_socket_ha_1:/var/run/redis/" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" + - "./docker-compose-cluster/awx-1-receptor.conf:/etc/receptor/receptor.conf" ports: - "2222:2222" - "5899-5999:5899-5999" + awx-2: user: ${CURRENT_UID} container_name: tools_awx_2_1 @@ -51,9 +54,11 @@ services: - "../:/awx_devel" - "./redis/redis_socket_ha_2:/var/run/redis/" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" + - "./docker-compose-cluster/awx-2-receptor.conf:/etc/receptor/receptor.conf" ports: - "2223:2222" - "7899-7999:7899-7999" + awx-3: user: ${CURRENT_UID} container_name: tools_awx_3_1 @@ -71,9 +76,11 @@ services: - "../:/awx_devel" - "./redis/redis_socket_ha_3:/var/run/redis/" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" + - "./docker-compose-cluster/awx-3-receptor.conf:/etc/receptor/receptor.conf" ports: - "2224:2222" - "8899-8999:8899-8999" + redis_1: user: ${CURRENT_UID} image: redis:latest @@ -82,6 +89,7 @@ services: volumes: - "./redis/redis.conf:/usr/local/etc/redis/redis.conf" - "./redis/redis_socket_ha_1:/var/run/redis/" + redis_2: user: ${CURRENT_UID} image: redis:latest @@ -98,6 +106,14 @@ services: volumes: - "./redis/redis.conf:/usr/local/etc/redis/redis.conf" - "./redis/redis_socket_ha_3:/var/run/redis/" + postgres: image: postgres:12 container_name: tools_postgres_1 + environment: + POSTGRES_HOST_AUTH_METHOD: trust + volumes: + - "awx_db:/var/lib/postgresql/data" + +volumes: + awx_db: diff --git a/tools/docker-compose-cluster/awx-1-receptor.conf b/tools/docker-compose-cluster/awx-1-receptor.conf new file mode 100644 index 0000000000..dcaca8263f --- /dev/null +++ b/tools/docker-compose-cluster/awx-1-receptor.conf @@ -0,0 +1,23 @@ +--- +- log-level: debug + +- control-service: + service: control + filename: /var/run/receptor/receptor.sock + +- tcp-listener: + port: 2200 + +- tcp-peer: + address: awx-2:2200 + +- work-command: + worktype: worker + command: ansible-runner + params: worker + +- work-kubernetes: + worktype: ocp + namespace: receptor + image: quay.io/shanemcd/ee + authmethod: kubeconfig diff --git a/tools/docker-compose-cluster/awx-2-receptor.conf b/tools/docker-compose-cluster/awx-2-receptor.conf new file mode 100644 index 0000000000..bf9d4889a0 --- /dev/null +++ b/tools/docker-compose-cluster/awx-2-receptor.conf @@ -0,0 +1,23 @@ +--- +- log-level: debug + +- control-service: + service: control + filename: /var/run/receptor/receptor.sock + +- tcp-listener: + port: 2200 + +- tcp-peer: + address: awx-3:2200 + +- work-command: + worktype: worker + command: ansible-runner + params: worker + +- work-kubernetes: + worktype: ocp + namespace: receptor + image: quay.io/shanemcd/ee + authmethod: kubeconfig diff --git a/tools/docker-compose-cluster/awx-3-receptor.conf b/tools/docker-compose-cluster/awx-3-receptor.conf new file mode 100644 index 0000000000..ac5db0d284 --- /dev/null +++ b/tools/docker-compose-cluster/awx-3-receptor.conf @@ -0,0 +1,23 @@ +--- +- log-level: debug + +- control-service: + service: control + filename: /var/run/receptor/receptor.sock + +- tcp-listener: + port: 2200 + +- tcp-peer: + address: awx-1:2200 + +- work-command: + worktype: worker + command: ansible-runner + params: worker + +- work-kubernetes: + worktype: ocp + namespace: receptor + image: quay.io/shanemcd/ee + authmethod: kubeconfig diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index 1a71b8018e..fc2eb2d028 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -90,6 +90,9 @@ autorestart = true stopsignal = KILL stopasgroup = true killasgroup = true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 [group:tower-processes] programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast,awx-rsyslogd From fd92ba0c0b32271bfc3e6e63a73524511d9efe3b Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 19 Nov 2020 12:44:28 -0500 Subject: [PATCH 077/157] Actually cancel things --- awx/main/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 7a408265d9..615eb90a4f 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1569,9 +1569,11 @@ class BaseTask(object): res = list(first_future.done)[0].result() if res.status == 'canceled': + receptor_ctl.simple_command(f"work cancel {result['unitid']}") resultsock.shutdown(socket.SHUT_RDWR) resultfile.close() + receptor_ctl.simple_command(f"work release {result['unitid']}") status = res.status rc = res.rc From be8168b555cbe7d978fb54d247e29931eb63104e Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 21 Nov 2020 11:01:09 -0500 Subject: [PATCH 078/157] Surface errors when launching jobs through Receptor This will raise errors such as: exec: "ansible-runner": executable file not found in $PATH --- awx/main/tasks.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 615eb90a4f..afa6982ada 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1529,10 +1529,12 @@ class BaseTask(object): receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') result = receptor_ctl.submit_work(worktype=worktype, payload=sockout.makefile('rb')) + unit_id = result['unitid'] + sockin.close() sockout.close() - resultsock, resultfile = receptor_ctl.get_work_results(result['unitid'], + resultsock, resultfile = receptor_ctl.get_work_results(unit_id, return_socket=True, return_sockfile=True) @@ -1547,7 +1549,7 @@ class BaseTask(object): def cancel_watcher(processor_future): while True: if processor_future.done(): - return + return processor_future.result() if self.cancel_callback(): result = namedtuple('result', ['status', 'rc']) @@ -1569,11 +1571,15 @@ class BaseTask(object): res = list(first_future.done)[0].result() if res.status == 'canceled': - receptor_ctl.simple_command(f"work cancel {result['unitid']}") + receptor_ctl.simple_command(f"work cancel {unit_id}") resultsock.shutdown(socket.SHUT_RDWR) resultfile.close() + elif res.status == 'error': + # TODO: There should be a more efficient way of getting this information + receptor_work_list = receptor_ctl.simple_command("work list") + raise RuntimeError(receptor_work_list[unit_id]['Detail']) - receptor_ctl.simple_command(f"work release {result['unitid']}") + receptor_ctl.simple_command(f"work release {unit_id}") status = res.status rc = res.rc From cf96275f1b52f089892a1d648212d500841cca90 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 23 Nov 2020 14:56:50 -0500 Subject: [PATCH 079/157] Pull awx -> receptor job code into its own class --- awx/main/tasks.py | 163 ++++++++++-------- .../test_inventory_source_injectors.py | 8 +- 2 files changed, 90 insertions(+), 81 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index afa6982ada..74a414026c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1433,7 +1433,6 @@ class BaseTask(object): passwords = self.build_passwords(self.instance, kwargs) self.build_extra_vars_file(self.instance, private_data_dir) args = self.build_args(self.instance, private_data_dir, passwords) - cwd = self.build_cwd(self.instance, private_data_dir) resource_profiling_params = self.build_params_resource_profiling(self.instance, private_data_dir) execution_environment_params = self.build_execution_environment_params(self.instance) @@ -1497,89 +1496,18 @@ class BaseTask(object): self.dispatcher = CallbackQueueDispatcher() if not isinstance(self.instance, ProjectUpdate): - worktype='worker' + work_type='worker' # TODO: container group jobs will not work with container isolation settings # but both will run with same settings when worker_in and worker_out are added params['settings'].update(execution_environment_params) else: - worktype='worker' + work_type='worker' params['settings'].update(execution_environment_params) - # Create a socketpair. Where the left side will be used for writing our payload - # (private data dir, kwargs). The right side will be passed to Receptor for - # reading. - sockin, sockout = socket.socketpair() - - # Spawned in a thread so Receptor can start reading before we finish writing, we - # write our payload to the left side of our socketpair. - def transmit(_socket): - ansible_runner.interface.run(streamer='transmit', - _output=_socket.makefile('wb'), - **params) - - # Socket must be shutdown here, or the reader will hang forever. - _socket.shutdown(socket.SHUT_WR) - - threading.Thread(target=transmit, args=[sockin]).start() - - self.instance.log_lifecycle("running_playbook") - # We establish a connection to the Receptor socket and submit our work, passing - # in the right side of our socketpair for reading. - receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') - result = receptor_ctl.submit_work(worktype=worktype, - payload=sockout.makefile('rb')) - unit_id = result['unitid'] + receptor_job = AWXReceptorJob(self, work_type, params) + res = receptor_job.run() - sockin.close() - sockout.close() - - resultsock, resultfile = receptor_ctl.get_work_results(unit_id, - return_socket=True, - return_sockfile=True) - - def processor(): - return ansible_runner.interface.run(streamer='process', - quiet=True, - _input=resultfile, - event_handler=self.event_handler, - finished_callback=self.finished_callback, - status_handler=self.status_handler) - - def cancel_watcher(processor_future): - while True: - if processor_future.done(): - return processor_future.result() - - if self.cancel_callback(): - result = namedtuple('result', ['status', 'rc']) - return result('canceled', 1) - time.sleep(1) - - # Both "processor" and "cancel_watcher" are spawned in separate threads. - # We wait for the first one to return. If cancel_watcher returns first, - # we yank the socket out from underneath the processor, which will cause it - # to exit. A reference to the processor_future is passed into the cancel_watcher_future, - # Which exits if the job has finished normally. The context manager ensures we do not - # leave any threads laying around. - with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: - processor_future = executor.submit(processor) - cancel_watcher_future = executor.submit(cancel_watcher, processor_future) - futures = [processor_future, cancel_watcher_future] - first_future = concurrent.futures.wait(futures, - return_when=concurrent.futures.FIRST_COMPLETED) - - res = list(first_future.done)[0].result() - if res.status == 'canceled': - receptor_ctl.simple_command(f"work cancel {unit_id}") - resultsock.shutdown(socket.SHUT_RDWR) - resultfile.close() - elif res.status == 'error': - # TODO: There should be a more efficient way of getting this information - receptor_work_list = receptor_ctl.simple_command("work list") - raise RuntimeError(receptor_work_list[unit_id]['Detail']) - - receptor_ctl.simple_command(f"work release {unit_id}") status = res.status rc = res.rc @@ -3201,3 +3129,86 @@ def deep_copy_model_obj( permission_check_func(creater, copy_mapping.values()) if isinstance(new_obj, Inventory): update_inventory_computed_fields.delay(new_obj.id) + + +class AWXReceptorJob: + def __init__(self, task, work_type, runner_params): + self.task = task + self.work_type = work_type + self.runner_params = runner_params + + def run(self): + # Create a socketpair. Where the left side will be used for writing our payload + # (private data dir, kwargs). The right side will be passed to Receptor for + # reading. + sockin, sockout = socket.socketpair() + + threading.Thread(target=self.transmit, args=[sockin]).start() + + # We establish a connection to the Receptor socket and submit our work, passing + # in the right side of our socketpair for reading. + receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') + result = receptor_ctl.submit_work(worktype=self.work_type, + payload=sockout.makefile('rb')) + unit_id = result['unitid'] + + sockin.close() + sockout.close() + + resultsock, resultfile = receptor_ctl.get_work_results(unit_id, + return_socket=True, + return_sockfile=True) + # Both "processor" and "cancel_watcher" are spawned in separate threads. + # We wait for the first one to return. If cancel_watcher returns first, + # we yank the socket out from underneath the processor, which will cause it + # to exit. A reference to the processor_future is passed into the cancel_watcher_future, + # Which exits if the job has finished normally. The context manager ensures we do not + # leave any threads laying around. + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + processor_future = executor.submit(self.processor, resultfile) + cancel_watcher_future = executor.submit(self.cancel_watcher, processor_future) + futures = [processor_future, cancel_watcher_future] + first_future = concurrent.futures.wait(futures, + return_when=concurrent.futures.FIRST_COMPLETED) + + res = list(first_future.done)[0].result() + if res.status == 'canceled': + receptor_ctl.simple_command(f"work cancel {unit_id}") + resultsock.shutdown(socket.SHUT_RDWR) + resultfile.close() + elif res.status == 'error': + # TODO: There should be a more efficient way of getting this information + receptor_work_list = receptor_ctl.simple_command("work list") + raise RuntimeError(receptor_work_list[unit_id]['Detail']) + + + receptor_ctl.simple_command(f"work release {unit_id}") + return res + + # Spawned in a thread so Receptor can start reading before we finish writing, we + # write our payload to the left side of our socketpair. + def transmit(self, _socket): + ansible_runner.interface.run(streamer='transmit', + _output=_socket.makefile('wb'), + **self.runner_params) + + # Socket must be shutdown here, or the reader will hang forever. + _socket.shutdown(socket.SHUT_WR) + + def processor(self, resultfile): + return ansible_runner.interface.run(streamer='process', + quiet=True, + _input=resultfile, + event_handler=self.task.event_handler, + finished_callback=self.task.finished_callback, + status_handler=self.task.status_handler) + + def cancel_watcher(self, processor_future): + while True: + if processor_future.done(): + return processor_future.result() + + if self.task.cancel_callback(): + result = namedtuple('result', ['status', 'rc']) + return result('canceled', 1) + time.sleep(1) diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index c4f7e6a17d..f9edfdcd22 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -200,15 +200,13 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential inventory_update = inventory_source.create_unified_job() task = RunInventoryUpdate() - def substitute_run(envvars=None, **_kw): + def substitute_run(awx_receptor_job): """This method will replace run_pexpect instead of running, it will read the private data directory contents It will make assertions that the contents are correct If MAKE_INVENTORY_REFERENCE_FILES is set, it will produce reference files """ - if _kw.get('streamer') != 'transmit': - Res = namedtuple('Result', ['status', 'rc']) - return Res('successful', 0) + envvars = awx_receptor_job.runner_params['envvars'] private_data_dir = envvars.pop('AWX_PRIVATE_DATA_DIR') assert envvars.pop('ANSIBLE_INVENTORY_ENABLED') == 'auto' @@ -260,6 +258,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): # The point of this test is that we replace run with assertions - with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): + with mock.patch('awx.main.tasks.AWXReceptorJob.run', substitute_run): # so this sets up everything for a run and then yields control over to substitute_run task.run(inventory_update.pk) From e453afa0643cadbcabfe5ad060f4455ec4efbf19 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 23 Nov 2020 18:38:06 -0500 Subject: [PATCH 080/157] FOLLOW UP ON THIS: Fix fact_cache directory location The part where we pass in the runner params to the processor phase is legit. Need to investigate why the fact_cache directory is no longer nested under job.id. --- awx/main/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 74a414026c..f96050616b 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1994,7 +1994,7 @@ class RunJob(BaseTask): return if job.use_fact_cache: job.finish_job_fact_cache( - os.path.join(private_data_dir, 'artifacts', str(job.id), 'fact_cache'), + os.path.join(private_data_dir, 'artifacts', 'fact_cache'), fact_modification_times, ) if isolated_manager_instance and not job.is_containerized: @@ -3201,7 +3201,8 @@ class AWXReceptorJob: _input=resultfile, event_handler=self.task.event_handler, finished_callback=self.task.finished_callback, - status_handler=self.task.status_handler) + status_handler=self.task.status_handler, + **self.runner_params) def cancel_watcher(self, processor_future): while True: From 7b7465f168bcd202aa423e920a014b68eadd9322 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 11 Jan 2021 19:55:03 -0500 Subject: [PATCH 081/157] Update receptor config to allow for runtime options --- tools/docker-compose/receptor.conf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/docker-compose/receptor.conf b/tools/docker-compose/receptor.conf index 7df861f6ca..ba975b8820 100644 --- a/tools/docker-compose/receptor.conf +++ b/tools/docker-compose/receptor.conf @@ -14,6 +14,7 @@ - work-kubernetes: worktype: ocp - namespace: receptor - image: quay.io/shanemcd/ee - authmethod: kubeconfig + authmethod: runtime + allowruntimeauth: true + allowruntimepod: true + allowruntimeparams: true From 286b1d4e25a82efe6ecd74b39ba713b6a5b878b7 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 19:38:03 -0500 Subject: [PATCH 082/157] InstanceGroup#is_containerized -> InstanceGroup#is_container_group --- awx/api/serializers.py | 12 ++++++------ awx/api/views/__init__.py | 2 +- awx/main/models/ad_hoc_commands.py | 4 ++-- awx/main/models/ha.py | 8 ++++---- awx/main/models/jobs.py | 6 +++--- awx/main/scheduler/task_manager.py | 12 ++++++------ awx/main/tasks.py | 4 ++-- awx/main/tests/functional/api/test_instance_group.py | 5 ++--- .../task_management/test_container_groups.py | 2 +- awx/ui_next/SEARCH.md | 8 ++++---- .../ContainerGroupDetails.test.jsx | 2 +- .../ContainerGroupEdit/ContainerGroupEdit.test.jsx | 2 +- .../InstanceGroupAdd/InstanceGroupAdd.test.jsx | 2 +- .../InstanceGroupDetails/InstanceGroupDetails.jsx | 2 +- .../InstanceGroupDetails.test.jsx | 4 ++-- .../InstanceGroupEdit/InstanceGroupEdit.test.jsx | 2 +- .../InstanceGroupList/InstanceGroupList.jsx | 2 +- .../InstanceGroupList/InstanceGroupListItem.jsx | 2 +- .../InstanceGroupList/InstanceGroupListItem.test.jsx | 4 ++-- .../InstanceGroup/shared/ContainerGroupForm.test.jsx | 2 +- .../InstanceGroup/shared/InstanceGroupForm.test.jsx | 2 +- 21 files changed, 44 insertions(+), 45 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 68e507a6d2..3a28239b0d 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -131,7 +131,7 @@ SUMMARIZABLE_FK_FIELDS = { 'source_script': DEFAULT_SUMMARY_FIELDS, 'role': ('id', 'role_field'), 'notification_template': DEFAULT_SUMMARY_FIELDS, - 'instance_group': ('id', 'name', 'controller_id', 'is_containerized'), + 'instance_group': ('id', 'name', 'controller_id', 'is_container_group'), 'insights_credential': DEFAULT_SUMMARY_FIELDS, 'source_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'), 'target_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'), @@ -4768,7 +4768,7 @@ class InstanceGroupSerializer(BaseSerializer): 'Isolated groups have a designated controller group.'), read_only=True ) - is_containerized = serializers.BooleanField( + is_container_group = serializers.BooleanField( help_text=_('Indicates whether instances in this group are containerized.' 'Containerized groups have a designated Openshift or Kubernetes cluster.'), read_only=True @@ -4798,7 +4798,7 @@ class InstanceGroupSerializer(BaseSerializer): fields = ("id", "type", "url", "related", "name", "created", "modified", "capacity", "committed_capacity", "consumed_capacity", "percent_capacity_remaining", "jobs_running", "jobs_total", - "instances", "controller", "is_controller", "is_isolated", "is_containerized", "credential", + "instances", "controller", "is_controller", "is_isolated", "is_container_group", "credential", "policy_instance_percentage", "policy_instance_minimum", "policy_instance_list", "pod_spec_override", "summary_fields") @@ -4823,17 +4823,17 @@ class InstanceGroupSerializer(BaseSerializer): raise serializers.ValidationError(_('Isolated instances may not be added or removed from instances groups via the API.')) if self.instance and self.instance.controller_id is not None: raise serializers.ValidationError(_('Isolated instance group membership may not be managed via the API.')) - if value and self.instance and self.instance.is_containerized: + if value and self.instance and self.instance.is_container_group: raise serializers.ValidationError(_('Containerized instances may not be managed via the API')) return value def validate_policy_instance_percentage(self, value): - if value and self.instance and self.instance.is_containerized: + if value and self.instance and self.instance.is_container_group: raise serializers.ValidationError(_('Containerized instances may not be managed via the API')) return value def validate_policy_instance_minimum(self, value): - if value and self.instance and self.instance.is_containerized: + if value and self.instance and self.instance.is_container_group: raise serializers.ValidationError(_('Containerized instances may not be managed via the API')) return value diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 7ec932c9ce..01ad6d98ed 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -397,7 +397,7 @@ class InstanceGroupDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAP permission_classes = (InstanceGroupTowerPermission,) def update_raw_data(self, data): - if self.get_object().is_containerized: + if self.get_object().is_container_group: data.pop('policy_instance_percentage', None) data.pop('policy_instance_minimum', None) data.pop('policy_instance_list', None) diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 54269fff1e..f327e2a7e6 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -151,8 +151,8 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin): return True @property - def is_containerized(self): - return bool(self.instance_group and self.instance_group.is_containerized) + def is_container_group_task(self): + return bool(self.instance_group and self.instance_group.is_container_group) @property def can_run_containerized(self): diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index 5071786653..4f96bdc5b1 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -247,7 +247,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin): return bool(self.controller) @property - def is_containerized(self): + def is_container_group(self): return bool(self.credential and self.credential.kubernetes) ''' @@ -306,9 +306,9 @@ def schedule_policy_task(): @receiver(post_save, sender=InstanceGroup) def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs): if created or instance.has_policy_changes(): - if not instance.is_containerized: + if not instance.is_container_group: schedule_policy_task() - elif created or instance.is_containerized: + elif created or instance.is_container_group: instance.set_default_policy_fields() @@ -320,7 +320,7 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs): @receiver(post_delete, sender=InstanceGroup) def on_instance_group_deleted(sender, instance, using, **kwargs): - if not instance.is_containerized: + if not instance.is_container_group: schedule_policy_task() diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 31f4784962..81e17cdebf 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -768,11 +768,11 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana @property def can_run_containerized(self): - return any([ig for ig in self.preferred_instance_groups if ig.is_containerized]) + return any([ig for ig in self.preferred_instance_groups if ig.is_container_group]) @property - def is_containerized(self): - return bool(self.instance_group and self.instance_group.is_containerized) + def is_container_group_task(self): + return bool(self.instance_group and self.instance_group.is_container_group) @property def preferred_instance_groups(self): diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index f06f93834a..8731ca3109 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -283,12 +283,12 @@ class TaskManager(): task.controller_node = controller_node logger.debug('Submitting isolated {} to queue {} controlled by {}.'.format( task.log_format, task.execution_node, controller_node)) - elif rampart_group.is_containerized: + elif rampart_group.is_container_group: # find one real, non-containerized instance with capacity to # act as the controller for k8s API interaction match = None for group in InstanceGroup.objects.all(): - if group.is_containerized or group.controller_id: + if group.is_container_group or group.controller_id: continue match = group.fit_task_to_most_remaining_capacity_instance(task, group.instances.all()) if match: @@ -521,14 +521,14 @@ class TaskManager(): self.start_task(task, None, task.get_jobs_fail_chain(), None) continue for rampart_group in preferred_instance_groups: - if task.can_run_containerized and rampart_group.is_containerized: + if task.can_run_containerized and rampart_group.is_container_group: self.graph[rampart_group.name]['graph'].add_job(task) self.start_task(task, rampart_group, task.get_jobs_fail_chain(), None) found_acceptable_queue = True break remaining_capacity = self.get_remaining_capacity(rampart_group.name) - if not rampart_group.is_containerized and self.get_remaining_capacity(rampart_group.name) <= 0: + if not rampart_group.is_container_group and self.get_remaining_capacity(rampart_group.name) <= 0: logger.debug("Skipping group {}, remaining_capacity {} <= 0".format( rampart_group.name, remaining_capacity)) continue @@ -536,8 +536,8 @@ class TaskManager(): execution_instance = InstanceGroup.fit_task_to_most_remaining_capacity_instance(task, self.graph[rampart_group.name]['instances']) or \ InstanceGroup.find_largest_idle_instance(self.graph[rampart_group.name]['instances']) - if execution_instance or rampart_group.is_containerized: - if not rampart_group.is_containerized: + if execution_instance or rampart_group.is_container_group: + if not rampart_group.is_container_group: execution_instance.remaining_capacity = max(0, execution_instance.remaining_capacity - task.task_impact) execution_instance.jobs_running += 1 logger.debug("Starting {} in group {} instance {} (remaining_capacity={})".format( diff --git a/awx/main/tasks.py b/awx/main/tasks.py index f96050616b..efc4f4594f 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -262,7 +262,7 @@ def apply_cluster_membership_policies(): # On a differential basis, apply instances to non-isolated groups with transaction.atomic(): for g in actual_groups: - if g.obj.is_containerized: + if g.obj.is_container_group: logger.debug('Skipping containerized group {} for policy calculation'.format(g.obj.name)) continue instances_to_add = set(g.instances) - set(g.prior_instances) @@ -507,7 +507,7 @@ def cluster_node_heartbeat(): def awx_k8s_reaper(): from awx.main.scheduler.kubernetes import PodManager # prevent circular import for group in InstanceGroup.objects.filter(credential__isnull=False).iterator(): - if group.is_containerized: + if group.is_container_group: logger.debug("Checking for orphaned k8s pods for {}.".format(group)) for job in UnifiedJob.objects.filter( pk__in=list(PodManager.list_active_jobs(group)) diff --git a/awx/main/tests/functional/api/test_instance_group.py b/awx/main/tests/functional/api/test_instance_group.py index 43c7d51960..61c1054912 100644 --- a/awx/main/tests/functional/api/test_instance_group.py +++ b/awx/main/tests/functional/api/test_instance_group.py @@ -255,7 +255,7 @@ def test_instance_group_update_fields(patch, instance, instance_group, admin, co # policy_instance_ variables can only be updated in instance groups that are NOT containerized # instance group (not containerized) ig_url = reverse("api:instance_group_detail", kwargs={'pk': instance_group.pk}) - assert not instance_group.is_containerized + assert not instance_group.is_container_group assert not containerized_instance_group.is_isolated resp = patch(ig_url, {'policy_instance_percentage':15}, admin, expect=200) assert 15 == resp.data['policy_instance_percentage'] @@ -266,7 +266,7 @@ def test_instance_group_update_fields(patch, instance, instance_group, admin, co # containerized instance group cg_url = reverse("api:instance_group_detail", kwargs={'pk': containerized_instance_group.pk}) - assert containerized_instance_group.is_containerized + assert containerized_instance_group.is_container_group assert not containerized_instance_group.is_isolated resp = patch(cg_url, {'policy_instance_percentage':15}, admin, expect=400) assert ["Containerized instances may not be managed via the API"] == resp.data['policy_instance_percentage'] @@ -291,4 +291,3 @@ def test_containerized_group_default_fields(instance_group, kube_credential): assert ig.policy_instance_list == [] assert ig.policy_instance_minimum == 0 assert ig.policy_instance_percentage == 0 - \ No newline at end of file diff --git a/awx/main/tests/functional/task_management/test_container_groups.py b/awx/main/tests/functional/task_management/test_container_groups.py index 47d982a725..c9028efacb 100644 --- a/awx/main/tests/functional/task_management/test_container_groups.py +++ b/awx/main/tests/functional/task_management/test_container_groups.py @@ -30,7 +30,7 @@ def containerized_job(default_instance_group, kube_credential, job_template_fact @pytest.mark.django_db def test_containerized_job(containerized_job): assert containerized_job.is_containerized - assert containerized_job.instance_group.is_containerized + assert containerized_job.instance_group.is_container_group assert containerized_job.instance_group.credential.kubernetes diff --git a/awx/ui_next/SEARCH.md b/awx/ui_next/SEARCH.md index 111dfb2f56..131e4fe277 100644 --- a/awx/ui_next/SEARCH.md +++ b/awx/ui_next/SEARCH.md @@ -86,7 +86,7 @@ Instances of orgs list include: **Instance Groups list** - Name - search is ?name=ig - - ? is_containerized boolean choice (doesn't work right now in API but will soon) - search is ?is_containerized=true + - ? is_container_group boolean choice (doesn't work right now in API but will soon) - search is ?is_container_group=true - ? credential name - search is ?credentials__name=kubey Instance of instance groups list include: @@ -136,7 +136,7 @@ Instance of team lists include: **Credentials list** - Name - - ? Type (dropdown on right with different types) + - ? Type (dropdown on right with different types) - ? Created by (username) - ? Modified by (username) @@ -273,7 +273,7 @@ For the UI url params, we want to only encode those params that aren't defaults, #### mergeParams vs. replaceParams -**mergeParams** is used to suppport putting values with the same key +**mergeParams** is used to suppport putting values with the same key From a UX perspective, we wanted to be able to support searching on the same key multiple times (i.e. searching for things like `?foo=bar&foo=baz`). We do this by creating an array of all values. i.e.: @@ -361,7 +361,7 @@ Smart search will be able to craft the tag through various states. Note that th "instance_groups__search" ], ``` - + PHASE 3: keys, give by object key names for data.actions.GET - type is given for each key which we could use to help craft the value diff --git a/awx/ui_next/src/screens/InstanceGroup/ContainerGroupDetails/ContainerGroupDetails.test.jsx b/awx/ui_next/src/screens/InstanceGroup/ContainerGroupDetails/ContainerGroupDetails.test.jsx index aa979ef730..68ca23f80a 100644 --- a/awx/ui_next/src/screens/InstanceGroup/ContainerGroupDetails/ContainerGroupDetails.test.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/ContainerGroupDetails/ContainerGroupDetails.test.jsx @@ -32,7 +32,7 @@ const instanceGroup = { controller: null, is_controller: false, is_isolated: false, - is_containerized: true, + is_container_group: true, credential: 71, policy_instance_percentage: 0, policy_instance_minimum: 0, diff --git a/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.test.jsx b/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.test.jsx index 937aa15adb..860c6363c5 100644 --- a/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.test.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.test.jsx @@ -31,7 +31,7 @@ const instanceGroup = { controller: null, is_controller: false, is_isolated: false, - is_containerized: true, + is_container_group: true, credential: 71, policy_instance_percentage: 0, policy_instance_minimum: 0, diff --git a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupAdd/InstanceGroupAdd.test.jsx b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupAdd/InstanceGroupAdd.test.jsx index 4b2d879398..a4ae1e74fc 100644 --- a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupAdd/InstanceGroupAdd.test.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupAdd/InstanceGroupAdd.test.jsx @@ -29,7 +29,7 @@ const instanceGroupData = { controller: null, is_controller: false, is_isolated: false, - is_containerized: false, + is_container_group: false, credential: null, policy_instance_percentage: 46, policy_instance_minimum: 12, diff --git a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupDetails/InstanceGroupDetails.jsx b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupDetails/InstanceGroupDetails.jsx index 748e92702d..05ad3277dc 100644 --- a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupDetails/InstanceGroupDetails.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupDetails/InstanceGroupDetails.jsx @@ -78,7 +78,7 @@ function InstanceGroupDetails({ instanceGroup, i18n }) { { - return item.is_containerized + return item.is_container_group ? `${match.url}/container_group/${item.id}/details` : `${match.url}/${item.id}/details`; }; diff --git a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.jsx b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.jsx index 8bfcf05325..4c47269074 100644 --- a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.jsx @@ -32,7 +32,7 @@ function InstanceGroupListItem({ const labelId = `check-action-${instanceGroup.id}`; const isContainerGroup = item => { - return item.is_containerized; + return item.is_container_group; }; function usedCapacity(item) { diff --git a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.test.jsx b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.test.jsx index 0f22a4b6d7..579aa36dcc 100644 --- a/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.test.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/InstanceGroupList/InstanceGroupListItem.test.jsx @@ -17,7 +17,7 @@ describe('', () => { policy_instance_minimum: 10, policy_instance_percentage: 50, percent_capacity_remaining: 60, - is_containerized: false, + is_container_group: false, summary_fields: { user_capabilities: { edit: true, @@ -34,7 +34,7 @@ describe('', () => { policy_instance_minimum: 0, policy_instance_percentage: 0, percent_capacity_remaining: 0, - is_containerized: true, + is_container_group: true, summary_fields: { user_capabilities: { edit: false, diff --git a/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.test.jsx b/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.test.jsx index 62709df53e..3e48389195 100644 --- a/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.test.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.test.jsx @@ -27,7 +27,7 @@ const instanceGroup = { controller: null, is_controller: false, is_isolated: false, - is_containerized: false, + is_container_group: false, credential: 3, policy_instance_percentage: 46, policy_instance_minimum: 12, diff --git a/awx/ui_next/src/screens/InstanceGroup/shared/InstanceGroupForm.test.jsx b/awx/ui_next/src/screens/InstanceGroup/shared/InstanceGroupForm.test.jsx index 233ce7f849..0dad4fe6d1 100644 --- a/awx/ui_next/src/screens/InstanceGroup/shared/InstanceGroupForm.test.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/shared/InstanceGroupForm.test.jsx @@ -27,7 +27,7 @@ const instanceGroup = { controller: null, is_controller: false, is_isolated: false, - is_containerized: false, + is_container_group: false, credential: null, policy_instance_percentage: 46, policy_instance_minimum: 12, From 373bb443aacb7a2c8b0abe44ea51417afcc1206b Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 19:39:56 -0500 Subject: [PATCH 083/157] UnifiedJob#is_containerized -> UnifiedJob#is_container_group_task --- awx/main/isolated/manager.py | 2 +- awx/main/managers.py | 2 +- awx/main/models/unified_jobs.py | 2 +- awx/main/scheduler/task_manager.py | 2 +- awx/main/tasks.py | 6 +++--- .../functional/task_management/test_container_groups.py | 2 +- awx/main/tests/unit/test_capacity.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/awx/main/isolated/manager.py b/awx/main/isolated/manager.py index de4783e277..ffeb6af908 100644 --- a/awx/main/isolated/manager.py +++ b/awx/main/isolated/manager.py @@ -48,7 +48,7 @@ class IsolatedManager(object): self.pod_manager = pod_manager def build_inventory(self, hosts): - if self.instance and self.instance.is_containerized: + if self.instance and self.instance.is_container_group_task: inventory = {'all': {'hosts': {}}} fd, path = tempfile.mkstemp( prefix='.kubeconfig', dir=self.private_data_dir diff --git a/awx/main/managers.py b/awx/main/managers.py index ae93a552a0..1af57a9423 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -237,7 +237,7 @@ class InstanceGroupManager(models.Manager): elif t.status == 'running': # Subtract capacity from all groups that contain the instance if t.execution_node not in instance_ig_mapping: - if not t.is_containerized: + if not t.is_container_group_task: logger.warning('Detected %s running inside lost instance, ' 'may still be waiting for reaper.', t.log_format) if t.instance_group: diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 7970de1b22..cf22430f7b 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -1490,7 +1490,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return bool(self.controller_node) @property - def is_containerized(self): + def is_container_group_task(self): return False def log_lifecycle(self, state, blocked_by=None): diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index 8731ca3109..50345e5bb7 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -594,7 +594,7 @@ class TaskManager(): ).exclude( execution_node__in=Instance.objects.values_list('hostname', flat=True) ): - if j.execution_node and not j.is_containerized: + if j.execution_node and not j.is_container_group_task: logger.error(f'{j.execution_node} is not a registered instance; reaping {j.log_format}') reap_job(j, 'failed') diff --git a/awx/main/tasks.py b/awx/main/tasks.py index efc4f4594f..5dd6b0fd5c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1867,7 +1867,7 @@ class RunJob(BaseTask): ''' Return whether this task should use proot. ''' - if job.is_containerized: + if job.is_container_group_task: return False return getattr(settings, 'AWX_PROOT_ENABLED', False) @@ -1997,7 +1997,7 @@ class RunJob(BaseTask): os.path.join(private_data_dir, 'artifacts', 'fact_cache'), fact_modification_times, ) - if isolated_manager_instance and not job.is_containerized: + if isolated_manager_instance and not job.is_container_group_task: isolated_manager_instance.cleanup() try: @@ -2994,7 +2994,7 @@ class RunAdHocCommand(BaseTask): ''' Return whether this task should use proot. ''' - if ad_hoc_command.is_containerized: + if ad_hoc_command.is_container_group_task: return False return getattr(settings, 'AWX_PROOT_ENABLED', False) diff --git a/awx/main/tests/functional/task_management/test_container_groups.py b/awx/main/tests/functional/task_management/test_container_groups.py index c9028efacb..84dcaf12d7 100644 --- a/awx/main/tests/functional/task_management/test_container_groups.py +++ b/awx/main/tests/functional/task_management/test_container_groups.py @@ -29,7 +29,7 @@ def containerized_job(default_instance_group, kube_credential, job_template_fact @pytest.mark.django_db def test_containerized_job(containerized_job): - assert containerized_job.is_containerized + assert containerized_job.is_container_group_task assert containerized_job.instance_group.is_container_group assert containerized_job.instance_group.credential.kubernetes diff --git a/awx/main/tests/unit/test_capacity.py b/awx/main/tests/unit/test_capacity.py index 16fe81053c..1da05ec1f3 100644 --- a/awx/main/tests/unit/test_capacity.py +++ b/awx/main/tests/unit/test_capacity.py @@ -11,7 +11,7 @@ class FakeObject(object): class Job(FakeObject): task_impact = 43 - is_containerized = False + is_container_group_task = False def log_format(self): return 'job 382 (fake)' From 1d9f01a201818b065398f4ed228676de9dbfde3c Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 19:40:35 -0500 Subject: [PATCH 084/157] Deleted unused build_params_process_isolation method --- awx/main/tasks.py | 40 -------------------- awx/main/tests/unit/test_tasks.py | 61 ------------------------------- 2 files changed, 101 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 5dd6b0fd5c..256d477323 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1005,46 +1005,6 @@ class BaseTask(object): Build ansible yaml file filled with extra vars to be passed via -e@file.yml ''' - def build_params_process_isolation(self, instance, private_data_dir, cwd): - ''' - Build ansible runner .run() parameters for process isolation. - ''' - process_isolation_params = dict() - if self.should_use_proot(instance): - local_paths = [private_data_dir] - if cwd != private_data_dir and Path(private_data_dir) not in Path(cwd).parents: - local_paths.append(cwd) - show_paths = self.proot_show_paths + local_paths + \ - settings.AWX_PROOT_SHOW_PATHS - - pi_path = settings.AWX_PROOT_BASE_PATH - if not self.instance.is_isolated() and not self.instance.is_containerized: - pi_path = tempfile.mkdtemp( - prefix='ansible_runner_pi_', - dir=settings.AWX_PROOT_BASE_PATH - ) - os.chmod(pi_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - self.cleanup_paths.append(pi_path) - - process_isolation_params = { - 'process_isolation': True, - 'process_isolation_path': pi_path, - 'process_isolation_show_paths': show_paths, - 'process_isolation_hide_paths': [ - settings.AWX_PROOT_BASE_PATH, - '/etc/tower', - '/etc/ssh', - '/var/lib/awx', - '/var/log', - settings.PROJECTS_ROOT, - settings.JOBOUTPUT_ROOT, - ] + getattr(settings, 'AWX_PROOT_HIDE_PATHS', None) or [], - 'process_isolation_ro_paths': [settings.ANSIBLE_VENV_PATH, settings.AWX_VENV_PATH], - } - if getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH) != settings.ANSIBLE_VENV_PATH: - process_isolation_params['process_isolation_ro_paths'].append(instance.ansible_virtualenv_path) - return process_isolation_params - def build_params_resource_profiling(self, instance, private_data_dir): resource_profiling_params = {} if self.should_use_resource_profiling(instance): diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 8586ad11c7..3acdd7ead9 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -547,44 +547,6 @@ class TestGenericRun(): job_cwd='/foobar', job_env={'switch': 'blade', 'foot': 'ball', 'secret_key': 'redacted_value'}) - def test_uses_process_isolation(self, settings): - job = Job(project=Project(), inventory=Inventory()) - task = tasks.RunJob() - task.should_use_proot = lambda instance: True - task.instance = job - - private_data_dir = '/foo' - cwd = '/bar' - - settings.AWX_PROOT_HIDE_PATHS = ['/AWX_PROOT_HIDE_PATHS1', '/AWX_PROOT_HIDE_PATHS2'] - settings.ANSIBLE_VENV_PATH = '/ANSIBLE_VENV_PATH' - settings.AWX_VENV_PATH = '/AWX_VENV_PATH' - - process_isolation_params = task.build_params_process_isolation(job, private_data_dir, cwd) - assert True is process_isolation_params['process_isolation'] - assert process_isolation_params['process_isolation_path'].startswith(settings.AWX_PROOT_BASE_PATH), \ - "Directory where a temp directory will be created for the remapping to take place" - assert private_data_dir in process_isolation_params['process_isolation_show_paths'], \ - "The per-job private data dir should be in the list of directories the user can see." - assert cwd in process_isolation_params['process_isolation_show_paths'], \ - "The current working directory should be in the list of directories the user can see." - - for p in [settings.AWX_PROOT_BASE_PATH, - '/etc/tower', - '/etc/ssh', - '/var/lib/awx', - '/var/log', - settings.PROJECTS_ROOT, - settings.JOBOUTPUT_ROOT, - '/AWX_PROOT_HIDE_PATHS1', - '/AWX_PROOT_HIDE_PATHS2']: - assert p in process_isolation_params['process_isolation_hide_paths'] - assert 9 == len(process_isolation_params['process_isolation_hide_paths']) - assert '/ANSIBLE_VENV_PATH' in process_isolation_params['process_isolation_ro_paths'] - assert '/AWX_VENV_PATH' in process_isolation_params['process_isolation_ro_paths'] - assert 2 == len(process_isolation_params['process_isolation_ro_paths']) - - @mock.patch('os.makedirs') def test_build_params_resource_profiling(self, os_makedirs): job = Job(project=Project(), inventory=Inventory()) @@ -1962,29 +1924,6 @@ class TestProjectUpdateCredentials(TestJobExecution): ] } - def test_process_isolation_exposes_projects_root(self, private_data_dir, project_update): - task = tasks.RunProjectUpdate() - task.revision_path = 'foobar' - task.instance = project_update - ssh = CredentialType.defaults['ssh']() - project_update.scm_type = 'git' - project_update.credential = Credential( - pk=1, - credential_type=ssh, - ) - process_isolation = task.build_params_process_isolation(job, private_data_dir, 'cwd') - - assert process_isolation['process_isolation'] is True - assert settings.PROJECTS_ROOT in process_isolation['process_isolation_show_paths'] - - task._write_extra_vars_file = mock.Mock() - - with mock.patch.object(Licenser, 'validate', lambda *args, **kw: {}): - task.build_extra_vars_file(project_update, private_data_dir) - - call_args, _ = task._write_extra_vars_file.call_args_list[0] - _, extra_vars = call_args - def test_username_and_password_auth(self, project_update, scm_type): task = tasks.RunProjectUpdate() ssh = CredentialType.defaults['ssh']() From d37cb64aaf0197ede2b336de23e5ff679a5a35c6 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 19:46:48 -0500 Subject: [PATCH 085/157] Delete some old container group v1 code --- awx/main/isolated/manager.py | 26 ++++---------------- awx/main/tasks.py | 47 ------------------------------------ 2 files changed, 5 insertions(+), 68 deletions(-) diff --git a/awx/main/isolated/manager.py b/awx/main/isolated/manager.py index ffeb6af908..61a55ff7a9 100644 --- a/awx/main/isolated/manager.py +++ b/awx/main/isolated/manager.py @@ -32,7 +32,7 @@ def set_pythonpath(venv_libdir, env): class IsolatedManager(object): - def __init__(self, event_handler, canceled_callback=None, check_callback=None, pod_manager=None): + def __init__(self, event_handler, canceled_callback=None, check_callback=None): """ :param event_handler: a callable used to persist event data from isolated nodes :param canceled_callback: a callable - which returns `True` or `False` @@ -45,28 +45,12 @@ class IsolatedManager(object): self.started_at = None self.captured_command_artifact = False self.instance = None - self.pod_manager = pod_manager def build_inventory(self, hosts): - if self.instance and self.instance.is_container_group_task: - inventory = {'all': {'hosts': {}}} - fd, path = tempfile.mkstemp( - prefix='.kubeconfig', dir=self.private_data_dir - ) - with open(path, 'wb') as temp: - temp.write(yaml.dump(self.pod_manager.kube_config).encode()) - temp.flush() - os.chmod(temp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - for host in hosts: - inventory['all']['hosts'][host] = { - "ansible_connection": "kubectl", - "ansible_kubectl_config": path, - } - else: - inventory = '\n'.join([ - '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME) - for host in hosts - ]) + inventory = '\n'.join([ + '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME) + for host in hosts + ]) return inventory diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 256d477323..2da516aacd 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1180,12 +1180,6 @@ class BaseTask(object): if os.path.isdir(job_profiling_dir): shutil.copytree(job_profiling_dir, os.path.join(awx_profiling_dir, str(instance.pk))) - if instance.is_containerized: - from awx.main.scheduler.kubernetes import PodManager # prevent circular import - pm = PodManager(instance) - logger.debug(f"Deleting pod {pm.pod_name}") - pm.delete() - def event_handler(self, event_data): # @@ -1325,16 +1319,6 @@ class BaseTask(object): Run the job/task and capture its output. ''' self.instance = self.model.objects.get(pk=pk) - containerized = self.instance.is_containerized - pod_manager = None - if containerized: - # Here we are trying to launch a pod before transitioning the job into a running - # state. For some scenarios (like waiting for resources to become available) we do this - # rather than marking the job as error or failed. This is not always desirable. Cases - # such as invalid authentication should surface as an error. - pod_manager = self.deploy_container_group_pod(self.instance) - if not pod_manager: - return # self.instance because of the update_model pattern and when it's used in callback handlers self.instance = self.update_model(pk, status='running', @@ -1517,37 +1501,6 @@ class BaseTask(object): raise AwxTaskError.TaskError(self.instance, rc) - def deploy_container_group_pod(self, task): - from awx.main.scheduler.kubernetes import PodManager # Avoid circular import - pod_manager = PodManager(self.instance) - try: - log_name = task.log_format - logger.debug(f"Launching pod for {log_name}.") - pod_manager.deploy() - except (ApiException, Exception) as exc: - if isinstance(exc, ApiException) and exc.status == 403: - try: - if 'exceeded quota' in json.loads(exc.body)['message']: - # If the k8s cluster does not have capacity, we move the - # job back into pending and wait until the next run of - # the task manager. This does not exactly play well with - # our current instance group precendence logic, since it - # will just sit here forever if kubernetes returns this - # error. - logger.warn(exc.body) - logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.") - self.update_model(task.pk, status='pending') - return - except Exception: - logger.exception(f"Unable to handle response from Kubernetes API for {log_name}.") - - logger.exception(f"Error when launching pod for {log_name}") - self.update_model(task.pk, status='error', result_traceback=traceback.format_exc()) - return - - self.update_model(task.pk, execution_node=pod_manager.pod_name) - return pod_manager - From 9df29e8fc41824dd78839909d50bc25ba24dfa0e Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 20:02:09 -0500 Subject: [PATCH 086/157] Use official awx-ee by default --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 7812b3af31..abf9d9014f 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -60,7 +60,7 @@ DATABASES = { } # TODO: remove this setting in favor of a default execution environment -AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/shanemcd/ee' +AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee' AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 From 70f7a082bba06ab0d2229eeb04741284264e66a5 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 20:02:45 -0500 Subject: [PATCH 087/157] Minimally functional container group v2 w/ receptor --- awx/main/tasks.py | 147 ++++++++++++++++++++++++----- awx/main/utils/common.py | 21 ++++- tools/docker-compose/receptor.conf | 2 +- 3 files changed, 146 insertions(+), 24 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 2da516aacd..ee630dcf2e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -78,7 +78,8 @@ from awx.main.dispatch import get_local_queuename, reaper from awx.main.utils import (update_scm_url, ignore_inventory_computed_fields, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, - get_awx_version) + get_awx_version, + deepmerge) from awx.main.utils.ansible import read_ansible_config from awx.main.utils.external_logging import reconfigure_rsyslog from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja @@ -1379,7 +1380,6 @@ class BaseTask(object): args = self.build_args(self.instance, private_data_dir, passwords) resource_profiling_params = self.build_params_resource_profiling(self.instance, private_data_dir) - execution_environment_params = self.build_execution_environment_params(self.instance) env = self.build_env(self.instance, private_data_dir, isolated, private_data_files=private_data_files) self.safe_env = build_safe_env(env) @@ -1413,12 +1413,6 @@ class BaseTask(object): }, } - if containerized: - # We don't want HOME passed through to container groups. - # TODO: remove this conditional after everything is containerized - params['envvars'].pop('HOME', None) - - if isinstance(self.instance, AdHocCommand): params['module'] = self.build_module_name(self.instance) params['module_args'] = self.build_module_args(self.instance) @@ -1439,17 +1433,8 @@ class BaseTask(object): self.dispatcher = CallbackQueueDispatcher() - if not isinstance(self.instance, ProjectUpdate): - work_type='worker' - # TODO: container group jobs will not work with container isolation settings - # but both will run with same settings when worker_in and worker_out are added - params['settings'].update(execution_environment_params) - else: - work_type='worker' - params['settings'].update(execution_environment_params) - self.instance.log_lifecycle("running_playbook") - receptor_job = AWXReceptorJob(self, work_type, params) + receptor_job = AWXReceptorJob(self, params) res = receptor_job.run() status = res.status @@ -3045,11 +3030,14 @@ def deep_copy_model_obj( class AWXReceptorJob: - def __init__(self, task, work_type, runner_params): + def __init__(self, task, runner_params): self.task = task - self.work_type = work_type self.runner_params = runner_params + if not self.task.instance.is_container_group_task: + execution_environment_params = self.task.build_execution_environment_params(self.task.instance) + self.runner_params['settings'].update(execution_environment_params) + def run(self): # Create a socketpair. Where the left side will be used for writing our payload # (private data dir, kwargs). The right side will be passed to Receptor for @@ -3062,7 +3050,8 @@ class AWXReceptorJob: # in the right side of our socketpair for reading. receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') result = receptor_ctl.submit_work(worktype=self.work_type, - payload=sockout.makefile('rb')) + payload=sockout.makefile('rb'), + params=self.receptor_params) unit_id = result['unitid'] sockin.close() @@ -3094,7 +3083,6 @@ class AWXReceptorJob: receptor_work_list = receptor_ctl.simple_command("work list") raise RuntimeError(receptor_work_list[unit_id]['Detail']) - receptor_ctl.simple_command(f"work release {unit_id}") return res @@ -3117,6 +3105,31 @@ class AWXReceptorJob: status_handler=self.task.status_handler, **self.runner_params) + @property + def receptor_params(self): + receptor_params = {} + if self.task.instance.is_container_group_task: + spec_yaml = yaml.dump(self.pod_definition, explicit_start=True) + kubeconfig_yaml = yaml.dump(self.kube_config, explicit_start=True) + + receptor_params = { + "secret_kube_pod": spec_yaml, + "secret_kube_config": kubeconfig_yaml + } + + return receptor_params + + + + @property + def work_type(self): + if self.task.instance.is_container_group_task: + work_type = 'ocp' + else: + work_type = 'local' + + return work_type + def cancel_watcher(self, processor_future): while True: if processor_future.done(): @@ -3126,3 +3139,93 @@ class AWXReceptorJob: result = namedtuple('result', ['status', 'rc']) return result('canceled', 1) time.sleep(1) + + @property + def pod_definition(self): + default_pod_spec = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE + }, + "spec": { + "containers": [{ + "image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE, + "name": 'worker', + "args": ['ansible-runner', 'worker'] + }] + } + } + + pod_spec_override = {} + if self.task and self.task.instance.instance_group.pod_spec_override: + pod_spec_override = parse_yaml_or_json( + self.task.instance_group.pod_spec_override) + pod_spec = {**default_pod_spec, **pod_spec_override} + + if self.task: + pod_spec['metadata'] = deepmerge( + pod_spec.get('metadata', {}), + dict(name=self.pod_name, + labels={ + 'ansible-awx': settings.INSTALL_UUID, + 'ansible-awx-job-id': str(self.task.instance.id) + })) + + return pod_spec + + @property + def pod_name(self): + return f"awx-job-{self.task.instance.id}" + + @property + def credential(self): + return self.task.instance.instance_group.credential + + @property + def namespace(self): + return self.pod_definition['metadata']['namespace'] + + @property + def kube_config(self): + host_input = self.credential.get_input('host') + config = { + "apiVersion": "v1", + "kind": "Config", + "preferences": {}, + "clusters": [ + { + "name": host_input, + "cluster": { + "server": host_input + } + } + ], + "users": [ + { + "name": host_input, + "user": { + "token": self.credential.get_input('bearer_token') + } + } + ], + "contexts": [ + { + "name": host_input, + "context": { + "cluster": host_input, + "user": host_input, + "namespace": self.namespace + } + } + ], + "current-context": host_input + } + + if self.credential.get_input('verify_ssl') and 'ssl_ca_cert' in self.credential.inputs: + config["clusters"][0]["cluster"]["certificate-authority-data"] = b64encode( + self.credential.get_input('ssl_ca_cert').encode() # encode to bytes + ).decode() # decode the base64 data into a str + else: + config["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = True + return config diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 283a028f3f..ad90d5e4ec 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -55,7 +55,8 @@ __all__ = [ 'model_instance_diff', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices', 'get_external_account', 'task_manager_bulk_reschedule', - 'schedule_task_manager', 'classproperty', 'create_temporary_fifo', 'truncate_stdout' + 'schedule_task_manager', 'classproperty', 'create_temporary_fifo', 'truncate_stdout', + 'deepmerge' ] @@ -1079,3 +1080,21 @@ def truncate_stdout(stdout, size): set_count += 1 return stdout + u'\u001b[0m' * (set_count - reset_count) + + +def deepmerge(a, b): + """ + Merge dict structures and return the result. + + >>> a = {'first': {'all_rows': {'pass': 'dog', 'number': '1'}}} + >>> b = {'first': {'all_rows': {'fail': 'cat', 'number': '5'}}} + >>> import pprint; pprint.pprint(deepmerge(a, b)) + {'first': {'all_rows': {'fail': 'cat', 'number': '5', 'pass': 'dog'}}} + """ + if isinstance(a, dict) and isinstance(b, dict): + return dict([(k, deepmerge(a.get(k), b.get(k))) + for k in set(a.keys()).union(b.keys())]) + elif b is None: + return a + else: + return b diff --git a/tools/docker-compose/receptor.conf b/tools/docker-compose/receptor.conf index ba975b8820..4c498243e2 100644 --- a/tools/docker-compose/receptor.conf +++ b/tools/docker-compose/receptor.conf @@ -8,7 +8,7 @@ - local-only: - work-command: - worktype: worker + worktype: local command: ansible-runner params: worker From fe9b24cde2aa06fc0328411264fa6c8ddd586ccf Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 13 Jan 2021 20:20:16 -0500 Subject: [PATCH 088/157] flake8 --- awx/main/isolated/manager.py | 1 - awx/main/tasks.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/awx/main/isolated/manager.py b/awx/main/isolated/manager.py index 61a55ff7a9..abcd41c5c1 100644 --- a/awx/main/isolated/manager.py +++ b/awx/main/isolated/manager.py @@ -6,7 +6,6 @@ import stat import tempfile import time import logging -import yaml import datetime from django.conf import settings diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ee630dcf2e..7f5dede047 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -26,6 +26,7 @@ import urllib.parse as urlparse import socket import threading import concurrent.futures +from base64 import b64encode # Django from django.conf import settings @@ -39,9 +40,6 @@ from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist from django_guid.middleware import GuidMiddleware -# Kubernetes -from kubernetes.client.rest import ApiException - # Django-CRUM from crum import impersonate @@ -79,7 +77,8 @@ from awx.main.utils import (update_scm_url, ignore_inventory_computed_fields, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, get_awx_version, - deepmerge) + deepmerge, + parse_yaml_or_json) from awx.main.utils.ansible import read_ansible_config from awx.main.utils.external_logging import reconfigure_rsyslog from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja From 70755a395b17e13d7f102df69713a35b33b83d9a Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 14 Jan 2021 19:34:36 -0500 Subject: [PATCH 089/157] Make receptorctl easier to use in dev env --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 1 + .../ansible/roles/sources/templates/docker-compose.yml.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 0d48f2ea56..bf04f8fd00 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -270,6 +270,7 @@ ENV HOME="/var/lib/awx" ENV PATH="/usr/pgsql-10/bin:${PATH}" {% if build_dev|bool %} +ENV PATH="/var/lib/awx/venv/awx/bin/:${PATH}" EXPOSE 8043 8013 8080 22 diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index db81b4ba39..00f136ef49 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -13,6 +13,7 @@ services: SDB_HOST: 0.0.0.0 SDB_PORT: 7899 AWX_GROUP_QUEUES: tower + RECEPTORCTL_SOCKET: /var/run/receptor/receptor.sock ports: - "2222:2222" - "8888:8888" From 341e1e34e336f85042aebe04a0ff45e830922d04 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 14 Jan 2021 19:35:28 -0500 Subject: [PATCH 090/157] Dont zip/unzip private data dir for local jobs --- awx/main/tasks.py | 9 ++++++++- tools/docker-compose/receptor.conf | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 7f5dede047..f660cf1813 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3088,6 +3088,9 @@ class AWXReceptorJob: # Spawned in a thread so Receptor can start reading before we finish writing, we # write our payload to the left side of our socketpair. def transmit(self, _socket): + if self.work_type == 'local': + self.runner_params['only_transmit_kwargs'] = True + ansible_runner.interface.run(streamer='transmit', _output=_socket.makefile('wb'), **self.runner_params) @@ -3106,7 +3109,6 @@ class AWXReceptorJob: @property def receptor_params(self): - receptor_params = {} if self.task.instance.is_container_group_task: spec_yaml = yaml.dump(self.pod_definition, explicit_start=True) kubeconfig_yaml = yaml.dump(self.kube_config, explicit_start=True) @@ -3115,6 +3117,11 @@ class AWXReceptorJob: "secret_kube_pod": spec_yaml, "secret_kube_config": kubeconfig_yaml } + else: + private_data_dir = self.runner_params['private_data_dir'] + receptor_params = { + "params": f"--private-data-dir={private_data_dir}" + } return receptor_params diff --git a/tools/docker-compose/receptor.conf b/tools/docker-compose/receptor.conf index 4c498243e2..f86d50f523 100644 --- a/tools/docker-compose/receptor.conf +++ b/tools/docker-compose/receptor.conf @@ -11,6 +11,7 @@ worktype: local command: ansible-runner params: worker + allowruntimeparams: true - work-kubernetes: worktype: ocp From c6be92cdf63e1b8e5cb2986770811394e38730a5 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sun, 17 Jan 2021 13:20:39 -0500 Subject: [PATCH 091/157] Create awx group in container --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 1 + tools/docker-compose/entrypoint.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index bf04f8fd00..8c7e685736 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -234,6 +234,7 @@ RUN for dir in \ for file in \ /etc/subuid \ /etc/subgid \ + /etc/group \ /etc/passwd \ /var/lib/awx/rsyslog/rsyslog.conf ; \ do touch $file ; chmod g+rw $file ; chgrp root $file ; done diff --git a/tools/docker-compose/entrypoint.sh b/tools/docker-compose/entrypoint.sh index c155f022e4..006435000a 100755 --- a/tools/docker-compose/entrypoint.sh +++ b/tools/docker-compose/entrypoint.sh @@ -7,6 +7,10 @@ root:x:0:0:root:/root:/bin/bash awx:x:`id -u`:`id -g`:,,,:/var/lib/awx:/bin/bash EOF +cat <> /etc/group +awx:x:`id -u`:awx +EOF + cat < /etc/subuid awx:100000:50001 EOF From e63383bde6f4f34307a256ea4222d61100ee6a53 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sun, 17 Jan 2021 13:31:39 -0500 Subject: [PATCH 092/157] Add PATH to blocked inventory source vars This used to be skiped because PATH was already present in the env we constructed for runner. --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index abf9d9014f..9d21ac68bb 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -685,7 +685,7 @@ AD_HOC_COMMANDS = [ 'win_user', ] -INV_ENV_VARIABLE_BLOCKED = ("HOME", "USER", "_", "TERM") +INV_ENV_VARIABLE_BLOCKED = ("HOME", "USER", "_", "TERM", "PATH") # ---------------- # -- Amazon EC2 -- From 3f76499c565b10bbf698213a496fe915c2efe2a9 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 18 Jan 2021 10:44:26 -0500 Subject: [PATCH 093/157] Use the fully qualified inventory plugin name only for foreman --- awx/main/models/inventory.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 28f4b09948..4e671af7df 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1373,6 +1373,7 @@ class PluginFileInjector(object): collection = None collection_migration = '2.9' # Starting with this version, we use collections + # TODO: delete this method and update unit tests @classmethod def get_proper_name(cls): if cls.plugin_name is None: @@ -1397,13 +1398,12 @@ class PluginFileInjector(object): def inventory_as_dict(self, inventory_update, private_data_dir): source_vars = dict(inventory_update.source_vars_dict) # make a copy - proper_name = self.get_proper_name() ''' None conveys that we should use the user-provided plugin. Note that a plugin value of '' should still be overridden. ''' - if proper_name is not None: - source_vars['plugin'] = proper_name + if self.plugin_name is not None: + source_vars['plugin'] = self.plugin_name return source_vars def build_env(self, inventory_update, env, private_data_dir, private_data_files): @@ -1573,6 +1573,12 @@ class satellite6(PluginFileInjector): ret['FOREMAN_PASSWORD'] = credential.get_input('password', default='') return ret + def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(satellite6, self).inventory_as_dict(inventory_update, private_data_dir) + # this inventory plugin requires the fully qualified inventory plugin name + ret['plugin'] = f'{cls.namespace}.{cls.collection}.{cls.plugin_name}' + return ret + class tower(PluginFileInjector): plugin_name = 'tower' From b05b6b2e0368d2ae87d7e53c897dc32b1e66cf3a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 19 Jan 2021 14:48:55 -0500 Subject: [PATCH 094/157] Fix minor syntax error failing AdHocCommands --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index f660cf1813..0fb1f92732 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3166,7 +3166,7 @@ class AWXReceptorJob: pod_spec_override = {} if self.task and self.task.instance.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( - self.task.instance_group.pod_spec_override) + self.task.instance.instance_group.pod_spec_override) pod_spec = {**default_pod_spec, **pod_spec_override} if self.task: From c29d476919417bdc2d4ba632eebbbb2ba66f36c5 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 21 Jan 2021 10:55:41 -0500 Subject: [PATCH 095/157] Fix obvious code error with foreman inventory --- awx/main/models/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 4e671af7df..9000fe41c4 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1576,7 +1576,7 @@ class satellite6(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super(satellite6, self).inventory_as_dict(inventory_update, private_data_dir) # this inventory plugin requires the fully qualified inventory plugin name - ret['plugin'] = f'{cls.namespace}.{cls.collection}.{cls.plugin_name}' + ret['plugin'] = f'{self.namespace}.{self.collection}.{self.plugin_name}' return ret From f850f8d3e0ebc97e3efed7d5001395bae85c997b Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 21 Jan 2021 12:25:34 -0500 Subject: [PATCH 096/157] introduce a new global flag for denoating K8S-based deployments - In K8S-based installs, only container groups are intended to be used for playbook execution (JTs, adhoc, inventory updates), so in this scenario, other job types have a task impact of zero. - In K8S-based installs, traditional instances have *zero* capacity (because they're only members of the control plane where services - http/s, local control plane execution - run) - This commit also includes some changes that allow for the task manager to launch tasks with task_impact=0 on instances that have capacity=0 (previously, an instance with zero capacity would never be selected as the "execution node" This means that when IS_K8S=True, any Job Template associated with an Instance Group will never actually go from pending -> running (because there's no capacity - all playbooks must run through Container Groups). For an improved ux, our intention is to introduce logic into the operator install process such that the *default* group that's created at install time is a *Container Group* that's configured to point at the K8S cluster where awx itself is deployed. --- awx/main/models/ha.py | 7 +++++++ awx/main/models/jobs.py | 2 ++ awx/main/models/projects.py | 2 ++ awx/main/scheduler/task_manager.py | 9 ++++++--- awx/settings/defaults.py | 7 +++++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index 4f96bdc5b1..6dd72861cc 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -147,6 +147,13 @@ class Instance(HasPolicyEditsMixin, BaseModel): return self.rampart_groups.filter(controller__isnull=False).exists() def refresh_capacity(self): + if settings.IS_K8S: + self.capacity = self.cpu = self.memory = self.cpu_capacity = self.mem_capacity = 0 # noqa + self.version = awx_application_version + self.save(update_fields=['capacity', 'version', 'modified', 'cpu', + 'memory', 'cpu_capacity', 'mem_capacity']) + return + cpu = get_cpu_capacity() mem = get_mem_capacity() if self.enabled: diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 81e17cdebf..70cdfa363a 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -1286,6 +1286,8 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin): @property def task_impact(self): + if settings.IS_K8S: + return 0 return 5 @property diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index ec14a2ef76..fb948916d0 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -563,6 +563,8 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage @property def task_impact(self): + if settings.IS_K8S: + return 0 return 0 if self.job_type == 'run' else 1 @property diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index 50345e5bb7..8d8df5eee2 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -70,7 +70,7 @@ class TaskManager(): ''' Init AFTER we know this instance of the task manager will run because the lock is acquired. ''' - instances = Instance.objects.filter(~Q(hostname=None), capacity__gt=0, enabled=True) + instances = Instance.objects.filter(~Q(hostname=None), enabled=True) self.real_instances = {i.hostname: i for i in instances} instances_partial = [SimpleNamespace(obj=instance, @@ -86,7 +86,7 @@ class TaskManager(): capacity_total=rampart_group.capacity, consumed_capacity=0, instances=[]) - for instance in rampart_group.instances.filter(capacity__gt=0, enabled=True).order_by('hostname'): + for instance in rampart_group.instances.filter(enabled=True).order_by('hostname'): if instance.hostname in instances_by_hostname: self.graph[rampart_group.name]['instances'].append(instances_by_hostname[instance.hostname]) @@ -528,7 +528,10 @@ class TaskManager(): break remaining_capacity = self.get_remaining_capacity(rampart_group.name) - if not rampart_group.is_container_group and self.get_remaining_capacity(rampart_group.name) <= 0: + if ( + task.task_impact > 0 and # project updates have a cost of zero + not rampart_group.is_container_group and + self.get_remaining_capacity(rampart_group.name) <= 0): logger.debug("Skipping group {}, remaining_capacity {} <= 0".format( rampart_group.name, remaining_capacity)) continue diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 9d21ac68bb..845bbe74d8 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -59,6 +59,13 @@ DATABASES = { } } +# Whether or not the deployment is a K8S-based deployment +# In K8S-based deployments, instances have zero capacity - all playbook +# automation is intended to flow through defined Container Groups that +# interface with some (or some set of) K8S api (which may or may not include +# the K8S cluster where awx itself is running) +IS_K8S = False + # TODO: remove this setting in favor of a default execution environment AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee' From a435843f23260cefae59664603759c9791f21134 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 25 Jan 2021 15:15:14 -0500 Subject: [PATCH 097/157] Exception handling to always release work units --- awx/main/tasks.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0fb1f92732..686bcf7526 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3038,6 +3038,17 @@ class AWXReceptorJob: self.runner_params['settings'].update(execution_environment_params) def run(self): + # We establish a connection to the Receptor socket + receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') + + try: + return self._run_internal(receptor_ctl) + finally: + # Make sure to always release the work unit if we established it + if self.unit_id is not None: + receptor_ctl.simple_command(f"work release {self.unit_id}") + + def _run_internal(self, receptor_ctl): # Create a socketpair. Where the left side will be used for writing our payload # (private data dir, kwargs). The right side will be passed to Receptor for # reading. @@ -3045,18 +3056,17 @@ class AWXReceptorJob: threading.Thread(target=self.transmit, args=[sockin]).start() - # We establish a connection to the Receptor socket and submit our work, passing + # submit our work, passing # in the right side of our socketpair for reading. - receptor_ctl = ReceptorControl('/var/run/receptor/receptor.sock') result = receptor_ctl.submit_work(worktype=self.work_type, payload=sockout.makefile('rb'), params=self.receptor_params) - unit_id = result['unitid'] + self.unit_id = result['unitid'] sockin.close() sockout.close() - resultsock, resultfile = receptor_ctl.get_work_results(unit_id, + resultsock, resultfile = receptor_ctl.get_work_results(self.unit_id, return_socket=True, return_sockfile=True) # Both "processor" and "cancel_watcher" are spawned in separate threads. @@ -3074,15 +3084,14 @@ class AWXReceptorJob: res = list(first_future.done)[0].result() if res.status == 'canceled': - receptor_ctl.simple_command(f"work cancel {unit_id}") + receptor_ctl.simple_command(f"work cancel {self.unit_id}") resultsock.shutdown(socket.SHUT_RDWR) resultfile.close() elif res.status == 'error': # TODO: There should be a more efficient way of getting this information receptor_work_list = receptor_ctl.simple_command("work list") - raise RuntimeError(receptor_work_list[unit_id]['Detail']) + raise RuntimeError(receptor_work_list[self.unit_id]['Detail']) - receptor_ctl.simple_command(f"work release {unit_id}") return res # Spawned in a thread so Receptor can start reading before we finish writing, we From fd0c4ec869425bffab02274b4aebd443788fbd12 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 26 Jan 2021 13:47:44 -0500 Subject: [PATCH 098/157] Pin to latest version of PyYAML Fixes https://github.com/yaml/pyyaml/issues/478 --- requirements/requirements.in | 2 +- requirements/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 93b5b4f72e..f54094ae72 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -40,7 +40,7 @@ pygerduty pyparsing python3-saml python-ldap>=3.3.1 # https://github.com/python-ldap/python-ldap/issues/270 -pyyaml>=5.3.1 # minimum version to pull in new pyyaml for CVE-2017-18342 +pyyaml>=5.4.1 # minimum to fix https://github.com/yaml/pyyaml/issues/478 schedule==0.6.0 social-auth-core==3.3.1 # see UPGRADE BLOCKERs social-auth-app-django==3.1.0 # see UPGRADE BLOCKERs diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a6901fd3ef..6256afd061 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -100,7 +100,7 @@ python-string-utils==1.0.0 # via openshift python3-openid==3.1.0 # via social-auth-core python3-saml==1.9.0 # via -r /awx_devel/requirements/requirements.in pytz==2019.3 # via django, irc, tempora, twilio -pyyaml==5.3.1 # via -r /awx_devel/requirements/requirements.in, ansible-runner, djangorestframework-yaml, kubernetes +pyyaml==5.4.1 # via -r /awx_devel/requirements/requirements.in, ansible-runner, djangorestframework-yaml, kubernetes redis==3.4.1 # via -r /awx_devel/requirements/requirements.in, django-redis requests-oauthlib==1.3.0 # via kubernetes, msrest, social-auth-core requests==2.23.0 # via -r /awx_devel/requirements/requirements.in, adal, azure-keyvault, django-oauth-toolkit, kubernetes, msrest, requests-oauthlib, slackclient, social-auth-core, twilio From 8ab7745e3abea3476beda4a293f788b203ed96d7 Mon Sep 17 00:00:00 2001 From: Yago Marques Date: Tue, 26 Jan 2021 16:57:01 -0300 Subject: [PATCH 099/157] WIP Inclusion of the EE option in the payloads within the Organization and Projects. (#9145) * add ee option on factories for organizations * add new lines * remove inventory from the options * remove line * remove line from projects * fix the tuple * fix lint problems --- awxkit/awxkit/api/pages/organizations.py | 12 +++++++++++- awxkit/awxkit/api/pages/projects.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/organizations.py b/awxkit/awxkit/api/pages/organizations.py index e1d13a0013..fe34eb07e7 100644 --- a/awxkit/awxkit/api/pages/organizations.py +++ b/awxkit/awxkit/api/pages/organizations.py @@ -39,10 +39,20 @@ class Organization(HasCreate, HasInstanceGroups, HasNotifications, base.Base): "disassociate": True, }) - def payload(self, **kwargs): payload = PseudoNamespace(name=kwargs.get('name') or 'Organization - {}'.format(random_title()), description=kwargs.get('description') or random_title(10)) + + for fk_field in ('default_environment',): + rel_obj = kwargs.get(fk_field) + if rel_obj is None: + continue + elif isinstance(rel_obj, int): + payload.update(**{fk_field: int(rel_obj)}) + elif hasattr(rel_obj, 'id'): + payload.update(**{fk_field: rel_obj.id}) + else: + raise AttributeError(f'Related field {fk_field} must be either integer of pkid or object') return payload def create_payload(self, name='', description='', **kwargs): diff --git a/awxkit/awxkit/api/pages/projects.py b/awxkit/awxkit/api/pages/projects.py index e40191260c..bc46f5edb3 100644 --- a/awxkit/awxkit/api/pages/projects.py +++ b/awxkit/awxkit/api/pages/projects.py @@ -43,6 +43,16 @@ class Project(HasCopy, HasCreate, HasNotifications, UnifiedJobTemplate): 'allow_override') update_payload(payload, fields, kwargs) + for fk_field in ('execution_environment', 'default_environment'): + rel_obj = kwargs.get(fk_field) + if rel_obj is None: + continue + elif isinstance(rel_obj, int): + payload.update(**{fk_field: int(rel_obj)}) + elif hasattr(rel_obj, 'id'): + payload.update(**{fk_field: rel_obj.id}) + else: + raise AttributeError(f'Related field {fk_field} must be either integer of pkid or object') return payload def create_payload( From c7e0e30f932a4c3043f59e39b31db0f7cfa4c647 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 27 Jan 2021 11:55:32 -0500 Subject: [PATCH 100/157] Make sure project updates run in default EE (#9172) * Make sure project updates run in default EE * Remove project execution_environment field from collection --- awx/api/serializers.py | 2 +- awx/main/models/mixins.py | 14 +++++++++----- awx/main/models/projects.py | 8 ++++++++ awx_collection/plugins/modules/tower_project.py | 8 -------- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 3a28239b0d..eb13f150d3 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1394,7 +1394,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): class Meta: model = Project - fields = ('*', 'organization', 'scm_update_on_launch', + fields = ('*', '-execution_environment', 'organization', 'scm_update_on_launch', 'scm_update_cache_timeout', 'allow_override', 'custom_virtualenv', 'default_environment') + \ ('last_update_failed', 'last_updated') # Backwards compatibility diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 459eadabcf..549c93607d 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -455,12 +455,17 @@ class ExecutionEnvironmentMixin(models.Model): help_text=_('The container image to be used for execution.'), ) + def get_execution_environment_default(self): + from awx.main.models.execution_environments import ExecutionEnvironment + + if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: + return settings.DEFAULT_EXECUTION_ENVIRONMENT + return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() + def resolve_execution_environment(self): """ Return the execution environment that should be used when creating a new job. """ - from awx.main.models.execution_environments import ExecutionEnvironment - if self.execution_environment is not None: return self.execution_environment if getattr(self, 'project_id', None) and self.project.default_environment is not None: @@ -470,9 +475,8 @@ class ExecutionEnvironmentMixin(models.Model): if getattr(self, 'inventory', None) and self.inventory.organization is not None: if self.inventory.organization.default_environment is not None: return self.inventory.organization.default_environment - if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: - return settings.DEFAULT_EXECUTION_ENVIRONMENT - return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() + + return self.get_execution_environment_default() class CustomVirtualEnvMixin(models.Model): diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index fb948916d0..c9bf87f408 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -187,6 +187,14 @@ class ProjectOptions(models.Model): pass return cred + def resolve_execution_environment(self): + """ + Project updates, themselves, will use the default execution environment. + Jobs using the project can use the default_environment, but the project updates + are not flexible enough to allow customizing the image they use. + """ + return self.get_execution_environment_default() + def get_project_path(self, check_if_exists=True): local_path = os.path.basename(self.local_path) if local_path and not local_path.startswith('.'): diff --git a/awx_collection/plugins/modules/tower_project.py b/awx_collection/plugins/modules/tower_project.py index 1a8248c1fa..acbf3833b5 100644 --- a/awx_collection/plugins/modules/tower_project.py +++ b/awx_collection/plugins/modules/tower_project.py @@ -106,10 +106,6 @@ options: description: - Default Execution Environment to use for jobs relating to the project. type: str - execution_environment: - description: - - Execution Environment to use for project updates. - type: str organization: description: - Name of organization for project. @@ -248,7 +244,6 @@ def main(): timeout=dict(type='int', default=0, aliases=['job_timeout']), custom_virtualenv=dict(), default_environment=dict(), - execution_environment=dict(), organization=dict(), notification_templates_started=dict(type="list", elements='str'), notification_templates_success=dict(type="list", elements='str'), @@ -281,7 +276,6 @@ def main(): timeout = module.params.get('timeout') custom_virtualenv = module.params.get('custom_virtualenv') default_ee = module.params.get('default_environment') - ee = module.params.get('execution_environment') organization = module.params.get('organization') state = module.params.get('state') wait = module.params.get('wait') @@ -347,8 +341,6 @@ def main(): project_fields['credential'] = credential if default_ee is not None: project_fields['default_environment'] = module.resolve_name_to_id('execution_environments', default_ee) - if ee is not None: - project_fields['execution_environment'] = module.resolve_name_to_id('execution_environments', ee) if allow_override is not None: project_fields['allow_override'] = allow_override if scm_type == '': From 8f66793276f92159b92eada75e2406d676ac3d20 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 27 Jan 2021 16:10:56 -0500 Subject: [PATCH 101/157] Assure that unit_id is always defined (#9180) --- awx/main/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 686bcf7526..db6b8644cd 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3032,6 +3032,7 @@ class AWXReceptorJob: def __init__(self, task, runner_params): self.task = task self.runner_params = runner_params + self.unit_id = None if not self.task.instance.is_container_group_task: execution_environment_params = self.task.build_execution_environment_params(self.task.instance) From 86363e260e756313ce95af00d5f250b80f4cec8d Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 27 Jan 2021 18:49:06 -0500 Subject: [PATCH 102/157] Provide new default pod defintion in CG metadata (#9181) --- awx/api/metadata.py | 4 ++-- awx/main/tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/api/metadata.py b/awx/api/metadata.py index 0b60f9a1ef..dedeeba8fb 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -24,7 +24,7 @@ from rest_framework.request import clone_request from awx.api.fields import ChoiceNullField from awx.main.fields import JSONField, ImplicitRoleField from awx.main.models import NotificationTemplate -from awx.main.scheduler.kubernetes import PodManager +from awx.main.tasks import AWXReceptorJob class Metadata(metadata.SimpleMetadata): @@ -209,7 +209,7 @@ class Metadata(metadata.SimpleMetadata): continue if field == "pod_spec_override": - meta['default'] = PodManager().pod_definition + meta['default'] = AWXReceptorJob().pod_definition # Add type choices if available from the serializer. if field == 'type' and hasattr(serializer, 'get_type_choices'): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index db6b8644cd..ef0bd7e6e5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3029,12 +3029,12 @@ def deep_copy_model_obj( class AWXReceptorJob: - def __init__(self, task, runner_params): + def __init__(self, task=None, runner_params=None): self.task = task self.runner_params = runner_params self.unit_id = None - if not self.task.instance.is_container_group_task: + if self.task and not self.task.instance.is_container_group_task: execution_environment_params = self.task.build_execution_environment_params(self.task.instance) self.runner_params['settings'].update(execution_environment_params) From 1a68df275c2878ef736e52201b4594e3dae70355 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sun, 31 Jan 2021 12:03:03 -0500 Subject: [PATCH 103/157] Set correct SDB_NOTIFY_HOST in minikube env --- tools/ansible/roles/dockerfile/files/launch_awx.sh | 2 +- tools/ansible/roles/dockerfile/files/launch_awx_task.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ansible/roles/dockerfile/files/launch_awx.sh b/tools/ansible/roles/dockerfile/files/launch_awx.sh index 839f7cf746..7b5e86f0bd 100644 --- a/tools/ansible/roles/dockerfile/files/launch_awx.sh +++ b/tools/ansible/roles/dockerfile/files/launch_awx.sh @@ -10,7 +10,7 @@ if [ -n "${AWX_KUBE_DEVEL}" ]; then make awx-link popd - export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}') + export SDB_NOTIFY_HOST=$MY_POD_IP fi source /etc/tower/conf.d/environment.sh diff --git a/tools/ansible/roles/dockerfile/files/launch_awx_task.sh b/tools/ansible/roles/dockerfile/files/launch_awx_task.sh index 120cc9e3f8..8b9774a477 100644 --- a/tools/ansible/roles/dockerfile/files/launch_awx_task.sh +++ b/tools/ansible/roles/dockerfile/files/launch_awx_task.sh @@ -10,7 +10,7 @@ if [ -n "${AWX_KUBE_DEVEL}" ]; then make awx-link popd - export SDB_NOTIFY_HOST=$(ip route | head -n1 | awk '{print $3}') + export SDB_NOTIFY_HOST=$MY_POD_IP fi source /etc/tower/conf.d/environment.sh From 9f97efece8f99e4a497b160599f9d76311d56feb Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sun, 31 Jan 2021 16:32:06 -0500 Subject: [PATCH 104/157] Stop and kill dispatcher and callback reciever as group --- .../roles/dockerfile/templates/supervisor_task.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/ansible/roles/dockerfile/templates/supervisor_task.conf.j2 b/tools/ansible/roles/dockerfile/templates/supervisor_task.conf.j2 index b9fe0be41a..994323e6a8 100644 --- a/tools/ansible/roles/dockerfile/templates/supervisor_task.conf.j2 +++ b/tools/ansible/roles/dockerfile/templates/supervisor_task.conf.j2 @@ -16,6 +16,8 @@ directory = /var/lib/awx autostart = true autorestart = true stopwaitsecs = 5 +stopasgroup=true +killasgroup=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr @@ -32,6 +34,8 @@ directory = /var/lib/awx autostart = true autorestart = true stopwaitsecs = 5 +stopasgroup=true +killasgroup=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr From 68f9c5137d10d1b58fb11eb642a889a8bdf7d118 Mon Sep 17 00:00:00 2001 From: nixocio Date: Tue, 2 Feb 2021 17:01:23 -0500 Subject: [PATCH 105/157] Mark string to translation Mark string to translation --- .../ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx index 7789facfbb..8e41c5a0f8 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx @@ -52,7 +52,7 @@ function ExecutionEnvironmentListItem({ ]} /> From 57b317d4404fffe1e4f2b3c0af1e937c78305cf8 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 3 Feb 2021 15:06:21 -0500 Subject: [PATCH 106/157] Get system jobs working under new deployment model (#9221) --- awx/main/models/ha.py | 3 +++ awx/main/tasks.py | 37 +++++++++++++++++++++++++++++++------ awx/settings/defaults.py | 2 +- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index 6dd72861cc..94d4b8d462 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -255,6 +255,9 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin): @property def is_container_group(self): + if settings.IS_K8S: + return True + return bool(self.credential and self.credential.kubernetes) ''' diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ef0bd7e6e5..0b915ab909 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -893,6 +893,9 @@ class BaseTask(object): return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) def build_execution_environment_params(self, instance): + if settings.IS_K8S: + return {} + if instance.execution_environment_id is None: from awx.main.signals import disable_activity_stream @@ -1423,6 +1426,9 @@ class BaseTask(object): # Disable Ansible fact cache. params['fact_cache_type'] = '' + if self.instance.is_container_group_task or settings.IS_K8S: + params['envvars'].pop('HOME', None) + ''' Delete parameters if the values are None or empty array ''' @@ -1433,8 +1439,16 @@ class BaseTask(object): self.dispatcher = CallbackQueueDispatcher() self.instance.log_lifecycle("running_playbook") - receptor_job = AWXReceptorJob(self, params) - res = receptor_job.run() + if isinstance(self.instance, SystemJob): + cwd = self.build_cwd(self.instance, private_data_dir) + res = ansible_runner.interface.run(project_dir=cwd, + event_handler=self.event_handler, + finished_callback=self.finished_callback, + status_handler=self.status_handler, + **params) + else: + receptor_job = AWXReceptorJob(self, params) + res = receptor_job.run() status = res.status rc = res.rc @@ -1769,6 +1783,9 @@ class RunJob(BaseTask): return getattr(settings, 'AWX_PROOT_ENABLED', False) def build_execution_environment_params(self, instance): + if settings.IS_K8S: + return {} + params = super(RunJob, self).build_execution_environment_params(instance) # If this has an insights agent and it is not already mounted then show it insights_dir = os.path.dirname(settings.INSIGHTS_SYSTEM_ID_FILE) @@ -2398,6 +2415,9 @@ class RunProjectUpdate(BaseTask): return getattr(settings, 'AWX_PROOT_ENABLED', False) def build_execution_environment_params(self, instance): + if settings.IS_K8S: + return {} + params = super(RunProjectUpdate, self).build_execution_environment_params(instance) project_path = instance.get_project_path(check_if_exists=False) cache_path = instance.get_cache_path() @@ -3098,7 +3118,7 @@ class AWXReceptorJob: # Spawned in a thread so Receptor can start reading before we finish writing, we # write our payload to the left side of our socketpair. def transmit(self, _socket): - if self.work_type == 'local': + if not settings.IS_K8S and self.work_type == 'local': self.runner_params['only_transmit_kwargs'] = True ansible_runner.interface.run(streamer='transmit', @@ -3121,12 +3141,14 @@ class AWXReceptorJob: def receptor_params(self): if self.task.instance.is_container_group_task: spec_yaml = yaml.dump(self.pod_definition, explicit_start=True) - kubeconfig_yaml = yaml.dump(self.kube_config, explicit_start=True) receptor_params = { "secret_kube_pod": spec_yaml, - "secret_kube_config": kubeconfig_yaml } + + if self.credential: + kubeconfig_yaml = yaml.dump(self.kube_config, explicit_start=True) + receptor_params["secret_kube_config"] = kubeconfig_yaml else: private_data_dir = self.runner_params['private_data_dir'] receptor_params = { @@ -3140,7 +3162,10 @@ class AWXReceptorJob: @property def work_type(self): if self.task.instance.is_container_group_task: - work_type = 'ocp' + if self.credential: + work_type = 'kubernetes-runtime-auth' + else: + work_type = 'kubernetes-incluster-auth' else: work_type = 'local' diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 845bbe74d8..6b2f78479a 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -67,7 +67,7 @@ DATABASES = { IS_K8S = False # TODO: remove this setting in favor of a default execution environment -AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee' +AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/shanemcd/awx-ee' AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 From 428f8addf8145a0ceec6bbe39938b0ecd45593d3 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 3 Feb 2021 17:06:14 -0500 Subject: [PATCH 107/157] Create default EE as a part of create_preload_data --- .../commands/create_preload_data.py | 114 ++++++++++-------- tools/docker-compose/bootstrap_development.sh | 5 - 2 files changed, 66 insertions(+), 53 deletions(-) diff --git a/awx/main/management/commands/create_preload_data.py b/awx/main/management/commands/create_preload_data.py index 9b1d131735..05ed18b96c 100644 --- a/awx/main/management/commands/create_preload_data.py +++ b/awx/main/management/commands/create_preload_data.py @@ -2,22 +2,22 @@ # All Rights Reserved from django.core.management.base import BaseCommand +from django.conf import settings from crum import impersonate -from awx.main.models import User, Organization, Project, Inventory, CredentialType, Credential, Host, JobTemplate +from awx.main.models import ( + User, Organization, Project, Inventory, CredentialType, + Credential, Host, JobTemplate, ExecutionEnvironment +) from awx.main.signals import disable_computed_fields class Command(BaseCommand): """Create preloaded data, intended for new installs """ - help = 'Creates a preload tower data iff there is none.' + help = 'Creates a preload tower data if there is none.' def handle(self, *args, **kwargs): - # Sanity check: Is there already an organization in the system? - if Organization.objects.count(): - print('An organization is already in the system, exiting.') - print('(changed: False)') - return + changed = False # Create a default organization as the first superuser found. try: @@ -26,44 +26,62 @@ class Command(BaseCommand): superuser = None with impersonate(superuser): with disable_computed_fields(): - o = Organization.objects.create(name='Default') - p = Project(name='Demo Project', - scm_type='git', - scm_url='https://github.com/ansible/ansible-tower-samples', - scm_update_on_launch=True, - scm_update_cache_timeout=0, - organization=o) - p.save(skip_update=True) - ssh_type = CredentialType.objects.filter(namespace='ssh').first() - c = Credential.objects.create(credential_type=ssh_type, - name='Demo Credential', - inputs={ - 'username': superuser.username - }, - created_by=superuser) - c.admin_role.members.add(superuser) - public_galaxy_credential = Credential( - name='Ansible Galaxy', - managed_by_tower=True, - credential_type=CredentialType.objects.get(kind='galaxy'), - inputs = { - 'url': 'https://galaxy.ansible.com/' - } - ) - public_galaxy_credential.save() - o.galaxy_credentials.add(public_galaxy_credential) - i = Inventory.objects.create(name='Demo Inventory', - organization=o, - created_by=superuser) - Host.objects.create(name='localhost', - inventory=i, - variables="ansible_connection: local\nansible_python_interpreter: '{{ ansible_playbook_python }}'", - created_by=superuser) - jt = JobTemplate.objects.create(name='Demo Job Template', - playbook='hello_world.yml', - project=p, - inventory=i) - jt.credentials.add(c) - print('Default organization added.') - print('Demo Credential, Inventory, and Job Template added.') - print('(changed: True)') + if not Organization.objects.exists(): + o = Organization.objects.create(name='Default') + + p = Project(name='Demo Project', + scm_type='git', + scm_url='https://github.com/ansible/ansible-tower-samples', + scm_update_on_launch=True, + scm_update_cache_timeout=0, + organization=o) + p.save(skip_update=True) + + ssh_type = CredentialType.objects.filter(namespace='ssh').first() + c = Credential.objects.create(credential_type=ssh_type, + name='Demo Credential', + inputs={ + 'username': superuser.username + }, + created_by=superuser) + + c.admin_role.members.add(superuser) + + public_galaxy_credential = Credential(name='Ansible Galaxy', + managed_by_tower=True, + credential_type=CredentialType.objects.get(kind='galaxy'), + inputs={'url': 'https://galaxy.ansible.com/'}) + public_galaxy_credential.save() + o.galaxy_credentials.add(public_galaxy_credential) + + i = Inventory.objects.create(name='Demo Inventory', + organization=o, + created_by=superuser) + + Host.objects.create(name='localhost', + inventory=i, + variables="ansible_connection: local\nansible_python_interpreter: '{{ ansible_playbook_python }}'", + created_by=superuser) + + jt = JobTemplate.objects.create(name='Demo Job Template', + playbook='hello_world.yml', + project=p, + inventory=i) + jt.credentials.add(c) + + print('Default organization added.') + print('Demo Credential, Inventory, and Job Template added.') + changed = True + + default_ee = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE + ee, created = ExecutionEnvironment.objects.get_or_create(name='Default EE', defaults={'image': default_ee, + 'managed_by_tower': True}) + + if created: + changed = True + print('Default Execution Environment registered.') + + if changed: + print('(changed: True)') + else: + print('(changed: False)') diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index 3cc937c9c6..1d3e399bf7 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -26,8 +26,3 @@ make init mkdir -p /awx_devel/awx/public/static mkdir -p /awx_devel/awx/ui/static mkdir -p /awx_devel/awx/ui_next/build/static - -echo "ee, created = ExecutionEnvironment.objects.get_or_create(name='Default EE', \ - defaults={'image': 'quay.io/ansible/awx-ee', \ - 'managed_by_tower': True}); \ - print('Already exists' if not created else 'Created')" | awx-manage shell_plus --quiet-load From 8eb4dafb17be1196b77994869efdadc5881b722f Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 4 Feb 2021 17:39:14 -0500 Subject: [PATCH 108/157] Fix missing postgresql module --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 8c7e685736..22e725a216 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -139,7 +139,7 @@ RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if buil pip3 install --no-cache-dir https://github.com/ansible/ansible/archive/${ANSIBLE_BRANCH}.tar.gz # TODO: Remove this once launch script removes need for postres modules -RUN ansible-galaxy collection install --collections-path /usr/share/ansible/collections community.general +RUN ansible-galaxy collection install --collections-path /usr/share/ansible/collections community.general community.postgresql RUN rm -rf /root/.cache && rm -rf /tmp/* From 9f39bab2b877cf8be2a206091003cfdefbcb511b Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sun, 7 Feb 2021 10:39:13 -0500 Subject: [PATCH 109/157] Quick fix for jobs failing in dev environment / VMs The other alternative here is to go all the way with https://github.com/ansible/ansible-runner/pull/617, which is proving to be difficult if not impossible. --- awx/main/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0b915ab909..9fa0456fff 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -906,7 +906,8 @@ class BaseTask(object): image = instance.execution_environment.image params = { "container_image": image, - "process_isolation": True + "process_isolation": True, + "container_options": ['--user=root'], } if settings.AWX_PROOT_SHOW_PATHS: params['container_volume_mounts'] = [] From 44d7d68322b8a58ee172f20886a50ad056e89237 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 8 Feb 2021 14:10:04 -0500 Subject: [PATCH 110/157] Update default ee image --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 6b2f78479a..845bbe74d8 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -67,7 +67,7 @@ DATABASES = { IS_K8S = False # TODO: remove this setting in favor of a default execution environment -AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/shanemcd/awx-ee' +AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee' AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 From ea39cbce73a25c90633149a4779a1a7ad95edbf9 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 8 Feb 2021 14:10:13 -0500 Subject: [PATCH 111/157] Update receptor.conf in dev env --- tools/docker-compose/receptor.conf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/docker-compose/receptor.conf b/tools/docker-compose/receptor.conf index f86d50f523..d5ac25cf2d 100644 --- a/tools/docker-compose/receptor.conf +++ b/tools/docker-compose/receptor.conf @@ -14,8 +14,15 @@ allowruntimeparams: true - work-kubernetes: - worktype: ocp + worktype: kubernetes-runtime-auth authmethod: runtime allowruntimeauth: true allowruntimepod: true allowruntimeparams: true + +- work-kubernetes: + worktype: kubernetes-incluster-auth + authmethod: incluster + allowruntimeauth: true + allowruntimepod: true + allowruntimeparams: true From ddcbc408b908626d42f8d5393bf61abaf5ecb310 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 8 Feb 2021 14:29:32 -0500 Subject: [PATCH 112/157] Remove Ansible from control plane Execution Environments or bust! --- .../roles/dockerfile/templates/Dockerfile.j2 | 28 ++----------------- tools/docker-compose/bootstrap_development.sh | 8 ------ 2 files changed, 3 insertions(+), 33 deletions(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 22e725a216..78eebc2977 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -4,15 +4,9 @@ ### DO NOT EDIT ### -# Locations - set globally to be used across stages -ARG COLLECTION_BASE="/var/lib/awx/vendor/awx_ansible_collections" -ARG ANSIBLE_BRANCH=devel - # Build container FROM centos:8 as builder -ARG ANSIBLE_BRANCH - ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 @@ -49,8 +43,7 @@ RUN dnf -y update && \ xmlsec1-devel \ xmlsec1-openssl-devel -RUN python3 -m ensurepip && pip3 install "virtualenv < 20" && \ - pip3 install --no-cache-dir https://github.com/ansible/ansible/archive/${ANSIBLE_BRANCH}.tar.gz +RUN python3 -m ensurepip && pip3 install "virtualenv < 20" # Install & build requirements ADD Makefile /tmp/Makefile @@ -61,11 +54,9 @@ ADD requirements/requirements_ansible.txt \ requirements/requirements.txt \ requirements/requirements_tower_uninstall.txt \ requirements/requirements_git.txt \ - requirements/collections_requirements.yml \ /tmp/requirements/ RUN cd /tmp && make requirements_awx requirements_ansible_py3 -RUN cd /tmp && make requirements_collections {% if (build_dev|bool) or (kube_dev|bool) %} ADD requirements/requirements_dev.txt /tmp/requirements @@ -84,9 +75,6 @@ RUN make sdist && \ # Final container(s) FROM centos:8 -ARG COLLECTION_BASE -ARG ANSIBLE_BRANCH - ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 @@ -100,7 +88,6 @@ RUN dnf -y update && \ dnf module -y enable 'postgresql:12' && \ dnf config-manager --set-enabled powertools && \ dnf -y install acl \ - bubblewrap \ git-core \ git-lfs \ glibc-langpack-en \ @@ -135,11 +122,7 @@ RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master RUN curl -L -o /usr/bin/tini https://github.com/krallin/tini/releases/download/v0.19.0/tini-{{ tini_architecture | default('amd64') }} && \ chmod +x /usr/bin/tini -RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if build_dev|bool %}flake8{% endif %} && \ - pip3 install --no-cache-dir https://github.com/ansible/ansible/archive/${ANSIBLE_BRANCH}.tar.gz - -# TODO: Remove this once launch script removes need for postres modules -RUN ansible-galaxy collection install --collections-path /usr/share/ansible/collections community.general community.postgresql +RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor {% if build_dev|bool %}flake8{% endif %} RUN rm -rf /root/.cache && rm -rf /tmp/* @@ -185,7 +168,7 @@ COPY --from=builder /var/lib/awx /var/lib/awx RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage {%if build_dev|bool %} -COPY --from=quay.io/shanemcd/receptor /usr/bin/receptor /usr/bin/receptor +COPY --from=quay.io/shanemcd/receptor:latest /usr/bin/receptor /usr/bin/receptor RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr \ -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \ openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \ @@ -239,11 +222,6 @@ RUN for dir in \ /var/lib/awx/rsyslog/rsyslog.conf ; \ do touch $file ; chmod g+rw $file ; chgrp root $file ; done -# Adjust any remaining permissions -RUN chmod u+s /usr/bin/bwrap ; \ - chgrp -R root ${COLLECTION_BASE} ; \ - chmod -R g+rw ${COLLECTION_BASE} - {% if (build_dev|bool) or (kube_dev|bool) %} RUN for dir in \ /var/lib/awx/venv \ diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index 1d3e399bf7..88a7e62941 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -1,14 +1,6 @@ #!/bin/bash set +x -# Wait for the databases to come up -ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all -ansible -i "127.0.0.1," -c local -v -m wait_for -a "path=/var/run/redis/redis.sock" all - -# In case AWX in the container wants to connect to itself, use "docker exec" to attach to the container otherwise -# TODO: FIX -#/etc/init.d/ssh start - # Move to the source directory so we can bootstrap if [ -f "/awx_devel/manage.py" ]; then cd /awx_devel From f2dfa132a77814a72340024655e4e830c0732ce2 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 8 Feb 2021 15:50:06 -0500 Subject: [PATCH 113/157] Install Ansible only for collection tests --- Makefile | 3 ++- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3d14d4ba9..858a6f67e4 100644 --- a/Makefile +++ b/Makefile @@ -383,7 +383,8 @@ test_collection: rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ - fi; \ + fi && \ + pip install ansible && \ py.test $(COLLECTION_TEST_DIRS) -v # The python path needs to be modified so that the tests can find Ansible within the container # First we will use anything expility set as PYTHONPATH diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 78eebc2977..15c25c2d3a 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -225,7 +225,9 @@ RUN for dir in \ {% if (build_dev|bool) or (kube_dev|bool) %} RUN for dir in \ /var/lib/awx/venv \ + /var/lib/awx/venv/awx/bin \ /var/lib/awx/venv/awx/lib/python3.6 \ + /var/lib/awx/venv/awx/lib/python3.6/site-packages \ /var/lib/awx/projects \ /var/lib/awx/rsyslog \ /var/run/awx-rsyslog \ From 6a7520d10fae18d43ee0d0f65b5d364ea5dcc0b6 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 9 Feb 2021 11:12:28 -0500 Subject: [PATCH 114/157] Handle quota exceeded in Container Groups v2 --- awx/main/tasks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 9fa0456fff..ab9484eba9 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1451,6 +1451,9 @@ class BaseTask(object): receptor_job = AWXReceptorJob(self, params) res = receptor_job.run() + if not res: + return + status = res.status rc = res.rc @@ -3112,7 +3115,15 @@ class AWXReceptorJob: elif res.status == 'error': # TODO: There should be a more efficient way of getting this information receptor_work_list = receptor_ctl.simple_command("work list") - raise RuntimeError(receptor_work_list[self.unit_id]['Detail']) + detail = receptor_work_list[self.unit_id]['Detail'] + if 'exceeded quota' in detail: + logger.warn(detail) + log_name = self.task.instance.log_format + logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.") + self.task.update_model(self.task.instance.pk, status='pending') + return + + raise RuntimeError(detail) return res From 92f0af684c12ae2dd2ae0c6839f80d899c47e2c4 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 27 Jan 2021 16:24:00 -0500 Subject: [PATCH 115/157] execution model pull container options added --- awx/main/models/execution_environments.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index 51c7c251ea..84b6cd8c0a 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -12,6 +12,12 @@ class ExecutionEnvironment(CommonModel): class Meta: ordering = ('-created',) + PULL_CHOICES = [ + ('always', _("Always pull container before running.")), + ('missing', _("No pull option has been selected")), + ('never', _("Never cull container before running")) + ] + organization = models.ForeignKey( 'Organization', null=True, @@ -35,6 +41,11 @@ class ExecutionEnvironment(CommonModel): default=None, on_delete=models.SET_NULL, ) + container_options = models.CharField( + max_length=1024, + choices=PULL_CHOICES, + help_text=_('Pull image before running?'), + ) def get_absolute_url(self, request=None): return reverse('api:execution_environment_detail', kwargs={'pk': self.pk}, request=request) From 31e7e10f303e885ed2406b2e74fcb4b241593f74 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 27 Jan 2021 17:32:01 -0500 Subject: [PATCH 116/157] migration for container options for EE model Co-authored-by: Shane McDonald --- ...6_executionenvironment_container_options.py | 18 ++++++++++++++++++ awx/main/models/execution_environments.py | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 awx/main/migrations/0126_executionenvironment_container_options.py diff --git a/awx/main/migrations/0126_executionenvironment_container_options.py b/awx/main/migrations/0126_executionenvironment_container_options.py new file mode 100644 index 0000000000..27c667f9f0 --- /dev/null +++ b/awx/main/migrations/0126_executionenvironment_container_options.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-01-27 22:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0125_more_ee_modeling_changes'), + ] + + operations = [ + migrations.AddField( + model_name='executionenvironment', + name='container_options', + field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected'), ('never', 'Never pull container before running')], default='missing', help_text='Pull image before running?', max_length=1024), + ), + ] diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index 84b6cd8c0a..353b6cdbc7 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -15,7 +15,7 @@ class ExecutionEnvironment(CommonModel): PULL_CHOICES = [ ('always', _("Always pull container before running.")), ('missing', _("No pull option has been selected")), - ('never', _("Never cull container before running")) + ('never', _("Never pull container before running")) ] organization = models.ForeignKey( @@ -44,6 +44,7 @@ class ExecutionEnvironment(CommonModel): container_options = models.CharField( max_length=1024, choices=PULL_CHOICES, + default='missing', help_text=_('Pull image before running?'), ) From 4ca33579a5d5daf887b2f426683ab592efea4a7d Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Fri, 5 Feb 2021 12:48:49 -0500 Subject: [PATCH 117/157] Add an interface for new ee options --- awx/api/serializers.py | 2 +- .../ExecutionEnvironmentAdd.test.jsx | 26 +++++- .../ExecutionEnvironmentDetails.jsx | 22 ++++- .../ExecutionEnvironmentDetails.test.jsx | 1 + .../ExecutionEnvironmentEdit.test.jsx | 20 +++++ .../shared/ExecutionEnvironmentForm.jsx | 90 ++++++++++++++++++- .../shared/ExecutionEnvironmentForm.test.jsx | 33 ++++++- 7 files changed, 186 insertions(+), 8 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index eb13f150d3..941b465157 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1365,7 +1365,7 @@ class ExecutionEnvironmentSerializer(BaseSerializer): class Meta: model = ExecutionEnvironment - fields = ('*', 'organization', 'image', 'managed_by_tower', 'credential') + fields = ('*', 'organization', 'image', 'managed_by_tower', 'credential', 'container_options') def get_related(self, obj): res = super(ExecutionEnvironmentSerializer, self).get_related(obj) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx index 5396746223..ede58e5d58 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx @@ -2,7 +2,10 @@ import React from 'react'; import { act } from 'react-dom/test-utils'; import { createMemoryHistory } from 'history'; -import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; +import { + mountWithContexts, + waitForElement, +} from '../../../../testUtils/enzymeHelpers'; import { ExecutionEnvironmentsAPI } from '../../../api'; import ExecutionEnvironmentAdd from './ExecutionEnvironmentAdd'; @@ -14,11 +17,30 @@ const mockMe = { }; const executionEnvironmentData = { + name: 'Test EE', credential: 4, description: 'A simple EE', image: 'https://registry.com/image/container', + container_options: 'one', }; +const mockOptions = { + data: { + actions: { + POST: { + container_options: { + choices: [ + ['one', 'One'], + ['two', 'Two'], + ['three', 'Three'], + ], + }, + }, + }, + }, +}; + +ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(mockOptions); ExecutionEnvironmentsAPI.create.mockResolvedValue({ data: { id: 42, @@ -61,6 +83,8 @@ describe('', () => { }); test('handleCancel should return the user back to the execution environments list', async () => { + await waitForElement(wrapper, 'ContentLoading', el => el.length === 0); + wrapper.find('Button[aria-label="Cancel"]').simulate('click'); expect(history.location.pathname).toEqual('/execution_environments'); }); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index abb6cf5ddc..64925df1cb 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -13,11 +13,18 @@ import { UserDateDetail, } from '../../../components/DetailList'; import useRequest, { useDismissableError } from '../../../util/useRequest'; +import { toTitleCase } from '../../../util/strings'; import { ExecutionEnvironmentsAPI } from '../../../api'; function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { const history = useHistory(); - const { id, image, description } = executionEnvironment; + const { + id, + name, + image, + description, + container_options, + } = executionEnvironment; const { request: deleteExecutionEnvironment, @@ -35,12 +42,25 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { return ( + + {executionEnvironment.summary_fields.credential && ( ', () => { let wrapper; let history; diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx index 5d7a16d217..4dd1b695a4 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx @@ -1,18 +1,28 @@ -import React, { useCallback } from 'react'; +import React, { useCallback, useEffect } from 'react'; import { func, shape } from 'prop-types'; import { Formik, useField, useFormikContext } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; -import { Form } from '@patternfly/react-core'; +import { Form, FormGroup } from '@patternfly/react-core'; +import { ExecutionEnvironmentsAPI } from '../../../api'; import CredentialLookup from '../../../components/Lookup/CredentialLookup'; import FormActionGroup from '../../../components/FormActionGroup'; import FormField, { FormSubmitError } from '../../../components/FormField'; +import AnsibleSelect from '../../../components/AnsibleSelect'; import { FormColumnLayout } from '../../../components/FormLayout'; import { OrganizationLookup } from '../../../components/Lookup'; +import ContentError from '../../../components/ContentError'; +import ContentLoading from '../../../components/ContentLoading'; import { required, url } from '../../../util/validators'; +import useRequest from '../../../util/useRequest'; -function ExecutionEnvironmentFormFields({ i18n, me, executionEnvironment }) { +function ExecutionEnvironmentFormFields({ + i18n, + me, + options, + executionEnvironment, +}) { const [credentialField] = useField('credential'); const [organizationField, organizationMeta, organizationHelpers] = useField({ name: 'organization', @@ -37,8 +47,28 @@ function ExecutionEnvironmentFormFields({ i18n, me, executionEnvironment }) { [setFieldValue] ); + const [ + containerOptionsField, + containerOptionsMeta, + containerOptionsHelpers, + ] = useField({ + name: 'container_options', + }); + + const containerPullChoices = options?.actions?.POST?.container_options?.choices.map( + ([value, label]) => ({ value, label, key: value }) + ); + return ( <> + + + { + containerOptionsHelpers.setValue(value); + }} + /> + { + const res = await ExecutionEnvironmentsAPI.readOptions(); + const { data } = res; + return data; + }, []), + null + ); + + useEffect(() => { + fetchOptions(); + }, [fetchOptions]); + + if (isLoading || !options) { + return ; + } + + if (error) { + return ; + } + const initialValues = { + name: executionEnvironment.name || '', image: executionEnvironment.image || '', + container_options: executionEnvironment?.container_options || '', description: executionEnvironment.description || '', credential: executionEnvironment.summary_fields?.credential || null, organization: executionEnvironment.summary_fields?.organization || null, @@ -101,7 +178,12 @@ function ExecutionEnvironmentForm({ {formik => ( - + {submitError && } ', () => { let wrapper; let onCancel; @@ -46,16 +68,19 @@ describe('', () => { beforeEach(async () => { onCancel = jest.fn(); onSubmit = jest.fn(); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(mockOptions); await act(async () => { wrapper = mountWithContexts( ); }); + await waitForElement(wrapper, 'ContentLoading', el => el.length === 0); }); afterEach(() => { @@ -83,6 +108,12 @@ describe('', () => { test('should update form values', async () => { await act(async () => { + wrapper.find('input#execution-environment-image').simulate('change', { + target: { + value: 'Updated EE Name', + name: 'name', + }, + }); wrapper.find('input#execution-environment-image').simulate('change', { target: { value: 'https://registry.com/image/container2', From b0265b060ba38316eaac3154ebd9e643b03c90ba Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Fri, 5 Feb 2021 16:53:38 -0500 Subject: [PATCH 118/157] Remove client-side url validator --- .../ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx index 4dd1b695a4..364d86e58f 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx @@ -14,7 +14,7 @@ import { FormColumnLayout } from '../../../components/FormLayout'; import { OrganizationLookup } from '../../../components/Lookup'; import ContentError from '../../../components/ContentError'; import ContentLoading from '../../../components/ContentLoading'; -import { required, url } from '../../../util/validators'; +import { required } from '../../../util/validators'; import useRequest from '../../../util/useRequest'; function ExecutionEnvironmentFormFields({ @@ -74,7 +74,7 @@ function ExecutionEnvironmentFormFields({ label={i18n._(t`Image name`)} name="image" type="text" - validate={url(i18n)} + validate={required(null, i18n)} isRequired tooltip={i18n._( t`The registry location where the container is stored.` From 4b40cb3abbb788bee0e88db49122cbdfb2e0111a Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Tue, 9 Feb 2021 11:40:44 -0500 Subject: [PATCH 119/157] changed the field name from 'container_options' to simply 'pull' --- awx/api/serializers.py | 2 +- ...126_executionenvironment_container_options.py | 4 ++-- awx/main/models/execution_environments.py | 6 +++--- .../ExecutionEnvironmentAdd.test.jsx | 4 ++-- .../ExecutionEnvironmentDetails.jsx | 16 +++------------- .../ExecutionEnvironmentEdit.test.jsx | 4 ++-- .../shared/ExecutionEnvironmentForm.jsx | 6 +++--- .../shared/ExecutionEnvironmentForm.test.jsx | 4 ++-- .../modules/tower_execution_environment.py | 11 +++++++++++ 9 files changed, 29 insertions(+), 28 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 941b465157..22b7ab3a7a 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1365,7 +1365,7 @@ class ExecutionEnvironmentSerializer(BaseSerializer): class Meta: model = ExecutionEnvironment - fields = ('*', 'organization', 'image', 'managed_by_tower', 'credential', 'container_options') + fields = ('*', 'organization', 'image', 'managed_by_tower', 'credential', 'pull') def get_related(self, obj): res = super(ExecutionEnvironmentSerializer, self).get_related(obj) diff --git a/awx/main/migrations/0126_executionenvironment_container_options.py b/awx/main/migrations/0126_executionenvironment_container_options.py index 27c667f9f0..c2fd2e77b5 100644 --- a/awx/main/migrations/0126_executionenvironment_container_options.py +++ b/awx/main/migrations/0126_executionenvironment_container_options.py @@ -12,7 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name='executionenvironment', - name='container_options', - field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected'), ('never', 'Never pull container before running')], default='missing', help_text='Pull image before running?', max_length=1024), + name='pull', + field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected.'), ('never', 'Never pull container before running.')], default='missing', help_text='Pull image before running?', max_length=1024), ), ] diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index 353b6cdbc7..681cdf94db 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -14,8 +14,8 @@ class ExecutionEnvironment(CommonModel): PULL_CHOICES = [ ('always', _("Always pull container before running.")), - ('missing', _("No pull option has been selected")), - ('never', _("Never pull container before running")) + ('missing', _("No pull option has been selected.")), + ('never', _("Never pull container before running.")) ] organization = models.ForeignKey( @@ -41,7 +41,7 @@ class ExecutionEnvironment(CommonModel): default=None, on_delete=models.SET_NULL, ) - container_options = models.CharField( + pull = models.CharField( max_length=1024, choices=PULL_CHOICES, default='missing', diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx index ede58e5d58..92f18c7d33 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentAdd/ExecutionEnvironmentAdd.test.jsx @@ -21,14 +21,14 @@ const executionEnvironmentData = { credential: 4, description: 'A simple EE', image: 'https://registry.com/image/container', - container_options: 'one', + pull: 'one', }; const mockOptions = { data: { actions: { POST: { - container_options: { + pull: { choices: [ ['one', 'One'], ['two', 'Two'], diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index 64925df1cb..866b74f5fb 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -18,13 +18,7 @@ import { ExecutionEnvironmentsAPI } from '../../../api'; function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { const history = useHistory(); - const { - id, - name, - image, - description, - container_options, - } = executionEnvironment; + const { id, name, image, description, pull } = executionEnvironment; const { request: deleteExecutionEnvironment, @@ -54,12 +48,8 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { /> {executionEnvironment.summary_fields.credential && ( ({ value, label, key: value }) ); @@ -168,7 +168,7 @@ function ExecutionEnvironmentForm({ const initialValues = { name: executionEnvironment.name || '', image: executionEnvironment.image || '', - container_options: executionEnvironment?.container_options || '', + pull: executionEnvironment?.pull || '', description: executionEnvironment.description || '', credential: executionEnvironment.summary_fields?.credential || null, organization: executionEnvironment.summary_fields?.organization || null, diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx index 1253852640..cddef9ffce 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.test.jsx @@ -19,7 +19,7 @@ const executionEnvironment = { id: 16, name: 'Test EE', type: 'execution_environment', - container_options: 'one', + pull: 'one', url: '/api/v2/execution_environments/16/', related: { created_by: '/api/v2/users/1/', @@ -48,7 +48,7 @@ const mockOptions = { data: { actions: { POST: { - container_options: { + pull: { choices: [ ['one', 'One'], ['two', 'Two'], diff --git a/awx_collection/plugins/modules/tower_execution_environment.py b/awx_collection/plugins/modules/tower_execution_environment.py index 280408f72a..862f4e3ab6 100644 --- a/awx_collection/plugins/modules/tower_execution_environment.py +++ b/awx_collection/plugins/modules/tower_execution_environment.py @@ -50,6 +50,12 @@ options: choices: ["present", "absent"] default: "present" type: str + pull: + description: + - determine image pull behavior + choices: ["always", "missing", "never"] + default: "missing" + type: str extends_documentation_fragment: awx.awx.auth ''' @@ -75,6 +81,7 @@ def main(): organization=dict(), credential=dict(default=''), state=dict(choices=['present', 'absent'], default='present'), + pull=dict(choices=['always', 'missing', 'never'], default='missing') ) # Create a module for ourselves @@ -85,6 +92,7 @@ def main(): image = module.params.get('image') description = module.params.get('description') state = module.params.get('state') + pull = module.params.get('pull') existing_item = module.get_one('execution_environments', name_or_id=name) @@ -98,6 +106,9 @@ def main(): if description: new_fields['description'] = description + if pull: + new_fields['pull'] = pull + # Attempt to look up the related items the user specified (these will fail the module if not found) organization = module.params.get('organization') if organization: From 0381a3ac8c7bd091d53433b2c6259c443633da47 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 9 Feb 2021 20:20:33 -0500 Subject: [PATCH 120/157] Container Pull Option -> Pull Co-authored-by: Jake McDermott --- .../ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx index 548946afeb..a8e0e33a0f 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/shared/ExecutionEnvironmentForm.jsx @@ -88,7 +88,7 @@ function ExecutionEnvironmentFormFields({ ? 'default' : 'error' } - label={i18n._(t`Container Pull Option`)} + label={i18n._(t`Pull`)} > Date: Mon, 8 Feb 2021 17:07:12 -0500 Subject: [PATCH 121/157] Update Job Detail container group variable `is_containerized` was updated on the API side to be `is_container_group`. --- awx/ui_next/src/screens/Job/JobDetail/JobDetail.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/ui_next/src/screens/Job/JobDetail/JobDetail.jsx b/awx/ui_next/src/screens/Job/JobDetail/JobDetail.jsx index 35b48f17fb..ad0be4c9e6 100644 --- a/awx/ui_next/src/screens/Job/JobDetail/JobDetail.jsx +++ b/awx/ui_next/src/screens/Job/JobDetail/JobDetail.jsx @@ -252,13 +252,13 @@ function JobDetail({ job, i18n }) { - {instanceGroup && !instanceGroup?.is_containerized && ( + {instanceGroup && !instanceGroup?.is_container_group && ( )} - {instanceGroup && instanceGroup?.is_containerized && ( + {instanceGroup && instanceGroup?.is_container_group && ( Date: Mon, 8 Feb 2021 16:35:28 -0500 Subject: [PATCH 122/157] Migrate EE list to tables Migrate EE list to tables. See:https://github.com/ansible/awx/issues/7884 --- .../ExecutionEnviromentList.test.jsx | 60 ++++++----- .../ExecutionEnvironmentList.jsx | 20 +++- .../ExecutionEnvironmentListItem.jsx | 102 +++++++++--------- .../ExecutionEnvironmentListItem.test.jsx | 54 +++++++--- 4 files changed, 135 insertions(+), 101 deletions(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx index 475dd1a8b5..1490ff49e3 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnviromentList.test.jsx @@ -15,6 +15,7 @@ const executionEnvironments = { data: { results: [ { + name: 'Foo', id: 1, image: 'https://registry.com/r/image/manifest', organization: null, @@ -23,6 +24,7 @@ const executionEnvironments = { summary_fields: { user_capabilities: { edit: true, delete: true } }, }, { + name: 'Bar', id: 2, image: 'https://registry.com/r/image2/manifest', organization: null, @@ -38,6 +40,14 @@ const executionEnvironments = { const options = { data: { actions: { POST: true } } }; describe('', () => { + beforeEach(() => { + ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); + ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); let wrapper; test('should mount successfully', async () => { @@ -52,9 +62,6 @@ describe('', () => { }); test('should have data fetched and render 2 rows', async () => { - ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); - ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); - await act(async () => { wrapper = mountWithContexts(); }); @@ -69,10 +76,7 @@ describe('', () => { expect(ExecutionEnvironmentsAPI.readOptions).toBeCalled(); }); - test('should delete item successfully', async () => { - ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); - ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); - + test('should delete items successfully', async () => { await act(async () => { wrapper = mountWithContexts(); }); @@ -82,27 +86,25 @@ describe('', () => { el => el.length > 0 ); - wrapper - .find('input#select-execution-environment-1') - .simulate('change', executionEnvironments.data.results[0]); - wrapper.update(); - - expect( - wrapper.find('input#select-execution-environment-1').prop('checked') - ).toBe(true); - await act(async () => { - wrapper.find('Button[aria-label="Delete"]').prop('onClick')(); + wrapper + .find('ExecutionEnvironmentListItem') + .at(0) + .invoke('onSelect')(); }); wrapper.update(); - await act(async () => { - wrapper.find('Button[aria-label="confirm delete"]').prop('onClick')(); + wrapper + .find('ExecutionEnvironmentListItem') + .at(1) + .invoke('onSelect')(); + }); + wrapper.update(); + await act(async () => { + wrapper.find('ToolbarDeleteButton').invoke('onDelete')(); }); - expect(ExecutionEnvironmentsAPI.destroy).toBeCalledWith( - executionEnvironments.data.results[0].id - ); + expect(ExecutionEnvironmentsAPI.destroy).toHaveBeenCalledTimes(2); }); test('should render deletion error modal', async () => { @@ -117,19 +119,24 @@ describe('', () => { }, }) ); - ExecutionEnvironmentsAPI.read.mockResolvedValue(executionEnvironments); - ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); await act(async () => { wrapper = mountWithContexts(); }); waitForElement(wrapper, 'ExecutionEnvironmentList', el => el.length > 0); wrapper - .find('input#select-execution-environment-1') + .find('ExecutionEnvironmentListItem') + .at(0) + .find('input') .simulate('change', 'a'); wrapper.update(); + expect( - wrapper.find('input#select-execution-environment-1').prop('checked') + wrapper + .find('ExecutionEnvironmentListItem') + .at(0) + .find('input') + .prop('checked') ).toBe(true); await act(async () => @@ -156,7 +163,6 @@ describe('', () => { }, }) ); - ExecutionEnvironmentsAPI.readOptions.mockResolvedValue(options); await act(async () => { wrapper = mountWithContexts(); }); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx index 749fe8893e..9d94085520 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx @@ -8,10 +8,14 @@ import { ExecutionEnvironmentsAPI } from '../../../api'; import { getQSConfig, parseQueryString } from '../../../util/qs'; import useRequest, { useDeleteItems } from '../../../util/useRequest'; import useSelected from '../../../util/useSelected'; -import PaginatedDataList, { +import { ToolbarDeleteButton, ToolbarAddButton, } from '../../../components/PaginatedDataList'; +import PaginatedTable, { + HeaderRow, + HeaderCell, +} from '../../../components/PaginatedTable'; import ErrorDetail from '../../../components/ErrorDetail'; import AlertModal from '../../../components/AlertModal'; import DatalistToolbar from '../../../components/DataListToolbar'; @@ -21,7 +25,7 @@ import ExecutionEnvironmentsListItem from './ExecutionEnvironmentListItem'; const QS_CONFIG = getQSConfig('execution_environments', { page: 1, page_size: 20, - order_by: 'image', + order_by: 'name', }); function ExecutionEnvironmentList({ i18n }) { @@ -106,7 +110,7 @@ function ExecutionEnvironmentList({ i18n }) { <> - + {i18n._(t`Name`)} + {i18n._(t`Image`)} + {i18n._(t`Organization`)} + + } renderToolbar={props => ( )} - renderItem={executionEnvironment => ( + renderRow={(executionEnvironment, index) => ( handleSelect(executionEnvironment)} diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx index 8e41c5a0f8..fbc8387d84 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx @@ -3,18 +3,11 @@ import { string, bool, func } from 'prop-types'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import { Link } from 'react-router-dom'; -import { - Button, - DataListAction, - DataListCheck, - DataListItem, - DataListItemRow, - DataListItemCells, - Tooltip, -} from '@patternfly/react-core'; +import { Button } from '@patternfly/react-core'; +import { Tr, Td } from '@patternfly/react-table'; import { PencilAltIcon } from '@patternfly/react-icons'; -import DataListCell from '../../../components/DataListCell'; +import { ActionsTd, ActionItem } from '../../../components/PaginatedTable'; import { ExecutionEnvironment } from '../../../types'; function ExecutionEnvironmentListItem({ @@ -23,55 +16,56 @@ function ExecutionEnvironmentListItem({ isSelected, onSelect, i18n, + rowIndex, }) { const labelId = `check-action-${executionEnvironment.id}`; return ( - - - - - - {executionEnvironment.image} - - , - ]} - /> - - + + + + {executionEnvironment.name} + + + + {executionEnvironment.image} + + + {executionEnvironment.organization ? ( + - - - - - + {executionEnvironment?.summary_fields?.organization?.name} + + ) : ( + i18n._(t`Globally Available`) + )} + + + + + + + ); } diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx index 4f51a51672..0e7c037aed 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.test.jsx @@ -8,21 +8,27 @@ import ExecutionEnvironmentListItem from './ExecutionEnvironmentListItem'; describe('', () => { let wrapper; const executionEnvironment = { + name: 'Foo', id: 1, image: 'https://registry.com/r/image/manifest', organization: null, credential: null, + summary_fields: { user_capabilities: { edit: true } }, }; test('should mount successfully', async () => { await act(async () => { wrapper = mountWithContexts( - {}} - /> + + + {}} + /> + +
); }); expect(wrapper.find('ExecutionEnvironmentListItem').length).toBe(1); @@ -31,22 +37,38 @@ describe('', () => { test('should render the proper data', async () => { await act(async () => { wrapper = mountWithContexts( - {}} - /> + + + {}} + /> + +
); }); expect( wrapper - .find('DataListCell[aria-label="execution environment image"]') + .find('Td') + .at(1) + .text() + ).toBe(executionEnvironment.name); + expect( + wrapper + .find('Td') + .at(2) .text() ).toBe(executionEnvironment.image); - expect(wrapper.find('PencilAltIcon').length).toBe(1); + expect( - wrapper.find('input#select-execution-environment-1').prop('checked') - ).toBe(false); + wrapper + .find('Td') + .at(3) + .text() + ).toBe('Globally Available'); + + expect(wrapper.find('PencilAltIcon').exists()).toBeTruthy(); }); }); From 05ef51f710dad8f8036bc5acee4097db4adc0d71 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 15 Feb 2021 17:10:43 -0500 Subject: [PATCH 123/157] Add migration to reset custom pod specs --- .../migrations/0127_reset_pod_spec_override.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 awx/main/migrations/0127_reset_pod_spec_override.py diff --git a/awx/main/migrations/0127_reset_pod_spec_override.py b/awx/main/migrations/0127_reset_pod_spec_override.py new file mode 100644 index 0000000000..c3ebe0b504 --- /dev/null +++ b/awx/main/migrations/0127_reset_pod_spec_override.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-02-15 22:02 + +from django.db import migrations + +def reset_pod_specs(apps, schema_editor): + InstanceGroup = apps.get_model('main', 'InstanceGroup') + InstanceGroup.objects.update(pod_spec_override="") + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0126_executionenvironment_container_options'), + ] + + operations = [ + migrations.RunPython(reset_pod_specs) + ] From 0bd8012fd9b72ec3cc45bb8633c8bfedf4bd17c0 Mon Sep 17 00:00:00 2001 From: nixocio Date: Wed, 17 Feb 2021 16:55:22 -0500 Subject: [PATCH 124/157] Update selectors on EE details page to ease testing Update selectors on EE details page to ease testing. --- .../ExecutionEnvironmentDetails.jsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index 866b74f5fb..13839bb95b 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -46,10 +46,15 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { value={image} dataCy="execution-environment-detail-image" /> - + {executionEnvironment.summary_fields.credential && ( @@ -78,6 +85,7 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { aria-label={i18n._(t`edit`)} component={Link} to={`/execution_environments/${id}/edit`} + ouiaId="edit-button" > {i18n._(t`Edit`)} @@ -86,6 +94,7 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { modalTitle={i18n._(t`Delete Execution Environment`)} onConfirm={deleteExecutionEnvironment} isDisabled={isLoading} + ouiaId="delete-button" > {i18n._(t`Delete`)} From 20ee73ce73160fe723cd82cb16aa18a03067f70a Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Tue, 16 Feb 2021 14:11:56 -0500 Subject: [PATCH 125/157] default pull options for container images to None, also adding pull options to awxkit --- .../0128_set_default_pull_to_none.py | 18 ++++++++++++++++++ awx/main/models/execution_environments.py | 2 +- .../modules/tower_execution_environment.py | 2 +- .../awxkit/api/pages/execution_environments.py | 7 ++++--- 4 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 awx/main/migrations/0128_set_default_pull_to_none.py diff --git a/awx/main/migrations/0128_set_default_pull_to_none.py b/awx/main/migrations/0128_set_default_pull_to_none.py new file mode 100644 index 0000000000..3643c610e1 --- /dev/null +++ b/awx/main/migrations/0128_set_default_pull_to_none.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-02-16 20:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0127_reset_pod_spec_override'), + ] + + operations = [ + migrations.AlterField( + model_name='executionenvironment', + name='pull', + field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected.'), ('never', 'Never pull container before running.')], default=None, help_text='Pull image before running?', max_length=1024), + ), + ] diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index 681cdf94db..cddd8c41e5 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -44,7 +44,7 @@ class ExecutionEnvironment(CommonModel): pull = models.CharField( max_length=1024, choices=PULL_CHOICES, - default='missing', + default=None, help_text=_('Pull image before running?'), ) diff --git a/awx_collection/plugins/modules/tower_execution_environment.py b/awx_collection/plugins/modules/tower_execution_environment.py index 862f4e3ab6..e738f9a3bf 100644 --- a/awx_collection/plugins/modules/tower_execution_environment.py +++ b/awx_collection/plugins/modules/tower_execution_environment.py @@ -54,7 +54,7 @@ options: description: - determine image pull behavior choices: ["always", "missing", "never"] - default: "missing" + default: None type: str extends_documentation_fragment: awx.awx.auth ''' diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py index a01aa91011..11e3e2f881 100644 --- a/awxkit/awxkit/api/pages/execution_environments.py +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -21,9 +21,9 @@ class ExecutionEnvironment(HasCreate, base.Base): NATURAL_KEY = ('name',) # fields are name, image, organization, managed_by_tower, credential - def create(self, name='', image='quay.io/ansible/ansible-runner:devel', credential=None, **kwargs): + def create(self, name='', image='quay.io/ansible/ansible-runner:devel', credential=None, pull=None, **kwargs): # we do not want to make a credential by default - payload = self.create_payload(name=name, image=image, credential=credential, **kwargs) + payload = self.create_payload(name=name, image=image, credential=credential, pull=pull, **kwargs) ret = self.update_identity(ExecutionEnvironments(self.connection).post(payload)) return ret @@ -33,12 +33,13 @@ class ExecutionEnvironment(HasCreate, base.Base): payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) return payload - def payload(self, name='', image=None, organization=None, credential=None, **kwargs): + def payload(self, name='', image=None, organization=None, credential=None, pull=None, **kwargs): payload = PseudoNamespace( name=name or "EE - {}".format(random_title()), image=image or random_title(10), organization=organization.id if organization else None, credential=credential.id if credential else None, + pull=pull if pull else None, **kwargs ) From b1361c8fe28a3682b1d27e652eb9acc6f4716f52 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Tue, 16 Feb 2021 17:41:44 -0500 Subject: [PATCH 126/157] edit original migration file, add blank string as acceptable to model --- ...6_executionenvironment_container_options.py | 2 +- .../0128_set_default_pull_to_none.py | 18 ------------------ awx/main/models/execution_environments.py | 5 +++-- .../modules/tower_execution_environment.py | 2 +- .../awxkit/api/pages/execution_environments.py | 6 +++--- 5 files changed, 8 insertions(+), 25 deletions(-) delete mode 100644 awx/main/migrations/0128_set_default_pull_to_none.py diff --git a/awx/main/migrations/0126_executionenvironment_container_options.py b/awx/main/migrations/0126_executionenvironment_container_options.py index c2fd2e77b5..d26fcb9298 100644 --- a/awx/main/migrations/0126_executionenvironment_container_options.py +++ b/awx/main/migrations/0126_executionenvironment_container_options.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='executionenvironment', name='pull', - field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected.'), ('never', 'Never pull container before running.')], default='missing', help_text='Pull image before running?', max_length=1024), + field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected.'), ('never', 'Never pull container before running.')], blank=True, default='', help_text='Pull image before running?', max_length=16), ), ] diff --git a/awx/main/migrations/0128_set_default_pull_to_none.py b/awx/main/migrations/0128_set_default_pull_to_none.py deleted file mode 100644 index 3643c610e1..0000000000 --- a/awx/main/migrations/0128_set_default_pull_to_none.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.16 on 2021-02-16 20:38 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0127_reset_pod_spec_override'), - ] - - operations = [ - migrations.AlterField( - model_name='executionenvironment', - name='pull', - field=models.CharField(choices=[('always', 'Always pull container before running.'), ('missing', 'No pull option has been selected.'), ('never', 'Never pull container before running.')], default=None, help_text='Pull image before running?', max_length=1024), - ), - ] diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index cddd8c41e5..eabd0cce7c 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -42,9 +42,10 @@ class ExecutionEnvironment(CommonModel): on_delete=models.SET_NULL, ) pull = models.CharField( - max_length=1024, + max_length=16, choices=PULL_CHOICES, - default=None, + blank=True, + default='', help_text=_('Pull image before running?'), ) diff --git a/awx_collection/plugins/modules/tower_execution_environment.py b/awx_collection/plugins/modules/tower_execution_environment.py index e738f9a3bf..320141721d 100644 --- a/awx_collection/plugins/modules/tower_execution_environment.py +++ b/awx_collection/plugins/modules/tower_execution_environment.py @@ -54,7 +54,7 @@ options: description: - determine image pull behavior choices: ["always", "missing", "never"] - default: None + default: '' type: str extends_documentation_fragment: awx.awx.auth ''' diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py index 11e3e2f881..94f53c1094 100644 --- a/awxkit/awxkit/api/pages/execution_environments.py +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -21,7 +21,7 @@ class ExecutionEnvironment(HasCreate, base.Base): NATURAL_KEY = ('name',) # fields are name, image, organization, managed_by_tower, credential - def create(self, name='', image='quay.io/ansible/ansible-runner:devel', credential=None, pull=None, **kwargs): + def create(self, name='', image='quay.io/ansible/ansible-runner:devel', credential=None, pull='', **kwargs): # we do not want to make a credential by default payload = self.create_payload(name=name, image=image, credential=credential, pull=pull, **kwargs) ret = self.update_identity(ExecutionEnvironments(self.connection).post(payload)) @@ -33,13 +33,13 @@ class ExecutionEnvironment(HasCreate, base.Base): payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) return payload - def payload(self, name='', image=None, organization=None, credential=None, pull=None, **kwargs): + def payload(self, name='', image=None, organization=None, credential=None, pull='', **kwargs): payload = PseudoNamespace( name=name or "EE - {}".format(random_title()), image=image or random_title(10), organization=organization.id if organization else None, credential=credential.id if credential else None, - pull=pull if pull else None, + pull=pull, **kwargs ) From ae5a1117d4960e16188128a7aabc251406c1e881 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 22 Feb 2021 14:51:12 -0500 Subject: [PATCH 127/157] Use official Receptor 0.9.5 release --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 15c25c2d3a..e17a0b04d8 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -168,7 +168,7 @@ COPY --from=builder /var/lib/awx /var/lib/awx RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage {%if build_dev|bool %} -COPY --from=quay.io/shanemcd/receptor:latest /usr/bin/receptor /usr/bin/receptor +COPY --from=quay.io/project-receptor/receptor:0.9.5 /usr/bin/receptor /usr/bin/receptor RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr \ -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \ openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \ From 05dded397d67541b66a77d38600581e0e0887831 Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Mon, 22 Feb 2021 14:00:33 -0500 Subject: [PATCH 128/157] make sure we use built in credential type this way we can pass kind="registry" to akit creat method and we get the correct built in type --- awxkit/awxkit/api/pages/credentials.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awxkit/awxkit/api/pages/credentials.py b/awxkit/awxkit/api/pages/credentials.py index f964ae38e4..e5b80cf7d0 100644 --- a/awxkit/awxkit/api/pages/credentials.py +++ b/awxkit/awxkit/api/pages/credentials.py @@ -82,6 +82,7 @@ def config_cred_from_kind(kind): credential_type_name_to_config_kind_map = { 'amazon web services': 'aws', + 'container registry': 'registry', 'ansible galaxy/automation hub api token': 'galaxy', 'ansible tower': 'tower', 'google compute engine': 'gce', From a39e1a528b6ff40372c7e9ae04728b96bd01fac7 Mon Sep 17 00:00:00 2001 From: nixocio Date: Mon, 5 Oct 2020 16:45:57 -0400 Subject: [PATCH 129/157] Add execution environment list to Organizations Add execution environment list to Organizations See: https://github.com/ansible/awx/issues/8210 --- awx/ui_next/src/api/models/Organizations.js | 11 ++ .../src/screens/Organization/Organization.jsx | 11 ++ .../Organization/Organization.test.jsx | 4 +- .../OrganizationExecEnvList.jsx | 126 ++++++++++++++++++ .../OrganizationExecEnvList.test.jsx | 116 ++++++++++++++++ .../OrganizationExecEnvListItem.jsx | 51 +++++++ .../OrganizationExecEnvListItem.test.jsx | 44 ++++++ .../OrganizationExecEnvList/index.js | 1 + .../screens/Organization/Organizations.jsx | 3 + 9 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx create mode 100644 awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.test.jsx create mode 100644 awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx create mode 100644 awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx create mode 100644 awx/ui_next/src/screens/Organization/OrganizationExecEnvList/index.js diff --git a/awx/ui_next/src/api/models/Organizations.js b/awx/ui_next/src/api/models/Organizations.js index ce236067b4..ec4a14c549 100644 --- a/awx/ui_next/src/api/models/Organizations.js +++ b/awx/ui_next/src/api/models/Organizations.js @@ -29,6 +29,17 @@ class Organizations extends InstanceGroupsMixin(NotificationsMixin(Base)) { params, }); } + readExecutionEnvironments(id, params) { + return this.http.get(`${this.baseUrl}${id}/execution_environments/`, { + params, + }); + } + + readExecutionEnvironmentsOptions(id, params) { + return this.http.options(`${this.baseUrl}${id}/execution_environments/`, { + params, + }); + } createUser(id, data) { return this.http.post(`${this.baseUrl}${id}/users/`, data); diff --git a/awx/ui_next/src/screens/Organization/Organization.jsx b/awx/ui_next/src/screens/Organization/Organization.jsx index 1a2ee641c9..a826ef641d 100644 --- a/awx/ui_next/src/screens/Organization/Organization.jsx +++ b/awx/ui_next/src/screens/Organization/Organization.jsx @@ -22,6 +22,7 @@ import OrganizationDetail from './OrganizationDetail'; import OrganizationEdit from './OrganizationEdit'; import OrganizationTeams from './OrganizationTeams'; import { OrganizationsAPI } from '../../api'; +import OrganizationExecEnvList from './OrganizationExecEnvList'; function Organization({ i18n, setBreadcrumb, me }) { const location = useLocation(); @@ -122,6 +123,11 @@ function Organization({ i18n, setBreadcrumb, me }) { { name: i18n._(t`Details`), link: `${match.url}/details`, id: 0 }, { name: i18n._(t`Access`), link: `${match.url}/access`, id: 1 }, { name: i18n._(t`Teams`), link: `${match.url}/teams`, id: 2 }, + { + name: i18n._(t`Execution Environments`), + link: `${match.url}/execution_environments`, + id: 4, + }, ]; if (canSeeNotificationsTab) { @@ -208,6 +214,11 @@ function Organization({ i18n, setBreadcrumb, me }) { /> )} + {organization && ( + + + + )} {!organizationLoading && !rolesLoading && ( diff --git a/awx/ui_next/src/screens/Organization/Organization.test.jsx b/awx/ui_next/src/screens/Organization/Organization.test.jsx index 10982505d5..487ebff36b 100644 --- a/awx/ui_next/src/screens/Organization/Organization.test.jsx +++ b/awx/ui_next/src/screens/Organization/Organization.test.jsx @@ -68,7 +68,7 @@ describe('', () => { const tabs = await waitForElement( wrapper, '.pf-c-tabs__item', - el => el.length === 5 + el => el.length === 6 ); expect(tabs.last().text()).toEqual('Notifications'); wrapper.unmount(); @@ -92,7 +92,7 @@ describe('', () => { const tabs = await waitForElement( wrapper, '.pf-c-tabs__item', - el => el.length === 4 + el => el.length === 5 ); tabs.forEach(tab => expect(tab.text()).not.toEqual('Notifications')); wrapper.unmount(); diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx new file mode 100644 index 0000000000..48caaff802 --- /dev/null +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx @@ -0,0 +1,126 @@ +import React, { useEffect, useCallback } from 'react'; +import { useLocation } from 'react-router-dom'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Card } from '@patternfly/react-core'; + +import { OrganizationsAPI } from '../../../api'; +import { getQSConfig, parseQueryString } from '../../../util/qs'; +import useRequest from '../../../util/useRequest'; +import PaginatedDataList from '../../../components/PaginatedDataList'; +import DatalistToolbar from '../../../components/DataListToolbar'; + +import OrganizationExecEnvListItem from './OrganizationExecEnvListItem'; + +const QS_CONFIG = getQSConfig('organizations', { + page: 1, + page_size: 20, + order_by: 'image', +}); + +function OrganizationExecEnvList({ i18n, organization }) { + const { id } = organization; + const location = useLocation(); + + const { + error: contentError, + isLoading, + request: fetchExecutionEnvironments, + result: { + executionEnvironments, + executionEnvironmentsCount, + relatedSearchableKeys, + searchableKeys, + }, + } = useRequest( + useCallback(async () => { + const params = parseQueryString(QS_CONFIG, location.search); + + const [response, responseActions] = await Promise.all([ + OrganizationsAPI.readExecutionEnvironments(id, params), + OrganizationsAPI.readExecutionEnvironmentsOptions(id, params), + ]); + + return { + executionEnvironments: response.data.results, + executionEnvironmentsCount: response.data.count, + actions: responseActions.data.actions, + relatedSearchableKeys: ( + responseActions?.data?.related_search_fields || [] + ).map(val => val.slice(0, -8)), + searchableKeys: Object.keys( + responseActions.data.actions?.GET || {} + ).filter(key => responseActions.data.actions?.GET[key].filterable), + }; + }, [location, id]), + { + executionEnvironments: [], + executionEnvironmentsCount: 0, + actions: {}, + relatedSearchableKeys: [], + searchableKeys: [], + } + ); + + useEffect(() => { + fetchExecutionEnvironments(); + }, [fetchExecutionEnvironments]); + + return ( + <> + + ( + + )} + renderItem={executionEnvironment => ( + + )} + /> + + + ); +} + +export default withI18n()(OrganizationExecEnvList); diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.test.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.test.jsx new file mode 100644 index 0000000000..07e8a53ea5 --- /dev/null +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.test.jsx @@ -0,0 +1,116 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { + mountWithContexts, + waitForElement, +} from '../../../../testUtils/enzymeHelpers'; + +import { OrganizationsAPI } from '../../../api'; +import OrganizationExecEnvList from './OrganizationExecEnvList'; + +jest.mock('../../../api/'); + +const executionEnvironments = { + data: { + count: 3, + results: [ + { + id: 1, + type: 'execution_environment', + url: '/api/v2/execution_environments/1/', + related: { + organization: '/api/v2/organizations/1/', + }, + organization: 1, + image: 'https://localhost.com/image/disk', + managed_by_tower: false, + credential: null, + }, + { + id: 2, + type: 'execution_environment', + url: '/api/v2/execution_environments/2/', + related: { + organization: '/api/v2/organizations/1/', + }, + organization: 1, + image: 'test/image123', + managed_by_tower: false, + credential: null, + }, + { + id: 3, + type: 'execution_environment', + url: '/api/v2/execution_environments/3/', + related: { + organization: '/api/v2/organizations/1/', + }, + organization: 1, + image: 'test/test', + managed_by_tower: false, + credential: null, + }, + ], + }, +}; + +const mockOrganization = { + id: 1, + type: 'organization', + name: 'Default', +}; + +const options = { data: { actions: { POST: {}, GET: {} } } }; + +describe('', () => { + let wrapper; + + test('should mount successfully', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + await waitForElement( + wrapper, + 'OrganizationExecEnvList', + el => el.length > 0 + ); + }); + + test('should have data fetched and render 3 rows', async () => { + OrganizationsAPI.readExecutionEnvironments.mockResolvedValue( + executionEnvironments + ); + + OrganizationsAPI.readExecutionEnvironmentsOptions.mockResolvedValue( + options + ); + + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + await waitForElement( + wrapper, + 'OrganizationExecEnvList', + el => el.length > 0 + ); + + expect(wrapper.find('OrganizationExecEnvListItem').length).toBe(3); + expect(OrganizationsAPI.readExecutionEnvironments).toBeCalled(); + expect(OrganizationsAPI.readExecutionEnvironmentsOptions).toBeCalled(); + }); + + test('should not render add button', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + waitForElement(wrapper, 'OrganizationExecEnvList', el => el.length > 0); + expect(wrapper.find('ToolbarAddButton').length).toBe(0); + }); +}); diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx new file mode 100644 index 0000000000..1998e8c783 --- /dev/null +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx @@ -0,0 +1,51 @@ +import React from 'react'; +import { string } from 'prop-types'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Link } from 'react-router-dom'; +import { + DataListItem, + DataListItemRow, + DataListItemCells, +} from '@patternfly/react-core'; + +import DataListCell from '../../../components/DataListCell'; +import { ExecutionEnvironment } from '../../../types'; + +function OrganizationExecEnvListItem({ + executionEnvironment, + detailUrl, + i18n, +}) { + const labelId = `check-action-${executionEnvironment.id}`; + + return ( + + + + + {executionEnvironment.image} + + , + ]} + /> + + + ); +} + +OrganizationExecEnvListItem.prototype = { + executionEnvironment: ExecutionEnvironment.isRequired, + detailUrl: string.isRequired, +}; + +export default withI18n()(OrganizationExecEnvListItem); diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx new file mode 100644 index 0000000000..9e4a2492aa --- /dev/null +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx @@ -0,0 +1,44 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; + +import OrganizationExecEnvListItem from './OrganizationExecEnvListItem'; + +describe('', () => { + let wrapper; + const executionEnvironment = { + id: 1, + image: 'https://registry.com/r/image/manifest', + organization: 1, + credential: null, + }; + + test('should mount successfully', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + expect(wrapper.find('OrganizationExecEnvListItem').length).toBe(1); + }); + + test('should render the proper data', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + expect( + wrapper + .find('DataListCell[aria-label="Execution environment image"]') + .text() + ).toBe(executionEnvironment.image); + }); +}); diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/index.js b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/index.js new file mode 100644 index 0000000000..668a3beb61 --- /dev/null +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/index.js @@ -0,0 +1 @@ +export { default } from './OrganizationExecEnvList'; diff --git a/awx/ui_next/src/screens/Organization/Organizations.jsx b/awx/ui_next/src/screens/Organization/Organizations.jsx index 6c7b17dc69..fcf1b8398b 100644 --- a/awx/ui_next/src/screens/Organization/Organizations.jsx +++ b/awx/ui_next/src/screens/Organization/Organizations.jsx @@ -34,6 +34,9 @@ function Organizations({ i18n }) { [`/organizations/${organization.id}/notifications`]: i18n._( t`Notifications` ), + [`/organizations/${organization.id}/execution_environments`]: i18n._( + t`Execution Environments` + ), }; setBreadcrumbConfig(breadcrumb); }, From cf513b33ee641ed8d5a4bec21eae57a186f5bc8f Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Mon, 22 Feb 2021 16:19:21 -0500 Subject: [PATCH 130/157] Add name field --- awx/ui_next/src/api/models/Organizations.js | 1 + .../OrganizationExecEnvList.jsx | 13 +++++++++++-- .../OrganizationExecEnvListItem.jsx | 12 +++++++++--- .../OrganizationExecEnvListItem.test.jsx | 2 ++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/awx/ui_next/src/api/models/Organizations.js b/awx/ui_next/src/api/models/Organizations.js index ec4a14c549..fd980fece8 100644 --- a/awx/ui_next/src/api/models/Organizations.js +++ b/awx/ui_next/src/api/models/Organizations.js @@ -29,6 +29,7 @@ class Organizations extends InstanceGroupsMixin(NotificationsMixin(Base)) { params, }); } + readExecutionEnvironments(id, params) { return this.http.get(`${this.baseUrl}${id}/execution_environments/`, { params, diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx index 48caaff802..9f2c4ae817 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx @@ -15,7 +15,7 @@ import OrganizationExecEnvListItem from './OrganizationExecEnvListItem'; const QS_CONFIG = getQSConfig('organizations', { page: 1, page_size: 20, - order_by: 'image', + order_by: 'name', }); function OrganizationExecEnvList({ i18n, organization }) { @@ -79,10 +79,15 @@ function OrganizationExecEnvList({ i18n, organization }) { toolbarSearchableKeys={searchableKeys} toolbarRelatedSearchableKeys={relatedSearchableKeys} toolbarSearchColumns={[ + { + name: i18n._(t`Name`), + key: 'name__icontains', + isDefault: true, + }, { name: i18n._(t`Image`), key: 'image__icontains', - isDefault: true, + isDefault: false, }, { name: i18n._(t`Created By (Username)`), @@ -94,6 +99,10 @@ function OrganizationExecEnvList({ i18n, organization }) { }, ]} toolbarSortColumns={[ + { + name: i18n._(t`Name`), + key: 'name', + }, { name: i18n._(t`Image`), key: 'image', diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx index 1998e8c783..0d2715d7a6 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.jsx @@ -28,13 +28,19 @@ function OrganizationExecEnvListItem({ + + {executionEnvironment.name} + + , - - {executionEnvironment.image} - + {executionEnvironment.image} , ]} /> diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx index 9e4a2492aa..29181f4ec3 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvListItem.test.jsx @@ -10,8 +10,10 @@ describe('', () => { const executionEnvironment = { id: 1, image: 'https://registry.com/r/image/manifest', + name: 'foo', organization: 1, credential: null, + pull: 'always', }; test('should mount successfully', async () => { From eaa74b40c106f63c69f4766713f5f8522b4dd044 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 17 Feb 2021 14:54:17 -0500 Subject: [PATCH 131/157] add org admins as able to control EEs even if they don't have the ee_admin role for the specific ee and prevent managed_by_tower EEs from being edited/deleted --- awx/main/access.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index a8a110e9c3..8817e7b917 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1325,7 +1325,7 @@ class ExecutionEnvironmentAccess(BaseAccess): def filtered_queryset(self): return ExecutionEnvironment.objects.filter( - Q(organization__in=Organization.accessible_pk_qs(self.user, 'execution_environment_admin_role')) | + Q(organization__in=Organization.accessible_pk_qs(self.user, 'member_role')) | Q(organization__isnull=True) ).distinct() @@ -1337,9 +1337,11 @@ class ExecutionEnvironmentAccess(BaseAccess): @check_superuser def can_change(self, obj, data): + if obj.managed_by_tower is True: + raise PermissionDenied if obj and obj.organization_id is None: raise PermissionDenied - if self.user not in obj.organization.execution_environment_admin_role: + if self.user not in obj.organization.execution_environment_admin_role and self.user not in obj.organization.admin_role: raise PermissionDenied org_pk = get_pk_from_dict(data, 'organization') if obj and obj.organization_id != org_pk: From 41fb21911e684a0c6363d77c19459571051f611c Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Thu, 18 Feb 2021 17:24:17 -0500 Subject: [PATCH 132/157] add execution_environment_admin_role to the an organizations read role, which access.py uses for determining access to reading an ee within an organization, add migration file for execution_env_admin role addition to read_roles within an organization, and set check related to mandatory --- awx/main/access.py | 6 +++--- .../0128_organiaztion_read_roles_ee_admin.py | 20 +++++++++++++++++++ awx/main/models/organization.py | 3 ++- 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 awx/main/migrations/0128_organiaztion_read_roles_ee_admin.py diff --git a/awx/main/access.py b/awx/main/access.py index 8817e7b917..9e9631751c 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1325,7 +1325,7 @@ class ExecutionEnvironmentAccess(BaseAccess): def filtered_queryset(self): return ExecutionEnvironment.objects.filter( - Q(organization__in=Organization.accessible_pk_qs(self.user, 'member_role')) | + Q(organization__in=Organization.accessible_pk_qs(self.user, 'read_role')) | Q(organization__isnull=True) ).distinct() @@ -1333,7 +1333,7 @@ class ExecutionEnvironmentAccess(BaseAccess): def can_add(self, data): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'execution_environment_admin_role').exists() - return self.check_related('organization', Organization, data) + return self.check_related('organization', Organization, data, mandatory=True) @check_superuser def can_change(self, obj, data): @@ -1341,7 +1341,7 @@ class ExecutionEnvironmentAccess(BaseAccess): raise PermissionDenied if obj and obj.organization_id is None: raise PermissionDenied - if self.user not in obj.organization.execution_environment_admin_role and self.user not in obj.organization.admin_role: + if self.user not in obj.organization.execution_environment_admin_role: raise PermissionDenied org_pk = get_pk_from_dict(data, 'organization') if obj and obj.organization_id != org_pk: diff --git a/awx/main/migrations/0128_organiaztion_read_roles_ee_admin.py b/awx/main/migrations/0128_organiaztion_read_roles_ee_admin.py new file mode 100644 index 0000000000..f03a4e0ba2 --- /dev/null +++ b/awx/main/migrations/0128_organiaztion_read_roles_ee_admin.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.16 on 2021-02-18 22:57 + +import awx.main.fields +from django.db import migrations +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0127_reset_pod_spec_override'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='read_role', + field=awx.main.fields.ImplicitRoleField(editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['member_role', 'auditor_role', 'execute_role', 'project_admin_role', 'inventory_admin_role', 'workflow_admin_role', 'notification_admin_role', 'credential_admin_role', 'job_template_admin_role', 'approval_role', 'execution_environment_admin_role'], related_name='+', to='main.Role'), + ), + ] diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index bdf1e38d7d..f0ecfea5c7 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -109,7 +109,8 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi 'execute_role', 'project_admin_role', 'inventory_admin_role', 'workflow_admin_role', 'notification_admin_role', 'credential_admin_role', - 'job_template_admin_role', 'approval_role',], + 'job_template_admin_role', 'approval_role', + 'execution_environment_admin_role',], ) approval_role = ImplicitRoleField( parent_role='admin_role', From 5b2adc89cfa85ec9e88aa961596f7ea9bddf53a1 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 19 Feb 2021 14:06:01 -0500 Subject: [PATCH 133/157] Make the managed_by_tower field read-only for EEs (similar to how we deal with it not being settable for Credentials) and add permissions checking for Org EE Admins. can_add: gets an explicit role to check against, `'execution_environment_admin_role'` can_change: leverages `self.check_related()` for the case where the Org is not changing, but also adds an explicit check for the EE Admin Role when the Org is changing to an explicit different Org. --- awx/api/serializers.py | 1 + awx/main/access.py | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 22b7ab3a7a..523171e43d 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1362,6 +1362,7 @@ class ProjectOptionsSerializer(BaseSerializer): class ExecutionEnvironmentSerializer(BaseSerializer): show_capabilities = ['edit', 'delete'] + managed_by_tower = serializers.ReadOnlyField() class Meta: model = ExecutionEnvironment diff --git a/awx/main/access.py b/awx/main/access.py index 9e9631751c..7da6709c41 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1329,29 +1329,31 @@ class ExecutionEnvironmentAccess(BaseAccess): Q(organization__isnull=True) ).distinct() - @check_superuser def can_add(self, data): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'execution_environment_admin_role').exists() - return self.check_related('organization', Organization, data, mandatory=True) - - @check_superuser - def can_change(self, obj, data): - if obj.managed_by_tower is True: + if obj.managed_by_tower: raise PermissionDenied + if self.user.is_superuser: + return True + return self.check_related('organization', Organization, data, mandatory=True, + role_field='execution_environment_admin_role') + + def can_change(self, obj, data): + if obj.managed_by_tower: + raise PermissionDenied + if self.user.is_superuser: + return True if obj and obj.organization_id is None: raise PermissionDenied if self.user not in obj.organization.execution_environment_admin_role: raise PermissionDenied - org_pk = get_pk_from_dict(data, 'organization') - if obj and obj.organization_id != org_pk: - # Prevent moving an EE to a different organization, unless a superuser or admin on both orgs. - if obj.organization_id is None or org_pk is None: - raise PermissionDenied - if self.user not in Organization.objects.get(id=org_pk).execution_environment_admin_role: - raise PermissionDenied - - return True + if data and 'organization' in data: + new_org = get_object_from_data('organization', Organization, data, obj=obj) + if not new_org or self.user not in new_org.execution_environment_admin_role: + return False + return self.check_related('organization', Organization, data, obj=obj, mandatory=True, + role_field='execution_environment_admin_role') def can_delete(self, obj): return self.can_change(obj, None) From b417fc38032809ce5155a6a746c22cfe527b609d Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 22 Feb 2021 17:07:43 -0500 Subject: [PATCH 134/157] Turn off permissions check bypassing for admins when hitting the execution environment list and detail views. --- awx/api/views/__init__.py | 2 ++ awx/main/access.py | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 01ad6d98ed..855d9e54e0 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -688,6 +688,7 @@ class TeamAccessList(ResourceAccessList): class ExecutionEnvironmentList(ListCreateAPIView): + always_allow_superuser = False model = models.ExecutionEnvironment serializer_class = serializers.ExecutionEnvironmentSerializer swagger_topic = "Execution Environments" @@ -695,6 +696,7 @@ class ExecutionEnvironmentList(ListCreateAPIView): class ExecutionEnvironmentDetail(RetrieveUpdateDestroyAPIView): + always_allow_superuser = False model = models.ExecutionEnvironment serializer_class = serializers.ExecutionEnvironmentSerializer swagger_topic = "Execution Environments" diff --git a/awx/main/access.py b/awx/main/access.py index 7da6709c41..d2a2aa9f3b 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1329,13 +1329,10 @@ class ExecutionEnvironmentAccess(BaseAccess): Q(organization__isnull=True) ).distinct() + @check_superuser def can_add(self, data): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'execution_environment_admin_role').exists() - if obj.managed_by_tower: - raise PermissionDenied - if self.user.is_superuser: - return True return self.check_related('organization', Organization, data, mandatory=True, role_field='execution_environment_admin_role') From 86a3a79be4fb603089ab2224d3840626f8314ea0 Mon Sep 17 00:00:00 2001 From: beeankha Date: Tue, 16 Feb 2021 17:40:53 -0500 Subject: [PATCH 135/157] Enable utilized EE Collections name and version info to be detected --- .../0129_unifiedjob_installed_collections.py | 19 +++++++++++++++++++ awx/main/models/unified_jobs.py | 8 +++++++- awx/main/tasks.py | 8 +++++++- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/0129_unifiedjob_installed_collections.py diff --git a/awx/main/migrations/0129_unifiedjob_installed_collections.py b/awx/main/migrations/0129_unifiedjob_installed_collections.py new file mode 100644 index 0000000000..897708a631 --- /dev/null +++ b/awx/main/migrations/0129_unifiedjob_installed_collections.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2021-02-16 20:27 + +import awx.main.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0128_organiaztion_read_roles_ee_admin'), + ] + + operations = [ + migrations.AddField( + model_name='unifiedjob', + name='installed_collections', + field=awx.main.fields.JSONBField(blank=True, default=dict, editable=False, help_text='The Collections names and versions installed in the execution environment.'), + ), + ] diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index cf22430f7b..45d7739ee3 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -50,7 +50,7 @@ from awx.main.utils import ( from awx.main.constants import ACTIVE_STATES, CAN_CANCEL from awx.main.redact import UriCleaner, REPLACE_STR from awx.main.consumers import emit_channel_notification -from awx.main.fields import JSONField, AskForField, OrderedManyToManyField +from awx.main.fields import JSONField, JSONBField, AskForField, OrderedManyToManyField __all__ = ['UnifiedJobTemplate', 'UnifiedJob', 'StdoutMaxBytesExceeded'] @@ -722,6 +722,12 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique 'Credential', related_name='%(class)ss', ) + installed_collections = JSONBField( + blank=True, + default=dict, + editable=False, + help_text=_("The Collections names and versions installed in the execution environment."), + ) def get_absolute_url(self, request=None): RealClass = self.get_real_instance_class() diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ab9484eba9..bdfcf18c90 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1179,11 +1179,17 @@ class BaseTask(object): instance.log_lifecycle("finalize_run") job_profiling_dir = os.path.join(private_data_dir, 'artifacts/playbook_profiling') awx_profiling_dir = '/var/log/tower/playbook_profiling/' + collections_info = os.path.join(private_data_dir, 'artifacts/') + if not os.path.exists(awx_profiling_dir): os.mkdir(awx_profiling_dir) if os.path.isdir(job_profiling_dir): shutil.copytree(job_profiling_dir, os.path.join(awx_profiling_dir, str(instance.pk))) - + if os.path.exists(collections_info): + with open(collections_info + 'collections.json') as ee_json_info: + ee_collections_info = ee_json_info.read() + instance.installed_collections = ee_collections_info + instance.save(update_fields=['installed_collections']) def event_handler(self, event_data): # From cb95de0862d84d1055b298bfc67606a0a3069b4d Mon Sep 17 00:00:00 2001 From: beeankha Date: Thu, 18 Feb 2021 15:54:07 -0500 Subject: [PATCH 136/157] Assign entire file path to variable --- awx/main/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index bdfcf18c90..55074683d0 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1179,14 +1179,14 @@ class BaseTask(object): instance.log_lifecycle("finalize_run") job_profiling_dir = os.path.join(private_data_dir, 'artifacts/playbook_profiling') awx_profiling_dir = '/var/log/tower/playbook_profiling/' - collections_info = os.path.join(private_data_dir, 'artifacts/') + collections_info = os.path.join(private_data_dir, 'artifacts/', 'collections.json') if not os.path.exists(awx_profiling_dir): os.mkdir(awx_profiling_dir) if os.path.isdir(job_profiling_dir): shutil.copytree(job_profiling_dir, os.path.join(awx_profiling_dir, str(instance.pk))) if os.path.exists(collections_info): - with open(collections_info + 'collections.json') as ee_json_info: + with open(collections_info) as ee_json_info: ee_collections_info = ee_json_info.read() instance.installed_collections = ee_collections_info instance.save(update_fields=['installed_collections']) From 0e80f663ab47528d154474eccb9c390deeef82ec Mon Sep 17 00:00:00 2001 From: beeankha Date: Mon, 22 Feb 2021 16:02:15 -0500 Subject: [PATCH 137/157] Add installed_collections column to unified job query --- awx/main/analytics/collectors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index b0ac43cc65..58b78c83a6 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -334,7 +334,8 @@ def unified_jobs_table(since, full_path, until, **kwargs): main_unifiedjob.finished, main_unifiedjob.elapsed, main_unifiedjob.job_explanation, - main_unifiedjob.instance_group_id + main_unifiedjob.instance_group_id, + main_unifiedjob.installed_collections FROM main_unifiedjob JOIN django_content_type ON main_unifiedjob.polymorphic_ctype_id = django_content_type.id LEFT JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id From 5b17ab6873d806e518d88458aafe7f9c0a5bb735 Mon Sep 17 00:00:00 2001 From: beeankha Date: Tue, 23 Feb 2021 11:54:30 -0500 Subject: [PATCH 138/157] Enable EE collections info to be loaded as valid JSON vs stringified JSON --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 55074683d0..10e07312c1 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1187,7 +1187,7 @@ class BaseTask(object): shutil.copytree(job_profiling_dir, os.path.join(awx_profiling_dir, str(instance.pk))) if os.path.exists(collections_info): with open(collections_info) as ee_json_info: - ee_collections_info = ee_json_info.read() + ee_collections_info = json.loads(ee_json_info.read()) instance.installed_collections = ee_collections_info instance.save(update_fields=['installed_collections']) From 60827143bbfe2e72faf8a37c74de1d795697e8eb Mon Sep 17 00:00:00 2001 From: beeankha Date: Tue, 23 Feb 2021 12:50:05 -0500 Subject: [PATCH 139/157] Bump up unified_jobs_table version for new column addition --- awx/main/analytics/collectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 58b78c83a6..89bc28ea56 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -311,7 +311,7 @@ def events_table(since, full_path, until, **kwargs): return _copy_table(table='events', query=events_query, path=full_path) -@register('unified_jobs_table', '1.1', format='csv', description=_('Data on jobs run'), expensive=True) +@register('unified_jobs_table', '1.2', format='csv', description=_('Data on jobs run'), expensive=True) def unified_jobs_table(since, full_path, until, **kwargs): unified_job_query = '''COPY (SELECT main_unifiedjob.id, main_unifiedjob.polymorphic_ctype_id, From 883fa4906aea59dd28d43ec80ed47c8bb03d0de2 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 23 Feb 2021 16:25:26 -0500 Subject: [PATCH 140/157] Fix receptor.conf path in dev env --- .../ansible/roles/sources/templates/docker-compose.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 00f136ef49..1d0799c7b9 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -33,10 +33,10 @@ services: - "../../docker-compose/_sources/websocket_secret.py:/etc/tower/conf.d/websocket_secret.py" - "../../docker-compose/_sources/local_settings.py:/etc/tower/conf.d/local_settings.py" - "../../docker-compose/_sources/SECRET_KEY:/etc/tower/SECRET_KEY" + - "../../docker-compose/receptor.conf:/etc/receptor/receptor.conf" - "redis_socket:/var/run/redis/:rw" - "receptor:/var/run/receptor/" - "/sys/fs/cgroup:/sys/fs/cgroup" - - "./docker-compose/receptor.conf:/etc/receptor/receptor.conf" - "~/.kube/config:/var/lib/awx/.kube/config" privileged: true tty: true From befc658042a755164ff4fb37633caa915c864e02 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 23 Feb 2021 16:48:50 -0500 Subject: [PATCH 141/157] Wire up --pull option for EEs --- awx/main/tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 10e07312c1..0f02f3a507 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -909,6 +909,11 @@ class BaseTask(object): "process_isolation": True, "container_options": ['--user=root'], } + + pull = instance.execution_environment.pull + if pull: + params['container_options'].append(f'--pull={pull}') + if settings.AWX_PROOT_SHOW_PATHS: params['container_volume_mounts'] = [] for this_path in settings.AWX_PROOT_SHOW_PATHS: From f2801e0c034773dc7f53ad1c854e7d4b50fb9551 Mon Sep 17 00:00:00 2001 From: nixocio Date: Tue, 23 Feb 2021 19:55:29 -0500 Subject: [PATCH 142/157] Minor update EE tables * Add table header `actions` * Add `name` as default search See: https://github.com/ansible/awx/issues/7884 Also: https://github.com/ansible/awx/issues/9087 --- .../ExecutionEnvironmentList/ExecutionEnvironmentList.jsx | 7 ++++++- .../ExecutionEnvironmentListItem.jsx | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx index 9d94085520..312b18f2cf 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentList.jsx @@ -121,10 +121,14 @@ function ExecutionEnvironmentList({ i18n }) { toolbarSearchableKeys={searchableKeys} toolbarRelatedSearchableKeys={relatedSearchableKeys} toolbarSearchColumns={[ + { + name: i18n._(t`Name`), + key: 'name__icontains', + isDefault: true, + }, { name: i18n._(t`Image`), key: 'image__icontains', - isDefault: true, }, ]} toolbarSortColumns={[ @@ -150,6 +154,7 @@ function ExecutionEnvironmentList({ i18n }) { {i18n._(t`Name`)} {i18n._(t`Image`)} {i18n._(t`Organization`)} + {i18n._(t`Actions`)} } renderToolbar={props => ( diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx index fbc8387d84..bb814a1921 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentList/ExecutionEnvironmentListItem.jsx @@ -50,7 +50,7 @@ function ExecutionEnvironmentListItem({ i18n._(t`Globally Available`) )} - + Date: Wed, 10 Feb 2021 16:52:23 -0500 Subject: [PATCH 143/157] adding needed url endpoint for copy functionality and the beginning of some testing that can be fleshed out more fully in later work --- awx/api/urls/execution_environments.py | 2 ++ awx/api/views/__init__.py | 6 ++++++ awx/main/models/execution_environments.py | 17 +++++++++++++++++ awx/main/tests/functional/test_rbac_team.py | 8 +++++++- awx/main/tests/functional/test_rbac_user.py | 2 +- .../awxkit/api/pages/execution_environments.py | 4 ++-- 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/awx/api/urls/execution_environments.py b/awx/api/urls/execution_environments.py index 08f852be08..99b9cb3ddc 100644 --- a/awx/api/urls/execution_environments.py +++ b/awx/api/urls/execution_environments.py @@ -4,6 +4,7 @@ from awx.api.views import ( ExecutionEnvironmentList, ExecutionEnvironmentDetail, ExecutionEnvironmentJobTemplateList, + ExecutionEnvironmentCopy, ExecutionEnvironmentActivityStreamList, ) @@ -12,6 +13,7 @@ urls = [ url(r'^$', ExecutionEnvironmentList.as_view(), name='execution_environment_list'), url(r'^(?P[0-9]+)/$', ExecutionEnvironmentDetail.as_view(), name='execution_environment_detail'), url(r'^(?P[0-9]+)/unified_job_templates/$', ExecutionEnvironmentJobTemplateList.as_view(), name='execution_environment_job_template_list'), + url(r'^(?P[0-9]+)/copy/$', ExecutionEnvironmentCopy.as_view(), name='execution_environment_copy'), url(r'^(?P[0-9]+)/activity_stream/$', ExecutionEnvironmentActivityStreamList.as_view(), name='execution_environment_activity_stream_list'), ] diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 855d9e54e0..2d29519de8 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -710,6 +710,12 @@ class ExecutionEnvironmentJobTemplateList(SubListAPIView): relationship = 'unifiedjobtemplates' +class ExecutionEnvironmentCopy(CopyAPIView): + + model = models.ExecutionEnvironment + copy_return_serializer_class = serializers.ExecutionEnvironmentSerializer + + class ExecutionEnvironmentActivityStreamList(SubListAPIView): model = models.ActivityStream diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index eabd0cce7c..b0fd2bb857 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -3,6 +3,7 @@ from django.utils.translation import ugettext_lazy as _ from awx.api.versioning import reverse from awx.main.models.base import CommonModel +from awx.main.utils import copy_model_by_class, copy_m2m_relationships __all__ = ['ExecutionEnvironment'] @@ -49,5 +50,21 @@ class ExecutionEnvironment(CommonModel): help_text=_('Pull image before running?'), ) + def copy_execution_environment(self): + ''' + Returns saved object, including related fields. + Create a copy of this unified job template. + ''' + execution_environment_class = self.__class__ + fields = (f.name for f in self.Meta.fields) + execution_environment_copy = copy_model_by_class(self, execution_environment_class, fields, {}) + + time_now = now() + execution_environment_copy.name = execution_environment_copy.name.split('@', 1)[0] + ' @ ' + time_now.strftime('%I:%M:%S %p') + + execution_environment_copy.save() + copy_m2m_relationships(self, execution_environment_copy, fields) + return execution_environment_copy + def get_absolute_url(self, request=None): return reverse('api:execution_environment_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/tests/functional/test_rbac_team.py b/awx/main/tests/functional/test_rbac_team.py index a18a69a94b..ed76e7e4a8 100644 --- a/awx/main/tests/functional/test_rbac_team.py +++ b/awx/main/tests/functional/test_rbac_team.py @@ -2,7 +2,7 @@ import pytest from unittest import mock from awx.main.access import TeamAccess -from awx.main.models import Project, Organization, Team +from awx.main.models import Project, Organization, Team, ExecutionEnvironment @pytest.mark.django_db @@ -143,6 +143,12 @@ def test_team_member_org_role_access_inventory(team, rando, inventory, organizat team.member_role.children.add(organization.inventory_admin_role) assert rando in inventory.admin_role +# @pytest.mark.django_db +# def test_team_member_org_role_access_execution_environment(team, rando, execution_environment, organization): +# team.member_role.members.add(rando) +# assert rando not in execution_environment.read_role +# team.member_role.children.add(organization.execution_environment_admin_role) +# assert rando in execution_environment.admin_role @pytest.mark.django_db def test_org_admin_team_access(organization, team, user, project): diff --git a/awx/main/tests/functional/test_rbac_user.py b/awx/main/tests/functional/test_rbac_user.py index b62a0db25f..376272bdfe 100644 --- a/awx/main/tests/functional/test_rbac_user.py +++ b/awx/main/tests/functional/test_rbac_user.py @@ -4,7 +4,7 @@ from unittest import mock from django.test import TransactionTestCase from awx.main.access import UserAccess, RoleAccess, TeamAccess -from awx.main.models import User, Organization, Inventory, Role +from awx.main.models import User, Organization, Inventory, Role, ExecutionEnvironment class TestSysAuditorTransactional(TransactionTestCase): diff --git a/awxkit/awxkit/api/pages/execution_environments.py b/awxkit/awxkit/api/pages/execution_environments.py index 94f53c1094..0471b1f1d3 100644 --- a/awxkit/awxkit/api/pages/execution_environments.py +++ b/awxkit/awxkit/api/pages/execution_environments.py @@ -1,6 +1,6 @@ import logging -from awxkit.api.mixins import DSAdapter, HasCreate +from awxkit.api.mixins import DSAdapter, HasCreate, HasCopy from awxkit.api.pages import ( Credential, Organization, @@ -15,7 +15,7 @@ from . import page log = logging.getLogger(__name__) -class ExecutionEnvironment(HasCreate, base.Base): +class ExecutionEnvironment(HasCreate, HasCopy, base.Base): dependencies = [Organization, Credential] NATURAL_KEY = ('name',) From 4d2fcfd8c1b9678e1b9d6a9e317e28d0083474e8 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Fri, 12 Feb 2021 15:55:32 -0500 Subject: [PATCH 144/157] add a functional test for creating an EE, remove bum copy function because it's not needed, copy works from the base class moved AWXKit pull additions to separate PR and made some changes that were causing linting errors in tests and add copy to show_capabilities for the ee serializer --- awx/api/serializers.py | 2 +- awx/main/models/execution_environments.py | 17 ----------------- awx/main/tests/functional/conftest.py | 6 ++++++ .../functional/test_execution_environments.py | 19 +++++++++++++++++++ awx/main/tests/functional/test_rbac_team.py | 8 +------- awx/main/tests/functional/test_rbac_user.py | 2 +- 6 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 awx/main/tests/functional/test_execution_environments.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 523171e43d..dce312a5a8 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1361,7 +1361,7 @@ class ProjectOptionsSerializer(BaseSerializer): class ExecutionEnvironmentSerializer(BaseSerializer): - show_capabilities = ['edit', 'delete'] + show_capabilities = ['edit', 'delete', 'copy'] managed_by_tower = serializers.ReadOnlyField() class Meta: diff --git a/awx/main/models/execution_environments.py b/awx/main/models/execution_environments.py index b0fd2bb857..eabd0cce7c 100644 --- a/awx/main/models/execution_environments.py +++ b/awx/main/models/execution_environments.py @@ -3,7 +3,6 @@ from django.utils.translation import ugettext_lazy as _ from awx.api.versioning import reverse from awx.main.models.base import CommonModel -from awx.main.utils import copy_model_by_class, copy_m2m_relationships __all__ = ['ExecutionEnvironment'] @@ -50,21 +49,5 @@ class ExecutionEnvironment(CommonModel): help_text=_('Pull image before running?'), ) - def copy_execution_environment(self): - ''' - Returns saved object, including related fields. - Create a copy of this unified job template. - ''' - execution_environment_class = self.__class__ - fields = (f.name for f in self.Meta.fields) - execution_environment_copy = copy_model_by_class(self, execution_environment_class, fields, {}) - - time_now = now() - execution_environment_copy.name = execution_environment_copy.name.split('@', 1)[0] + ' @ ' + time_now.strftime('%I:%M:%S %p') - - execution_environment_copy.save() - copy_m2m_relationships(self, execution_environment_copy, fields) - return execution_environment_copy - def get_absolute_url(self, request=None): return reverse('api:execution_environment_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 7111950003..4cbd5a40d3 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -52,6 +52,7 @@ from awx.main.models.events import ( from awx.main.models.workflow import WorkflowJobTemplate from awx.main.models.ad_hoc_commands import AdHocCommand from awx.main.models.oauth import OAuth2Application as Application +from awx.main.models.execution_environments import ExecutionEnvironment __SWAGGER_REQUESTS__ = {} @@ -850,3 +851,8 @@ def slice_job_factory(slice_jt_factory): node.save() return slice_job return r + + +@pytest.fixture +def execution_environment(organization): + return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", organization=organization) diff --git a/awx/main/tests/functional/test_execution_environments.py b/awx/main/tests/functional/test_execution_environments.py new file mode 100644 index 0000000000..5f1e430fe8 --- /dev/null +++ b/awx/main/tests/functional/test_execution_environments.py @@ -0,0 +1,19 @@ +import pytest + +from awx.main.models import (ExecutionEnvironment) + + +@pytest.mark.django_db +def test_execution_environment_creation(execution_environment, organization): + execution_env = ExecutionEnvironment.objects.create( + name='Hello Environment', + image='', + organization=organization, + managed_by_tower=False, + credential=None, + pull='missing' + ) + assert type(execution_env) is type(execution_environment) + assert execution_env.organization == organization + assert execution_env.name == 'Hello Environment' + assert execution_env.pull == 'missing' diff --git a/awx/main/tests/functional/test_rbac_team.py b/awx/main/tests/functional/test_rbac_team.py index ed76e7e4a8..a18a69a94b 100644 --- a/awx/main/tests/functional/test_rbac_team.py +++ b/awx/main/tests/functional/test_rbac_team.py @@ -2,7 +2,7 @@ import pytest from unittest import mock from awx.main.access import TeamAccess -from awx.main.models import Project, Organization, Team, ExecutionEnvironment +from awx.main.models import Project, Organization, Team @pytest.mark.django_db @@ -143,12 +143,6 @@ def test_team_member_org_role_access_inventory(team, rando, inventory, organizat team.member_role.children.add(organization.inventory_admin_role) assert rando in inventory.admin_role -# @pytest.mark.django_db -# def test_team_member_org_role_access_execution_environment(team, rando, execution_environment, organization): -# team.member_role.members.add(rando) -# assert rando not in execution_environment.read_role -# team.member_role.children.add(organization.execution_environment_admin_role) -# assert rando in execution_environment.admin_role @pytest.mark.django_db def test_org_admin_team_access(organization, team, user, project): diff --git a/awx/main/tests/functional/test_rbac_user.py b/awx/main/tests/functional/test_rbac_user.py index 376272bdfe..b62a0db25f 100644 --- a/awx/main/tests/functional/test_rbac_user.py +++ b/awx/main/tests/functional/test_rbac_user.py @@ -4,7 +4,7 @@ from unittest import mock from django.test import TransactionTestCase from awx.main.access import UserAccess, RoleAccess, TeamAccess -from awx.main.models import User, Organization, Inventory, Role, ExecutionEnvironment +from awx.main.models import User, Organization, Inventory, Role class TestSysAuditorTransactional(TransactionTestCase): From adf708366a112f502551233afa5071b475947796 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 23 Feb 2021 18:47:47 -0500 Subject: [PATCH 145/157] Add "copy" to EE related links --- awx/api/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index dce312a5a8..6f8dcd3fad 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1373,6 +1373,7 @@ class ExecutionEnvironmentSerializer(BaseSerializer): res.update( activity_stream=self.reverse('api:execution_environment_activity_stream_list', kwargs={'pk': obj.pk}), unified_job_templates=self.reverse('api:execution_environment_job_template_list', kwargs={'pk': obj.pk}), + copy=self.reverse('api:execution_environment_copy', kwargs={'pk': obj.pk}), ) if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) From 6e67ae68fd416ee63a715e69472bc7f9122aa8c2 Mon Sep 17 00:00:00 2001 From: nixocio Date: Tue, 9 Feb 2021 17:23:47 -0500 Subject: [PATCH 146/157] Add Execution Environments into a few screens Add EE to the following screens: * Job Template * Organization * Project * Workflow Job Template Also, add a new lookup component - ExecutionEnvironmentLoookup. See: https://github.com/ansible/awx/issues/9189 --- .../Lookup/ExecutionEnvironmentLookup.jsx | 167 ++++++++++++++++++ .../ExecutionEnvironmentLookup.test.jsx | 76 ++++++++ awx/ui_next/src/components/Lookup/index.js | 1 + .../OrganizationAdd/OrganizationAdd.jsx | 5 +- .../OrganizationAdd/OrganizationAdd.test.jsx | 7 +- .../OrganizationDetail/OrganizationDetail.jsx | 6 + .../OrganizationDetail.test.jsx | 11 +- .../OrganizationEdit/OrganizationEdit.jsx | 5 +- .../OrganizationEdit.test.jsx | 8 + .../Organization/shared/OrganizationForm.jsx | 35 +++- .../shared/OrganizationForm.test.jsx | 20 ++- .../screens/Project/ProjectAdd/ProjectAdd.jsx | 1 + .../Project/ProjectAdd/ProjectAdd.test.jsx | 8 +- .../Project/ProjectDetail/ProjectDetail.jsx | 9 + .../ProjectDetail/ProjectDetail.test.jsx | 12 +- .../Project/ProjectEdit/ProjectEdit.jsx | 1 + .../screens/Project/shared/ProjectForm.jsx | 30 ++++ .../JobTemplateAdd/JobTemplateAdd.jsx | 5 +- .../JobTemplateAdd/JobTemplateAdd.test.jsx | 17 +- .../JobTemplateDetail/JobTemplateDetail.jsx | 6 + .../JobTemplateDetail.test.jsx | 10 ++ .../JobTemplateEdit/JobTemplateEdit.jsx | 5 +- .../WorkflowJobTemplateAdd.jsx | 5 +- .../WorkflowJobTemplateDetail.jsx | 6 + .../WorkflowJobTemplateDetail.test.jsx | 12 ++ .../WorkflowJobTemplateEdit.jsx | 7 +- .../Template/shared/JobTemplateForm.jsx | 68 ++++++- .../shared/WorkflowJobTemplateForm.jsx | 29 ++- .../Template/shared/data.job_template.json | 9 +- awx/ui_next/src/types.js | 1 + 30 files changed, 558 insertions(+), 24 deletions(-) create mode 100644 awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx create mode 100644 awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.test.jsx diff --git a/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx new file mode 100644 index 0000000000..ecab4f3b44 --- /dev/null +++ b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx @@ -0,0 +1,167 @@ +import React, { useCallback, useEffect } from 'react'; +import { string, func, bool } from 'prop-types'; +import { withRouter, useLocation } from 'react-router-dom'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { FormGroup, Tooltip } from '@patternfly/react-core'; + +import { ExecutionEnvironmentsAPI } from '../../api'; +import { ExecutionEnvironment } from '../../types'; +import { getQSConfig, parseQueryString, mergeParams } from '../../util/qs'; +import Popover from '../Popover'; +import OptionsList from '../OptionsList'; +import useRequest from '../../util/useRequest'; + +import Lookup from './Lookup'; +import LookupErrorMessage from './shared/LookupErrorMessage'; + +const QS_CONFIG = getQSConfig('execution_environments', { + page: 1, + page_size: 5, + order_by: 'name', +}); + +function ExecutionEnvironmentLookup({ + globallyAvailable, + i18n, + isDefaultEnvironment, + isDisabled, + onChange, + organizationId, + popoverContent, + tooltip, + value, + onBlur, +}) { + const location = useLocation(); + + const { + result: { + executionEnvironments, + count, + relatedSearchableKeys, + searchableKeys, + }, + request: fetchExecutionEnvironments, + error, + isLoading, + } = useRequest( + useCallback(async () => { + const params = parseQueryString(QS_CONFIG, location.search); + const globallyAvailableParams = globallyAvailable + ? { or__organization__isnull: 'True' } + : {}; + const organizationIdParams = organizationId + ? { or__organization__id: organizationId } + : {}; + const [{ data }, actionsResponse] = await Promise.all([ + ExecutionEnvironmentsAPI.read( + mergeParams(params, { + ...globallyAvailableParams, + ...organizationIdParams, + }) + ), + ExecutionEnvironmentsAPI.readOptions(), + ]); + return { + executionEnvironments: data.results, + count: data.count, + relatedSearchableKeys: ( + actionsResponse?.data?.related_search_fields || [] + ).map(val => val.slice(0, -8)), + searchableKeys: Object.keys( + actionsResponse.data.actions?.GET || {} + ).filter(key => actionsResponse.data.actions?.GET[key].filterable), + }; + }, [location, globallyAvailable, organizationId]), + { + executionEnvironments: [], + count: 0, + relatedSearchableKeys: [], + searchableKeys: [], + } + ); + + useEffect(() => { + fetchExecutionEnvironments(); + }, [fetchExecutionEnvironments]); + + const renderLookup = () => ( + <> + ( + dispatch({ type: 'SELECT_ITEM', item })} + deselectItem={item => dispatch({ type: 'DESELECT_ITEM', item })} + /> + )} + /> + + ); + + return ( + } + > + {isDisabled ? ( + {renderLookup()} + ) : ( + renderLookup() + )} + + + + ); +} + +ExecutionEnvironmentLookup.propTypes = { + value: ExecutionEnvironment, + popoverContent: string, + onChange: func.isRequired, + isDefaultEnvironment: bool, +}; + +ExecutionEnvironmentLookup.defaultProps = { + popoverContent: '', + isDefaultEnvironment: false, + value: null, +}; + +export default withI18n()(withRouter(ExecutionEnvironmentLookup)); diff --git a/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.test.jsx b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.test.jsx new file mode 100644 index 0000000000..783d43707b --- /dev/null +++ b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.test.jsx @@ -0,0 +1,76 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { mountWithContexts } from '../../../testUtils/enzymeHelpers'; +import ExecutionEnvironmentLookup from './ExecutionEnvironmentLookup'; +import { ExecutionEnvironmentsAPI } from '../../api'; + +jest.mock('../../api'); + +const mockedExecutionEnvironments = { + count: 1, + results: [ + { + id: 2, + name: 'Foo', + image: 'quay.io/ansible/awx-ee', + pull: 'missing', + }, + ], +}; + +const executionEnvironment = { + id: 42, + name: 'Bar', + image: 'quay.io/ansible/bar', + pull: 'missing', +}; + +describe('ExecutionEnvironmentLookup', () => { + let wrapper; + + beforeEach(() => { + ExecutionEnvironmentsAPI.read.mockResolvedValue( + mockedExecutionEnvironments + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + wrapper.unmount(); + }); + + test('should render successfully', async () => { + ExecutionEnvironmentsAPI.readOptions.mockReturnValue({ + data: { + actions: { + GET: {}, + POST: {}, + }, + related_search_fields: [], + }, + }); + await act(async () => { + wrapper = mountWithContexts( + {}} + /> + ); + }); + wrapper.update(); + expect(ExecutionEnvironmentsAPI.read).toHaveBeenCalledTimes(1); + expect(wrapper.find('ExecutionEnvironmentLookup')).toHaveLength(1); + }); + + test('should fetch execution environments', async () => { + await act(async () => { + wrapper = mountWithContexts( + {}} + /> + ); + }); + expect(ExecutionEnvironmentsAPI.read).toHaveBeenCalledTimes(1); + }); +}); diff --git a/awx/ui_next/src/components/Lookup/index.js b/awx/ui_next/src/components/Lookup/index.js index a2fcfbe570..7c8b6845b1 100644 --- a/awx/ui_next/src/components/Lookup/index.js +++ b/awx/ui_next/src/components/Lookup/index.js @@ -7,3 +7,4 @@ export { default as CredentialLookup } from './CredentialLookup'; export { default as ApplicationLookup } from './ApplicationLookup'; export { default as HostFilterLookup } from './HostFilterLookup'; export { default as OrganizationLookup } from './OrganizationLookup'; +export { default as ExecutionEnvironmentLookup } from './ExecutionEnvironmentLookup'; diff --git a/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.jsx b/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.jsx index d9c14765ac..adbe04820d 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.jsx @@ -13,7 +13,10 @@ function OrganizationAdd() { const handleSubmit = async (values, groupsToAssociate) => { try { - const { data: response } = await OrganizationsAPI.create(values); + const { data: response } = await OrganizationsAPI.create({ + ...values, + default_environment: values.default_environment?.id, + }); await Promise.all( groupsToAssociate .map(id => OrganizationsAPI.associateInstanceGroup(response.id, id)) diff --git a/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.test.jsx b/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.test.jsx index 8fa4e2cbc2..d99634ea09 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.test.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationAdd/OrganizationAdd.test.jsx @@ -17,13 +17,18 @@ describe('', () => { description: 'new description', custom_virtualenv: 'Buzz', galaxy_credentials: [], + default_environment: { id: 1, name: 'Foo' }, }; OrganizationsAPI.create.mockResolvedValueOnce({ data: {} }); await act(async () => { const wrapper = mountWithContexts(); wrapper.find('OrganizationForm').prop('onSubmit')(updatedOrgData, []); }); - expect(OrganizationsAPI.create).toHaveBeenCalledWith(updatedOrgData); + expect(OrganizationsAPI.create).toHaveBeenCalledWith({ + ...updatedOrgData, + default_environment: 1, + }); + expect(OrganizationsAPI.create).toHaveBeenCalledTimes(1); }); test('should navigate to organizations list when cancel is clicked', async () => { diff --git a/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx b/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx index 6b55780333..a1abee28a8 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx @@ -94,6 +94,12 @@ function OrganizationDetail({ i18n, organization }) { label={i18n._(t`Ansible Environment`)} value={custom_virtualenv} /> + {summary_fields?.default_environment?.name && ( + + )} ', () => { const mockOrganization = { + id: 12, name: 'Foo', description: 'Bar', custom_virtualenv: 'Fizz', @@ -24,7 +25,14 @@ describe('', () => { edit: true, delete: true, }, + default_environment: { + id: 1, + name: 'Default EE', + description: '', + image: 'quay.io/ansible/awx-ee', + }, }, + default_environment: 1, }; const mockInstanceGroups = { data: { @@ -43,7 +51,7 @@ describe('', () => { jest.clearAllMocks(); }); - test('initially renders succesfully', async () => { + test('initially renders successfully', async () => { await act(async () => { mountWithContexts(); }); @@ -86,6 +94,7 @@ describe('', () => { { label: 'Created', value: '7/7/2015, 5:21:26 PM' }, { label: 'Last Modified', value: '8/11/2019, 7:47:37 PM' }, { label: 'Max Hosts', value: '0' }, + { label: 'Default Execution Environment', value: 'Default EE' }, ]; for (let i = 0; i < testParams.length; i++) { const { label, value } = testParams[i]; diff --git a/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.jsx b/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.jsx index 849a273ef5..3297d2fd6f 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.jsx @@ -28,7 +28,10 @@ function OrganizationEdit({ organization }) { const addedCredentialIds = addedCredentials.map(({ id }) => id); const removedCredentialIds = removedCredentials.map(({ id }) => id); - await OrganizationsAPI.update(organization.id, values); + await OrganizationsAPI.update(organization.id, { + ...values, + default_environment: values.default_environment?.id || null, + }); await Promise.all( groupsToAssociate .map(id => diff --git a/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.test.jsx b/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.test.jsx index ea62e38c9e..5556ee05d5 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.test.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationEdit/OrganizationEdit.test.jsx @@ -19,6 +19,13 @@ describe('', () => { related: { instance_groups: '/api/v2/organizations/1/instance_groups', }, + default_environment: 1, + summary_fields: { + default_environment: { + id: 1, + name: 'Baz', + }, + }, }; test('onSubmit should call api update', async () => { @@ -31,6 +38,7 @@ describe('', () => { name: 'new name', description: 'new description', custom_virtualenv: 'Buzz', + default_environment: null, }; wrapper.find('OrganizationForm').prop('onSubmit')(updatedOrgData, [], []); diff --git a/awx/ui_next/src/screens/Organization/shared/OrganizationForm.jsx b/awx/ui_next/src/screens/Organization/shared/OrganizationForm.jsx index 094e6ac5b6..eb46f8c5cc 100644 --- a/awx/ui_next/src/screens/Organization/shared/OrganizationForm.jsx +++ b/awx/ui_next/src/screens/Organization/shared/OrganizationForm.jsx @@ -12,16 +12,21 @@ import ContentError from '../../../components/ContentError'; import ContentLoading from '../../../components/ContentLoading'; import FormField, { FormSubmitError } from '../../../components/FormField'; import FormActionGroup from '../../../components/FormActionGroup/FormActionGroup'; -import { InstanceGroupsLookup } from '../../../components/Lookup'; +import { + InstanceGroupsLookup, + ExecutionEnvironmentLookup, +} from '../../../components/Lookup'; import { getAddedAndRemoved } from '../../../util/lists'; import { required, minMaxValue } from '../../../util/validators'; import { FormColumnLayout } from '../../../components/FormLayout'; import CredentialLookup from '../../../components/Lookup/CredentialLookup'; function OrganizationFormFields({ i18n, instanceGroups, setInstanceGroups }) { + const { license_info = {}, me = {} } = useConfig(); + const { custom_virtualenvs } = useContext(ConfigContext); + const { setFieldValue } = useFormikContext(); const [venvField] = useField('custom_virtualenv'); - const { license_info = {}, me = {} } = useConfig(); const [ galaxyCredentialsField, @@ -29,12 +34,19 @@ function OrganizationFormFields({ i18n, instanceGroups, setInstanceGroups }) { galaxyCredentialsHelpers, ] = useField('galaxy_credentials'); + const [ + executionEnvironmentField, + executionEnvironmentMeta, + executionEnvironmentHelpers, + ] = useField({ + name: 'default_environment', + }); + const defaultVenv = { label: i18n._(t`Use Default Ansible Environment`), value: '/var/lib/awx/venv/ansible/', key: 'default', }; - const { custom_virtualenvs } = useContext(ConfigContext); const handleCredentialUpdate = useCallback( value => { @@ -100,6 +112,20 @@ function OrganizationFormFields({ i18n, instanceGroups, setInstanceGroups }) { t`Select the Instance Groups for this Organization to run on.` )} /> + executionEnvironmentHelpers.setTouched()} + value={executionEnvironmentField.value} + onChange={value => executionEnvironmentHelpers.setValue(value)} + popoverContent={i18n._( + t`Select the default execution environment for this organization.` + )} + globallyAvailable + isDefaultEnvironment + /> @@ -221,6 +249,7 @@ OrganizationForm.defaultProps = { description: '', max_hosts: '0', custom_virtualenv: '', + default_environment: '', }, submitError: null, }; diff --git a/awx/ui_next/src/screens/Organization/shared/OrganizationForm.test.jsx b/awx/ui_next/src/screens/Organization/shared/OrganizationForm.test.jsx index 67cf0a60d6..7dfbca620c 100644 --- a/awx/ui_next/src/screens/Organization/shared/OrganizationForm.test.jsx +++ b/awx/ui_next/src/screens/Organization/shared/OrganizationForm.test.jsx @@ -4,7 +4,7 @@ import { mountWithContexts, waitForElement, } from '../../../../testUtils/enzymeHelpers'; -import { OrganizationsAPI } from '../../../api'; +import { OrganizationsAPI, ExecutionEnvironmentsAPI } from '../../../api'; import OrganizationForm from './OrganizationForm'; @@ -32,6 +32,8 @@ describe('', () => { { name: 'Two', id: 2 }, ]; + const mockExecutionEnvironment = [{ name: 'EE' }]; + afterEach(() => { jest.clearAllMocks(); }); @@ -132,6 +134,11 @@ describe('', () => { results: mockInstanceGroups, }, }); + ExecutionEnvironmentsAPI.read.mockReturnValue({ + data: { + results: mockExecutionEnvironment, + }, + }); let wrapper; const onSubmit = jest.fn(); await act(async () => { @@ -155,10 +162,15 @@ describe('', () => { wrapper.find('input#org-max_hosts').simulate('change', { target: { value: 134, name: 'max_hosts' }, }); + wrapper.find('ExecutionEnvironmentLookup').invoke('onChange')({ + id: 1, + name: 'Test EE', + }); }); await act(async () => { wrapper.find('button[aria-label="Save"]').simulate('click'); }); + wrapper.update(); expect(onSubmit).toHaveBeenCalledTimes(1); expect(onSubmit.mock.calls[0][0]).toEqual({ name: 'new foo', @@ -166,6 +178,7 @@ describe('', () => { galaxy_credentials: [], custom_virtualenv: 'Fizz', max_hosts: 134, + default_environment: { id: 1, name: 'Test EE' }, }); }); @@ -209,12 +222,16 @@ describe('', () => { results: mockInstanceGroups, }, }); + ExecutionEnvironmentsAPI.read.mockReturnValue({ + data: { results: mockExecutionEnvironment }, + }); const mockDataForm = { name: 'Foo', description: 'Bar', galaxy_credentials: [], max_hosts: 1, custom_virtualenv: 'Fizz', + default_environment: '', }; const onSubmit = jest.fn(); OrganizationsAPI.update.mockResolvedValue(1, mockDataForm); @@ -320,6 +337,7 @@ describe('', () => { galaxy_credentials: [], max_hosts: 0, custom_virtualenv: 'Fizz', + default_environment: '', }, [], [] diff --git a/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.jsx b/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.jsx index d0190830e6..eaaa4274f3 100644 --- a/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.jsx +++ b/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.jsx @@ -27,6 +27,7 @@ function ProjectAdd() { } = await ProjectsAPI.create({ ...values, organization: values.organization.id, + default_environment: values.default_environment?.id, }); history.push(`/projects/${id}/details`); } catch (error) { diff --git a/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx b/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx index 8bc136b889..76bfd49256 100644 --- a/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx +++ b/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx @@ -20,11 +20,12 @@ describe('', () => { scm_clean: true, credential: 100, local_path: '', - organization: 2, + organization: { id: 2, name: 'Bar' }, scm_update_on_launch: true, scm_update_cache_timeout: 3, allow_override: false, custom_virtualenv: '/var/lib/awx/venv/custom-env', + default_environment: { id: 1, name: 'Foo' }, }; const projectOptionsResolve = { @@ -102,6 +103,11 @@ describe('', () => { await waitForElement(wrapper, 'ContentLoading', el => el.length === 0); wrapper.find('ProjectForm').invoke('handleSubmit')(projectData); expect(ProjectsAPI.create).toHaveBeenCalledTimes(1); + expect(ProjectsAPI.create).toHaveBeenCalledWith({ + ...projectData, + organization: 2, + default_environment: 1, + }); }); test('handleSubmit should throw an error', async () => { diff --git a/awx/ui_next/src/screens/Project/ProjectDetail/ProjectDetail.jsx b/awx/ui_next/src/screens/Project/ProjectDetail/ProjectDetail.jsx index 4c92c9695e..431e6e02b3 100644 --- a/awx/ui_next/src/screens/Project/ProjectDetail/ProjectDetail.jsx +++ b/awx/ui_next/src/screens/Project/ProjectDetail/ProjectDetail.jsx @@ -124,10 +124,18 @@ function ProjectDetail({ project, i18n }) { label={i18n._(t`Cache Timeout`)} value={`${scm_update_cache_timeout} ${i18n._(t`Seconds`)}`} /> + + {summary_fields?.default_environment?.name && ( + + )} + {({ project_base_dir }) => ( + ', () => { id: 10, name: 'Foo', }, + default_environment: { + id: 12, + name: 'Bar', + image: 'quay.io/ansible/awx-ee', + }, credential: { id: 1000, name: 'qux', @@ -67,9 +72,10 @@ describe('', () => { scm_update_cache_timeout: 5, allow_override: true, custom_virtualenv: '/custom-venv', + default_environment: 1, }; - test('initially renders succesfully', () => { + test('initially renders successfully', () => { mountWithContexts(); }); @@ -95,6 +101,10 @@ describe('', () => { `${mockProject.scm_update_cache_timeout} Seconds` ); assertDetail('Ansible Environment', mockProject.custom_virtualenv); + assertDetail( + 'Execution Environment', + mockProject.summary_fields.default_environment.name + ); const dateDetails = wrapper.find('UserDateDetail'); expect(dateDetails).toHaveLength(2); expect(dateDetails.at(0).prop('label')).toEqual('Created'); diff --git a/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.jsx b/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.jsx index 6642e03503..3682a01cdc 100644 --- a/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.jsx +++ b/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.jsx @@ -26,6 +26,7 @@ function ProjectEdit({ project }) { } = await ProjectsAPI.update(project.id, { ...values, organization: values.organization.id, + default_environment: values.default_environment?.id || null, }); history.push(`/projects/${id}/details`); } catch (error) { diff --git a/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx b/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx index 8c52218c1e..b2b5b80486 100644 --- a/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx +++ b/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx @@ -12,6 +12,7 @@ import ContentLoading from '../../../components/ContentLoading'; import FormActionGroup from '../../../components/FormActionGroup/FormActionGroup'; import FormField, { FormSubmitError } from '../../../components/FormField'; import OrganizationLookup from '../../../components/Lookup/OrganizationLookup'; +import ExecutionEnvironmentLookup from '../../../components/Lookup/ExecutionEnvironmentLookup'; import { CredentialTypesAPI, ProjectsAPI } from '../../../api'; import { required } from '../../../util/validators'; import { @@ -101,6 +102,14 @@ function ProjectFormFields({ validate: required(i18n._(t`Select a value for this field`), i18n), }); + const [ + executionEnvironmentField, + executionEnvironmentMeta, + executionEnvironmentHelpers, + ] = useField({ + name: 'default_environment', + }); + /* Save current scm subform field values to state */ const saveSubFormState = form => { const currentScmFormFields = { ...scmFormFields }; @@ -178,6 +187,25 @@ function ProjectFormFields({ required autoPopulate={!project?.id} /> + executionEnvironmentHelpers.setTouched()} + value={executionEnvironmentField.value} + onChange={value => executionEnvironmentHelpers.setValue(value)} + popoverContent={i18n._( + t`Select the default execution environment for this project.` + )} + tooltip={i18n._( + t`Select an organization before editing the default execution environment.` + )} + globallyAvailable + isDisabled={!organizationField.value} + organizationId={organizationField.value?.id} + isDefaultEnvironment + /> diff --git a/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.jsx b/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.jsx index 562987ba4b..bbb5e47b84 100644 --- a/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.jsx +++ b/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.jsx @@ -27,7 +27,10 @@ function JobTemplateAdd() { try { const { data: { id, type }, - } = await JobTemplatesAPI.create(remainingValues); + } = await JobTemplatesAPI.create({ + ...remainingValues, + execution_environment: values.execution_environment?.id, + }); await Promise.all([ submitLabels(id, labels, values.project.summary_fields.organization.id), submitInstanceGroups(id, instanceGroups), diff --git a/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.test.jsx b/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.test.jsx index 7b8675cc65..9860d968c5 100644 --- a/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.test.jsx +++ b/awx/ui_next/src/screens/Template/JobTemplateAdd/JobTemplateAdd.test.jsx @@ -58,6 +58,7 @@ const jobTemplateData = { timeout: 0, use_fact_cache: false, verbosity: '0', + execution_environment: { id: 1, name: 'Foo' }, }; describe('', () => { @@ -77,6 +78,12 @@ describe('', () => { beforeEach(() => { LabelsAPI.read.mockResolvedValue({ data: { results: [] } }); + ProjectsAPI.readDetail.mockReturnValue({ + name: 'foo', + id: 1, + allow_override: true, + organization: 1, + }); }); afterEach(() => { @@ -126,12 +133,13 @@ describe('', () => { ...jobTemplateData, }, }); + let wrapper; await act(async () => { wrapper = mountWithContexts(); }); await waitForElement(wrapper, 'EmptyStateBody', el => el.length === 0); - act(() => { + await act(() => { wrapper.find('input#template-name').simulate('change', { target: { value: 'Bar', name: 'name' }, }); @@ -144,6 +152,10 @@ describe('', () => { name: 'project', summary_fields: { organization: { id: 1, name: 'Org Foo' } }, }); + wrapper.find('ExecutionEnvironmentLookup').invoke('onChange')({ + id: 1, + name: 'Foo', + }); wrapper.update(); wrapper.find('Select#template-playbook').prop('onToggle')(); wrapper.update(); @@ -170,6 +182,7 @@ describe('', () => { inventory: 2, webhook_credential: undefined, webhook_service: '', + execution_environment: 1, }); }); @@ -190,7 +203,7 @@ describe('', () => { }); }); await waitForElement(wrapper, 'EmptyStateBody', el => el.length === 0); - act(() => { + await act(async () => { wrapper.find('input#template-name').simulate('change', { target: { value: 'Foo', name: 'name' }, }); diff --git a/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx b/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx index 3feed6bb7d..171891fe37 100644 --- a/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx +++ b/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx @@ -206,6 +206,12 @@ function JobTemplateDetail({ i18n, template }) { ) : ( )} + {summary_fields?.execution_environment && ( + + )} ', () => { el => el.length === 0 ); }); + test('webhook fields should render properly', () => { expect(wrapper.find('Detail[label="Webhook Service"]').length).toBe(1); expect(wrapper.find('Detail[label="Webhook Service"]').prop('value')).toBe( @@ -154,4 +155,13 @@ describe('', () => { expect(wrapper.find('Detail[label="Webhook Key"]').length).toBe(1); expect(wrapper.find('Detail[label="Webhook Credential"]').length).toBe(1); }); + + test('execution environment field should render properly', () => { + expect(wrapper.find('Detail[label="Execution Environment"]').length).toBe( + 1 + ); + expect( + wrapper.find('Detail[label="Execution Environment"]').prop('value') + ).toBe('Default EE'); + }); }); diff --git a/awx/ui_next/src/screens/Template/JobTemplateEdit/JobTemplateEdit.jsx b/awx/ui_next/src/screens/Template/JobTemplateEdit/JobTemplateEdit.jsx index 213900d40d..6f109604b5 100644 --- a/awx/ui_next/src/screens/Template/JobTemplateEdit/JobTemplateEdit.jsx +++ b/awx/ui_next/src/screens/Template/JobTemplateEdit/JobTemplateEdit.jsx @@ -57,7 +57,10 @@ function JobTemplateEdit({ template }) { remainingValues.project = values.project.id; remainingValues.webhook_credential = webhook_credential?.id || null; try { - await JobTemplatesAPI.update(template.id, remainingValues); + await JobTemplatesAPI.update(template.id, { + ...remainingValues, + execution_environment: values.execution_environment?.id, + }); await Promise.all([ submitLabels(labels, template?.organization), submitInstanceGroups(instanceGroups, initialInstanceGroups), diff --git a/awx/ui_next/src/screens/Template/WorkflowJobTemplateAdd/WorkflowJobTemplateAdd.jsx b/awx/ui_next/src/screens/Template/WorkflowJobTemplateAdd/WorkflowJobTemplateAdd.jsx index d9edd77030..3656c73c02 100644 --- a/awx/ui_next/src/screens/Template/WorkflowJobTemplateAdd/WorkflowJobTemplateAdd.jsx +++ b/awx/ui_next/src/screens/Template/WorkflowJobTemplateAdd/WorkflowJobTemplateAdd.jsx @@ -28,7 +28,10 @@ function WorkflowJobTemplateAdd() { try { const { data: { id }, - } = await WorkflowJobTemplatesAPI.create(templatePayload); + } = await WorkflowJobTemplatesAPI.create({ + ...templatePayload, + execution_environment: values.execution_environment?.id, + }); await Promise.all(await submitLabels(id, labels, organizationId)); history.push(`/templates/workflow_job_template/${id}/visualizer`); } catch (err) { diff --git a/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx b/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx index 0ce65ca092..60331770ca 100644 --- a/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx +++ b/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx @@ -125,6 +125,12 @@ function WorkflowJobTemplateDetail({ template, i18n }) { } /> )} + {summary_fields?.execution_environment && ( + + )} {summary_fields.inventory && ( ', () => { created_by: { id: 1, username: 'Athena' }, modified_by: { id: 1, username: 'Apollo' }, organization: { id: 1, name: 'Org' }, + execution_environment: { + id: 4, + name: 'Demo EE', + description: '', + image: 'quay.io/ansible/awx-ee', + }, inventory: { kind: 'Foo', id: 1, name: 'Bar' }, labels: { results: [ @@ -40,6 +46,7 @@ describe('', () => { }, webhook_service: 'Github', webhook_key: 'Foo webhook key', + execution_environment: 4, }; beforeEach(async () => { @@ -127,6 +134,11 @@ describe('', () => { prop: 'value', value: 'Workflow Job Template', }, + { + element: 'Detail[label="Execution Environment"]', + prop: 'value', + value: 'Demo EE', + }, ]; const organization = wrapper diff --git a/awx/ui_next/src/screens/Template/WorkflowJobTemplateEdit/WorkflowJobTemplateEdit.jsx b/awx/ui_next/src/screens/Template/WorkflowJobTemplateEdit/WorkflowJobTemplateEdit.jsx index be61a0ee43..cb963e634f 100644 --- a/awx/ui_next/src/screens/Template/WorkflowJobTemplateEdit/WorkflowJobTemplateEdit.jsx +++ b/awx/ui_next/src/screens/Template/WorkflowJobTemplateEdit/WorkflowJobTemplateEdit.jsx @@ -20,7 +20,7 @@ function WorkflowJobTemplateEdit({ template }) { ...templatePayload } = values; templatePayload.inventory = inventory?.id || null; - templatePayload.organization = organization?.id; + templatePayload.organization = organization?.id || null; templatePayload.webhook_credential = webhook_credential?.id || null; const formOrgId = @@ -29,7 +29,10 @@ function WorkflowJobTemplateEdit({ template }) { await Promise.all( await submitLabels(labels, formOrgId, template.organization) ); - await WorkflowJobTemplatesAPI.update(template.id, templatePayload); + await WorkflowJobTemplatesAPI.update(template.id, { + ...templatePayload, + execution_environment: values.execution_environment?.id, + }); history.push(`/templates/workflow_job_template/${template.id}/details`); } catch (err) { setFormSubmitError(err); diff --git a/awx/ui_next/src/screens/Template/shared/JobTemplateForm.jsx b/awx/ui_next/src/screens/Template/shared/JobTemplateForm.jsx index f504224507..c267e157bb 100644 --- a/awx/ui_next/src/screens/Template/shared/JobTemplateForm.jsx +++ b/awx/ui_next/src/screens/Template/shared/JobTemplateForm.jsx @@ -37,9 +37,10 @@ import { InstanceGroupsLookup, ProjectLookup, MultiCredentialsLookup, + ExecutionEnvironmentLookup, } from '../../../components/Lookup'; import Popover from '../../../components/Popover'; -import { JobTemplatesAPI } from '../../../api'; +import { JobTemplatesAPI, ProjectsAPI } from '../../../api'; import LabelSelect from './LabelSelect'; import PlaybookSelect from './PlaybookSelect'; import WebhookSubForm from './WebhookSubForm'; @@ -101,10 +102,40 @@ function JobTemplateForm({ 'webhook_credential' ); + const [ + executionEnvironmentField, + executionEnvironmentMeta, + executionEnvironmentHelpers, + ] = useField({ name: 'execution_environment' }); + + const projectId = projectField.value?.id; + + const { + request: fetchProject, + error: fetchProjectError, + isLoading: fetchProjectLoading, + result: projectData, + } = useRequest( + useCallback(async () => { + if (!projectId) { + return {}; + } + const { data } = await ProjectsAPI.readDetail(projectId); + return data; + }, [projectId]), + { + projectData: null, + } + ); + + useEffect(() => { + fetchProject(); + }, [fetchProject]); + const { request: loadRelatedInstanceGroups, error: instanceGroupError, - contentLoading: instanceGroupLoading, + isLoading: instanceGroupLoading, } = useRequest( useCallback(async () => { if (!template?.id) { @@ -182,12 +213,16 @@ function JobTemplateForm({ callbackUrl = `${origin}${path}`; } - if (instanceGroupLoading) { + if (instanceGroupLoading || fetchProjectLoading) { return ; } - if (contentError || instanceGroupError) { - return ; + if (contentError || instanceGroupError || fetchProjectError) { + return ( + + ); } return ( @@ -258,6 +293,7 @@ function JobTemplateForm({ isOverrideDisabled={isOverrideDisabledLookup} /> + projectHelpers.setTouched()} @@ -270,6 +306,26 @@ function JobTemplateForm({ autoPopulate={!template?.id} isOverrideDisabled={isOverrideDisabledLookup} /> + + executionEnvironmentHelpers.setTouched()} + value={executionEnvironmentField.value} + onChange={value => executionEnvironmentHelpers.setValue(value)} + popoverContent={i18n._( + t`Select the execution environment for this job template.` + )} + tooltip={i18n._( + t`Select a project before editing the execution environment.` + )} + globallyAvailable + isDisabled={!projectField.value} + organizationId={projectData?.organization} + /> + {projectField.value?.allow_override && ( { diff --git a/awx/ui_next/src/screens/Template/shared/WorkflowJobTemplateForm.jsx b/awx/ui_next/src/screens/Template/shared/WorkflowJobTemplateForm.jsx index fd15c92e28..5ee34bbe12 100644 --- a/awx/ui_next/src/screens/Template/shared/WorkflowJobTemplateForm.jsx +++ b/awx/ui_next/src/screens/Template/shared/WorkflowJobTemplateForm.jsx @@ -22,7 +22,10 @@ import { SubFormLayout, } from '../../../components/FormLayout'; import OrganizationLookup from '../../../components/Lookup/OrganizationLookup'; -import { InventoryLookup } from '../../../components/Lookup'; +import { + InventoryLookup, + ExecutionEnvironmentLookup, +} from '../../../components/Lookup'; import { VariablesField } from '../../../components/CodeMirrorInput'; import FormActionGroup from '../../../components/FormActionGroup'; import ContentError from '../../../components/ContentError'; @@ -63,6 +66,14 @@ function WorkflowJobTemplateForm({ 'webhook_credential' ); + const [ + executionEnvironmentField, + executionEnvironmentMeta, + executionEnvironmentHelpers, + ] = useField({ + name: 'execution_environment', + }); + useEffect(() => { if (enableWebhooks) { webhookServiceHelpers.setValue(webhookServiceMeta.initialValue); @@ -178,6 +189,20 @@ function WorkflowJobTemplateForm({ }} /> + executionEnvironmentHelpers.setTouched()} + value={executionEnvironmentField.value} + onChange={value => executionEnvironmentHelpers.setValue(value)} + tooltip={i18n._( + t`Select the default execution environment for this organization to run on.` + )} + globallyAvailable + organizationId={organizationField.value?.id} + /> { diff --git a/awx/ui_next/src/screens/Template/shared/data.job_template.json b/awx/ui_next/src/screens/Template/shared/data.job_template.json index 804c3b72a2..fa516d46db 100644 --- a/awx/ui_next/src/screens/Template/shared/data.job_template.json +++ b/awx/ui_next/src/screens/Template/shared/data.job_template.json @@ -133,6 +133,12 @@ "id": "1", "name": "Webhook Credential" + }, + "execution_environment": { + "id": 1, + "name": "Default EE", + "description": "", + "image": "quay.io/ansible/awx-ee" } }, "created": "2019-09-30T16:18:34.564820Z", @@ -177,5 +183,6 @@ "job_slice_count": 1, "webhook_credential": 1, "webhook_key": "asertdyuhjkhgfd234567kjgfds", - "webhook_service": "github" + "webhook_service": "github", + "execution_environment": 1 } diff --git a/awx/ui_next/src/types.js b/awx/ui_next/src/types.js index 53c7ed3b3d..744360eaea 100644 --- a/awx/ui_next/src/types.js +++ b/awx/ui_next/src/types.js @@ -416,4 +416,5 @@ export const ExecutionEnvironment = shape({ url: string, summary_fields: shape({}), description: string, + pull: string, }); From 065b943870a4d8574c21432f774a71aa29652221 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 1 Mar 2021 20:20:25 -0500 Subject: [PATCH 147/157] Fix mode for k8s launch scripts --- tools/ansible/roles/dockerfile/files/launch_awx.sh | 0 tools/ansible/roles/dockerfile/files/launch_awx_task.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/ansible/roles/dockerfile/files/launch_awx.sh mode change 100644 => 100755 tools/ansible/roles/dockerfile/files/launch_awx_task.sh diff --git a/tools/ansible/roles/dockerfile/files/launch_awx.sh b/tools/ansible/roles/dockerfile/files/launch_awx.sh old mode 100644 new mode 100755 diff --git a/tools/ansible/roles/dockerfile/files/launch_awx_task.sh b/tools/ansible/roles/dockerfile/files/launch_awx_task.sh old mode 100644 new mode 100755 From 03c5cc779bac10cf250e766ce3a5865c6c14507a Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 1 Mar 2021 20:20:54 -0500 Subject: [PATCH 148/157] Only install podman in local dev env --- .../ansible/roles/dockerfile/templates/Dockerfile.j2 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index e17a0b04d8..3f7666a615 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -148,7 +148,6 @@ RUN dnf -y install \ nss \ make \ patch \ - podman \ socat \ tmux \ wget \ @@ -162,6 +161,11 @@ RUN dnf -y install \ RUN dnf --enablerepo=debuginfo -y install python3-debuginfo || : {% endif %} +{% if build_dev|bool %} +RUN dnf install -y podman +RUN echo -e 'cgroup_manager = "cgroupfs"\nevents_logger = "file"' > /etc/containers/libpod.conf +{% endif %} + # Copy app from builder COPY --from=builder /var/lib/awx /var/lib/awx @@ -245,8 +249,6 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log && \ ln -sf /dev/stderr /var/log/nginx/error.log {% endif %} -RUN echo -e 'cgroup_manager = "cgroupfs"\nevents_logger = "file"' > /etc/containers/libpod.conf - ENV HOME="/var/lib/awx" ENV PATH="/usr/pgsql-10/bin:${PATH}" @@ -264,6 +266,5 @@ EXPOSE 8052 ENTRYPOINT ["/usr/bin/tini", "--"] CMD /usr/bin/launch_awx.sh VOLUME /var/lib/nginx -{% endif %} - VOLUME /var/lib/awx/.local/share/containers/storage +{% endif %} From eba12a62072eeac1fa5caf3254978efc032c8232 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 2 Mar 2021 16:16:20 -0500 Subject: [PATCH 149/157] Add ui_next to .dockerignore --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 46c83b0467..07c13d382d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,3 @@ awx/ui/node_modules +awx/ui_next/node_modules +Dockerfile From e23a2b4506f788e7318e3f2a77192de888cb5f4e Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 2 Mar 2021 16:16:40 -0500 Subject: [PATCH 150/157] Use var instead of set_fact --- tools/ansible/roles/image_build/defaults/main.yml | 2 +- tools/ansible/roles/image_build/tasks/main.yml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/ansible/roles/image_build/defaults/main.yml b/tools/ansible/roles/image_build/defaults/main.yml index 0d45e047d8..076a4f47b3 100644 --- a/tools/ansible/roles/image_build/defaults/main.yml +++ b/tools/ansible/roles/image_build/defaults/main.yml @@ -1,5 +1,5 @@ --- -create_preload_data: true +awx_image: quay.io/ansible/awx # Helper vars to construct the proper download URL for the current architecture tini_architecture: '{{ { "x86_64": "amd64", "aarch64": "arm64", "armv7": "arm" }[ansible_facts.architecture] }}' diff --git a/tools/ansible/roles/image_build/tasks/main.yml b/tools/ansible/roles/image_build/tasks/main.yml index ae6e30b7a3..13c5c48a3d 100644 --- a/tools/ansible/roles/image_build/tasks/main.yml +++ b/tools/ansible/roles/image_build/tasks/main.yml @@ -17,10 +17,6 @@ dest: "../awx/ui_next/public/static/media/" when: awx_official|default(false)|bool -- name: Set awx image name - set_fact: - awx_image: "{{ awx_image|default('awx') }}" - # Calling Docker directly because docker-py doesnt support BuildKit - name: Build AWX image command: docker build -t {{ awx_image }}:{{ awx_version }} -f ../../{{ dockerfile_name }} ../.. From 609b17aa209ca368f6c15ac8ca46d63b16d175e6 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 2 Mar 2021 16:31:04 -0500 Subject: [PATCH 151/157] Use receptor 0.9.6 --- tools/ansible/roles/dockerfile/templates/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 3f7666a615..66a0eeebbd 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -172,7 +172,7 @@ COPY --from=builder /var/lib/awx /var/lib/awx RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage {%if build_dev|bool %} -COPY --from=quay.io/project-receptor/receptor:0.9.5 /usr/bin/receptor /usr/bin/receptor +COPY --from=quay.io/project-receptor/receptor:0.9.6 /usr/bin/receptor /usr/bin/receptor RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr \ -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \ openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \ From 33d7342ffec4fe967f14d06711012722fe4062b2 Mon Sep 17 00:00:00 2001 From: nixocio Date: Tue, 2 Mar 2021 13:04:30 -0500 Subject: [PATCH 152/157] Linkify reference to EE on details page Linkify reference to EE on a few details page. See: https://github.com/ansible/awx/issues/9189 --- .../OrganizationDetail/OrganizationDetail.jsx | 8 +++++++- .../src/screens/Project/ProjectDetail/ProjectDetail.jsx | 9 +++++++-- .../Template/JobTemplateDetail/JobTemplateDetail.jsx | 8 +++++++- .../JobTemplateDetail/JobTemplateDetail.test.jsx | 2 +- .../WorkflowJobTemplateDetail.jsx | 8 +++++++- .../WorkflowJobTemplateDetail.test.jsx | 9 ++++----- 6 files changed, 33 insertions(+), 11 deletions(-) diff --git a/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx b/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx index a1abee28a8..e3b544c091 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationDetail/OrganizationDetail.jsx @@ -97,7 +97,13 @@ function OrganizationDetail({ i18n, organization }) { {summary_fields?.default_environment?.name && ( + {summary_fields.default_environment.name} + + } /> )} + {summary_fields.default_environment.name} + + } /> )} - {({ project_base_dir }) => ( + {summary_fields.execution_environment.name} + + } /> )} ', () => { 1 ); expect( - wrapper.find('Detail[label="Execution Environment"]').prop('value') + wrapper.find(`Detail[label="Execution Environment"] dd`).text() ).toBe('Default EE'); }); }); diff --git a/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx b/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx index 60331770ca..4f8237e8c2 100644 --- a/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx +++ b/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.jsx @@ -128,7 +128,13 @@ function WorkflowJobTemplateDetail({ template, i18n }) { {summary_fields?.execution_environment && ( + {summary_fields.execution_environment.name} + + } /> )} diff --git a/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.test.jsx b/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.test.jsx index 17a6593fda..0c1be1e8e9 100644 --- a/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.test.jsx +++ b/awx/ui_next/src/screens/Template/WorkflowJobTemplateDetail/WorkflowJobTemplateDetail.test.jsx @@ -134,11 +134,6 @@ describe('', () => { prop: 'value', value: 'Workflow Job Template', }, - { - element: 'Detail[label="Execution Environment"]', - prop: 'value', - value: 'Demo EE', - }, ]; const organization = wrapper @@ -162,6 +157,10 @@ describe('', () => { }; renderedValues.map(value => assertValue(value)); + + expect( + wrapper.find(`Detail[label="Execution Environment"] dd`).text() + ).toBe('Demo EE'); }); test('link out resource have the correct url', () => { From 29ff69a7749d792d586a8f8e85349a657e9fbcc1 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 2 Mar 2021 19:32:15 -0500 Subject: [PATCH 153/157] For container group pods, use namespace Tower is deployed into by default --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 845bbe74d8..ba8b8c69cf 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -72,7 +72,7 @@ AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee' AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 -AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = 'default' +AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = os.getenv('MY_POD_NAMESPACE', 'default') # TODO: remove this setting in favor of a default execution environment AWX_CONTAINER_GROUP_DEFAULT_IMAGE = AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE From aab58f5ae7c7ae369dae7434a338d3534a0378bd Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 2 Mar 2021 19:11:56 -0500 Subject: [PATCH 154/157] Dont require credential for Container Groups Might want to follow up and make this only apply to Tower on Kubernetes --- .../InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.jsx | 2 +- .../src/screens/InstanceGroup/shared/ContainerGroupForm.jsx | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.jsx b/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.jsx index 3bf66a9d4d..9f4454c0a8 100644 --- a/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/ContainerGroupEdit/ContainerGroupEdit.jsx @@ -37,7 +37,7 @@ function ContainerGroupEdit({ instanceGroup }) { try { await InstanceGroupsAPI.update(instanceGroup.id, { name: values.name, - credential: values.credential.id, + credential: values.credential ? values.credential.id : null, pod_spec_override: values.override ? values.pod_spec_override : null, }); history.push(detailsIUrl); diff --git a/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.jsx b/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.jsx index fda18b73bc..e41ab0a1b0 100644 --- a/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.jsx +++ b/awx/ui_next/src/screens/InstanceGroup/shared/ContainerGroupForm.jsx @@ -25,7 +25,6 @@ function ContainerGroupFormFields({ i18n, instanceGroup }) { const { setFieldValue } = useFormikContext(); const [credentialField, credentialMeta, credentialHelpers] = useField({ name: 'credential', - validate: required(i18n._(t`Select a value for this field`), i18n), }); const [overrideField] = useField('override'); @@ -55,9 +54,8 @@ function ContainerGroupFormFields({ i18n, instanceGroup }) { onBlur={() => credentialHelpers.setTouched()} onChange={onCredentialChange} value={credentialField.value} - required tooltip={i18n._( - t`Credential to authenticate with Kubernetes or OpenShift. Must be of type "Kubernetes/OpenShift API Bearer Token”.` + t`Credential to authenticate with Kubernetes or OpenShift. Must be of type "Kubernetes/OpenShift API Bearer Token". If left blank, the underlying Pod's service account will be used.` )} autoPopulate={!instanceGroup?.id} /> From fd21603c0ecdab6bb95d537a0845ec04a72f6c17 Mon Sep 17 00:00:00 2001 From: nixocio Date: Mon, 1 Mar 2021 11:10:31 -0500 Subject: [PATCH 155/157] Update EE breadcrumb to use name instead of image Update EE breadcrumb to use name instead of image to be consistent with the other screens. See: https://github.com/ansible/awx/issues/9087 --- .../src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx index d95c15959f..802e78a679 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironments.jsx @@ -22,7 +22,7 @@ function ExecutionEnvironments({ i18n }) { setBreadcrumbConfig({ '/execution_environments': i18n._(t`Execution environments`), '/execution_environments/add': i18n._(t`Create Execution environments`), - [`/execution_environments/${executionEnvironments.id}`]: `${executionEnvironments.image}`, + [`/execution_environments/${executionEnvironments.id}`]: `${executionEnvironments.name}`, [`/execution_environments/${executionEnvironments.id}/edit`]: i18n._( t`Edit details` ), From 62215ca432f00a863a1a2f61ed85decd0aedbd7e Mon Sep 17 00:00:00 2001 From: nixocio Date: Mon, 1 Mar 2021 10:49:51 -0500 Subject: [PATCH 156/157] Add organization to EE details page Add organization to EE details page. See: https://github.com/ansible/awx/issues/9432 --- .../ExecutionEnvironmentDetails.jsx | 25 +++++++++++- .../ExecutionEnvironmentDetails.test.jsx | 39 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index 13839bb95b..68b9f9879d 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -18,7 +18,15 @@ import { ExecutionEnvironmentsAPI } from '../../../api'; function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { const history = useHistory(); - const { id, name, image, description, pull } = executionEnvironment; + const { + id, + name, + image, + description, + pull, + organization, + summary_fields, + } = executionEnvironment; const { request: deleteExecutionEnvironment, @@ -51,6 +59,21 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { value={description} dataCy="execution-environment-detail-description" /> + + {summary_fields.organization.name} + + ) : ( + i18n._(t`Globally Available`) + ) + } + dataCy="execution-environment-detail-organization" + /> ', () => { expect(wrapper.find('Detail[label="Description"]').prop('value')).toEqual( 'Foo' ); + expect(wrapper.find('Detail[label="Organization"]').prop('value')).toEqual( + 'Globally Available' + ); + expect( + wrapper.find('Detail[label="Credential"]').prop('value').props.children + ).toEqual(executionEnvironment.summary_fields.credential.name); + const dates = wrapper.find('UserDateDetail'); + expect(dates).toHaveLength(2); + expect(dates.at(0).prop('date')).toEqual(executionEnvironment.created); + expect(dates.at(1).prop('date')).toEqual(executionEnvironment.modified); + }); + + test('should render organization detail', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + wrapper.update(); + + expect(wrapper.find('Detail[label="Image"]').prop('value')).toEqual( + executionEnvironment.image + ); + expect(wrapper.find('Detail[label="Description"]').prop('value')).toEqual( + 'Foo' + ); + expect(wrapper.find(`Detail[label="Organization"] dd`).text()).toBe('Bar'); expect( wrapper.find('Detail[label="Credential"]').prop('value').props.children ).toEqual(executionEnvironment.summary_fields.credential.name); From 0a4a1bed0aebd2745fac9291c678483242e3b9f3 Mon Sep 17 00:00:00 2001 From: nixocio Date: Tue, 2 Mar 2021 09:22:09 -0500 Subject: [PATCH 157/157] Add EE to inventory sources Add EE to inventory sources See: https://github.com/ansible/awx/issues/9189 --- .../Lookup/ExecutionEnvironmentLookup.jsx | 4 +- .../InventorySourceAdd/InventorySourceAdd.jsx | 9 ++-- .../InventorySourceAdd.test.jsx | 42 ++++++++++++++----- .../InventorySourceDetail.jsx | 13 ++++++ .../InventorySourceEdit.jsx | 9 ++-- .../InventorySourceEdit.test.jsx | 20 ++++++--- .../InventorySources/InventorySources.jsx | 2 +- .../Inventory/shared/InventorySourceForm.jsx | 30 ++++++++++++- .../shared/InventorySourceForm.test.jsx | 1 + 9 files changed, 104 insertions(+), 26 deletions(-) diff --git a/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx index ecab4f3b44..86f659c430 100644 --- a/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx +++ b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect } from 'react'; import { string, func, bool } from 'prop-types'; -import { withRouter, useLocation } from 'react-router-dom'; +import { useLocation } from 'react-router-dom'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import { FormGroup, Tooltip } from '@patternfly/react-core'; @@ -164,4 +164,4 @@ ExecutionEnvironmentLookup.defaultProps = { value: null, }; -export default withI18n()(withRouter(ExecutionEnvironmentLookup)); +export default withI18n()(ExecutionEnvironmentLookup); diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.jsx index 0db70f2dbf..67ea90f4ea 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.jsx @@ -1,14 +1,14 @@ import React, { useCallback, useEffect } from 'react'; -import { useHistory, useParams } from 'react-router-dom'; +import { useHistory } from 'react-router-dom'; import { Card } from '@patternfly/react-core'; import { InventorySourcesAPI } from '../../../api'; import useRequest from '../../../util/useRequest'; import { CardBody } from '../../../components/Card'; import InventorySourceForm from '../shared/InventorySourceForm'; -function InventorySourceAdd() { +function InventorySourceAdd({ inventory }) { const history = useHistory(); - const { id } = useParams(); + const { id, organization } = inventory; const { error, request, result } = useRequest( useCallback(async values => { @@ -31,6 +31,7 @@ function InventorySourceAdd() { source_path, source_project, source_script, + execution_environment, ...remainingForm } = form; @@ -46,6 +47,7 @@ function InventorySourceAdd() { credential: credential?.id || null, inventory: id, source_script: source_script?.id || null, + execution_environment: execution_environment?.id || null, ...sourcePath, ...sourceProject, ...remainingForm, @@ -63,6 +65,7 @@ function InventorySourceAdd() { onCancel={handleCancel} onSubmit={handleSubmit} submitError={error} + organizationId={organization} /> diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.test.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.test.jsx index afc4d69d0e..c186d4dcb7 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.test.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceAdd/InventorySourceAdd.test.jsx @@ -35,6 +35,12 @@ describe('', () => { verbosity: 1, }; + const mockInventory = { + id: 111, + name: 'Foo', + organization: 2, + }; + InventorySourcesAPI.readOptions.mockResolvedValue({ data: { actions: { @@ -72,9 +78,12 @@ describe('', () => { custom_virtualenvs: ['venv/foo', 'venv/bar'], }; await act(async () => { - wrapper = mountWithContexts(, { - context: { config }, - }); + wrapper = mountWithContexts( + , + { + context: { config }, + } + ); }); await waitForElement(wrapper, 'ContentLoading', el => el.length === 0); expect(wrapper.find('FormGroup[label="Name"]')).toHaveLength(1); @@ -88,9 +97,12 @@ describe('', () => { test('should navigate to inventory sources list when cancel is clicked', async () => { const history = createMemoryHistory({}); await act(async () => { - wrapper = mountWithContexts(, { - context: { router: { history } }, - }); + wrapper = mountWithContexts( + , + { + context: { router: { history } }, + } + ); }); await act(async () => { wrapper.find('InventorySourceForm').invoke('onCancel')(); @@ -103,7 +115,9 @@ describe('', () => { test('should post to the api when submit is clicked', async () => { InventorySourcesAPI.create.mockResolvedValueOnce({ data: {} }); await act(async () => { - wrapper = mountWithContexts(); + wrapper = mountWithContexts( + + ); }); await act(async () => { wrapper.find('InventorySourceForm').invoke('onSubmit')(invSourceData); @@ -114,6 +128,7 @@ describe('', () => { credential: 222, source_project: 999, source_script: null, + execution_environment: null, }); }); @@ -123,9 +138,12 @@ describe('', () => { data: { id: 123, inventory: 111 }, }); await act(async () => { - wrapper = mountWithContexts(, { - context: { router: { history } }, - }); + wrapper = mountWithContexts( + , + { + context: { router: { history } }, + } + ); }); await act(async () => { wrapper.find('InventorySourceForm').invoke('onSubmit')(invSourceData); @@ -143,7 +161,9 @@ describe('', () => { }; InventorySourcesAPI.create.mockImplementation(() => Promise.reject(error)); await act(async () => { - wrapper = mountWithContexts(); + wrapper = mountWithContexts( + + ); }); expect(wrapper.find('FormSubmitError').length).toBe(0); await act(async () => { diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx index f3fcdebfca..4383998994 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx @@ -50,6 +50,7 @@ function InventorySourceDetail({ inventorySource, i18n }) { organization, source_project, user_capabilities, + execution_environment, }, } = inventorySource; const [deletionError, setDeletionError] = useState(false); @@ -214,6 +215,18 @@ function InventorySourceDetail({ inventorySource, i18n }) { } /> )} + {execution_environment?.name && ( + + {execution_environment.name} + + } + /> + )} {source === 'scm' ? ( diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceEdit/InventorySourceEdit.test.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceEdit/InventorySourceEdit.test.jsx index 87ec0288c1..cc8ad163b2 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceEdit/InventorySourceEdit.test.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceEdit/InventorySourceEdit.test.jsx @@ -18,7 +18,7 @@ jest.mock('react-router-dom', () => ({ }), })); -describe('', () => { +describe('', () => { let wrapper; let history; const mockInvSrc = { @@ -37,6 +37,11 @@ describe('', () => { update_on_project_update: false, verbosity: 1, }; + const mockInventory = { + id: 1, + name: 'Foo', + organization: 1, + }; InventorySourcesAPI.readOptions.mockResolvedValue({ data: { actions: { @@ -89,9 +94,12 @@ describe('', () => { beforeAll(async () => { history = createMemoryHistory(); await act(async () => { - wrapper = mountWithContexts(, { - context: { router: { history } }, - }); + wrapper = mountWithContexts( + , + { + context: { router: { history } }, + } + ); }); await waitForElement(wrapper, 'ContentLoading', el => el.length === 0); }); @@ -133,7 +141,9 @@ describe('', () => { }; InventorySourcesAPI.replace.mockImplementation(() => Promise.reject(error)); await act(async () => { - wrapper = mountWithContexts(); + wrapper = mountWithContexts( + + ); }); expect(wrapper.find('FormSubmitError').length).toBe(0); await act(async () => { diff --git a/awx/ui_next/src/screens/Inventory/InventorySources/InventorySources.jsx b/awx/ui_next/src/screens/Inventory/InventorySources/InventorySources.jsx index 2e8f1a3785..e55125187d 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySources/InventorySources.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySources/InventorySources.jsx @@ -9,7 +9,7 @@ function InventorySources({ inventory, setBreadcrumb }) { return ( - + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx index 3bc824fca8..05bbd4e370 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx @@ -31,6 +31,7 @@ import { VMwareSubForm, VirtualizationSubForm, } from './InventorySourceSubForms'; +import { ExecutionEnvironmentLookup } from '../../../components/Lookup'; const buildSourceChoiceOptions = options => { const sourceChoices = options.actions.GET.source.choices.map( @@ -39,7 +40,12 @@ const buildSourceChoiceOptions = options => { return sourceChoices.filter(({ key }) => key !== 'file'); }; -const InventorySourceFormFields = ({ source, sourceOptions, i18n }) => { +const InventorySourceFormFields = ({ + source, + sourceOptions, + organizationId, + i18n, +}) => { const { values, initialValues, @@ -51,6 +57,13 @@ const InventorySourceFormFields = ({ source, sourceOptions, i18n }) => { name: 'source', validate: required(i18n._(t`Set a value for this field`), i18n), }); + const [ + executionEnvironmentField, + executionEnvironmentMeta, + executionEnvironmentHelpers, + ] = useField({ + name: 'execution_environment', + }); const { custom_virtualenvs } = useContext(ConfigContext); const [venvField] = useField('custom_virtualenv'); const defaultVenv = { @@ -111,6 +124,17 @@ const InventorySourceFormFields = ({ source, sourceOptions, i18n }) => { name="description" type="text" /> + executionEnvironmentHelpers.setTouched()} + value={executionEnvironmentField.value} + onChange={value => executionEnvironmentHelpers.setValue(value)} + globallyAvailable + organizationId={organizationId} + /> { const initialValues = { credential: source?.summary_fields?.credential || null, @@ -264,6 +289,8 @@ const InventorySourceForm = ({ enabled_var: source?.enabled_var || '', enabled_value: source?.enabled_value || '', host_filter: source?.host_filter || '', + execution_environment: + source?.summary_fields?.execution_environment || null, }; const { @@ -306,6 +333,7 @@ const InventorySourceForm = ({ i18n={i18n} source={source} sourceOptions={sourceOptions} + organizationId={organizationId} /> {submitError && } ', () => { expect( wrapper.find('FormGroup[label="Ansible Environment"]') ).toHaveLength(1); + expect(wrapper.find('ExecutionEnvironmentLookup')).toHaveLength(1); }); test('should display subform when source dropdown has a value', async () => {