From 43450c26943b5fd9a8a961db80385b872cc507dc Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 9 Nov 2016 12:15:56 -0500 Subject: [PATCH 01/10] WorkflowJob should set the workflow launch_type for spawned jobs --- awx/main/models/unified_jobs.py | 1 + awx/main/models/workflow.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 2c46157007..520c8fe4de 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -394,6 +394,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique ('callback', _('Callback')), # Job was started via host callback. ('scheduled', _('Scheduled')), # Job was started from a schedule. ('dependency', _('Dependency')), # Job was started as a dependency of another job. + ('workflow', _('Workflow')), # Job was started from a workflow job. ] PASSWORD_FIELDS = ('start_args',) diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 6d99be326f..426e2ac6b5 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -256,6 +256,8 @@ class WorkflowJobNode(WorkflowNodeBase): extra_vars.update(functional_aa_dict) if extra_vars: data['extra_vars'] = extra_vars + # ensure that unified jobs created by WorkflowJobs are marked + data['launch_type'] = 'workflow' return data class WorkflowJobOptions(BaseModel): From 42f1a491c16f6c8737e5ea1ebdb8f93248db0e8e Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 9 Nov 2016 12:26:21 -0500 Subject: [PATCH 02/10] add WorkflowJob properties to UnifiedJob --- awx/main/models/unified_jobs.py | 10 ++++++++++ awx/main/models/workflow.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 520c8fe4de..b75b100d22 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -758,6 +758,16 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=False): return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive, escape_ascii=True) + @property + def spawned_by_workflow(self): + return self.launch_type == 'workflow' + + @property + def workflow_job_id(self): + if self.spawned_by_workflow(): + return self.unified_job_node.workflow_job.pk + return None + @property def celery_task(self): try: diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 426e2ac6b5..f51394afa3 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -185,9 +185,9 @@ class WorkflowJobTemplateNode(WorkflowNodeBase): return WorkflowJobNode.objects.create(**create_kwargs) class WorkflowJobNode(WorkflowNodeBase): - job = models.ForeignKey( + job = models.OneToOneField( 'UnifiedJob', - related_name='unified_job_nodes', + related_name='unified_job_node', blank=True, null=True, default=None, From 5bf00fe96bcf4062a2a0d36745a79be58083289e Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Mon, 14 Nov 2016 14:28:56 -0500 Subject: [PATCH 03/10] Output WorkflowEvents on a dedicated channel --- awx/api/serializers.py | 2 ++ awx/main/models/unified_jobs.py | 2 +- awx/main/signals.py | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 986f5c8c1e..3012c722c6 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2452,6 +2452,8 @@ class JobEventSerializer(BaseSerializer): try: d['job']['job_template_id'] = obj.job.job_template.id d['job']['job_template_name'] = obj.job.job_template.name + d['job']['spawned_by_workflow'] = obj.job.spawned_by_workflow + d['job']['workflow_job_id'] = obj.job.workflow_job_id except (KeyError, AttributeError): pass return d diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index b75b100d22..c373db5adc 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -764,7 +764,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique @property def workflow_job_id(self): - if self.spawned_by_workflow(): + if self.spawned_by_workflow: return self.unified_job_node.workflow_job.pk return None diff --git a/awx/main/signals.py b/awx/main/signals.py index 7a06934d27..cb0b51273c 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -35,7 +35,6 @@ logger = logging.getLogger('awx.main.signals') def emit_job_event_detail(sender, **kwargs): instance = kwargs['instance'] created = kwargs['created'] - print("before created job_event_detail") if created: event_serialized = JobEventSerializer(instance).data event_serialized['id'] = instance.id @@ -45,6 +44,10 @@ def emit_job_event_detail(sender, **kwargs): event_serialized["group_name"] = "job_events" emit_channel_notification('job_events-' + str(instance.job.id), event_serialized) + if instance.job.spawned_by_workflow: + event_serialized['group_name'] = "workflow_events" + emit_channel_notification('workflow_events-' + str(instance.job.workflow_job_id), event_serialized) + def emit_ad_hoc_command_event_detail(sender, **kwargs): instance = kwargs['instance'] created = kwargs['created'] From 324d2dca72dd6744454634840384c5c737c7af34 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Mon, 14 Nov 2016 14:29:52 -0500 Subject: [PATCH 04/10] add JobEvent and UnifiedJob workflow property unit tests --- .../serializers/test_job_event_serializers.py | 28 +++++++++++++++++++ .../tests/unit/models/test_unified_job.py | 16 +++++++++++ 2 files changed, 44 insertions(+) create mode 100644 awx/main/tests/unit/api/serializers/test_job_event_serializers.py create mode 100644 awx/main/tests/unit/models/test_unified_job.py diff --git a/awx/main/tests/unit/api/serializers/test_job_event_serializers.py b/awx/main/tests/unit/api/serializers/test_job_event_serializers.py new file mode 100644 index 0000000000..a9c7ee7f08 --- /dev/null +++ b/awx/main/tests/unit/api/serializers/test_job_event_serializers.py @@ -0,0 +1,28 @@ +import pytest +import mock + +from awx.api.serializers import JobEventSerializer +from awx.main.models import ( + Job, + JobTemplate, + JobEvent, +) + + +@pytest.fixture +def job_event(mocker): + job_event = mocker.MagicMock(spec=JobEvent) + + job = Job(id=1, name="job-1") + job.job_template = JobTemplate(id=1, name="job-template-1") + job_event.job = job + + return job_event + + +def test_summary_field_workflow_exists(job_event): + with mock.patch('awx.api.serializers.BaseSerializer.get_summary_fields', lambda x,y: {'job':{}}): + serializer = JobEventSerializer(job_event) + summary_fields = serializer.get_summary_fields(job_event) + assert 'spawned_by_workflow' in summary_fields['job'] + assert 'workflow_job_id' in summary_fields['job'] diff --git a/awx/main/tests/unit/models/test_unified_job.py b/awx/main/tests/unit/models/test_unified_job.py new file mode 100644 index 0000000000..af8833482a --- /dev/null +++ b/awx/main/tests/unit/models/test_unified_job.py @@ -0,0 +1,16 @@ +import mock + +from awx.main.models import ( + UnifiedJob, + WorkflowJob, + WorkflowJobNode, +) + + +def test_unified_job_workflow_attributes(): + with mock.patch('django.db.ConnectionRouter.db_for_write'): + job = UnifiedJob(id=1, name="job-1", launch_type="workflow") + job.unified_job_node = WorkflowJobNode(workflow_job=WorkflowJob(pk=1)) + + assert job.spawned_by_workflow is True + assert job.workflow_job_id == 1 From 9fb4e75711eae84e711965f0adeda0e2c91a525f Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 16 Nov 2016 09:43:56 -0500 Subject: [PATCH 05/10] update websockets documentation --- docs/websockets.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/websockets.md b/docs/websockets.md index 960926e2d7..43c903f7b1 100644 --- a/docs/websockets.md +++ b/docs/websockets.md @@ -21,8 +21,9 @@ Once you''ve connected you are not subscribed to any event groups. You subscribe 'groups': { 'jobs': ['status_changed', 'summary'], 'schedules': ['changed'], - 'ad_hoc_command_events': [ids,], - 'job_events': [ids,], + 'ad_hoc_command_events': [ids...], + 'job_events': [ids...], + 'workflow_events': [ids...] 'control': ['limit_reached'], } @@ -39,12 +40,13 @@ production and development deployments that I will point out, but the actual ser between the two environments. ### Services -| Name | Details | -|:---------:|:-----------------------------------------------------------------------------------------------------------:| -| nginx | listens on ports 80/443, handles HTTPS proxying, serves static assets, routes requests for daphne and uwsgi | -| uwsgi | listens on port 8050, handles API requests | -| daphne | listens on port 8051, handles Websocket requests | -| runworker | no listening port, watches and processes the message queue | +| Name | Details | +|:-----------:|:-----------------------------------------------------------------------------------------------------------:| +| nginx | listens on ports 80/443, handles HTTPS proxying, serves static assets, routes requests for daphne and uwsgi | +| uwsgi | listens on port 8050, handles API requests | +| daphne | listens on port 8051, handles Websocket requests | +| runworker | no listening port, watches and processes the message queue | +| supervisord | (production-only) handles the process management of all the services except nginx | When a request comes in to *nginx* and have the `Upgrade` header and is for the path `/websocket`, then *nginx* knows that it should be routing that request to our *daphne* service. From 6d14af336c8cd90254615b440c4ea4db928aeac2 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 16 Nov 2016 09:51:31 -0500 Subject: [PATCH 06/10] added Workflow migrations for channels integrations --- .../migrations/0050_v310_workflow_channels.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 awx/main/migrations/0050_v310_workflow_channels.py diff --git a/awx/main/migrations/0050_v310_workflow_channels.py b/awx/main/migrations/0050_v310_workflow_channels.py new file mode 100644 index 0000000000..d35231b161 --- /dev/null +++ b/awx/main/migrations/0050_v310_workflow_channels.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0049_v310_workflow_surveys'), + ] + + operations = [ + migrations.AlterField( + model_name='unifiedjob', + name='launch_type', + field=models.CharField(default=b'manual', max_length=20, editable=False, choices=[(b'manual', 'Manual'), (b'relaunch', 'Relaunch'), (b'callback', 'Callback'), (b'scheduled', 'Scheduled'), (b'dependency', 'Dependency'), (b'workflow', 'Workflow')]), + ), + migrations.AlterField( + model_name='workflowjobnode', + name='job', + field=models.OneToOneField(related_name='unified_job_node', null=True, on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJob'), + ), + ] From 9dda4d452f325b5bb600c39eafa761b9d16faf21 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 16 Nov 2016 11:02:35 -0500 Subject: [PATCH 07/10] renamed Workflow channel migration --- ...v310_workflow_channels.py => 0051_v310_workflow_channels.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename awx/main/migrations/{0050_v310_workflow_channels.py => 0051_v310_workflow_channels.py} (94%) diff --git a/awx/main/migrations/0050_v310_workflow_channels.py b/awx/main/migrations/0051_v310_workflow_channels.py similarity index 94% rename from awx/main/migrations/0050_v310_workflow_channels.py rename to awx/main/migrations/0051_v310_workflow_channels.py index d35231b161..4406abe08f 100644 --- a/awx/main/migrations/0050_v310_workflow_channels.py +++ b/awx/main/migrations/0051_v310_workflow_channels.py @@ -8,7 +8,7 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('main', '0049_v310_workflow_surveys'), + ('main', '0050_v310_JSONField_changes'), ] operations = [ From cd084f5c60897cb77ca7792095c894d47efc9eab Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Thu, 17 Nov 2016 17:03:41 -0500 Subject: [PATCH 08/10] removed JobEventSerializer additions, consolidated migrations, swapped job_events and jobs channel updates --- awx/api/serializers.py | 2 -- .../migrations/0034_v310_add_workflows.py | 7 ++++- .../migrations/0051_v310_workflow_channels.py | 25 ----------------- awx/main/models/unified_jobs.py | 7 ++++- awx/main/signals.py | 4 --- .../serializers/test_job_event_serializers.py | 28 ------------------- 6 files changed, 12 insertions(+), 61 deletions(-) delete mode 100644 awx/main/migrations/0051_v310_workflow_channels.py delete mode 100644 awx/main/tests/unit/api/serializers/test_job_event_serializers.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 2c5d4257bb..4fea827ec0 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2476,8 +2476,6 @@ class JobEventSerializer(BaseSerializer): try: d['job']['job_template_id'] = obj.job.job_template.id d['job']['job_template_name'] = obj.job.job_template.name - d['job']['spawned_by_workflow'] = obj.job.spawned_by_workflow - d['job']['workflow_job_id'] = obj.job.workflow_job_id except (KeyError, AttributeError): pass return d diff --git a/awx/main/migrations/0034_v310_add_workflows.py b/awx/main/migrations/0034_v310_add_workflows.py index c80a4ecc88..4dfb84177a 100644 --- a/awx/main/migrations/0034_v310_add_workflows.py +++ b/awx/main/migrations/0034_v310_add_workflows.py @@ -15,6 +15,11 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AlterField( + model_name='unifiedjob', + name='launch_type', + field=models.CharField(default=b'manual', max_length=20, editable=False, choices=[(b'manual', 'Manual'), (b'relaunch', 'Relaunch'), (b'callback', 'Callback'), (b'scheduled', 'Scheduled'), (b'dependency', 'Dependency'), (b'workflow', 'Workflow')]), + ), migrations.CreateModel( name='WorkflowJob', fields=[ @@ -34,7 +39,7 @@ class Migration(migrations.Migration): ('modified', models.DateTimeField(default=None, editable=False)), ('always_nodes', models.ManyToManyField(related_name='workflowjobnodes_always', to='main.WorkflowJobNode', blank=True)), ('failure_nodes', models.ManyToManyField(related_name='workflowjobnodes_failure', to='main.WorkflowJobNode', blank=True)), - ('job', models.ForeignKey(related_name='unified_job_nodes', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJob', null=True)), + ('job', models.OneToOneField(related_name='unified_job_node', on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJob', null=True)), ('success_nodes', models.ManyToManyField(related_name='workflowjobnodes_success', to='main.WorkflowJobNode', blank=True)), ], options={ diff --git a/awx/main/migrations/0051_v310_workflow_channels.py b/awx/main/migrations/0051_v310_workflow_channels.py deleted file mode 100644 index 4406abe08f..0000000000 --- a/awx/main/migrations/0051_v310_workflow_channels.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0050_v310_JSONField_changes'), - ] - - operations = [ - migrations.AlterField( - model_name='unifiedjob', - name='launch_type', - field=models.CharField(default=b'manual', max_length=20, editable=False, choices=[(b'manual', 'Manual'), (b'relaunch', 'Relaunch'), (b'callback', 'Callback'), (b'scheduled', 'Scheduled'), (b'dependency', 'Dependency'), (b'workflow', 'Workflow')]), - ), - migrations.AlterField( - model_name='workflowjobnode', - name='job', - field=models.OneToOneField(related_name='unified_job_node', null=True, on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to='main.UnifiedJob'), - ), - ] diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 2a5c6dd93f..5054270bfb 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -792,7 +792,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique def websocket_emit_data(self): ''' Return extra data that should be included when submitting data to the browser over the websocket connection ''' - return {} + return {'workflow_job_id': self.workflow_job_id} def websocket_emit_status(self, status): status_data = dict(unified_job_id=self.id, status=status) @@ -800,6 +800,11 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique status_data['group_name'] = 'jobs' emit_channel_notification('jobs-status_changed', status_data) + if self.spawned_by_workflow: + event_serialized['group_name'] = "workflow_events" + emit_channel_notification('workflow_events-' + str(self.workflow_job_id), status_data) + + def notification_data(self): return dict(id=self.id, name=self.name, diff --git a/awx/main/signals.py b/awx/main/signals.py index d0e84c904c..ceda8899b1 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -45,10 +45,6 @@ def emit_job_event_detail(sender, **kwargs): event_serialized["group_name"] = "job_events" emit_channel_notification('job_events-' + str(instance.job.id), event_serialized) - if instance.job.spawned_by_workflow: - event_serialized['group_name'] = "workflow_events" - emit_channel_notification('workflow_events-' + str(instance.job.workflow_job_id), event_serialized) - def emit_ad_hoc_command_event_detail(sender, **kwargs): instance = kwargs['instance'] diff --git a/awx/main/tests/unit/api/serializers/test_job_event_serializers.py b/awx/main/tests/unit/api/serializers/test_job_event_serializers.py deleted file mode 100644 index a9c7ee7f08..0000000000 --- a/awx/main/tests/unit/api/serializers/test_job_event_serializers.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -import mock - -from awx.api.serializers import JobEventSerializer -from awx.main.models import ( - Job, - JobTemplate, - JobEvent, -) - - -@pytest.fixture -def job_event(mocker): - job_event = mocker.MagicMock(spec=JobEvent) - - job = Job(id=1, name="job-1") - job.job_template = JobTemplate(id=1, name="job-template-1") - job_event.job = job - - return job_event - - -def test_summary_field_workflow_exists(job_event): - with mock.patch('awx.api.serializers.BaseSerializer.get_summary_fields', lambda x,y: {'job':{}}): - serializer = JobEventSerializer(job_event) - summary_fields = serializer.get_summary_fields(job_event) - assert 'spawned_by_workflow' in summary_fields['job'] - assert 'workflow_job_id' in summary_fields['job'] From d882a14d8761f614509443cf8867d5945203eab6 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Thu, 17 Nov 2016 17:22:31 -0500 Subject: [PATCH 09/10] fix group_name for workflow_events --- awx/main/models/unified_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 5054270bfb..7e95e5abd7 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -801,7 +801,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique emit_channel_notification('jobs-status_changed', status_data) if self.spawned_by_workflow: - event_serialized['group_name'] = "workflow_events" + status_data['group_name'] = "workflow_events" emit_channel_notification('workflow_events-' + str(self.workflow_job_id), status_data) From f14be79c69e1d27ff2ddd98168e711b69b2f54da Mon Sep 17 00:00:00 2001 From: jaredevantabor Date: Wed, 16 Nov 2016 16:45:18 -0800 Subject: [PATCH 10/10] adding workflow_event group for workflow results page --- awx/ui/client/src/shared/socket/socket.service.js | 3 +++ .../client/src/workflow-results/workflow-results.route.js | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index 64933f8fbd..b636cc1db8 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -215,6 +215,9 @@ export default if(state.data && state.data.socket && state.data.socket.groups.hasOwnProperty( "ad_hoc_command_events")){ state.data.socket.groups.ad_hoc_command_events = [id]; } + if(state.data && state.data.socket && state.data.socket.groups.hasOwnProperty( "workflow_events")){ + state.data.socket.groups.workflow_events = [id]; + } self.subscribe(state); } return true; diff --git a/awx/ui/client/src/workflow-results/workflow-results.route.js b/awx/ui/client/src/workflow-results/workflow-results.route.js index 4d15d50777..8e70daf17a 100644 --- a/awx/ui/client/src/workflow-results/workflow-results.route.js +++ b/awx/ui/client/src/workflow-results/workflow-results.route.js @@ -18,11 +18,7 @@ export default { data: { socket: { "groups":{ - "jobs": ["status_changed", "summary"], - // not sure if you're gonna need to use job_events - // or if y'all will come up w/ a new socket group specifically - // for workflows - // "job_events": [] + "workflow_events": [] } } },